From a4b6a1ee7da971cab2ab9d9005092285ab2cf67a Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 18:06:57 +0100 Subject: [PATCH 01/93] osbench: persist.write_4k_fsync_beside_1m As persist.write_4k_fsync, each iteration first writing 1 MiB over another 1 MiB file and leaving it unsynced: an fsync of one file that also makes another file's write durable pays for it here. It is the A/B base for a per-file fsync (issues/an-fsync-of-a-served-file-syncs-its-whole-volume.md), which asks for one file's fsync measured beside the volume's. Co-Authored-By: Claude Opus 5.5 --- tests/guest/osbench/src/storage.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/guest/osbench/src/storage.rs b/tests/guest/osbench/src/storage.rs index d5b7d371c27..4f02bc11f3f 100644 --- a/tests/guest/osbench/src/storage.rs +++ b/tests/guest/osbench/src/storage.rs @@ -56,6 +56,28 @@ fn durable(disk: &files::Target) { Err(e) => refuse("persist.write_4k_fsync", e), } + // As `write_4k_fsync`, each after 1 MiB written over another 1 MiB file + // and not synced: 100 after 5. An fsync of one file that makes another's + // write durable too pays for it here. + let beside = scratch.at("beside"); + let made = (|| -> std::io::Result<(File, File)> { + fs::write(&beside, vec![2u8; MIB])?; + let other = OpenOptions::new().write(true).open(&beside)?; + other.sync_all()?; + Ok((OpenOptions::new().write(true).open(&file)?, other)) + })(); + let other_bytes = vec![0x3cu8; MIB]; + match made { + Ok((mut f, mut other)) => each("persist.write_4k_fsync_beside_1m", 5, 100, &served, |_| { + io("seek", other.seek(SeekFrom::Start(0)))?; + io("write", other.write_all(&other_bytes))?; + io("seek", f.seek(SeekFrom::Start(0)))?; + io("write", f.write_all(&block))?; + io("fsync", f.sync_all()) + }), + Err(e) => refuse("persist.write_4k_fsync_beside_1m", e), + } + // 4 KiB appended, then `sync_data`: 100 after 5, so the file grows by 420 KiB. match OpenOptions::new().create(true).append(true).open(scratch.at("log")) { Ok(mut f) => each("persist.append_4k_fsync", 5, 100, &served, |_| { From 31206cbf4231d9e0473d5938839237e6d9548141 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 18:06:14 +0100 Subject: [PATCH 02/93] paged: a holder's read is atomic against its pager's write (XSH 2.9.7) A holder's SYS_PAGED_READ copied a pinned chunk of 16 pages at a time while the file server's write landed page by page in the same frames, so a read beside a write of several pages could return pages from before and after it, or a page torn at the copy (memory stage 7a's recorded gap). Every write of an object's now runs as a write span on it (kernel::paged::Object): each write call and each dirty supply begins one before its first byte lands and ends it after its last, and a resize moves the object's version. A holder's read begins only while no span is open and counts its bytes only where the version has not moved since; a read a write met answers WouldBlock, and the SDK, which already asks the server whenever a holder read fails, reads through the server, whose loop orders the read against the write. A server write takes many calls when a page is absent, in part or at the dirty ceiling. A call that does not take all it was given leaves the span open (held) until the new PagedOp::EndWrite, and later calls nest in it, so the file server's whole write is one span. The server's first call of a write is always a write: one that takes everything is the whole write at no extra syscall; otherwise the server ends the held span on every path, the refused one included. Holder reads never wait on a write: the pager is never made to wait on its readers, and a read that met one costs one round trip. Tests: - kernel::paged: a read counts only where no write began or was under way; a short call holds the write until its end. - fileserver pages: a write of many calls (an absent page supplied, the dirty ceiling written back, a last page in part) is read whole or not at all at every call boundary, and ends whether it lands or is refused. - guest a_served_files_read_never_tears_against_its_write: a writer puts 400 whole generations of a 64-page served file while a reader on its own description reads it whole through its holder; red on a torn read. Closes issues/a-read-and-a-write-of-a-paged-file-can-tear-across-pages.md. Co-Authored-By: Claude Opus 5.5 --- ...e-of-a-paged-file-can-tear-across-pages.md | 23 --- kernel/pure/paged.rs | 99 ++++++++++- kernel/src/object/paged.rs | 107 ++++++++++-- kernel/src/syscall/paged.rs | 1 + sdk/abi/src/syscall.rs | 16 +- sdk/toyos/src/paged.rs | 10 +- system/fileserver/src/pages.rs | 46 +++-- system/fileserver/src/pages/tests.rs | 160 +++++++++++++----- ...iles_read_never_tears_against_its_write.rs | 55 ++++++ 9 files changed, 423 insertions(+), 94 deletions(-) delete mode 100644 issues/a-read-and-a-write-of-a-paged-file-can-tear-across-pages.md create mode 100644 tests/guest/rust-tests/src/bin/a_served_files_read_never_tears_against_its_write.rs diff --git a/issues/a-read-and-a-write-of-a-paged-file-can-tear-across-pages.md b/issues/a-read-and-a-write-of-a-paged-file-can-tear-across-pages.md deleted file mode 100644 index a8b6236d619..00000000000 --- a/issues/a-read-and-a-write-of-a-paged-file-can-tear-across-pages.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-31 ---- - -# A read and a write of a paged file can tear across pages - -POSIX XSH 2.9.7 makes a `read` and a `write` of a regular file atomic with -respect to each other. A holder's `SYS_PAGED_READ` copies a pinned frame at a -time (`kernel/src/object/paged.rs`), and the file server's write lands page -by page under write pins, outside the object's lock. A read concurrent with a -write of several pages can return some pages before the write and some after -it, and a page can be torn at the bytes being copied. - -This is the contract memory stage 7a states (writes are visible at once, with -no atomicity beyond the hardware's), and it is not what POSIX asks. - -**Exit**: a read and a write of one file are atomic with respect to each -other, or each is atomic per call by a rule the file server and the kernel -state together, with a guest test that reds on a torn read. - -**Owner**: the storage track (`issues/storage-is-layers-and-a-role-is-a-filesystem.md`). diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index d3194de8fe9..b6129f9d227 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -17,6 +17,12 @@ //! it in the same step, so the size never covers bytes not yet in the object; //! a shrink removes every page past it, dirty or not. //! +//! **A holder's read is atomic against every write** (POSIX XSH 2.9.7): it +//! begins only while no write is under way ([`Object::read_begin`]), and its +//! bytes count only where no write has begun since ([`Object::unchanged`]). +//! A write is one call, or, where a call does not take all it was given, every +//! call until [`Object::end_write`]: the pager's whole write. +//! //! What a removed frame owes (the last pin's free, a zeroed tail) is the //! caller's: every removal answers its frames. @@ -204,15 +210,25 @@ struct Slot { } /// One object's part of the table: its size, whether its pager ended, the -/// list of its slots. +/// list of its slots, and its writes under way. pub struct Object { id: u32, size: u64, gone: bool, head: u32, pages: usize, + /// Moved at every write's begin and end, and at every resize. + version: u64, + /// Writes under way: each call's, and the held one. + writing: u32, + /// A call that did not take all it was given left its write under way. + held: bool, } +/// What a holder's read began at ([`Object::read_begin`]). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Version(u64); + impl Object { pub fn size(&self) -> u64 { self.size @@ -230,6 +246,43 @@ impl Object { fn page_end(&self) -> u64 { self.size.div_ceil(PAGE) } + + /// Where a holder's read begins: `None` while a write is under way. + pub fn read_begin(&self) -> Option { + (self.writing == 0).then_some(Version(self.version)) + } + + /// Whether no write or resize has begun since the read began at `v`. + pub fn unchanged(&self, v: Version) -> bool { + self.version == v.0 + } + + /// A write call begins, before its first byte lands. + pub fn write_begin(&mut self) { + self.writing = self.writing.checked_add(1).expect("an object's writes overflowed"); + self.version += 1; + } + + /// A write call ends, after its last byte landed. One that did not take + /// all it was given leaves the write under way until + /// [`end_write`](Self::end_write), unless an earlier call's already is. + pub fn write_end(&mut self, whole: bool) { + if !whole && !self.held { + self.held = true; + return; + } + self.writing = self.writing.checked_sub(1).expect("a write ended that never began"); + self.version += 1; + } + + /// The pager ends the write a short call left under way. + pub fn end_write(&mut self) -> Result<(), Refused> { + if !core::mem::take(&mut self.held) { + return Err(Refused::InvalidArgument); + } + self.write_end(true); + Ok(()) + } } /// A pager's table: `capacity` slots and a hash of them, both made here. @@ -307,7 +360,7 @@ impl Pages { /// [`release`](Self::release) takes every page it held. pub fn object(&mut self, id: u32, size: u64) -> Object { assert!(size <= LAST_END, "an object of {size} bytes past the last page"); - Object { id, size, gone: false, head: NONE, pages: 0 } + Object { id, size, gone: false, head: NONE, pages: 0, version: 0, writing: 0, held: false } } fn hash(&self, object: u32, page: u64) -> usize { @@ -617,6 +670,7 @@ impl Pages { /// answers the frame of a partial last page, whose tail past the size the /// caller zeroes. pub fn resize(&mut self, o: &mut Object, size: u64, out: &mut impl FnMut(F)) -> Option { + o.version += 1; let keep = size.div_ceil(PAGE); if size < o.size { let mut at = o.head; @@ -677,6 +731,47 @@ mod tests { |f| freed.push(f) } + /// XSH 2.9.7: a read that began before a write, or while one was under + /// way, never counts; one that began after it does. + #[test] + fn a_read_counts_only_where_no_write_began_or_was_under_way() { + let mut p = Pages::::new(8, 4).unwrap(); + let mut o = p.object(1, PAGE); + let before = o.read_begin().expect("nothing writes yet"); + o.write_begin(); + assert_eq!(o.read_begin(), None, "a read meets the write in flight"); + assert!(!o.unchanged(before), "a read that began before it meets it"); + o.write_end(true); + assert!(!o.unchanged(before), "and still does once it ended"); + let after = o.read_begin().expect("the write ended"); + assert!(o.unchanged(after)); + p.resize(&mut o, 0, &mut none); + assert!(!o.unchanged(after), "a shrink is a change a read meets"); + } + + /// A pager's write that takes several calls is one write: the first short + /// call leaves it under way, later ones nest in it, and only its end lets + /// a read begin. + #[test] + fn a_short_call_holds_the_write_until_its_end() { + let mut p = Pages::::new(8, 4).unwrap(); + let mut o = p.object(2, PAGE); + assert_eq!(o.end_write(), Err(Refused::InvalidArgument), "nothing is held"); + o.write_begin(); + o.write_end(false); + assert_eq!(o.read_begin(), None, "a short call's write is under way"); + o.write_begin(); + o.write_end(false); + o.write_begin(); + o.write_end(true); + assert_eq!(o.read_begin(), None, "a later call, short or whole, ends no held write"); + let held = o.version; + assert_eq!(o.end_write(), Ok(())); + let v = o.read_begin().expect("the end lets a read begin"); + assert!(o.unchanged(v) && v != Version(held), "the end moved the version"); + assert_eq!(o.end_write(), Err(Refused::InvalidArgument), "a write ends once"); + } + #[test] fn a_supplied_page_is_found_and_a_second_supply_changes_nothing() { let mut p = Pages::new(8, 4).unwrap(); diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index a7b74d6b77d..e0c3cfc4f52 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -22,6 +22,11 @@ //! A pager's own process is refused a holder's read, so no pager waits on a //! pager. //! +//! **A holder's read is atomic against its pager's writes** +//! ([`kernel::paged::Object::read_begin`]): one that meets a write is answered +//! `WouldBlock`, and the holder asks the pager, whose own reads its writes +//! order. +//! //! **Lock order.** The pager's lock is innermost but for the allocator's. A //! copy to or from user memory holds pins, never the lock; no wait holds a //! pin. @@ -41,7 +46,7 @@ use core::ptr; use core::sync::atomic::{AtomicU32, Ordering}; use kernel::chunked::{Chunked, Stack}; -use kernel::paged::{Core, Draw, Leave, Lookup, Object, Pages, Refused, Supplied, PAGE}; +use kernel::paged::{Core, Draw, Leave, Lookup, Object, Pages, Refused, Supplied, Version, PAGE}; use kernel::sched::task::WaitClass; use toyos_abi::syscall::{PagedNotice, SyscallError, NOTICE_NO_HOLDERS, NOTICE_REQUESTED}; @@ -450,21 +455,26 @@ impl PagerObject { let mut done = 0usize; let waiting = core::pin::pin!(Waiting::new(self, slot)); let mut pieces = [Piece::default(); CHUNK]; + // A holder's: what its bytes count against. + let mut since: Option = None; loop { let at = offset.saturating_add(done as u64); let step = { let mut state = self.state.lock(); + let size = state.entry(slot).object.size(); if state.gone || state.entry(slot).object.gone() { Step::Gone + } else if done == want || (done == 0 && at >= size) { + // Whole, each copy counted as it ended; or nothing to copy. + Step::Done + } else if waits && !Self::unwritten(&state.entry(slot).object, &mut since) { + Step::Written + } else if at >= size { + Step::Done } else { - let size = state.entry(slot).object.size(); - if done == want || at >= size { - Step::Done - } else { - let end = size.min(offset.saturating_add(want as u64)); - let waiter = waits.then_some(waiting.as_ref()); - self.pin_chunk(&mut state, slot, at..end, &mut pieces, waiter) - } + let end = size.min(offset.saturating_add(want as u64)); + let waiter = waits.then_some(waiting.as_ref()); + self.pin_chunk(&mut state, slot, at..end, &mut pieces, waiter) } }; match step { @@ -478,15 +488,20 @@ impl PagerObject { for piece in &pieces[..count] { pmm::unpin_range(piece.phys, PAGE as usize); } - let gone = { + let (gone, written) = { let state = self.state.lock(); - state.gone || state.entry(slot).object.gone() + let object = &state.entry(slot).object; + (state.gone || object.gone(), waits && !Self::unwritten(object, &mut since)) }; // A frame copied as its pager ended is no byte to report. if gone { return SyscallError::Io.to_u64(); } + if written { + return SyscallError::WouldBlock.to_u64(); + } } + Step::Written => return SyscallError::WouldBlock.to_u64(), Step::Done => return done as u64, Step::Gone if done > 0 => return done as u64, Step::Gone => return SyscallError::Gone.to_u64(), @@ -519,6 +534,18 @@ impl PagerObject { } } + /// Under the lock: whether no write met a holder's read of `object` that + /// began at `since`, or begins now where it is `None`. + fn unwritten(object: &Object, since: &mut Option) -> bool { + match *since { + Some(v) => object.unchanged(v), + None => { + *since = object.read_begin(); + since.is_some() + } + } + } + /// Under the lock: pin the present pages of `range`, at most [`CHUNK`], /// into `pieces`, or else say what the first page is, recording a /// waiter's request for every page from an absent one to the range's end. @@ -561,13 +588,55 @@ impl PagerObject { Step::Copy(count) } + /// A write call on object `id` begins: its slot, held by a reference until + /// [`Self::write_end`], so no release can hand it to another object. + fn write_begin(&self, id: u64) -> Result { + let mut state = self.state.lock(); + let slot = state.live(id)?; + let entry = state.entry_mut(slot); + entry.refs += 1; + entry.object.write_begin(); + Ok(slot) + } + + /// The write call [`Self::write_begin`] began ends, `whole` where it took + /// everything it was given: its reference goes. + fn write_end(&self, slot: u32, whole: bool) { + let mut state = self.state.lock(); + state.entry_mut(slot).object.write_end(whole); + self.unref_locked(&mut state, slot); + } + + /// The pager ends the write of object `id` a short call left under way. + pub fn end_write(&self, id: u64) -> u64 { + let mut state = self.state.lock(); + let ended = state.live(id).and_then(|slot| state.entry_mut(slot).object.end_write().map_err(word)); + match ended { + Ok(()) => 0, + Err(e) => e.to_u64(), + } + } + /// Copy `src` into fresh pages of object `id` from page-aligned `offset`, - /// clean or dirty: the pages installed. The caller checked the span is in - /// reach. + /// clean or dirty: the pages installed. A dirty supply is a write. The + /// caller checked the span is in reach. pub fn supply(&self, id: u64, offset: u64, src: &UserBytes, dirty: bool) -> u64 { if !offset.is_multiple_of(PAGE) || src.len() == 0 { return SyscallError::InvalidArgument.to_u64(); } + if !dirty { + return self.supply_pages(id, offset, src, false); + } + let slot = match self.write_begin(id) { + Ok(slot) => slot, + Err(e) => return e.to_u64(), + }; + let answer = self.supply_pages(id, offset, src, true); + self.write_end(slot, answer == (src.len() as u64).div_ceil(PAGE)); + answer + } + + fn supply_pages(&self, id: u64, offset: u64, src: &UserBytes, dirty: bool) -> u64 { let mut installed = 0u64; let mut done = 0usize; while done < src.len() { @@ -663,6 +732,16 @@ impl PagerObject { /// the bytes written, up to the first absent page. The caller checked the /// span is in reach. pub fn write(&self, id: u64, offset: u64, src: &UserBytes) -> u64 { + let slot = match self.write_begin(id) { + Ok(slot) => slot, + Err(e) => return e.to_u64(), + }; + let answer = self.write_pages(id, offset, src); + self.write_end(slot, answer == src.len() as u64); + answer + } + + fn write_pages(&self, id: u64, offset: u64, src: &UserBytes) -> u64 { let mut done = 0usize; while done < src.len() { let mut pinned = [(0u64, 0u64, 0usize, 0usize); CHUNK]; @@ -799,6 +878,8 @@ enum Step { /// This many pieces were pinned. Copy(usize), Absent(u64), + /// A write met a holder's read. + Written, Failed, Gone, Done, diff --git a/kernel/src/syscall/paged.rs b/kernel/src/syscall/paged.rs index cd7f1e5f78d..0a8aaa0f164 100644 --- a/kernel/src/syscall/paged.rs +++ b/kernel/src/syscall/paged.rs @@ -132,6 +132,7 @@ pub(super) fn sys_paged_op(ctx: &SyscallContext<'_>, h: RawHandle, op: u64, args x if x == PagedOp::Fail as u64 => pager.fail(id, args.offset, args.len), x if x == PagedOp::Resize as u64 => pager.resize(id, args.offset), x if x == PagedOp::Release as u64 => pager.release(id), + x if x == PagedOp::EndWrite as u64 => pager.end_write(id), x if x == PagedOp::WritebackBegin as u64 || x == PagedOp::WritebackEnd as u64 => { let end = (x == PagedOp::WritebackEnd as u64).then_some(args.flags & PAGED_WRITTEN != 0); if args.flags & !PAGED_WRITTEN != 0 || (end.is_none() && args.flags != 0) || !range_ok(args.len) { diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index d4c2f98e573..cf3436efc87 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -3311,15 +3311,18 @@ pub const PAGED_RANGE_PAGES: u64 = 512; pub enum PagedOp { /// Copy `len` bytes from `buf` into fresh pages at `offset`, a whole page /// in: clean, or dirty with [`PAGED_DIRTY`], which also grows the size to - /// the end of the bytes. A page already present is left as it is. - /// Answers the pages installed; `ResourceExhausted` where no page could - /// be had, `WouldBlock` at the dirty ceiling, `InvalidArgument` for a - /// clean page past the size or a dirty one past its end. + /// the end of the bytes and is a write ([`PagedOp::Write`]). A page + /// already present is left as it is. Answers the pages installed; + /// `ResourceExhausted` where no page could be had, `WouldBlock` at the + /// dirty ceiling, `InvalidArgument` for a clean page past the size or a + /// dirty one past its end. Supply = 0, /// Copy `len` bytes from `buf` into the present pages at `offset`, dirty, /// growing the size to their end as each page's bytes land. Answers the /// bytes written, stopping at the first absent page; `NotFound` when the - /// first is absent, `WouldBlock` at the dirty ceiling. + /// first is absent, `WouldBlock` at the dirty ceiling. A holder's read + /// that meets a write is answered `WouldBlock`; a write that does not take + /// all it was given stays under way until [`PagedOp::EndWrite`]. Write = 1, /// Remove the clean pages of `len` pages from page `offset` that no /// write-back or write holds. Answers how many went. @@ -3345,6 +3348,9 @@ pub enum PagedOp { /// Let the object go: its pages go, its holders answer `Gone`, and its id /// names another object once its last holder has gone. Release = 8, + /// End the write a short [`PagedOp::Write`] or dirty supply left under + /// way; `InvalidArgument` where none is. + EndWrite = 9, } /// [`PagedOp::Supply`]'s pages are dirty. diff --git a/sdk/toyos/src/paged.rs b/sdk/toyos/src/paged.rs index dfca567c87c..1f5bef1e30e 100644 --- a/sdk/toyos/src/paged.rs +++ b/sdk/toyos/src/paged.rs @@ -123,6 +123,13 @@ impl Pager { self.op(PagedOp::Write, args, Some(SyscallError::NotFound)).map(|n| n as usize) } + /// End the write of object `id` that a [`Self::write`] or dirty + /// [`Self::supply`] which did not take all it was given left under way: + /// until this, every holder's read of it is asked of this pager instead. + pub fn end_write(&self, id: u64) -> Result<(), Refused> { + self.op(PagedOp::EndWrite, PagedArgs { object: id, ..PagedArgs::default() }, None).map(drop) + } + /// Drop the clean pages of `count` from page `first` that nothing holds: /// how many went. pub fn withdraw(&self, id: u64, first: u64, count: u64) -> Result { @@ -200,7 +207,8 @@ impl Paged { /// Up to `out.len()` bytes at `offset`, 0 at or past the size, waiting for /// every absent page until it is supplied, failed (`Io`, or a short count /// after some bytes), cut off by a shrink, or its object or pager ends - /// (`Gone`). + /// (`Gone`). A read a write of the pager's met is `WouldBlock`: ask the + /// pager, whose writes order its own reads. pub fn read(&self, out: &mut [u8], offset: u64) -> Result { syscall::paged_read(self.0.raw(), out, offset) } diff --git a/system/fileserver/src/pages.rs b/system/fileserver/src/pages.rs index b93579abb6f..9e0c6e3b7e6 100644 --- a/system/fileserver/src/pages.rs +++ b/system/fileserver/src/pages.rs @@ -11,6 +11,10 @@ //! sake until room is wanted: then a file nobody holds and nothing dirties //! goes first, oldest first. //! +//! **A write is one write of the object's**, however many calls it takes: +//! a call that stops short leaves it under way until its end, so no holder +//! reads the file half written (`toyos::paged::Pager::end_write`). +//! //! **The volume's size is the object's.** A write reserves on the volume //! first, so the volume's size runs past the object's only for the bytes the //! write then puts in before it returns; nothing reads the volume for them @@ -36,6 +40,7 @@ pub trait Pager { fn read(&self, id: u64, out: Bytes<'_>, offset: u64) -> Result; fn supply(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result; fn write(&self, id: u64, offset: u64, bytes: &[u8]) -> Result; + fn end_write(&self, id: u64) -> Result<(), Refused>; fn withdraw(&self, id: u64, first: u64, count: u64) -> Result; fn fail(&self, id: u64, first: u64, count: u64) -> Result<(), Refused>; fn resize(&self, id: u64, size: u64) -> Result<(), Refused>; @@ -57,6 +62,9 @@ impl Pager for toyos::paged::Pager { fn write(&self, id: u64, offset: u64, bytes: &[u8]) -> Result { toyos::paged::Pager::write(self, id, offset, bytes) } + fn end_write(&self, id: u64) -> Result<(), Refused> { + toyos::paged::Pager::end_write(self, id) + } fn withdraw(&self, id: u64, first: u64, count: u64) -> Result { toyos::paged::Pager::withdraw(self, id, first, count) } @@ -369,31 +377,42 @@ impl Files

{ let reserved = volume.reserve(node, offset, data.len() as u64)?; let old = reserved.size; self.touch(identity); - let mut done = 0; - let put = self.put(volume, &Writing { identity, node, old, contents: reserved.contents(), offset, data }, &mut done); let id = self.files[&identity].object; - if let Err(e) = put { + let mut put = Put { done: 0, held: false }; + let written = self.put(volume, &Writing { identity, node, old, contents: reserved.contents(), offset, data }, &mut put); + if let Err(e) = &written { // What was reserved and not put would read as the blocks' old bytes. - let landed = if done > 0 { offset + done as u64 } else { 0 }; + let landed = if put.done > 0 { offset + put.done as u64 } else { 0 }; let cut = old.max(landed); let resized = self.pg().resize(id, cut).map_err(word).and(volume.unreserve(node, reserved, landed)); if let Err(again) = resized { - println!("fileserver: a refused write of file {identity} was not cut back to {cut} ({again:?})"); + println!("fileserver: a refused write of file {identity} was not cut back to {cut} ({again:?}), after {e:?}"); } - return Err(e); } - Ok(()) + if put.held { + match self.pg().end_write(id) { + Ok(()) | Err(Refused::Gone) => {} + Err(why) => println!("fileserver: file {identity}'s write would not end ({why:?}); its holders ask this server"), + } + } + written } /// The bytes of a write into the object, each absent page supplied dirty, - /// counting into `done` what landed. - fn put(&mut self, volume: &mut dyn Volume, w: &Writing<'_>, done: &mut usize) -> Result<(), Error> { + /// counting into `put` what landed and whether a call left the write + /// under way. + fn put(&mut self, volume: &mut dyn Volume, w: &Writing<'_>, put: &mut Put) -> Result<(), Error> { let Writing { identity, node, old, contents, offset, data } = *w; let end = offset + data.len() as u64; + let done = &mut put.done; while *done < data.len() { let at = offset + *done as u64; let id = self.files.get(&identity).ok_or(Error::Stale)?.object; - match self.pg().write(id, at, &data[*done..]) { + let wrote = self.pg().write(id, at, &data[*done..]); + // A first call that takes everything is the whole write; any later + // call is under the write a short first one left under way. + put.held |= wrote != Ok(data.len() - *done); + match wrote { Ok(n) => { self.landed(identity, at, n); *done += n; @@ -558,6 +577,13 @@ impl Files

{ } } +/// How far a write got: the bytes that landed, and whether a call left the +/// object's write under way. +struct Put { + done: usize, + held: bool, +} + /// A write under way: its file, at `offset`, which was `old` bytes long. #[derive(Clone, Copy)] struct Writing<'a> { diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index 49199789016..f6b26ad1596 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -1,5 +1,5 @@ use std::cell::RefCell; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::rc::Rc; use toyos_abi::syscall::SyscallError; @@ -23,6 +23,11 @@ struct Model { supplies: u32, /// Every supply from here is refused. refusing: bool, + /// Objects a call that did not take all it was given left written. + held: BTreeSet, + /// Each object's pages as a holder could read them after each change + /// that left no write under way. + views: Vec<(u64, BTreeMap>)>, } struct Page { @@ -35,6 +40,18 @@ impl Model { fn held(&self) -> usize { self.pages.values().filter(|p| p.dirty || p.writeback).count() } + + /// A change to `id` ended, `whole` where the call took all it was given: + /// the kernel's rule (`kernel::paged::Object::write_end`). + fn changed(&mut self, id: u64, whole: bool) { + if !whole { + self.held.insert(id); + } + if !self.held.contains(&id) { + let view = self.pages.iter().filter(|((c, _), _)| *c == id).map(|(&(_, p), page)| (p, page.bytes.clone())).collect(); + self.views.push((id, view)); + } + } } #[derive(Clone)] @@ -48,45 +65,8 @@ impl Fake { fn pages_of(&self, object: u64) -> usize { self.0.borrow().pages.keys().filter(|(c, _)| *c == object).count() } -} - -impl Pager for Fake { - fn object(&self, size: u64) -> Result { - let mut m = self.0.borrow_mut(); - if m.sizes.len() >= m.objects { - return Err(Refused::NoRoom); - } - m.next += 1; - let id = m.next; - m.sizes.insert(id, size); - Ok(id) - } - - fn release(&self, id: u64) -> Result<(), Refused> { - let mut m = self.0.borrow_mut(); - m.pages.retain(|(c, _), _| *c != id); - m.sizes.remove(&id).map(drop).ok_or(Refused::Gone) - } - - fn read(&self, id: u64, out: Bytes<'_>, offset: u64) -> Result { - let m = self.0.borrow(); - let size = m.sizes[&id]; - let end = size.min(offset + out.len() as u64); - let mut at = offset; - while at < end { - let Some(page) = m.pages.get(&(id, at / PAGE)) else { break }; - let within = (at % PAGE) as usize; - let n = ((PAGE as usize - within) as u64).min(end - at) as usize; - out.write((at - offset) as usize, &page.bytes[within..within + n]); - at += n as u64; - } - match at - offset { - 0 if offset < end => Err(Refused::Absent), - n => Ok(n as usize), - } - } - fn supply(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result { + fn supply_pages(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result { let mut m = self.0.borrow_mut(); m.supplies += 1; if m.refusing { @@ -123,7 +103,7 @@ impl Pager for Fake { Ok(installed) } - fn write(&self, id: u64, offset: u64, bytes: &[u8]) -> Result { + fn write_pages(&self, id: u64, offset: u64, bytes: &[u8]) -> Result { let mut m = self.0.borrow_mut(); let mut done = 0; while done < bytes.len() { @@ -144,6 +124,64 @@ impl Pager for Fake { } if done == 0 { Err(Refused::Absent) } else { Ok(done) } } +} + +impl Pager for Fake { + fn object(&self, size: u64) -> Result { + let mut m = self.0.borrow_mut(); + if m.sizes.len() >= m.objects { + return Err(Refused::NoRoom); + } + m.next += 1; + let id = m.next; + m.sizes.insert(id, size); + Ok(id) + } + + fn release(&self, id: u64) -> Result<(), Refused> { + let mut m = self.0.borrow_mut(); + m.pages.retain(|(c, _), _| *c != id); + m.sizes.remove(&id).map(drop).ok_or(Refused::Gone) + } + + fn read(&self, id: u64, out: Bytes<'_>, offset: u64) -> Result { + let m = self.0.borrow(); + let size = m.sizes[&id]; + let end = size.min(offset + out.len() as u64); + let mut at = offset; + while at < end { + let Some(page) = m.pages.get(&(id, at / PAGE)) else { break }; + let within = (at % PAGE) as usize; + let n = ((PAGE as usize - within) as u64).min(end - at) as usize; + out.write((at - offset) as usize, &page.bytes[within..within + n]); + at += n as u64; + } + match at - offset { + 0 if offset < end => Err(Refused::Absent), + n => Ok(n as usize), + } + } + + fn supply(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result { + let answer = self.supply_pages(id, offset, bytes, dirty); + if dirty { + self.0.borrow_mut().changed(id, answer == Ok(bytes.len().div_ceil(PAGE as usize) as u64)); + } + answer + } + + fn write(&self, id: u64, offset: u64, bytes: &[u8]) -> Result { + let answer = self.write_pages(id, offset, bytes); + self.0.borrow_mut().changed(id, answer == Ok(bytes.len())); + answer + } + + fn end_write(&self, id: u64) -> Result<(), Refused> { + let mut m = self.0.borrow_mut(); + assert!(m.held.remove(&id), "a write ended that no short call left under way"); + m.changed(id, true); + Ok(()) + } fn withdraw(&self, id: u64, first: u64, count: u64) -> Result { let mut m = self.0.borrow_mut(); @@ -165,6 +203,7 @@ impl Pager for Fake { page.bytes[(size % PAGE) as usize..].fill(0); } m.sizes.insert(id, size); + m.changed(id, true); Ok(()) } @@ -475,3 +514,44 @@ fn an_object_refused_for_room_is_made_by_a_closed_files_going() { let identity = v.identity("home/d").unwrap(); assert_eq!(files.opened(&mut v, identity, n), Err(Error::NoSpace), "both held open: refused, by name"); } + +/// XSH 2.9.7, the server's half: a write that takes many calls — an absent +/// page supplied, the dirty ceiling written back, a last page in part — is +/// one write of the object's, so every state a holder can read between calls +/// is the file before it or after it, and the write ends whether it lands +/// or is refused. +#[test] +fn a_write_of_many_calls_is_read_whole_or_not_at_all() { + const PAGES: usize = 8; + let page = PAGE as usize; + let old = pattern(PAGES * page, 10); + let mut v = volume(); + let n = v.open("home/f", CREATE).unwrap(); + v.write(n, 0, &old).unwrap(); + v.close(n).unwrap(); + let fake = Fake::new(64, 4); + let mut files = Files::new(Some(fake.clone())); + let at = open(&mut files, &mut v, "home/f"); + read(&mut files, &mut v, at, 0, PAGES * page); + let object = files.object(at.1).unwrap(); + assert_eq!(fake.withdraw(object, 3, 1), Ok(1)); + fake.0.borrow_mut().views.clear(); + let supplies = fake.0.borrow().supplies; + + let new = pattern(6 * page + 100, 11); + files.write(&mut v, at.1, at.0, 0, &new).unwrap(); + let mut after = old.clone(); + after[..new.len()].copy_from_slice(&new); + let m = fake.0.borrow(); + assert!(m.held.is_empty(), "the write was left under way"); + assert!(m.supplies > supplies, "the write supplied no absent page: one call, nothing to tear"); + for (k, (_, view)) in m.views.iter().filter(|(id, _)| *id == object).enumerate() { + let as_of = |file: &[u8]| view.iter().all(|(&p, bytes)| p as usize >= PAGES || bytes[..] == file[p as usize * page..][..page]); + assert!(as_of(&old) || as_of(&after), "view {k} of {} is neither the file before the write nor after it", m.views.len()); + } + drop(m); + + fake.0.borrow_mut().refusing = true; + assert!(files.write(&mut v, at.1, at.0, 7 * page as u64 + 50, &pattern(3 * page, 12)).is_err()); + assert!(fake.0.borrow().held.is_empty(), "a refused write was left under way"); +} diff --git a/tests/guest/rust-tests/src/bin/a_served_files_read_never_tears_against_its_write.rs b/tests/guest/rust-tests/src/bin/a_served_files_read_never_tears_against_its_write.rs new file mode 100644 index 00000000000..1e532242e2f --- /dev/null +++ b/tests/guest/rust-tests/src/bin/a_served_files_read_never_tears_against_its_write.rs @@ -0,0 +1,55 @@ +//! XSH 2.9.7: a read and a write of one regular file are atomic with respect +//! to each other. A writer puts whole generations of a served file, each byte +//! the generation's number, while a reader on its own description reads the +//! whole file through its paged object; every read is one generation. +//! +//! A guest test because the tear is an interleaving of the kernel's copy of a +//! holder's read with the file server's write into the same frames, on two +//! CPUs; the host suites hold each side's rule (`kernel::paged`, the file +//! server's `pages` tests). + +use std::fs::{self, OpenOptions}; +use std::os::toyos::fs::FileExt; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; + +const FILE: &str = "/home/a-served-files-read-never-tears"; +/// Sixty-four pages: four of the kernel's sixteen-page copies, inside one +/// request's window. +const BYTES: usize = 64 * 4096; +const GENERATIONS: u32 = 400; + +fn main() { + let _ = fs::remove_file(FILE); + let writer = OpenOptions::new().write(true).create(true).truncate(true).open(FILE).expect("open to write"); + assert_eq!(writer.write_at(&[0u8; BYTES], 0).expect("the first generation"), BYTES); + let reader = OpenOptions::new().read(true).open(FILE).expect("open to read"); + let done = AtomicBool::new(false); + + let (reads, seen) = thread::scope(|s| { + let reading = s.spawn(|| { + let mut buf = vec![0u8; BYTES]; + let (mut reads, mut seen) = (0u32, std::collections::BTreeSet::new()); + while !done.load(Ordering::Acquire) { + assert_eq!(reader.read_at(&mut buf, 0).expect("read"), BYTES, "a short read of a file no write shortens"); + if let Some(at) = buf.iter().position(|&b| b != buf[0]) { + panic!("read {reads} is torn: generation {} to byte {at}, then generation {}", buf[0], buf[at]); + } + seen.insert(buf[0]); + reads += 1; + } + (reads, seen) + }); + for generation in 1..=GENERATIONS { + let byte = generation as u8; + assert_eq!(writer.write_at(&[byte; BYTES], 0).expect("write"), BYTES, "generation {generation} written short"); + } + done.store(true, Ordering::Release); + reading.join().expect("the reader") + }); + // The premise: the reader ran beside the writes and met more than one. + assert!(seen.len() > 2, "{reads} reads met only generations {seen:?}: nothing ran beside the writes"); + drop((writer, reader)); + fs::remove_file(FILE).unwrap(); + println!("a_served_files_read_never_tears_against_its_write: {reads} reads, {} generations met, none torn", seen.len()); +} From b3897d50a3a5282fbc47254f2917d48ea4a19882 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 18:09:40 +0100 Subject: [PATCH 03/93] fileserver: an fsync commits its own file, and what the commit carries FSYNC synced the whole volume: every dirty page of every file written back and every open file's entry written before the format's commit. An fsync now makes durable the file it names and only what the volume's commit cannot leave out: - Volume::carried names the open files the next commit carries whatever it is asked: on DATA one whose entry was written since the last commit (a rename, a shrink) or whose reserve took blocks since, which the commit's allocator holds as taken; on FAT every changed open file, since a cache flush writes every chain the cache holds. - Files::fsync writes back the named file's pages and those of every carried file, then Volume::commit writes the named entry and each carried one and commits. A carried file whose pages are refused refuses the commit: its entry would name blocks without its bytes. - So after a stop no entry names a block holding an earlier tenant's bytes. A write over blocks a commit already named is carried by nothing, and waits for its own fsync or the volume's sync. A directory's fsync commits the carried files only: DATA writes every name change into the tree at once. SYNC and the write-back timer still sync the whole volume, and a per-file fsync leaves the timer due. Found while doing this: a sync commits the entry of a file whose pages were refused (issues/a-sync-commits-the-entry-of-a-file-whose-pages-were-refused.md). Tests (fileserver pages): an fsync makes its file durable across a stop and leaves another file's overwrite unwritten; a carried file's pages and entry go with another's fsync, so after a stop on a volume of stale blocks it reads its own bytes; a directory's fsync makes a new name durable. issues/an-fsync-of-a-served-file-syncs-its-whole-volume.md stays open until its measurement, osbench's persist.write_4k_fsync_beside_1m base against head on metal, is recorded. Co-Authored-By: Claude Opus 5.5 --- ...ntry-of-a-file-whose-pages-were-refused.md | 29 ++++++++ ...of-a-served-file-syncs-its-whole-volume.md | 12 +++- system/fileserver/src/absent.rs | 4 ++ system/fileserver/src/data.rs | 52 ++++++++++++-- system/fileserver/src/fat.rs | 24 +++++++ system/fileserver/src/main.rs | 30 ++++---- system/fileserver/src/pages.rs | 22 ++++++ system/fileserver/src/pages/tests.rs | 70 +++++++++++++++++++ system/fileserver/src/root.rs | 4 ++ system/fileserver/src/volume.rs | 16 +++++ tests/guest/osbench/README.md | 8 ++- 11 files changed, 245 insertions(+), 26 deletions(-) create mode 100644 issues/a-sync-commits-the-entry-of-a-file-whose-pages-were-refused.md diff --git a/issues/a-sync-commits-the-entry-of-a-file-whose-pages-were-refused.md b/issues/a-sync-commits-the-entry-of-a-file-whose-pages-were-refused.md new file mode 100644 index 00000000000..bcca2298d20 --- /dev/null +++ b/issues/a-sync-commits-the-entry-of-a-file-whose-pages-were-refused.md @@ -0,0 +1,29 @@ +--- +status: open +kind: defect +opened: 2026-11-01 +--- + +# A sync commits the entry of a file whose pages the volume refused + +`Files::sync` (`system/fileserver/src/pages.rs`) writes every dirty file's +pages back and then calls the volume's `sync` even where a file's write-back +was refused. DATA's `sync` writes every open file's entry, that file's +among them, so the commit makes durable an entry whose new extents name +blocks that never received its pages. In the same boot the file still reads +right: its pages stay dirty in its object and the next sync tries them +again. After a stop before that, it reads what those blocks last held, +which may be a deleted file's bytes +(`issues/a-reallocated-extent-on-data-keeps-the-deleted-files-bytes.md`). + +An fsync does not: a carried file whose pages are refused refuses the +commit (`Files::fsync`). + +Found by reading, while making an fsync commit one file; no test drives a +refused write-back into a stop. + +**Exit**: a sync's commit carries no entry of a file whose pages were +refused, with a host test that refuses one file's write-back, stops the +volume after the sync and reads that file as its last commit left it. + +**Owner**: the storage track (`issues/storage-is-layers-and-a-role-is-a-filesystem.md`). diff --git a/issues/an-fsync-of-a-served-file-syncs-its-whole-volume.md b/issues/an-fsync-of-a-served-file-syncs-its-whole-volume.md index e955882008a..c47c6158405 100644 --- a/issues/an-fsync-of-a-served-file-syncs-its-whole-volume.md +++ b/issues/an-fsync-of-a-served-file-syncs-its-whole-volume.md @@ -6,10 +6,14 @@ opened: 2026-09-27 # An fsync of a served file syncs its whole volume -The file server answers `FSYNC` by syncing the volume, every open file's +**Built, unmeasured.** `FSYNC` now commits its own file and what the +volume's commit carries (`Files::fsync`, `Volume::carried`), and no longer +syncs the volume. What follows is how it was. + +The file server answered `FSYNC` by syncing the volume, every open file's entry and every dirty block of the cache (`system/fileserver/src/main.rs`, -`FSYNC`, which calls `sync` with the file's node and syncs the whole volume). So an -`fsync` or `File::sync_all` of one file on `/home` syncs all of DATA, and a +`FSYNC`, which called `sync` with the file's node and synced the whole volume). So an +`fsync` or `File::sync_all` of one file on `/home` synced all of DATA, and a 4 KiB create and fsync cost 3.5–3.6 ms p50 on QEMU TCG against the kernel's 1.0–1.3 ms, measured on the branch that moved DATA to the file server. @@ -20,3 +24,5 @@ for no durability. **Exit**: `FSYNC` makes durable the file it names and what that file's entry depends on, with a guest measurement of one file's fsync beside the volume's. +What is owed is that measurement: osbench's +`persist.write_4k_fsync_beside_1m`, base against head on metal. diff --git a/system/fileserver/src/absent.rs b/system/fileserver/src/absent.rs index 219d9a77f09..04f5df12417 100644 --- a/system/fileserver/src/absent.rs +++ b/system/fileserver/src/absent.rs @@ -110,6 +110,10 @@ impl Volume for Absent { Ok(Vec::new()) } + fn commit(&mut self, _node: Option) -> Result<(), Error> { + Ok(()) + } + fn room(&mut self) -> Result { Ok(toyos::fs::Room { free: 0, unit: 1 }) } diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index 763409b815c..2bbe6a7af63 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -17,7 +17,9 @@ //! must) the block of each page a write will touch and moves the length; the //! write-back puts the pages; the new length and extents reach the file's //! entry at its close, an fsync, or the volume's sync, and a sync writes every -//! such entry before the cache is flushed. A length that +//! such entry before the cache is flushed. An fsync's commit writes its own +//! file's and those of the files it carries ([`Volume::carried`]): any whose +//! entry is written, or whose blocks were taken, since the last. A length that //! shrinks is recorded before the blocks past it are freed, so a failure //! between the two leaks blocks rather than leaving an entry naming freed ones. //! @@ -58,6 +60,9 @@ struct Open { mtime: u64, /// Length or extents changed since the entry was last written. dirty: bool, + /// The entry was written, or blocks taken, since the last commit + /// ([`Volume::carried`]). + carried: bool, /// Unlinked or renamed over: its blocks are no longer its own. gone: bool, holders: u32, @@ -336,12 +341,27 @@ impl DataVolume { } mapped("the entry", &open.path, self.fs.update_metadata(&open.path, &open.extents, open.size, open.mtime))?; open.dirty = false; + open.carried = true; if let Some(meta) = self.names.get_mut(&open.path) { (meta.size, meta.mtime) = (open.size, open.mtime); } Ok(()) } + /// The disk as a stop now leaves it: what the last flush wrote. + #[cfg(test)] + pub(crate) fn stopped(self) -> D { + drop(self.fs); + Rc::try_unwrap(self.cache).ok().expect("one owner").into_disk() + } + + /// A commit landed: no entry it made durable is owed to the next. + fn committed(&mut self) { + for open in self.open.values_mut().filter(|open| !open.dirty) { + open.carried = false; + } + } + /// Forget a node nobody holds and whose entry is written. fn release(&mut self, node: Node) { let open = self.open.remove(&node).expect("an open node"); @@ -362,7 +382,7 @@ impl DataVolume { fn new_node(&mut self, path: &str, extents: Vec, size: u64, mtime: u64) -> Node { let node = self.next; self.next += 1; - self.open.insert(node, Open { path: path.to_string(), extents, size, mtime, dirty: false, gone: false, holders: 0 }); + self.open.insert(node, Open { path: path.to_string(), extents, size, mtime, dirty: false, carried: false, gone: false, holders: 0 }); self.by_path.insert(path.to_string(), node); node } @@ -580,6 +600,7 @@ impl Volume for DataVolume { open.size = open.size.max(end); open.mtime = now; open.dirty = true; + open.carried |= open.extents.iter().map(|e| e.block_count as u64).sum::() > covered; noted(&mut self.names, open); Ok(reserved) } @@ -659,6 +680,7 @@ impl Volume for DataVolume { // Recorded first, freed second. mapped("a truncate", &open.path, self.fs.update_metadata(&open.path, &open.extents, size, now))?; open.dirty = false; + open.carried = true; if let Some(meta) = self.names.get_mut(&open.path) { (meta.size, meta.mtime) = (size, now); } @@ -828,9 +850,31 @@ impl Volume for DataVolume { } } mapped("sync", "", self.fs.sync())?; + self.committed(); Ok(unwritten) } + fn carried(&self, node: Node) -> bool { + self.open.get(&node).is_some_and(|open| open.carried && !open.gone) + } + + fn commit(&mut self, node: Option) -> Result<(), Error> { + if let Some(node) = node { + self.persist(node)?; + } + let carried: Vec = self.open.keys().copied().filter(|&n| self.carried(n)).collect(); + for n in carried { + match self.persist(n) { + Err(e) => println!("fileserver: node {n}'s entry was not written at a commit ({e:?}); the next one writes it"), + Ok(()) if self.open[&n].holders == 0 => self.release(n), + Ok(()) => {} + } + } + mapped("sync", "", self.fs.sync())?; + self.committed(); + Ok(()) + } + fn room(&mut self) -> Result { Ok(toyos::fs::Room { free: self.fs.free_bytes(), unit: BLOCK as u64 }) } @@ -1160,9 +1204,7 @@ mod tests { } fn remount(v: DataVolume) -> DataVolume { - let DataVolume { fs, cache, .. } = v; - drop(fs); - let disk = Rc::try_unwrap(cache).ok().expect("one owner").into_disk(); + let disk = v.stopped(); let Probed::Mounted(again) = DataVolume::probe(disk, CLEAN_LIMIT, &["home"], clock) else { panic!("remount") }; again } diff --git a/system/fileserver/src/fat.rs b/system/fileserver/src/fat.rs index b72741c92e4..a903c586e8e 100644 --- a/system/fileserver/src/fat.rs +++ b/system/fileserver/src/fat.rs @@ -433,6 +433,30 @@ impl Volume for FatVolume { Ok(unlevel) } + /// A flush writes every chain the cache holds, whichever file asked: an + /// open file changed since its entry was level is carried. + fn carried(&self, node: Node) -> bool { + self.open.get(&node).is_some_and(|open| !open.gone && (open.dirty || open.file.needs_reconcile())) + } + + fn commit(&mut self, node: Option) -> Result<(), fs::Error> { + if !self.writable { + return Ok(()); + } + if let Some(node) = node { + self.level(node)?; + } + let carried: Vec = self.open.keys().copied().filter(|&n| self.carried(n)).collect(); + for n in carried { + match self.level(n) { + Err(e) => println!("fileserver: node {n} was not brought level at a commit ({e:?}); the next one does it"), + Ok(()) if self.open[&n].holders == 0 => self.release(n), + Ok(()) => {} + } + } + self.cache.block_on(self.fs.sync()).map_err(|e| logged("sync", "", e)) + } + fn room(&mut self) -> Result { let unit = u64::from(self.fs.geometry().bytes_per_cluster()); if !self.writable { diff --git a/system/fileserver/src/main.rs b/system/fileserver/src/main.rs index 92133ef9669..732e2877bbd 100644 --- a/system/fileserver/src/main.rs +++ b/system/fileserver/src/main.rs @@ -40,10 +40,11 @@ //! client lends is read once per request into this process's memory, and //! what is acted on is that copy. //! -//! **What is written reaches the disk at a sync**: an fsync, a client's -//! `SYNC`, or [`fileserver::writeback::WRITEBACK`] after the first unsynced write when nothing asked -//! sooner. A kill loses what no sync covered, which is POSIX's promise and no -//! more. +//! **What is written reaches the disk at a sync**: a client's `SYNC`, or +//! [`fileserver::writeback::WRITEBACK`] after the first unsynced write when +//! nothing asked sooner; an fsync makes its own file durable, with what the +//! volume's commit carries beside it ([`fileserver::pages::Files::fsync`]). A +//! kill loses what neither covered, which is POSIX's promise and no more. use std::cell::RefCell; use std::collections::BTreeMap; @@ -544,7 +545,7 @@ impl Server { self.drop_client(id, "it never lent its window"); } if self.writeback.take_due(now) { - if let Err(e) = self.sync(None) { + if let Err(e) = self.sync() { println!("fileserver: the write-back sync failed: {e:?}"); } } @@ -928,14 +929,14 @@ impl Server { SETATTR => self.set_attr(id, &r), FSYNC => { let file = self.settled(id, r.fid)?; - let node = match file.borrow().held { - Held::File(node) => Some(node), - // A directory's names are made durable with the volume. + let named = match file.borrow().held { + Held::File(node) => Some((file.borrow().identity, node)), Held::Dir { .. } => None, }; - self.sync(node) + self.files.fsync(&mut *self.volume, named)?; + Ok(Answer::Reply(Reply::ok())) } - SYNC => self.sync(None), + SYNC => self.sync(), ROOM => { let room = self.volume.room()?; Ok(Answer::Reply(Reply { value: room.free, value2: room.unit, ..Reply::ok() })) @@ -1257,13 +1258,12 @@ impl Server { Ok(Answer::Reply(Reply::ok())) } - /// The volume synced, answered as `node`'s file fared, or with none as - /// every file did. A file whose entry was refused keeps the write-back - /// due, so the next one tries it again. - fn sync(&mut self, node: Option) -> Result { + /// The volume synced, answered as every file fared. A file whose entry + /// was refused keeps the write-back due, so the next one tries it again. + fn sync(&mut self) -> Result { let unwritten = self.files.sync(&mut *self.volume)?; self.writeback.synced(!unwritten.is_empty(), Instant::now()); - match unwritten.into_iter().find(|&(n, _)| node.is_none_or(|node| n == node)) { + match unwritten.into_iter().next() { Some((_, e)) => Err(e), None => Ok(Answer::Reply(Reply::ok())), } diff --git a/system/fileserver/src/pages.rs b/system/fileserver/src/pages.rs index 9e0c6e3b7e6..35a9df05ca7 100644 --- a/system/fileserver/src/pages.rs +++ b/system/fileserver/src/pages.rs @@ -549,6 +549,28 @@ impl Files

{ Ok(unwritten) } + /// An fsync: `file`'s pages to the volume, and every file's the volume's + /// commit carries, then the commit. A carried file whose pages would not + /// go refuses it, since its entry would name blocks without its bytes. + pub fn fsync(&mut self, volume: &mut dyn Volume, file: Option<(u64, Node)>) -> Result<(), Error> { + let named = file.map(|(identity, _)| identity); + let owed: Vec = self + .files + .iter() + .filter(|(id, f)| f.dirty.is_some() && (Some(**id) == named || volume.carried(f.node))) + .map(|(id, _)| *id) + .collect(); + for id in owed { + if let Err(e) = self.write_back(volume, id) { + if Some(id) != named { + println!("fileserver: file {id}'s pages would not go to the volume ({e:?}); an fsync of another is refused"); + } + return Err(e); + } + } + volume.commit(file.map(|(_, node)| node)) + } + /// What the pager's objects asked. pub fn notice(&mut self, volume: &mut dyn Volume, notice: Notice) { match notice { diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index f6b26ad1596..5da1f04ac51 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -555,3 +555,73 @@ fn a_write_of_many_calls_is_read_whole_or_not_at_all() { assert!(files.write(&mut v, at.1, at.0, 7 * page as u64 + 50, &pattern(3 * page, 12)).is_err()); assert!(fake.0.borrow().held.is_empty(), "a refused write was left under way"); } + +/// A volume whose every block holds what an earlier tenant left, 0xAB. +fn stale_volume() -> DataVolume { + let disk = Ram::new(4096); + let stale = vec![[0xAB; toyos_blockdev::BLOCK]; 4096]; + toyos_blockdev::block_on(toyos_blockdev::BlockDev::write(&disk, 0, &stale), || ()).unwrap(); + DataVolume::format(disk, 256, &["home"], clock).unwrap() +} + +/// `v` stopped now and mounted again: what its last commit left. +fn after_a_stop(v: DataVolume) -> DataVolume { + match DataVolume::probe(v.stopped(), 256, &["home"], clock) { + crate::data::Probed::Mounted(v) => v, + _ => panic!("the volume does not mount after a stop"), + } +} + +/// An fsync makes its own file durable, and leaves another file's write +/// over blocks the volume already named in the object, unwritten: nothing +/// the commit makes durable names those blocks anew. +#[test] +fn an_fsync_makes_its_file_durable_and_no_other_files_overwrite() { + let page = PAGE as usize; + let mut v = stale_volume(); + let mut files = Files::new(Some(Fake::new(64, 16))); + let b = open(&mut files, &mut v, "home/b"); + files.write(&mut v, b.1, b.0, 0, &pattern(2 * page, 1)).unwrap(); + assert_eq!(files.sync(&mut v).unwrap(), []); + let a = open(&mut files, &mut v, "home/a"); + files.write(&mut v, a.1, a.0, 0, &pattern(2 * page, 2)).unwrap(); + files.write(&mut v, b.1, b.0, 0, &pattern(2 * page, 3)).unwrap(); + + files.fsync(&mut v, Some((a.1, a.0))).unwrap(); + assert!(files.files[&b.1].dirty.is_some(), "the other file's overwrite was written back"); + let mut v = after_a_stop(v); + assert!(on_volume(&mut v, "home/a") == pattern(2 * page, 2), "the fsynced file is not as written"); + assert!(on_volume(&mut v, "home/b") == pattern(2 * page, 1), "the other file's overwrite reached the volume"); +} + +/// A file whose blocks the commit holds as taken is carried by an fsync of +/// another: its pages reach the volume first and its entry goes with them, +/// so after a stop it names no block that holds an earlier tenant's bytes, +/// and no block is held that nothing names. +#[test] +fn an_fsync_writes_first_the_pages_of_every_file_its_commit_carries() { + let page = PAGE as usize; + let mut v = stale_volume(); + let mut files = Files::new(Some(Fake::new(64, 16))); + let a = open(&mut files, &mut v, "home/a"); + let c = open(&mut files, &mut v, "home/c"); + files.write(&mut v, a.1, a.0, 0, &pattern(page, 4)).unwrap(); + files.write(&mut v, c.1, c.0, 0, &pattern(3 * page, 5)).unwrap(); + + files.fsync(&mut v, Some((a.1, a.0))).unwrap(); + let mut v = after_a_stop(v); + assert!(on_volume(&mut v, "home/a") == pattern(page, 4), "the fsynced file is not as written"); + let carried = on_volume(&mut v, "home/c"); + assert!(carried == pattern(3 * page, 5), "the carried file reads {} bytes, not its own", carried.len()); +} + +/// A directory's fsync commits the names every change already wrote. +#[test] +fn a_directorys_fsync_makes_a_new_name_durable() { + let mut v = stale_volume(); + let mut files = Files::new(Some(Fake::new(64, 16))); + v.mkdir("home/d").unwrap(); + files.fsync(&mut v, None).unwrap(); + let mut v = after_a_stop(v); + assert_eq!(v.lstat("home/d").map(|m| m.kind), Ok(crate::volume::Kind::Dir)); +} diff --git a/system/fileserver/src/root.rs b/system/fileserver/src/root.rs index 5160117f1f4..1fba2ea1604 100644 --- a/system/fileserver/src/root.rs +++ b/system/fileserver/src/root.rs @@ -250,6 +250,10 @@ impl Volume for RootVolume { Ok(Vec::new()) } + fn commit(&mut self, _node: Option) -> Result<(), Error> { + Ok(()) + } + /// None: ROOT is read-only. fn room(&mut self) -> Result { Ok(toyos::fs::Room { free: 0, unit: BLOCK as u64 }) diff --git a/system/fileserver/src/volume.rs b/system/fileserver/src/volume.rs index 25faa80d37a..713ef862539 100644 --- a/system/fileserver/src/volume.rs +++ b/system/fileserver/src/volume.rs @@ -190,6 +190,22 @@ pub trait Volume { /// volume's own flush refused. fn sync(&mut self) -> Result, Error>; + /// Whether the volume's next commit carries `node`'s entry whatever it is + /// asked to: one written since the last, or one naming blocks taken since, + /// which the commit holds as taken. The file's pages reach the volume + /// before that commit, so no entry it makes durable names a block that + /// does not hold the file's bytes. + fn carried(&self, _node: Node) -> bool { + false + } + + /// `node`'s entry durable, with every entry [`Self::carried`] names and + /// no other open file's: an fsync's commit. `None` commits only those, as + /// a directory's fsync does, whose names every change writes at once. + /// `Err` is `node`'s entry or the commit refused; another's refused entry + /// is said, and kept for the next commit. + fn commit(&mut self, node: Option) -> Result<(), Error>; + /// What more can be written to the volume: none, where nothing can be. fn room(&mut self) -> Result; diff --git a/tests/guest/osbench/README.md b/tests/guest/osbench/README.md index 6b488884ead..dbdd42f614e 100644 --- a/tests/guest/osbench/README.md +++ b/tests/guest/osbench/README.md @@ -220,9 +220,11 @@ What makes the disk rows one comparison, and what stays each system's own: `nobarrier` would show in `fs.disk.mount`'s superblock options, and the device's write-cache mode is in `served_by` (`write cache write back`). On ToyOS, the NVMe driver issues Flush where the controller has a volatile - write cache, which its boot log says. ToyOS's fsync makes its whole volume - durable, not just the file - (`issues/an-fsync-of-a-served-file-syncs-its-whole-volume.md`). + write cache, which its boot log says. ToyOS's fsync makes its file + durable with every file the volume's commit carries beside it: one whose + entry was written or whose blocks were taken since the last commit + (`persist.write_4k_fsync_beside_1m` writes over another file's blocks, + which no commit carries). - **One procedure for the cache.** Every row reads only files it wrote in its own run, and neither system's cache is dropped before it. What then reaches the disk is each system's own write-back and cache policy, and From 6ae35a46e9fcd7b7bb86fdec650dc6c12e3f7750 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 19:52:59 +0100 Subject: [PATCH 04/93] review r1 of #1141: a holder's WouldBlock is paged_read's, and the claims narrowed - R1: the holder's answer WouldBlock is documented on paged_read, the boundary a holder implements against; PagedOp::Write keeps only the EndWrite rule. - R3: a carried file's test no longer claims no block is held that no entry names: DataVolume::commit commits past a refused carried entry, whose taken blocks then leak after a stop. Filed with S3, one carried file's refused write-back refusing every fsync on the volume (issues/a-refused-carried-file-costs-every-other-files-fsync-or-leaks-its-blocks.md). - S2: the tear member's generations are its bytes mod 256. R2 is in the fsync commit: its issue stays open until the A/B records the exit's measurement. S1 (the version move at a write's end is redundant) is declined for this round: it changes the kernel the staged A/B images were built from. Comments, ABI docs and issue records only: no code changes, so the staged images at b4edede36 still measure this head. Co-Authored-By: Claude Opus 5.5 --- ...y-other-files-fsync-or-leaks-its-blocks.md | 37 +++++++++++++++++++ sdk/abi/src/syscall.rs | 9 +++-- system/fileserver/src/pages/tests.rs | 3 +- ...iles_read_never_tears_against_its_write.rs | 5 ++- 4 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 issues/a-refused-carried-file-costs-every-other-files-fsync-or-leaks-its-blocks.md diff --git a/issues/a-refused-carried-file-costs-every-other-files-fsync-or-leaks-its-blocks.md b/issues/a-refused-carried-file-costs-every-other-files-fsync-or-leaks-its-blocks.md new file mode 100644 index 00000000000..d1779ab1bb8 --- /dev/null +++ b/issues/a-refused-carried-file-costs-every-other-files-fsync-or-leaks-its-blocks.md @@ -0,0 +1,37 @@ +--- +status: open +kind: defect +opened: 2026-11-01 +--- + +# A carried file that is refused costs every other file's fsync, or leaks its blocks + +An fsync on DATA commits its own file and every file the commit carries +(`Volume::carried`): one whose entry was written, or whose reserve took +blocks, since the last commit (`system/fileserver/src/pages.rs`, +`Files::fsync`; `system/fileserver/src/data.rs`, `DataVolume::commit`). A +refusal on a carried file is paid by the files beside it: + +- **Its pages refused.** `Files::fsync` refuses the commit when a carried + file's write-back is refused, since its entry would name blocks without + its bytes. While that write-back stays refused, every fsync of every + file on the volume is refused with it. +- **Its entry refused.** `DataVolume::commit` says a carried entry whose + write was refused, and commits anyway. Where the file took blocks since + the last commit, the commit makes the allocator's take durable with no + entry naming them, and after a stop those blocks are leaked: not readable + by any file, and not free. This is not a disclosure. + +A file carried only by blocks taken (its entry unwritten since the last +commit) could be left out of the commit at the cost of that leak, rather +than refusing everyone's fsync. + +Found by review of #1141; no test drives either refusal into another file's +fsync or a stop. + +**Exit**: a refusal on one carried file costs only that file: an fsync of +another lands whole, and after a stop the volume holds as free every block +no entry names. A host test refuses one carried file's write-back, and +another's entry, fsyncs a third, stops, and counts the free blocks. + +**Owner**: the storage track (`issues/storage-is-layers-and-a-role-is-a-filesystem.md`). diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index cf3436efc87..dd0911e4bb2 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -3320,9 +3320,8 @@ pub enum PagedOp { /// Copy `len` bytes from `buf` into the present pages at `offset`, dirty, /// growing the size to their end as each page's bytes land. Answers the /// bytes written, stopping at the first absent page; `NotFound` when the - /// first is absent, `WouldBlock` at the dirty ceiling. A holder's read - /// that meets a write is answered `WouldBlock`; a write that does not take - /// all it was given stays under way until [`PagedOp::EndWrite`]. + /// first is absent, `WouldBlock` at the dirty ceiling. A write that does + /// not take all it was given stays under way until [`PagedOp::EndWrite`]. Write = 1, /// Remove the clean pages of `len` pages from page `offset` that no /// write-back or write holds. Answers how many went. @@ -3422,7 +3421,9 @@ pub fn paged_holder(pager: RawHandle, object: u64) -> Result Date: Mon, 2 Nov 2026 00:06:06 +0100 Subject: [PATCH 05/93] fileserver: an fsync that leaves nothing owed answers the write-back The volume-wide sync an fsync used to run cleared the write-back the server's loop keeps due. The per-file fsync never did: after it, the server waited every turn with a deadline, which every park and idle entry programs into the CPU's timer, and 2 s later walked the volume in a sync with nothing left to write. The volume now says whether an open file's entry is changed past what its last commit made durable (Volume::owes), and Files adds its dirty pages (Files::owes). An fsync that leaves neither answers the write-back as a sync does. One that leaves another file's write keeps it due from that write. Co-Authored-By: Claude Opus 5.5 --- system/fileserver/src/data.rs | 22 ++++++++++++++++++++++ system/fileserver/src/fat.rs | 18 ++++++++++++++++++ system/fileserver/src/main.rs | 4 ++++ system/fileserver/src/pages.rs | 6 ++++++ system/fileserver/src/pages/tests.rs | 28 ++++++++++++++++++++++++++++ system/fileserver/src/volume.rs | 6 ++++++ 6 files changed, 84 insertions(+) diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index 2bbe6a7af63..75389f99024 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -858,6 +858,10 @@ impl Volume for DataVolume { self.open.get(&node).is_some_and(|open| open.carried && !open.gone) } + fn owes(&self) -> bool { + self.open.values().any(|open| open.dirty && !open.gone) + } + fn commit(&mut self, node: Option) -> Result<(), Error> { if let Some(node) = node { self.persist(node)?; @@ -1186,6 +1190,24 @@ mod tests { 1_717_245_296_123_456_789 } + /// A commit leaves owed the entry of a file it did not carry, an overwrite + /// within the file's own blocks, and nothing once that entry is committed. + #[test] + fn a_commit_owes_the_entry_of_a_file_it_did_not_carry() { + let mut v = vol(); + let a = v.open("home/a", CREATE).unwrap(); + let b = v.open("home/b", CREATE).unwrap(); + v.write(a, 0, b"a").unwrap(); + v.write(b, 0, b"b").unwrap(); + v.commit(Some(a)).unwrap(); + assert!(!v.owes(), "b took blocks, so the commit carried its entry"); + v.write(b, 0, b"B").unwrap(); + v.commit(Some(a)).unwrap(); + assert!(v.owes(), "b's overwrite took no block and was not carried"); + v.commit(Some(b)).unwrap(); + assert!(!v.owes()); + } + /// DATA stamps a write with the nanosecond its clock read, and the entry /// keeps it whole across a remount: a stamp of whole seconds would round /// it away. diff --git a/system/fileserver/src/fat.rs b/system/fileserver/src/fat.rs index a903c586e8e..ec75dbf9f12 100644 --- a/system/fileserver/src/fat.rs +++ b/system/fileserver/src/fat.rs @@ -439,6 +439,10 @@ impl Volume for FatVolume { self.open.get(&node).is_some_and(|open| !open.gone && (open.dirty || open.file.needs_reconcile())) } + fn owes(&self) -> bool { + self.open.keys().any(|&node| self.carried(node)) + } + fn commit(&mut self, node: Option) -> Result<(), fs::Error> { if !self.writable { return Ok(()); @@ -573,6 +577,20 @@ mod tests { assert_eq!(v.lstat("a.txt").unwrap().mtime, 1_717_245_300 * NANOS_PER_SEC); } + /// A write is owed until a commit, and a commit naming another file + /// carries it: FAT's flush writes every chain it holds. + #[test] + fn a_commit_of_one_file_leaves_no_other_s_write_owed() { + const CREATE: OpenHow = OpenHow { create: true, create_new: false, truncate: false }; + let (mut v, _, _) = spec_fat(); + let a = v.open("a.txt", CREATE).unwrap(); + let b = v.open("b.txt", CREATE).unwrap(); + v.write(b, 0, &[2; 3000]).unwrap(); + assert!(v.owes()); + v.commit(Some(a)).unwrap(); + assert!(!v.owes()); + } + /// What [`a_sync_of_files_nothing_changed_asks_the_disk_nothing`]'s volume /// reads as the time, which only that test sets. static LATER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); diff --git a/system/fileserver/src/main.rs b/system/fileserver/src/main.rs index 732e2877bbd..e4d46290ecc 100644 --- a/system/fileserver/src/main.rs +++ b/system/fileserver/src/main.rs @@ -934,6 +934,10 @@ impl Server { Held::Dir { .. } => None, }; self.files.fsync(&mut *self.volume, named)?; + // One that left nothing to make durable did the write-back's work. + if !self.files.owes(&*self.volume) { + self.writeback.synced(false, Instant::now()); + } Ok(Answer::Reply(Reply::ok())) } SYNC => self.sync(), diff --git a/system/fileserver/src/pages.rs b/system/fileserver/src/pages.rs index 35a9df05ca7..490439887f8 100644 --- a/system/fileserver/src/pages.rs +++ b/system/fileserver/src/pages.rs @@ -571,6 +571,12 @@ impl Files

{ volume.commit(file.map(|(_, node)| node)) } + /// Whether a change is not yet durable: a dirty page, or an entry the + /// volume owes ([`Volume::owes`]). + pub fn owes(&self, volume: &dyn Volume) -> bool { + self.files.values().any(|f| f.dirty.is_some()) || volume.owes() + } + /// What the pager's objects asked. pub fn notice(&mut self, volume: &mut dyn Volume, notice: Notice) { match notice { diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index 741e092b727..a35bdceb392 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -23,6 +23,8 @@ struct Model { supplies: u32, /// Every supply from here is refused. refusing: bool, + /// This object's write-backs are refused. + unwritable: Option, /// Objects a call that did not take all it was given left written. held: BTreeSet, /// Each object's pages as a holder could read them after each change @@ -208,6 +210,9 @@ impl Pager for Fake { } fn writeback_begin(&self, id: u64, first: u64, count: u64) -> Result<[u64; 8], Refused> { + if self.0.borrow().unwritable == Some(id) { + return Err(Refused::Kernel(SyscallError::Io)); + } let mut bits = [0u64; 8]; for (&(c, p), page) in self.0.borrow_mut().pages.iter_mut() { if c == id && (first..first + count).contains(&p) && page.dirty { @@ -589,6 +594,7 @@ fn an_fsync_makes_its_file_durable_and_no_other_files_overwrite() { files.fsync(&mut v, Some((a.1, a.0))).unwrap(); assert!(files.files[&b.1].dirty.is_some(), "the other file's overwrite was written back"); + assert!(files.owes(&v), "the other file's overwrite is owed to a later sync"); let mut v = after_a_stop(v); assert!(on_volume(&mut v, "home/a") == pattern(2 * page, 2), "the fsynced file is not as written"); assert!(on_volume(&mut v, "home/b") == pattern(2 * page, 1), "the other file's overwrite reached the volume"); @@ -608,12 +614,34 @@ fn an_fsync_writes_first_the_pages_of_every_file_its_commit_carries() { files.write(&mut v, c.1, c.0, 0, &pattern(3 * page, 5)).unwrap(); files.fsync(&mut v, Some((a.1, a.0))).unwrap(); + assert!(!files.owes(&v), "the fsync left a change owed"); let mut v = after_a_stop(v); assert!(on_volume(&mut v, "home/a") == pattern(page, 4), "the fsynced file is not as written"); let carried = on_volume(&mut v, "home/c"); assert!(carried == pattern(3 * page, 5), "the carried file reads {} bytes, not its own", carried.len()); } +/// A file whose pages a sync could not write is owed after an fsync of +/// another, though that sync made its entry durable and the commit does not +/// carry it. +#[test] +fn a_file_a_sync_could_not_write_back_stays_owed_past_another_s_fsync() { + let page = PAGE as usize; + let mut v = volume(); + let fake = Fake::new(64, 16); + let mut files = Files::new(Some(fake.clone())); + let a = open(&mut files, &mut v, "home/a"); + let f = open(&mut files, &mut v, "home/f"); + files.write(&mut v, f.1, f.0, 0, &pattern(page, 6)).unwrap(); + fake.0.borrow_mut().unwritable = Some(files.files[&f.1].object); + assert_eq!(files.sync(&mut v).map(|u| u.len()), Ok(1), "the sync wrote the refused file's pages"); + assert!(!v.owes(), "the sync left f's entry owed"); + files.write(&mut v, a.1, a.0, 0, &pattern(page, 7)).unwrap(); + + files.fsync(&mut v, Some((a.1, a.0))).unwrap(); + assert!(files.owes(&v), "f's unwritten pages are owed to a later sync"); +} + /// A directory's fsync commits the names every change already wrote. #[test] fn a_directorys_fsync_makes_a_new_name_durable() { diff --git a/system/fileserver/src/volume.rs b/system/fileserver/src/volume.rs index 713ef862539..f6ada02f691 100644 --- a/system/fileserver/src/volume.rs +++ b/system/fileserver/src/volume.rs @@ -199,6 +199,12 @@ pub trait Volume { false } + /// Whether an open file's entry is changed past what a commit or sync + /// last made durable. Everything else written is durable after either. + fn owes(&self) -> bool { + false + } + /// `node`'s entry durable, with every entry [`Self::carried`] names and /// no other open file's: an fsync's commit. `None` commits only those, as /// a directory's fsync does, whose names every change writes at once. From ea23ee415ab6e904aa25ad0bd564de7b0c3edbf3 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 19:51:09 +0100 Subject: [PATCH 06/93] paged: a holder's fstat is its object's, without a request (FS stage 3a) A served file's fstat was a file server round trip. A paged object now publishes what a holder's fstat needs that can change: the kernel's own size and write-span version, and the mtime its pager sets. SYS_PAGED_STAT(holder) answers {size, mtime, version} under the object's lock, WouldBlock while a write is under way (atomic as a read is), and the SDK then asks the server. Kind, node, volume and nlink come from the open's reply, which no change to the file moves. The owner sets the mtime as a word it already sends: PagedArgs.time on Write and Resize, paged_create's mtime, and PagedOp::Stamp for a utimens. The file server hands the object the volume's own value after each change (reserve, truncate, an open that truncates, a set time, a refused write's cut back), at the volume's resolution. The version is the object's data generation; there is no second counter. FAT's mtime was its entry's last level, not its last write (issues/a-fat-files-mtime-through-its-handle-is-its-last-level.md), so it could not be published. A write or resize now records its even second, which the handle answers until the level stamps it. Tests: - kernel::paged: a stat publishes no version while a write is held. - fileserver pages: the object's size and mtime are the volume's after an open, a write, a refused write, a truncate and a stamp. - fileserver fat: a file's mtime is its last write's through its handle, off the disk a sync later and after a reopen (replaces the test that pinned the stamp to the flush). - guest a_held_files_fstat_is_its_servers_stat: the held fstat agrees with the server's stat of the path after each change. Closes issues/a-fat-files-mtime-through-its-handle-is-its-last-level.md. Co-Authored-By: Claude Opus 5.5 --- ...me-through-its-handle-is-its-last-level.md | 20 ----- kernel/pure/paged.rs | 56 ++++++++----- kernel/src/object/paged.rs | 63 ++++++++++---- kernel/src/syscall/dispatch.rs | 5 +- kernel/src/syscall/paged.rs | 39 +++++++-- sdk/abi/src/syscall.rs | 55 +++++++++--- sdk/toyos/src/files/owner.rs | 27 +++++- sdk/toyos/src/paged.rs | 44 ++++++---- system/fileserver/src/fat.rs | 66 ++++++++++----- system/fileserver/src/main.rs | 12 ++- system/fileserver/src/pages.rs | 83 ++++++++++++++----- system/fileserver/src/pages/tests.rs | 55 +++++++++++- tests/guest/pagerprobe/src/main.rs | 6 +- .../a_held_files_fstat_is_its_servers_stat.rs | 41 +++++++++ 14 files changed, 424 insertions(+), 148 deletions(-) delete mode 100644 issues/a-fat-files-mtime-through-its-handle-is-its-last-level.md create mode 100644 tests/guest/rust-tests/src/bin/a_held_files_fstat_is_its_servers_stat.rs diff --git a/issues/a-fat-files-mtime-through-its-handle-is-its-last-level.md b/issues/a-fat-files-mtime-through-its-handle-is-its-last-level.md deleted file mode 100644 index de11f4bb7e1..00000000000 --- a/issues/a-fat-files-mtime-through-its-handle-is-its-last-level.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-29 ---- - -# A FAT file's mtime through an open handle is its entry's last level, not its last write - -The file server's FAT volume answers a file's mtime off its directory entry -(`system/fileserver/src/fat.rs`, `node_meta` and `lstat`), and an entry is stamped -only when it is brought level: at the last close, an fsync or a sync. A write -moves nothing an `fstat` reads, so a writer that writes and `fstat`s reads the -stamp of the entry's last level, and the same file closed and opened again reads -the level that close made: one file, two mtimes. - -**Mechanism read off the code; not reproduced.** - -**Exit condition.** For a FAT file, the mtime `fstat` answers through a handle -after a write is the one it answers after the file is closed and opened again, -checked by a test that writes, `fstat`s, reopens and `fstat`s again. diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index b6129f9d227..d3a279be09e 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -223,6 +223,8 @@ pub struct Object { writing: u32, /// A call that did not take all it was given left its write under way. held: bool, + /// What its pager last set: nanoseconds, as the file's volume keeps it. + mtime: u64, } /// What a holder's read began at ([`Object::read_begin`]). @@ -247,6 +249,20 @@ impl Object { self.size.div_ceil(PAGE) } + pub fn mtime(&self) -> u64 { + self.mtime + } + + pub fn set_mtime(&mut self, mtime: u64) { + self.mtime = mtime; + } + + /// The version a holder's stat publishes: `None` while a write is under + /// way, as for a read. + pub fn published(&self) -> Option { + (self.writing == 0).then_some(self.version) + } + /// Where a holder's read begins: `None` while a write is under way. pub fn read_begin(&self) -> Option { (self.writing == 0).then_some(Version(self.version)) @@ -358,9 +374,9 @@ impl Pages { /// An object of `size` bytes, at most [`LAST_END`], named `id`: an id no /// live object of this table has, which a released one's may be, since /// [`release`](Self::release) takes every page it held. - pub fn object(&mut self, id: u32, size: u64) -> Object { + pub fn object(&mut self, id: u32, size: u64, mtime: u64) -> Object { assert!(size <= LAST_END, "an object of {size} bytes past the last page"); - Object { id, size, gone: false, head: NONE, pages: 0, version: 0, writing: 0, held: false } + Object { id, size, gone: false, head: NONE, pages: 0, version: 0, writing: 0, held: false, mtime } } fn hash(&self, object: u32, page: u64) -> usize { @@ -736,7 +752,7 @@ mod tests { #[test] fn a_read_counts_only_where_no_write_began_or_was_under_way() { let mut p = Pages::::new(8, 4).unwrap(); - let mut o = p.object(1, PAGE); + let mut o = p.object(1, PAGE, 0); let before = o.read_begin().expect("nothing writes yet"); o.write_begin(); assert_eq!(o.read_begin(), None, "a read meets the write in flight"); @@ -755,7 +771,7 @@ mod tests { #[test] fn a_short_call_holds_the_write_until_its_end() { let mut p = Pages::::new(8, 4).unwrap(); - let mut o = p.object(2, PAGE); + let mut o = p.object(2, PAGE, 0); assert_eq!(o.end_write(), Err(Refused::InvalidArgument), "nothing is held"); o.write_begin(); o.write_end(false); @@ -765,17 +781,19 @@ mod tests { o.write_begin(); o.write_end(true); assert_eq!(o.read_begin(), None, "a later call, short or whole, ends no held write"); + assert_eq!(o.published(), None, "nor does a stat publish one half written"); let held = o.version; assert_eq!(o.end_write(), Ok(())); let v = o.read_begin().expect("the end lets a read begin"); assert!(o.unchanged(v) && v != Version(held), "the end moved the version"); + assert_eq!(o.published(), Some(v.0), "a stat publishes the version a read begins at"); assert_eq!(o.end_write(), Err(Refused::InvalidArgument), "a write ends once"); } #[test] fn a_supplied_page_is_found_and_a_second_supply_changes_nothing() { let mut p = Pages::new(8, 4).unwrap(); - let mut o = p.object(1, 3 * PAGE); + let mut o = p.object(1, 3 * PAGE, 0); assert_eq!(p.lookup(&o, 1), Lookup::Absent); assert_eq!(p.supply(&mut o, 1, 100, false, 2 * PAGE), Ok(Supplied::Installed)); assert_eq!(p.supply(&mut o, 1, 200, false, 2 * PAGE), Ok(Supplied::Present)); @@ -786,10 +804,10 @@ mod tests { #[test] fn a_clean_supply_past_the_size_is_refused_and_a_dirty_one_extends_it() { let mut p = Pages::new(8, 4).unwrap(); - let mut o = p.object(2, PAGE); + let mut o = p.object(2, PAGE, 0); assert_eq!(p.supply(&mut o, 1, 1, false, 2 * PAGE), Err(Refused::PastEnd)); assert_eq!(o.size(), PAGE); - let mut short = p.object(3, 100); + let mut short = p.object(3, 100, 0); assert_eq!(p.supply(&mut short, 0, 2, false, PAGE), Ok(Supplied::Installed)); assert_eq!(short.size(), 100, "a clean page ending past the size leaves it"); assert_eq!(p.supply(&mut o, 1, 1, true, PAGE + 10), Ok(Supplied::Installed)); @@ -802,7 +820,7 @@ mod tests { #[test] fn a_reader_racing_an_extending_write_never_sees_its_unwritten_bytes() { let mut p = Pages::new(8, 4).unwrap(); - let mut o = p.object(4, 100); + let mut o = p.object(4, 100, 0); p.supply(&mut o, 0, 7, false, 100).unwrap(); assert_eq!(p.write_pin(&mut o, 0), Ok(7)); assert_eq!(o.size(), 100, "the copy is in flight: a reader still stops at the old end"); @@ -814,7 +832,7 @@ mod tests { #[test] fn an_unpin_after_a_shrink_removed_the_page_grows_nothing() { let mut p = Pages::new(8, 4).unwrap(); - let mut o = p.object(5, PAGE); + let mut o = p.object(5, PAGE, 0); p.supply(&mut o, 0, 7, false, PAGE).unwrap(); p.write_pin(&mut o, 0).unwrap(); p.resize(&mut o, 0, &mut none); @@ -829,7 +847,7 @@ mod tests { #[test] fn the_dirty_ceiling_refuses_and_a_write_back_frees_it() { let mut p = Pages::new(8, 2).unwrap(); - let mut o = p.object(6, 4 * PAGE); + let mut o = p.object(6, 4 * PAGE, 0); for page in 0..2 { p.supply(&mut o, page, page, true, (page + 1) * PAGE).unwrap(); } @@ -844,7 +862,7 @@ mod tests { #[test] fn kept_is_exactly_the_pages_dirtied_or_pinned_since_the_begin_or_unwritten() { let mut p = Pages::new(16, 16).unwrap(); - let mut o = p.object(7, 8 * PAGE); + let mut o = p.object(7, 8 * PAGE, 0); for page in 0..6 { p.supply(&mut o, page, page, true, (page + 1) * PAGE).unwrap(); } @@ -866,7 +884,7 @@ mod tests { #[test] fn a_range_must_be_ended_exactly_as_begun_and_once() { let mut p: Pages = Pages::new(8, 8).unwrap(); - let mut o = p.object(8, PAGE * 1024); + let mut o = p.object(8, PAGE * 1024, 0); p.writeback_begin(&mut o, 0, 4).unwrap(); assert_eq!(p.writeback_begin(&mut o, 3, 2), Err(Refused::InvalidArgument), "overlapping"); assert_eq!(p.writeback_end(&mut o, 0, 3, true), Err(Refused::InvalidArgument), "not the range begun"); @@ -879,7 +897,7 @@ mod tests { #[test] fn only_clean_unheld_pages_are_withdrawn() { let mut p = Pages::new(8, 8).unwrap(); - let mut o = p.object(9, 4 * PAGE); + let mut o = p.object(9, 4 * PAGE, 0); p.supply(&mut o, 0, 10, false, PAGE).unwrap(); p.supply(&mut o, 1, 11, true, 2 * PAGE).unwrap(); p.supply(&mut o, 2, 12, true, 3 * PAGE).unwrap(); @@ -937,7 +955,7 @@ mod tests { #[test] fn a_withdrawal_of_a_long_range_walks_the_objects_pages() { let mut p = Pages::new(8, 8).unwrap(); - let mut o = p.object(1, LAST_END); + let mut o = p.object(1, LAST_END, 0); p.supply(&mut o, 7, 1, false, 8 * PAGE).unwrap(); p.supply(&mut o, 1 << 40, 2, false, ((1 << 40) + 1) * PAGE).unwrap(); let mut freed = Vec::new(); @@ -959,7 +977,7 @@ mod tests { assert!(!within_reach(u64::MAX - PAGE + 1, PAGE), "the supply that wrapped `ends`"); assert!(!within_reach(1, u64::MAX)); let mut p = Pages::new(8, 8).unwrap(); - let mut o = p.object(1, LAST_END); + let mut o = p.object(1, LAST_END, 0); let last = LAST_END / PAGE - 1; assert_eq!(p.supply(&mut o, last, 1, true, LAST_END), Ok(Supplied::Installed), "the last page"); assert!(Pages::::new(Pages::::MAX_CAPACITY + 1, 0).is_none()); @@ -968,7 +986,7 @@ mod tests { #[test] fn a_shrink_removes_dirty_pages_releases_their_slots_and_names_the_tail() { let mut p = Pages::new(8, 8).unwrap(); - let mut o = p.object(11, 3 * PAGE); + let mut o = p.object(11, 3 * PAGE, 0); for page in 0..3 { p.supply(&mut o, page, page + 100, true, (page + 1) * PAGE).unwrap(); } @@ -984,7 +1002,7 @@ mod tests { #[test] fn a_failed_page_answers_failed_until_supplied_again() { let mut p = Pages::new(8, 8).unwrap(); - let mut o = p.object(12, 2 * PAGE); + let mut o = p.object(12, 2 * PAGE, 0); p.supply(&mut o, 0, 5, false, PAGE).unwrap(); let mut freed = Vec::new(); p.fail(&mut o, 0, 2, &mut sink(&mut freed)).unwrap(); @@ -997,7 +1015,7 @@ mod tests { #[test] fn an_invalidated_object_answers_gone_and_gives_every_frame_back() { let mut p = Pages::new(8, 8).unwrap(); - let mut o = p.object(13, 4 * PAGE); + let mut o = p.object(13, 4 * PAGE, 0); p.supply(&mut o, 0, 1, true, PAGE).unwrap(); p.supply(&mut o, 2, 3, false, 3 * PAGE).unwrap(); let mut freed = Vec::new(); @@ -1012,7 +1030,7 @@ mod tests { #[test] fn a_full_table_refuses_and_every_slot_is_reached_after_churn() { let mut p = Pages::new(64, 64).unwrap(); - let mut objects: Vec = (0..4).map(|i| p.object(i, 1 << 30)).collect(); + let mut objects: Vec = (0..4).map(|i| p.object(i, 1 << 30, 0)).collect(); let mut model: BTreeSet<(usize, u64)> = BTreeSet::new(); let mut x = 1u64; for round in 0..20_000u64 { diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index e0c3cfc4f52..ef87f83e408 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -48,7 +48,7 @@ use core::sync::atomic::{AtomicU32, Ordering}; use kernel::chunked::{Chunked, Stack}; use kernel::paged::{Core, Draw, Leave, Lookup, Object, Pages, Refused, Supplied, Version, PAGE}; use kernel::sched::task::WaitClass; -use toyos_abi::syscall::{PagedNotice, SyscallError, NOTICE_NO_HOLDERS, NOTICE_REQUESTED}; +use toyos_abi::syscall::{PagedNotice, PagedStat, SyscallError, NOTICE_NO_HOLDERS, NOTICE_REQUESTED}; use crate::mm::pmm; use crate::process::Pid; @@ -271,9 +271,9 @@ impl PagerObject { unreported || state.no_holders > 0 } - /// A new object of `size` bytes: its id. `ResourceExhausted` where every - /// slot holds one. - pub fn add(&self, size: u64) -> u64 { + /// A new object of `size` bytes last written at `mtime`: its id. + /// `ResourceExhausted` where every slot holds one. + pub fn add(&self, size: u64, mtime: u64) -> u64 { if size > kernel::paged::LAST_END { return SyscallError::InvalidArgument.to_u64(); } @@ -282,7 +282,7 @@ impl PagerObject { return SyscallError::Gone.to_u64(); } let Some(slot) = state.free.pop() else { return SyscallError::ResourceExhausted.to_u64() }; - let object = state.pages.object(slot, size); + let object = state.pages.object(slot, size, mtime); state.entries[slot as usize] = Some(Entry { object, refs: 1, holders: 0, owned: true, no_holders: false }); u64::from(slot) } @@ -588,17 +588,45 @@ impl PagerObject { Step::Copy(count) } - /// A write call on object `id` begins: its slot, held by a reference until - /// [`Self::write_end`], so no release can hand it to another object. - fn write_begin(&self, id: u64) -> Result { + /// A write call on object `id` begins, stamping it `mtime` where given: + /// its slot, held by a reference until [`Self::write_end`], so no release + /// can hand it to another object. + fn write_begin(&self, id: u64, mtime: Option) -> Result { let mut state = self.state.lock(); let slot = state.live(id)?; let entry = state.entry_mut(slot); entry.refs += 1; entry.object.write_begin(); + if let Some(mtime) = mtime { + entry.object.set_mtime(mtime); + } Ok(slot) } + /// Stamp object `id` last written at `mtime`. + pub fn stamp(&self, id: u64, mtime: u64) -> u64 { + let mut state = self.state.lock(); + match state.live(id) { + Ok(slot) => { + state.entry_mut(slot).object.set_mtime(mtime); + 0 + } + Err(e) => e.to_u64(), + } + } + + /// What the object at `slot` publishes to a holder: `WouldBlock` while a + /// write is under way. + fn stat(&self, slot: u32) -> Result { + let state = self.state.lock(); + let object = &state.entry(slot).object; + if state.gone || object.gone() { + return Err(SyscallError::Gone); + } + let version = object.published().ok_or(SyscallError::WouldBlock)?; + Ok(PagedStat { size: object.size(), mtime: object.mtime(), version }) + } + /// The write call [`Self::write_begin`] began ends, `whole` where it took /// everything it was given: its reference goes. fn write_end(&self, slot: u32, whole: bool) { @@ -627,7 +655,7 @@ impl PagerObject { if !dirty { return self.supply_pages(id, offset, src, false); } - let slot = match self.write_begin(id) { + let slot = match self.write_begin(id, None) { Ok(slot) => slot, Err(e) => return e.to_u64(), }; @@ -728,11 +756,11 @@ impl PagerObject { installed } - /// Copy `src` into the present pages of object `id` from `offset`, dirty: - /// the bytes written, up to the first absent page. The caller checked the - /// span is in reach. - pub fn write(&self, id: u64, offset: u64, src: &UserBytes) -> u64 { - let slot = match self.write_begin(id) { + /// Copy `src` into the present pages of object `id` from `offset`, dirty, + /// stamping it `mtime`: the bytes written, up to the first absent page. + /// The caller checked the span is in reach. + pub fn write(&self, id: u64, offset: u64, src: &UserBytes, mtime: u64) -> u64 { + let slot = match self.write_begin(id, Some(mtime)) { Ok(slot) => slot, Err(e) => return e.to_u64(), }; @@ -819,11 +847,12 @@ impl PagerObject { } } - pub fn resize(&self, id: u64, size: u64) -> u64 { + pub fn resize(&self, id: u64, size: u64, mtime: u64) -> u64 { if size > kernel::paged::LAST_END { return SyscallError::InvalidArgument.to_u64(); } let resized = self.removing(id, |pages, object, out| { + object.set_mtime(mtime); if let Some(phys) = pages.resize(object, size, &mut |f| out(f)) { let within = (size % PAGE) as usize; // SAFETY: the last page's frame, the object's while the lock is @@ -1088,6 +1117,10 @@ impl PagedHolderObject { pub fn read(&self, out: &mut UserBytesMut, offset: u64) -> u64 { self.named.pager.read(self.named.slot, out, offset, true) } + + pub fn stat(&self) -> Result { + self.named.pager.stat(self.named.slot) + } } impl ZeroHandles for PagedHolderObject { diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 2d5767926ff..a9e41623438 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -48,7 +48,7 @@ use super::machine::{ }; #[cfg(feature = "test-actuators")] use super::machine::lower_sysinfo_bound; -use super::paged::{sys_paged_create, sys_paged_holder, sys_paged_notices, sys_paged_op, sys_paged_read, sys_pager_create}; +use super::paged::{sys_paged_create, sys_paged_holder, sys_paged_notices, sys_paged_op, sys_paged_read, sys_paged_stat, sys_pager_create}; use super::proc::{ spawn_pager, spawn_place, sys_endowments, sys_exit, sys_nanosleep, sys_process_stats, sys_account_info, sys_job_create, sys_job_floor, sys_job_own, sys_process_quit, sys_process_wait, sys_rt_enter, sys_spawn, sys_thread_exit, sys_thread_get_affinity, sys_thread_join, sys_thread_set_affinity, sys_thread_spawn, sys_wake_latency_hold, @@ -684,13 +684,14 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> SYS_ACPI => sys_acpi(&ctx, RawHandle(a1 as u32), a2, a3), SYS_DEVICE_REG_READ => sys_device_reg_read(RawHandle(a1 as u32), a2, a3), SYS_PAGER_CREATE => sys_pager_create(a1, a2, a3, a4), - SYS_PAGED_CREATE => sys_paged_create(RawHandle(a1 as u32), a2), + SYS_PAGED_CREATE => sys_paged_create(RawHandle(a1 as u32), a2, a3), SYS_PAGED_HOLDER => sys_paged_holder(RawHandle(a1 as u32), a2), SYS_PAGED_READ => { let Some(mut buf) = ctx.user_bytes_mut(UserAddr::new(a2), a3) else { return bad_addr }; sys_paged_read(RawHandle(a1 as u32), &mut buf, a4) } SYS_PAGED_OP => sys_paged_op(&ctx, RawHandle(a1 as u32), a2, UserAddr::new(a3)), + SYS_PAGED_STAT => sys_paged_stat(&ctx, RawHandle(a1 as u32), UserAddr::new(a2)), SYS_PAGED_NOTICES => { let size = core::mem::size_of::(); let Some(len) = (a3 as usize).checked_mul(size) else { return SyscallError::InvalidArgument.to_u64() }; diff --git a/kernel/src/syscall/paged.rs b/kernel/src/syscall/paged.rs index 0a8aaa0f164..5f3fc5f71cc 100644 --- a/kernel/src/syscall/paged.rs +++ b/kernel/src/syscall/paged.rs @@ -10,7 +10,7 @@ use kernel::paged::within_reach; use toyos_abi::handle::{RawHandle, Rights}; use toyos_abi::syscall::{PagedArgs, PagedOp, SyscallError, PAGED_DIRTY, PAGED_RANGE_PAGES, PAGED_WRITTEN}; -use crate::object::paged::{PagerObject, Unmade}; +use crate::object::paged::{PagedHolderObject, PagerObject, Unmade}; use crate::object::{ops, KObjectRef}; use crate::process::{self, PagerRole}; use crate::user_ptr::{SyscallContext, UserBytesMut}; @@ -48,9 +48,9 @@ fn pager(h: RawHandle, rights: Rights) -> Result, u64> { process::with_process_data(|data| data.handles.get::(h, rights)).map_err(|e| e.refuse()) } -pub(super) fn sys_paged_create(h: RawHandle, size: u64) -> u64 { +pub(super) fn sys_paged_create(h: RawHandle, size: u64, mtime: u64) -> u64 { match pager(h, Rights::WRITE) { - Ok(pager) => pager.add(size), + Ok(pager) => pager.add(size, mtime), Err(refused) => refused, } } @@ -80,6 +80,28 @@ pub(super) fn sys_paged_holder(h: RawHandle, object: u64) -> u64 { /// on itself or on another pager. The handle is often a peer's, sent as a /// holder, so another kind is refused `InvalidArgument`, never a fault. pub(super) fn sys_paged_read(h: RawHandle, out: &mut UserBytesMut, offset: u64) -> u64 { + match holder(h) { + Ok(holder) => holder.read(out, offset), + Err(refused) => refused, + } +} + +/// A holder's stat, refused as its read is. +pub(super) fn sys_paged_stat(ctx: &SyscallContext<'_>, h: RawHandle, out: UserAddr) -> u64 { + let stat = match holder(h) { + Ok(holder) => holder.stat(), + Err(refused) => return refused, + }; + match stat { + Ok(stat) => match ctx.copy_out(out, &stat) { + Ok(()) => 0, + Err(_) => SyscallError::BadAddress.to_u64(), + }, + Err(e) => e.to_u64(), + } +} + +fn holder(h: RawHandle) -> Result, u64> { let resolved = process::with_process_data(|data| { let KObjectRef::Paged(holder) = data.handles.get_ref(h, Rights::NONE)? else { return Ok(Err(SyscallError::InvalidArgument)); @@ -91,9 +113,9 @@ pub(super) fn sys_paged_read(h: RawHandle, out: &mut UserBytesMut, offset: u64) Ok(if data.pager.is_pager() { Err(SyscallError::PermissionDenied) } else { Ok(holder) }) }); match resolved { - Ok(Ok(holder)) => holder.read(out, offset), - Ok(Err(e)) => e.to_u64(), - Err(e) => crate::object::HandleError::refuse(e), + Ok(Ok(holder)) => Ok(holder), + Ok(Err(e)) => Err(e.to_u64()), + Err(e) => Err(crate::object::HandleError::refuse(e)), } } @@ -125,12 +147,13 @@ pub(super) fn sys_paged_op(ctx: &SyscallContext<'_>, h: RawHandle, op: u64, args let Some(src) = ctx.user_bytes(UserAddr::new(args.buf), args.len) else { return SyscallError::BadAddress.to_u64() }; match x == PagedOp::Supply as u64 { true => pager.supply(id, args.offset, &src, args.flags & PAGED_DIRTY != 0), - false => pager.write(id, args.offset, &src), + false => pager.write(id, args.offset, &src, args.time), } } x if x == PagedOp::Withdraw as u64 => pager.withdraw(id, args.offset, args.len), x if x == PagedOp::Fail as u64 => pager.fail(id, args.offset, args.len), - x if x == PagedOp::Resize as u64 => pager.resize(id, args.offset), + x if x == PagedOp::Resize as u64 => pager.resize(id, args.offset, args.time), + x if x == PagedOp::Stamp as u64 => pager.stamp(id, args.time), x if x == PagedOp::Release as u64 => pager.release(id), x if x == PagedOp::EndWrite as u64 => pager.end_write(id), x if x == PagedOp::WritebackBegin as u64 || x == PagedOp::WritebackEnd as u64 => { diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index dd0911e4bb2..2378ac33d01 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -370,6 +370,8 @@ pub const SYS_THREAD_SET_AFFINITY: u64 = 161; pub const SYS_THREAD_GET_AFFINITY: u64 = 162; /// The CPU the caller is running on. See [`current_cpu`]. pub const SYS_CURRENT_CPU: u64 = 163; +/// What a holder's paged object publishes. See [`paged_stat`]. +pub const SYS_PAGED_STAT: u64 = 164; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -377,13 +379,13 @@ pub const SYS_CURRENT_CPU: u64 = 163; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 165; +pub const SYSCALL_PROFILE_BINS: usize = 166; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; -const _: () = assert!(SYS_CURRENT_CPU < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_PAGED_STAT < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; /// Every bit `SYS_PROCESS_WAIT`'s flag word defines; a wait carrying another @@ -3320,8 +3322,10 @@ pub enum PagedOp { /// Copy `len` bytes from `buf` into the present pages at `offset`, dirty, /// growing the size to their end as each page's bytes land. Answers the /// bytes written, stopping at the first absent page; `NotFound` when the - /// first is absent, `WouldBlock` at the dirty ceiling. A write that does - /// not take all it was given stays under way until [`PagedOp::EndWrite`]. + /// first is absent, `WouldBlock` at the dirty ceiling. `time` is the + /// file's mtime from this write, as its volume keeps it. A write that + /// does not take all it was given stays under way until + /// [`PagedOp::EndWrite`]. Write = 1, /// Remove the clean pages of `len` pages from page `offset` that no /// write-back or write holds. Answers how many went. @@ -3329,9 +3333,9 @@ pub enum PagedOp { /// Mark `len` pages from page `offset` failed: each read of one answers /// `Io` until it is supplied again. Fail = 3, - /// Set the size to `offset`. A shrink drops every page past it, dirty or - /// not, and zeroes the last page's tail; the new range of a growth reads - /// as the pager supplies it. + /// Set the size to `offset`, and the mtime to `time`. A shrink drops + /// every page past it, dirty or not, and zeroes the last page's tail; the + /// new range of a growth reads as the pager supplies it. Resize = 4, /// Begin writing back `len` pages from page `offset`, at most /// [`PAGED_RANGE_PAGES`]: each dirty page is cleaned and held until the @@ -3350,6 +3354,9 @@ pub enum PagedOp { /// End the write a short [`PagedOp::Write`] or dirty supply left under /// way; `InvalidArgument` where none is. EndWrite = 9, + /// Set the mtime to `time`, as a file's `utimens` does; the bytes and + /// the version stay. + Stamp = 10, } /// [`PagedOp::Supply`]'s pages are dirty. @@ -3370,6 +3377,21 @@ crate::user_safe! { pub len: u64, pub buf: u64, pub flags: u64, + /// [`PagedOp::Write`]'s, [`PagedOp::Resize`]'s and [`PagedOp::Stamp`]'s + /// mtime, nanoseconds since the Unix epoch. + pub time: u64, + } +} + +crate::user_safe! { + /// What a paged object publishes to its holders ([`paged_stat`]): the + /// size and version the kernel keeps, and the mtime its pager last set. + #[derive(Clone, Copy, Default, Debug, PartialEq, Eq)] + pub struct PagedStat { + pub size: u64, + pub mtime: u64, + /// Moved by every write and resize: the object's data generation. + pub version: u64, } } @@ -3404,11 +3426,12 @@ pub fn pager_create(capacity: u64, dirty: u64, objects: u64, report_ns: u64) -> check(syscall(SYS_PAGER_CREATE, capacity, dirty, objects, report_ns)).map(|raw| RawHandle(raw as u32)) } -/// A paged object of `size` bytes, its pages absent: its id, which `pager`'s -/// calls and notices name it by. `ResourceExhausted` where the pager serves -/// as many objects as it was made for. -pub fn paged_create(pager: RawHandle, size: u64) -> Result { - check(syscall(SYS_PAGED_CREATE, pager.0 as u64, size, 0, 0)) +/// A paged object of `size` bytes last written at `mtime`, its pages absent: +/// its id, which `pager`'s calls and notices name it by. +/// `ResourceExhausted` where the pager serves as many objects as it was made +/// for. +pub fn paged_create(pager: RawHandle, size: u64, mtime: u64) -> Result { + check(syscall(SYS_PAGED_CREATE, pager.0 as u64, size, mtime, 0)) } /// A holder handle on `pager`'s object `object`, carrying `READ`, which may @@ -3431,6 +3454,14 @@ pub fn paged_read(holder: RawHandle, out: &mut [u8], offset: u64) -> Result Result { + let mut out = PagedStat::default(); + check(syscall(SYS_PAGED_STAT, holder.0 as u64, &mut out as *mut PagedStat as u64, 0, 0)).map(|_| out) +} + /// One [`PagedOp`] on one of `pager`'s objects. pub fn paged_op(pager: RawHandle, op: PagedOp, args: &PagedArgs) -> Result { check(syscall(SYS_PAGED_OP, pager.0 as u64, op as u64, args as *const PagedArgs as u64, 0)) diff --git a/sdk/toyos/src/files/owner.rs b/sdk/toyos/src/files/owner.rs index 123eb963a5e..980cf660891 100644 --- a/sdk/toyos/src/files/owner.rs +++ b/sdk/toyos/src/files/owner.rs @@ -34,7 +34,7 @@ enum Place { enum Held { /// A file or directory a file server holds open, and the holder of its /// paged object where the server sent one. - Served { cap: Arc, fid: Fid, position: Position, holder: Option }, + Served { cap: Arc, fid: Fid, position: Position, holder: Option }, /// A directory the kernel serves, by its absolute path. KernelDir(String), } @@ -47,6 +47,22 @@ impl Drop for Held { } } +/// A served file's paged object, and what its open answered of it: an +/// `fstat` is the object's size and mtime over the rest of that, which no +/// change to the file moves. +struct Holder { + paged: crate::paged::Paged, + opened: fs::Stat, +} + +impl Holder { + /// `None` while a write is under way, and the server is asked. + fn stat(&self) -> Option { + let published = self.paged.stat().ok()?; + Some(fs::Stat { size: published.size, mtime: published.mtime, ..self.opened }) + } +} + /// A served file's offset: kept here while this process is the /// description's only holder, so a seek costs no request and a read or a /// write goes at it as a positioned one; the server's from the moment the @@ -302,7 +318,7 @@ fn on_fd_held( match named.map_err(Error::Fd)?? { Err(answer) => answer, Ok(held) => match &*held { - Held::Served { cap, fid, position, holder } => match holder.as_ref().and_then(|h| holding(h, position)) { + Held::Served { cap, fid, position, holder } => match holder.as_ref().and_then(|h| holding(&h.paged, position)) { Some(answer) => Ok(answer), None => served(&mut cap.dir.lock(), *fid, position).map_err(Error::Served), }, @@ -445,7 +461,7 @@ unsafe extern "C" fn toyos_fs_open(at: Fd, path: *const u8, len: usize, how: Ope // it; a pager's process reads through the server. let holder = match opened.holder.map(crate::paged::Paged::received) { None | Some(Err(crate::paged::NotAHolder::Refused | crate::paged::NotAHolder::Ended)) => None, - Some(Ok(holder)) => Some(holder), + Some(Ok(paged)) => Some(Holder { paged, opened: opened.stat }), Some(Err(crate::paged::NotAHolder::Other)) => { dir.close(opened.fid); return Err(Refused::Error(fs::Error::Malformed)); @@ -476,7 +492,10 @@ unsafe extern "C" fn toyos_fs_stat(at: Fd, path: *const u8, len: usize, follow: (At::Fd(fd), true) => match named(fd) { Ok(Named::Handle(handle)) => syscall::fstat(handle).map(Stat::of_kernel).map_err(Error::Kernel), Ok(Named::Held(held)) => match &*held { - Held::Served { cap, fid, .. } => cap.dir.lock().fstat(*fid).map(stat_of).map_err(Error::Served), + Held::Served { cap, fid, holder, .. } => match holder.as_ref().and_then(Holder::stat) { + Some(stat) => Ok(stat_of(stat)), + None => cap.dir.lock().fstat(*fid).map(stat_of).map_err(Error::Served), + }, Held::KernelDir(dir) => syscall::stat(dir.as_bytes(), true).map(Stat::of_kernel).map_err(Error::Kernel), }, Err(e) => Err(e), diff --git a/sdk/toyos/src/paged.rs b/sdk/toyos/src/paged.rs index 1f5bef1e30e..83b31156e38 100644 --- a/sdk/toyos/src/paged.rs +++ b/sdk/toyos/src/paged.rs @@ -12,7 +12,7 @@ use core::time::Duration; use toyos_abi::syscall::{self, PagedArgs, PagedOp, SyscallError, NOTICE_NO_HOLDERS, NOTICE_REQUESTED, PAGED_DIRTY, PAGED_WRITTEN}; -pub use toyos_abi::syscall::{PagedNotice, PAGED_RANGE_PAGES}; +pub use toyos_abi::syscall::{PagedNotice, PagedStat, PAGED_RANGE_PAGES}; use crate::shm::Bytes; use crate::{AsHandle, BorrowedHandle, OwnedHandle}; @@ -81,9 +81,10 @@ impl Pager { })) } - /// An object of `size` bytes, every page absent: its id. - pub fn object(&self, size: u64) -> Result { - syscall::paged_create(self.0.raw(), size).map_err(|e| Refused::of(e, None)) + /// An object of `size` bytes last written at `mtime`, every page absent: + /// its id. + pub fn object(&self, size: u64, mtime: u64) -> Result { + syscall::paged_create(self.0.raw(), size, mtime).map_err(|e| Refused::of(e, None)) } /// A holder handle on object `id`, to send. @@ -101,7 +102,7 @@ impl Pager { /// size or the first absent page after the first; never waits. `out` may /// be a window another process shares. pub fn read(&self, id: u64, out: Bytes<'_>, offset: u64) -> Result { - let args = PagedArgs { object: id, offset, len: out.len() as u64, buf: out.as_mut_ptr() as u64, flags: 0 }; + let args = PagedArgs { object: id, offset, len: out.len() as u64, buf: out.as_mut_ptr() as u64, ..PagedArgs::default() }; self.op(PagedOp::Read, args, Some(SyscallError::WouldBlock)).map(|n| n as usize) } @@ -111,15 +112,15 @@ impl Pager { /// kept. The pages installed. pub fn supply(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result { let flags = if dirty { PAGED_DIRTY } else { 0 }; - let args = PagedArgs { object: id, offset, len: bytes.len() as u64, buf: bytes.as_ptr() as u64, flags }; + let args = PagedArgs { object: id, offset, len: bytes.len() as u64, buf: bytes.as_ptr() as u64, flags, time: 0 }; self.op(PagedOp::Supply, args, None) } /// `bytes` into the present pages of object `id` from `offset`, dirty, - /// the size growing as they land: how many were written before the first - /// absent page. - pub fn write(&self, id: u64, offset: u64, bytes: &[u8]) -> Result { - let args = PagedArgs { object: id, offset, len: bytes.len() as u64, buf: bytes.as_ptr() as u64, flags: 0 }; + /// the size growing as they land and the object stamped `mtime`: how + /// many were written before the first absent page. + pub fn write(&self, id: u64, offset: u64, bytes: &[u8], mtime: u64) -> Result { + let args = PagedArgs { object: id, offset, len: bytes.len() as u64, buf: bytes.as_ptr() as u64, flags: 0, time: mtime }; self.op(PagedOp::Write, args, Some(SyscallError::NotFound)).map(|n| n as usize) } @@ -141,16 +142,21 @@ impl Pager { self.op(PagedOp::Fail, PagedArgs { object: id, offset: first, len: count, ..PagedArgs::default() }, None).map(drop) } - /// The size, as `ftruncate` sets it: a shrink drops every page past it, - /// dirty or not. - pub fn resize(&self, id: u64, size: u64) -> Result<(), Refused> { - self.op(PagedOp::Resize, PagedArgs { object: id, offset: size, ..PagedArgs::default() }, None).map(drop) + /// The size and mtime, as `ftruncate` sets them: a shrink drops every + /// page past it, dirty or not. + pub fn resize(&self, id: u64, size: u64, mtime: u64) -> Result<(), Refused> { + self.op(PagedOp::Resize, PagedArgs { object: id, offset: size, time: mtime, ..PagedArgs::default() }, None).map(drop) + } + + /// The mtime, as `utimens` sets it. + pub fn stamp(&self, id: u64, mtime: u64) -> Result<(), Refused> { + self.op(PagedOp::Stamp, PagedArgs { object: id, time: mtime, ..PagedArgs::default() }, None).map(drop) } /// Begin writing back `count` pages from `first`: a bit per page begun. pub fn writeback_begin(&self, id: u64, first: u64, count: u64) -> Result<[u64; 8], Refused> { let mut bits = [0u64; 8]; - let args = PagedArgs { object: id, offset: first, len: count, buf: bits.as_mut_ptr() as u64, flags: 0 }; + let args = PagedArgs { object: id, offset: first, len: count, buf: bits.as_mut_ptr() as u64, ..PagedArgs::default() }; self.op(PagedOp::WritebackBegin, args, None)?; Ok(bits) } @@ -160,7 +166,7 @@ impl Pager { pub fn writeback_end(&self, id: u64, first: u64, count: u64, written: bool) -> Result<[u64; 8], Refused> { let mut bits = [0u64; 8]; let flags = if written { PAGED_WRITTEN } else { 0 }; - let args = PagedArgs { object: id, offset: first, len: count, buf: bits.as_mut_ptr() as u64, flags }; + let args = PagedArgs { object: id, offset: first, len: count, buf: bits.as_mut_ptr() as u64, flags, time: 0 }; self.op(PagedOp::WritebackEnd, args, None)?; Ok(bits) } @@ -212,4 +218,10 @@ impl Paged { pub fn read(&self, out: &mut [u8], offset: u64) -> Result { syscall::paged_read(self.0.raw(), out, offset) } + + /// The size, mtime and version its object publishes; `WouldBlock` while + /// a write of the pager's is under way, and the pager is asked. + pub fn stat(&self) -> Result { + syscall::paged_stat(self.0.raw()) + } } diff --git a/system/fileserver/src/fat.rs b/system/fileserver/src/fat.rs index ec75dbf9f12..9bd1a71cea2 100644 --- a/system/fileserver/src/fat.rs +++ b/system/fileserver/src/fat.rs @@ -4,8 +4,8 @@ //! **A growing write is two writes: the chain, then the directory entry.** //! Between them the volume holds clusters its entry does not reach, which //! fatgen103 refuses, so an open file changed since it was last level is -//! brought level — its entry written and stamped, its chain reconciled — at -//! its last close, at an fsync and at a sync. **The two are a cache flush +//! brought level — its entry written and stamped with its last write, its +//! chain reconciled — at its last close, at an fsync and at a sync. **The two are a cache flush //! apart**: the cache writes lowest block first and the disk keeps no order //! until a flush, so an entry put into the cache beside its chain and data //! would reach the disk before the clusters it names. @@ -41,6 +41,9 @@ struct Open { /// Written, resized or opened truncating since its entry was last /// brought level. dirty: bool, + /// When it was last written or resized, which its level stamps: its + /// mtime meanwhile. + written: Option, } /// The disk's `ReadOnly` reaches `toyos-fat32` as a refused write, which it @@ -122,6 +125,12 @@ impl FatVolume { FatTime::from_unix_secs((self.clock)() / NANOS_PER_SEC) } + /// Now as an entry's write time keeps it: the even second at or before. + fn write_time(&self) -> FatTime { + let secs = (self.clock)() / NANOS_PER_SEC; + FatTime::from_unix_secs(secs - secs % 2) + } + fn entry(&mut self, node: Node) -> Result<&mut Open, fs::Error> { let open = self.open.get_mut(&node).ok_or(fs::Error::Stale)?; if open.gone { @@ -130,15 +139,17 @@ impl FatVolume { Ok(open) } - /// Bring one open file's entry level with its chain, stamped now, when - /// it changed since the entry was last written. + /// Bring one open file's entry level with its chain, stamped with its + /// last write (or now, for a chain alone), when it changed since the + /// entry was last written. fn level(&mut self, node: Node) -> Result<(), fs::Error> { match self.open.get(&node) { Some(open) if !open.gone && (open.dirty || open.file.needs_reconcile()) => {} _ => return Ok(()), } - let time = self.time(); + let now = self.time(); let open = self.open.get_mut(&node).expect("just found"); + let time = open.written.unwrap_or(now); // The chain and the data on the disk before the entry that reaches // them is in the cache. if open.file.needs_reconcile() { @@ -149,6 +160,7 @@ impl FatVolume { self.cache.block_on(self.fs.reconcile(&mut open.file, time)).map_err(|e| logged("a reconcile", &open.path, e))?; } open.dirty = false; + open.written = None; Ok(()) } @@ -185,10 +197,10 @@ impl Volume for FatVolume { if path.is_empty() { return Ok(Meta { kind: Kind::Dir, size: 0, mtime: 0 }); } - if let Some(open) = self.by_path.get(path).and_then(|n| self.open.get(n)) { - let len = open.file.len(); - let mtime = self.cache.block_on(self.fs.metadata(path)).map_err(|e| logged("metadata", path, e))?.modified_unix; - return Ok(Meta { kind: Kind::File, size: len, mtime: mtime * NANOS_PER_SEC }); + if let Some(&node) = self.by_path.get(path) { + if self.open.contains_key(&node) { + return self.node_meta(node); + } } let meta = self.cache.block_on(self.fs.metadata(path)).map_err(|e| match e { Error::NotADirectory => fs::Error::NotFound, @@ -245,7 +257,7 @@ impl Volume for FatVolume { }; let node = self.next; self.next += 1; - self.open.insert(node, Open { path: path.to_string(), file, holders: 0, gone: false, dirty: false }); + self.open.insert(node, Open { path: path.to_string(), file, holders: 0, gone: false, dirty: false, written: None }); self.by_path.insert(path.to_string(), node); node } @@ -253,7 +265,10 @@ impl Volume for FatVolume { self.open.get_mut(&node).expect("just found or made").holders += 1; if how.truncate { // POSIX's `open` stamps a truncated file even when it was empty. - self.open.get_mut(&node).expect("just found or made").dirty = true; + let now = self.write_time(); + let open = self.open.get_mut(&node).expect("just found or made"); + open.dirty = true; + open.written = Some(now); if let Err(e) = self.truncate(node, 0) { // A refused close is said, and its node kept for the next sync. let _ = self.close(node); @@ -281,6 +296,9 @@ impl Volume for FatVolume { fn node_meta(&mut self, node: Node) -> Result { let open = self.entry(node)?; + if let Some(written) = open.written { + return Ok(Meta { kind: Kind::File, size: open.file.len(), mtime: written.to_unix_secs() * NANOS_PER_SEC }); + } let (path, size) = (open.path.clone(), open.file.len()); let mtime = self.cache.block_on(self.fs.metadata(&path)).map_err(|e| logged("metadata", &path, e))?.modified_unix; Ok(Meta { kind: Kind::File, size, mtime: mtime * NANOS_PER_SEC }) @@ -306,11 +324,13 @@ impl Volume for FatVolume { } fn reserve(&mut self, node: Node, offset: u64, len: u64) -> Result { + let now = self.write_time(); let open = self.open.get_mut(&node).ok_or(fs::Error::Stale)?; if open.gone { return Err(fs::Error::Stale); } open.dirty = true; + open.written = Some(now); // A chain has no holes: it holds a block for every page of its size. let reserved = Reserved { size: open.file.len(), held: open.file.len().div_ceil(toyos_blockdev::BLOCK as u64) }; self.cache.block_on(self.fs.reserve(&mut open.file, offset, len)).map_err(|e| logged("reserve", &open.path, e))?; @@ -327,6 +347,7 @@ impl Volume for FatVolume { } fn truncate(&mut self, node: Node, size: u64) -> Result<(), fs::Error> { + let now = self.write_time(); let open = self.open.get_mut(&node).ok_or(fs::Error::Stale)?; if open.gone { return Err(fs::Error::Stale); @@ -335,6 +356,7 @@ impl Volume for FatVolume { return Ok(()); } open.dirty = true; + open.written = Some(now); self.cache.block_on(self.fs.set_len(&mut open.file, size)).map_err(|e| logged("set_len", &open.path, e)) } @@ -555,26 +577,32 @@ mod tests { (v, refused, spec) } - /// What [`the_flush_stamps_its_even_second_and_not_the_write_s`]'s volume - /// reads as the time, which only that test sets. + /// What [`a_files_mtime_is_its_last_write_s_through_its_handle_and_after_a_reopen`]'s + /// volume reads as the time, which only that test sets. static NOW: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - /// FAT stamps a file's entry with the flush that levels it, read back off - /// the disk: the whole second of that flush, kept to the even one at or - /// before it (`FatTime`'s two-second resolution), and never the write's. + /// A FAT file's mtime is its last write's, kept to the even second at or + /// before it (`FatTime`'s two-second resolution): through its handle + /// before its entry is level, and read back off the disk after a sync + /// later and a reopen, alike. #[test] - fn the_flush_stamps_its_even_second_and_not_the_write_s() { + fn a_files_mtime_is_its_last_write_s_through_its_handle_and_after_a_reopen() { use std::sync::atomic::Ordering::Relaxed; const CREATE: OpenHow = OpenHow { create: true, create_new: false, truncate: false }; let (mut v, _, _) = spec_fat_on(|| NOW.load(Relaxed)); - NOW.store(1_717_245_296 * NANOS_PER_SEC, Relaxed); + NOW.store(1_717_245_101 * NANOS_PER_SEC, Relaxed); let a = v.open("a.txt", CREATE).unwrap(); + NOW.store(1_717_245_297 * NANOS_PER_SEC, Relaxed); v.write(a, 0, &[1; 3000]).unwrap(); + let written = 1_717_245_296 * NANOS_PER_SEC; + assert_eq!(v.node_meta(a).unwrap().mtime, written, "through the handle"); NOW.store(1_717_245_301 * NANOS_PER_SEC + 500_000_000, Relaxed); assert_eq!(v.sync(), Ok(Vec::new())); v.close(a).unwrap(); evict(&v); - assert_eq!(v.lstat("a.txt").unwrap().mtime, 1_717_245_300 * NANOS_PER_SEC); + assert_eq!(v.lstat("a.txt").unwrap().mtime, written, "off the disk, a sync later"); + let again = v.open("a.txt", CREATE).unwrap(); + assert_eq!(v.node_meta(again).unwrap().mtime, written, "through a handle after a reopen"); } /// A write is owed until a commit, and a commit naming another file diff --git a/system/fileserver/src/main.rs b/system/fileserver/src/main.rs index e4d46290ecc..fbbf489718d 100644 --- a/system/fileserver/src/main.rs +++ b/system/fileserver/src/main.rs @@ -1134,7 +1134,7 @@ impl Server { // The volume truncated first, so a refused open changed // nothing; the object follows before anything writes back. if how.truncate && found.is_some() { - let truncated = self.volume.identity(&path).and_then(|identity| self.files.truncated(identity)); + let truncated = self.volume.identity(&path).and_then(|identity| self.files.truncated(&mut *self.volume, identity, node)); if let Err(e) = truncated { let _ = self.volume.close(node); return Err(e); @@ -1243,8 +1243,13 @@ impl Server { Time::At(nanos) => nanos, }; if held { - let node = self.fid(id, r.fid)?.borrow().node().map_err(|_| Error::Unsupported)?; + let file = self.fid(id, r.fid)?; + let (node, identity) = { + let f = file.borrow(); + (f.node().map_err(|_| Error::Unsupported)?, f.identity) + }; self.volume.set_mtime(node, mtime)?; + self.files.stamped(&mut *self.volume, identity, node)?; } else { let path = match self.resolve(id, r.fid, &rel, r.flags & FOLLOW_LAST != 0)? { Resolved::Absolute(p) => return self.follow(id, r, &p), @@ -1253,8 +1258,9 @@ impl Server { if self.volume.lstat(&path)?.kind != Kind::File { return Err(Error::Unsupported); } + let identity = self.volume.identity(&path)?; let node = self.volume.open(&path, OpenHow { create: false, create_new: false, truncate: false })?; - let set = self.volume.set_mtime(node, mtime); + let set = self.volume.set_mtime(node, mtime).and_then(|()| self.files.stamped(&mut *self.volume, identity, node)); let closed = self.volume.close(node); set.and(closed)?; } diff --git a/system/fileserver/src/pages.rs b/system/fileserver/src/pages.rs index 490439887f8..70f14537bbf 100644 --- a/system/fileserver/src/pages.rs +++ b/system/fileserver/src/pages.rs @@ -15,7 +15,9 @@ //! a call that stops short leaves it under way until its end, so no holder //! reads the file half written (`toyos::paged::Pager::end_write`). //! -//! **The volume's size is the object's.** A write reserves on the volume +//! **The volume's size and mtime are the object's**, which its holders +//! `fstat` without a request: each call that moves the volume's mtime hands +//! the object the volume's own value, at its resolution. A write reserves on the volume //! first, so the volume's size runs past the object's only for the bytes the //! write then puts in before it returns; nothing reads the volume for them //! meanwhile, and a write refused part way is cut back to what it put. @@ -36,22 +38,23 @@ const RANGE: u64 = PAGED_RANGE_PAGES; /// A pager's calls on its objects, by id, as `toyos::paged::Pager` makes them. pub trait Pager { - fn object(&self, size: u64) -> Result; + fn object(&self, size: u64, mtime: u64) -> Result; fn read(&self, id: u64, out: Bytes<'_>, offset: u64) -> Result; fn supply(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result; - fn write(&self, id: u64, offset: u64, bytes: &[u8]) -> Result; + fn write(&self, id: u64, offset: u64, bytes: &[u8], mtime: u64) -> Result; fn end_write(&self, id: u64) -> Result<(), Refused>; fn withdraw(&self, id: u64, first: u64, count: u64) -> Result; fn fail(&self, id: u64, first: u64, count: u64) -> Result<(), Refused>; - fn resize(&self, id: u64, size: u64) -> Result<(), Refused>; + fn resize(&self, id: u64, size: u64, mtime: u64) -> Result<(), Refused>; + fn stamp(&self, id: u64, mtime: u64) -> Result<(), Refused>; fn writeback_begin(&self, id: u64, first: u64, count: u64) -> Result<[u64; 8], Refused>; fn writeback_end(&self, id: u64, first: u64, count: u64, written: bool) -> Result<[u64; 8], Refused>; fn release(&self, id: u64) -> Result<(), Refused>; } impl Pager for toyos::paged::Pager { - fn object(&self, size: u64) -> Result { - toyos::paged::Pager::object(self, size) + fn object(&self, size: u64, mtime: u64) -> Result { + toyos::paged::Pager::object(self, size, mtime) } fn read(&self, id: u64, out: Bytes<'_>, offset: u64) -> Result { toyos::paged::Pager::read(self, id, out, offset) @@ -59,8 +62,8 @@ impl Pager for toyos::paged::Pager { fn supply(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result { toyos::paged::Pager::supply(self, id, offset, bytes, dirty) } - fn write(&self, id: u64, offset: u64, bytes: &[u8]) -> Result { - toyos::paged::Pager::write(self, id, offset, bytes) + fn write(&self, id: u64, offset: u64, bytes: &[u8], mtime: u64) -> Result { + toyos::paged::Pager::write(self, id, offset, bytes, mtime) } fn end_write(&self, id: u64) -> Result<(), Refused> { toyos::paged::Pager::end_write(self, id) @@ -71,8 +74,11 @@ impl Pager for toyos::paged::Pager { fn fail(&self, id: u64, first: u64, count: u64) -> Result<(), Refused> { toyos::paged::Pager::fail(self, id, first, count) } - fn resize(&self, id: u64, size: u64) -> Result<(), Refused> { - toyos::paged::Pager::resize(self, id, size) + fn resize(&self, id: u64, size: u64, mtime: u64) -> Result<(), Refused> { + toyos::paged::Pager::resize(self, id, size, mtime) + } + fn stamp(&self, id: u64, mtime: u64) -> Result<(), Refused> { + toyos::paged::Pager::stamp(self, id, mtime) } fn writeback_begin(&self, id: u64, first: u64, count: u64) -> Result<[u64; 8], Refused> { toyos::paged::Pager::writeback_begin(self, id, first, count) @@ -169,13 +175,14 @@ impl Files

{ Some(f) } - /// After an open of `identity` that truncated: its object holds no page - /// of before. The volume went first, so a refused open changed nothing, - /// and nothing writes back between the two. - pub fn truncated(&mut self, identity: u64) -> Result<(), Error> { + /// After an open of `identity` at `node` that truncated: its object holds + /// no page of before. The volume went first, so a refused open changed + /// nothing, and nothing writes back between the two. + pub fn truncated(&mut self, volume: &mut dyn Volume, identity: u64, node: Node) -> Result<(), Error> { let Some(f) = self.files.get(&identity) else { return Ok(()) }; let id = f.object; - self.pg().resize(id, 0).map_err(word)?; + let mtime = volume.node_meta(node)?.mtime; + self.pg().resize(id, 0, mtime).map_err(word)?; self.files.get_mut(&identity).expect("found").dirty = None; Ok(()) } @@ -196,11 +203,11 @@ impl Files

{ Some(_) => self.forget(volume, identity), None => {} } - let size = volume.node_meta(node)?.size; + let meta = volume.node_meta(node)?; // Objects are bounded by count as pages are: a file nobody holds goes // to make room for another's. let object = loop { - match self.pg().object(size) { + match self.pg().object(meta.size, meta.mtime) { Ok(object) => break object, Err(Refused::NoRoom) if self.drop_oldest(volume, identity) => {} Err(why) => return Err(word(why)), @@ -376,15 +383,30 @@ impl Files

{ } let reserved = volume.reserve(node, offset, data.len() as u64)?; let old = reserved.size; + let mtime = match volume.node_meta(node) { + Ok(meta) => meta.mtime, + Err(e) => { + if let Err(again) = volume.unreserve(node, reserved, 0) { + println!("fileserver: a write of file {identity} was not cut back to {old} ({again:?}), after {e:?}"); + } + return Err(e); + } + }; self.touch(identity); let id = self.files[&identity].object; let mut put = Put { done: 0, held: false }; - let written = self.put(volume, &Writing { identity, node, old, contents: reserved.contents(), offset, data }, &mut put); + let writing = Writing { identity, node, old, contents: reserved.contents(), offset, data, mtime }; + let written = self.put(volume, &writing, &mut put); if let Err(e) = &written { // What was reserved and not put would read as the blocks' old bytes. let landed = if put.done > 0 { offset + put.done as u64 } else { 0 }; let cut = old.max(landed); - let resized = self.pg().resize(id, cut).map_err(word).and(volume.unreserve(node, reserved, landed)); + let resized = self + .pg() + .resize(id, cut, mtime) + .map_err(word) + .and(volume.unreserve(node, reserved, landed)) + .and_then(|()| self.stamped(volume, identity, node)); if let Err(again) = resized { println!("fileserver: a refused write of file {identity} was not cut back to {cut} ({again:?}), after {e:?}"); } @@ -402,13 +424,13 @@ impl Files

{ /// counting into `put` what landed and whether a call left the write /// under way. fn put(&mut self, volume: &mut dyn Volume, w: &Writing<'_>, put: &mut Put) -> Result<(), Error> { - let Writing { identity, node, old, contents, offset, data } = *w; + let Writing { identity, node, old, contents, offset, data, mtime } = *w; let end = offset + data.len() as u64; let done = &mut put.done; while *done < data.len() { let at = offset + *done as u64; let id = self.files.get(&identity).ok_or(Error::Stale)?.object; - let wrote = self.pg().write(id, at, &data[*done..]); + let wrote = self.pg().write(id, at, &data[*done..], mtime); // A first call that takes everything is the whole write; any later // call is under the write a short first one left under way. put.held |= wrote != Ok(data.len() - *done); @@ -468,18 +490,31 @@ impl Files

{ } } + /// `identity`'s volume moved its mtime, which `node` reads: its object + /// follows. + pub fn stamped(&mut self, volume: &mut dyn Volume, identity: u64, node: Node) -> Result<(), Error> { + let Some(f) = self.files.get(&identity) else { return Ok(()) }; + let id = f.object; + let mtime = volume.node_meta(node)?.mtime; + self.pg().stamp(id, mtime).map_err(word) + } + /// `node`'s file cut or grown to `size`: the volume first, so a refusal /// changes nothing, then the object, before anything can write a page /// past the size back. pub fn truncate(&mut self, volume: &mut dyn Volume, identity: u64, node: Node, size: u64) -> Result<(), Error> { volume.truncate(node, size)?; - let Some(f) = self.files.get_mut(&identity) else { return Ok(()) }; + if !self.files.contains_key(&identity) { + return Ok(()); + } + let mtime = volume.node_meta(node)?.mtime; + let f = self.files.get_mut(&identity).expect("found"); if let Some((lo, hi)) = f.dirty { let hi = hi.min(size.div_ceil(PAGE)); f.dirty = (lo < hi).then_some((lo, hi)); } let id = f.object; - self.pg().resize(id, size).map_err(word) + self.pg().resize(id, size, mtime).map_err(word) } /// `identity`'s dirty pages written to the volume. A page dirtied again @@ -622,6 +657,8 @@ struct Writing<'a> { contents: u64, offset: u64, data: &'a [u8], + /// The file's mtime from this write, as the volume keeps it. + mtime: u64, } /// The runs of set bits among the first `count` of `bits`: where each starts diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index a35bdceb392..216f35ec08b 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -20,6 +20,7 @@ struct Model { next: u64, pages: BTreeMap<(u64, u64), Page>, sizes: BTreeMap, + mtimes: BTreeMap, supplies: u32, /// Every supply from here is refused. refusing: bool, @@ -129,7 +130,7 @@ impl Fake { } impl Pager for Fake { - fn object(&self, size: u64) -> Result { + fn object(&self, size: u64, mtime: u64) -> Result { let mut m = self.0.borrow_mut(); if m.sizes.len() >= m.objects { return Err(Refused::NoRoom); @@ -137,6 +138,7 @@ impl Pager for Fake { m.next += 1; let id = m.next; m.sizes.insert(id, size); + m.mtimes.insert(id, mtime); Ok(id) } @@ -172,7 +174,8 @@ impl Pager for Fake { answer } - fn write(&self, id: u64, offset: u64, bytes: &[u8]) -> Result { + fn write(&self, id: u64, offset: u64, bytes: &[u8], mtime: u64) -> Result { + self.0.borrow_mut().mtimes.insert(id, mtime); let answer = self.write_pages(id, offset, bytes); self.0.borrow_mut().changed(id, answer == Ok(bytes.len())); answer @@ -198,8 +201,14 @@ impl Pager for Fake { Ok(()) } - fn resize(&self, id: u64, size: u64) -> Result<(), Refused> { + fn stamp(&self, id: u64, mtime: u64) -> Result<(), Refused> { + self.0.borrow_mut().mtimes.insert(id, mtime); + Ok(()) + } + + fn resize(&self, id: u64, size: u64, mtime: u64) -> Result<(), Refused> { let mut m = self.0.borrow_mut(); + m.mtimes.insert(id, mtime); m.pages.retain(|(c, p), _| *c != id || *p < size.div_ceil(PAGE)); if let Some(page) = m.pages.get_mut(&(id, size / PAGE)) { page.bytes[(size % PAGE) as usize..].fill(0); @@ -464,7 +473,7 @@ fn a_truncating_open_leaves_the_object_no_page_of_before() { let truncate = OpenHow { create: false, create_new: false, truncate: true }; let again = v.open("home/f", truncate).unwrap(); - files.truncated(id).unwrap(); + files.truncated(&mut v, id, again).unwrap(); files.opened(&mut v, id, again).unwrap(); let object = files.object(id).unwrap(); assert_eq!((fake.0.borrow().sizes[&object], fake.pages_of(object)), (0, 0), "a holder reads the object, and finds nothing of before"); @@ -652,3 +661,41 @@ fn a_directorys_fsync_makes_a_new_name_durable() { let mut v = after_a_stop(v); assert_eq!(v.lstat("home/d").map(|m| m.kind), Ok(crate::volume::Kind::Dir)); } + +static NOW: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1_000_000_000); + +/// A clock a step ahead at each reading, so each change stamps a new time. +fn ticking() -> u64 { + NOW.fetch_add(1_000_000_007, std::sync::atomic::Ordering::Relaxed) +} + +/// What a holder's `fstat` reads is the object's size and mtime: after an +/// open, a write, one refused, a truncate and a stamp, each is the volume's. +#[test] +fn an_objects_size_and_mtime_are_the_volumes_after_every_change() { + let mut v = DataVolume::format(Ram::new(4096), 256, &["home"], ticking).unwrap(); + let fake = Fake::new(64, 16); + let mut files = Files::new(Some(fake.clone())); + let n = v.open("home/f", CREATE).unwrap(); + v.write(n, 0, b"before").unwrap(); + v.close(n).unwrap(); + let (node, id) = open(&mut files, &mut v, "home/f"); + let object = files.object(id).unwrap(); + let published = |v: &mut DataVolume, what: &str| { + let m = fake.0.borrow(); + let meta = v.node_meta(node).unwrap(); + assert_eq!((m.sizes[&object], m.mtimes[&object]), (meta.size, meta.mtime), "after {what}"); + }; + published(&mut v, "the open"); + files.write(&mut v, id, node, 3 * PAGE, b"after").unwrap(); + published(&mut v, "a write"); + fake.0.borrow_mut().refusing = true; + assert!(files.write(&mut v, id, node, 9 * PAGE, &pattern(2 * PAGE as usize, 1)).is_err()); + fake.0.borrow_mut().refusing = false; + published(&mut v, "a refused write"); + files.truncate(&mut v, id, node, 100).unwrap(); + published(&mut v, "a truncate"); + v.set_mtime(node, 42_000_000_000).unwrap(); + files.stamped(&mut v, id, node).unwrap(); + published(&mut v, "a stamp"); +} diff --git a/tests/guest/pagerprobe/src/main.rs b/tests/guest/pagerprobe/src/main.rs index 57f696e5320..b8a70a7515a 100644 --- a/tests/guest/pagerprobe/src/main.rs +++ b/tests/guest/pagerprobe/src/main.rs @@ -39,7 +39,7 @@ fn main() { return hold(); } let pager = Pager::create(LARGE + 16, 0, 4, Duration::from_secs(1)).expect("pagerprobe: its row starts it as a pager"); - let large = pager.object(LARGE * PAGE).expect("pagerprobe: the large object"); + let large = pager.object(LARGE * PAGE, 0).expect("pagerprobe: the large object"); fill(&pager, large); assert_eq!(pager.withdraw(large, 0, u64::MAX), Ok(LARGE), "pagerprobe: the whole object withdrawn at once"); fill(&pager, large); @@ -49,10 +49,10 @@ fn main() { Err(Refused::Kernel(SyscallError::InvalidArgument)), "pagerprobe: a second release of one object was taken" ); - let left = pager.object(LARGE * PAGE).expect("pagerprobe: the object the pager's end takes"); + let left = pager.object(LARGE * PAGE, 0).expect("pagerprobe: the object the pager's end takes"); fill(&pager, left); println!("pagerprobe: more than 512 MiB in one object withdrawn, released and left for the end"); - let object = pager.object(4096).expect("pagerprobe: an object"); + let object = pager.object(4096, 0).expect("pagerprobe: an object"); let own = pager.holder(object).expect("pagerprobe: a holder for itself"); // The handle is a holder, and this process a pager: refused by the kernel. assert_eq!( diff --git a/tests/guest/rust-tests/src/bin/a_held_files_fstat_is_its_servers_stat.rs b/tests/guest/rust-tests/src/bin/a_held_files_fstat_is_its_servers_stat.rs new file mode 100644 index 00000000000..5e760d8b63a --- /dev/null +++ b/tests/guest/rust-tests/src/bin/a_held_files_fstat_is_its_servers_stat.rs @@ -0,0 +1,41 @@ +//! A served file's `fstat` through a descriptor holding its paged object is +//! answered by the kernel from what the object publishes, without a request. +//! After a write, a truncate and a set time, it agrees with the server's +//! `stat` of the file's path on size and mtime, and on every field the open +//! answered. +//! +//! A guest test because the two answers come from two components, the +//! kernel's object and the file server's volume, which only a guest runs +//! together; each side's rule is host-tested (`fileserver` pages, `fat`). + +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::os::toyos::fs::MetadataExt; +use std::time::{Duration, SystemTime}; + +const FILE: &str = "/home/a-held-files-fstat"; + +fn agree(f: &File, what: &str) { + let held = f.metadata().expect("fstat"); + let served = fs::metadata(FILE).expect("stat"); + let fields = |m: &fs::Metadata| (m.len(), m.modified().ok(), m.is_file(), m.ino(), m.dev(), m.nlink()); + assert_eq!(fields(&held), fields(&served), "after {what}: the held fstat, then the server's stat"); +} + +fn main() { + let _ = fs::remove_file(FILE); + let mut f = OpenOptions::new().read(true).write(true).create(true).truncate(true).open(FILE).expect("open"); + agree(&f, "the open"); + f.write_all(&[7u8; 5000]).expect("write"); + agree(&f, "a write"); + assert_eq!(f.metadata().unwrap().len(), 5000, "the write's size"); + f.set_len(100).expect("truncate"); + agree(&f, "a truncate"); + let then = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000); + f.set_modified(then).expect("set the mtime"); + agree(&f, "a set time"); + assert_eq!(f.metadata().unwrap().modified().unwrap(), then, "the time set"); + drop(f); + fs::remove_file(FILE).unwrap(); + println!("a_held_files_fstat_is_its_servers_stat: passed"); +} From ca6340ef3b8ba8985a0a5ad75fe52defa7ead4e5 Mon Sep 17 00:00:00 2001 From: japabu Date: Sun, 1 Nov 2026 20:54:44 +0100 Subject: [PATCH 07/93] issues: FS stage 3b waits for memory stage 7b's mappings The orchestrator's ruling on the stage-3 split. The client name cache comes on mapped paged objects, with file slots, since stat by path of an existing file is the row that matters. Built before mappings, its slots would be read by syscall and it would need a names-only pager for SYSTEM and a dirty allowance for BOOT: an interim mechanism the mappings would replace. 3a, a held descriptor's fstat, is this PR. Co-Authored-By: Claude Opus 5.5 --- ...rage-is-layers-and-a-role-is-a-filesystem.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/issues/storage-is-layers-and-a-role-is-a-filesystem.md b/issues/storage-is-layers-and-a-role-is-a-filesystem.md index 077dce9a3dd..3252db436e7 100644 --- a/issues/storage-is-layers-and-a-role-is-a-filesystem.md +++ b/issues/storage-is-layers-and-a-role-is-a-filesystem.md @@ -140,6 +140,23 @@ keeps the volumes' structures alone. Mapping them is memory stage 7b, the process-memory track's (`issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md`). +**A served file's metadata without a request**, in two stages: + +- **3a:** a held descriptor's `fstat` is its paged object's size, mtime and + version (`SYS_PAGED_STAT`). The version is the object's write span, and the + one data generation. +- **3b, deferred until memory stage 7b's mappings:** a client cache of names, + validated by a per-connection page the file server writes. The page carries + generations of the directories in the connection's view and of the files + it has answered about, so that `stat` of an existing file's path hits the + cache; that row is what stands against Linux's 1.5 µs. Its slots carry + identity and generation, and a client compares both under acquire. Server + death clears the page through the paged object's `Gone`. + - Why deferred: building it before mappings means reading slots by + syscall, plus a names-only pager for SYSTEM and a dirty allowance for + BOOT. That is an interim mechanism the mappings would replace. + - Exit: `fsrtt.std.stat` and `open_close`, measured on both benches. + LOG stays its own FAT32 partition through the dev phase, because a Mac has to read the stick; folding it into DATA in the installed product is deferred by the owner's ruling of 2026-09-26, and no stage above owes it yet. From 4f2efdb462fb3b6b3cd42d2f0da97c9e7b605e26 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 16:45:35 +0100 Subject: [PATCH 08/93] WIP: move volatile files to anonymous userland objects Save in-progress implementation before integrating paged-object fixes. --- images/system.toml | 2 +- kernel/pure/dirnodes.rs | 207 ---------- kernel/pure/lib.rs | 1 - kernel/pure/paged.rs | 44 +- kernel/src/actuator.rs | 2 - kernel/src/file_cache.rs | 572 -------------------------- kernel/src/fs_rename.rs | 66 --- kernel/src/loader/mod.rs | 23 +- kernel/src/main.rs | 17 - kernel/src/object/file.rs | 39 -- kernel/src/object/mod.rs | 3 - kernel/src/object/ops.rs | 280 +------------ kernel/src/object/paged.rs | 4 +- kernel/src/revoke_selftest.rs | 55 --- kernel/src/syscall/dispatch.rs | 102 +---- kernel/src/syscall/fs.rs | 194 --------- kernel/src/syscall/mod.rs | 1 - kernel/src/syscall/paged.rs | 4 +- kernel/src/syscall/proc.rs | 11 +- kernel/src/tmpfs.rs | 247 ----------- kernel/src/user_ptr.rs | 25 -- kernel/src/vfs.rs | 699 -------------------------------- lib/manifest/src/lib.rs | 9 +- sdk/abi/src/syscall.rs | 6 + sdk/toyos/src/paged.rs | 6 + system/fileserver/src/lib.rs | 1 + system/fileserver/src/main.rs | 25 +- system/fileserver/src/memory.rs | 252 ++++++++++++ system/fileserver/src/pages.rs | 41 +- system/fileserver/src/volume.rs | 7 + system/supervisor/src/main.rs | 1 + 31 files changed, 405 insertions(+), 2541 deletions(-) delete mode 100644 kernel/pure/dirnodes.rs delete mode 100644 kernel/src/file_cache.rs delete mode 100644 kernel/src/fs_rename.rs delete mode 100644 kernel/src/object/file.rs delete mode 100644 kernel/src/revoke_selftest.rs delete mode 100644 kernel/src/syscall/fs.rs delete mode 100644 kernel/src/tmpfs.rs delete mode 100644 kernel/src/vfs.rs create mode 100644 system/fileserver/src/memory.rs diff --git a/images/system.toml b/images/system.toml index 08e6eb7d06b..6886bbb8aa3 100644 --- a/images/system.toml +++ b/images/system.toml @@ -376,5 +376,5 @@ receives = ["disk", "input"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] diff --git a/kernel/pure/dirnodes.rs b/kernel/pure/dirnodes.rs deleted file mode 100644 index dc745487cbd..00000000000 --- a/kernel/pure/dirnodes.rs +++ /dev/null @@ -1,207 +0,0 @@ -//! The numbers a kernel volume gives its directories -//! (`toyos_abi::syscall::Stat::node`). -//! -//! ROOT and `/tmp` keep no record of a directory: one is the prefix its names -//! share, or a name `mkdir` left in the VFS. So the VFS numbers each by its -//! absolute path, drawing the number on the first ask, and forgets it once a -//! removal beneath the path leaves no directory there: a number is never -//! another directory's while both exist. The paths are userland's choice, so -//! each is at most [`MAX_PATH`] bytes and charged to whoever caused it, the -//! charge `C` held beside its number until it is forgotten. - -#![forbid(unsafe_code)] - -use alloc::collections::BTreeMap; -use alloc::string::String; -use toyos_abi::syscall::{SyscallError, MAX_PATH}; - -pub struct Directories { - of: BTreeMap, -} - -impl Default for Directories { - fn default() -> Self { - Self::new() - } -} - -impl Directories { - pub const fn new() -> Self { - Self { of: BTreeMap::new() } - } - - /// The number of the directory at `path`, drawn by `draw` when this does - /// not hold one; `draw` answers a number nothing else on its volume holds. - /// A path longer than [`MAX_PATH`] is refused `InvalidArgument`, as `cd` - /// and `mkdir` refuse it, and a new one whose `charge` is refused with - /// that refusal; either draws nothing and holds nothing. - pub fn of( - &mut self, - path: &str, - draw: impl FnOnce() -> u64, - charge: impl FnOnce(&str) -> Result, - ) -> Result { - if let Some(&(number, _)) = self.of.get(path) { - return Ok(number); - } - if path.len() > MAX_PATH { - return Err(SyscallError::InvalidArgument); - } - let charge = charge(path)?; - let number = draw(); - self.of.insert(String::from(path), (number, charge)); - Ok(number) - } - - /// A name at `path` was removed or moved away: `path` and every directory - /// above it whose number this holds and that `exists` no longer answers for - /// is forgotten, so a directory made there later is drawn a new one. The - /// walk begins at the longest of them within [`MAX_PATH`], since nothing - /// longer is held, so its length is bounded whatever `path`'s is. - pub fn removed(&mut self, path: &str, mut exists: impl FnMut(&str) -> bool) { - let mut at = path; - if at.len() > MAX_PATH { - match at.as_bytes()[..=MAX_PATH].iter().rposition(|&b| b == b'/') { - Some(0) | None => return, - Some(slash) => at = &at[..slash], - } - } - loop { - if self.of.contains_key(at) && !exists(at) { - self.of.remove(at); - } - match at.rfind('/') { - Some(0) | None => return, - Some(slash) => at = &at[..slash], - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use alloc::format; - use std::cell::Cell; - use std::vec::Vec; - - /// A counter that never rewinds, as the file cache's is. - fn counter() -> impl FnMut() -> u64 { - let mut next = 0; - move || { - next += 1; - next - } - } - - fn never() -> u64 { - panic!("a held number is never drawn again, and a refused one never drawn") - } - - /// A charge that is never refused. - fn free(_: &str) -> Result<(), SyscallError> { - Ok(()) - } - - /// Asked twice, a directory answers one number; two directories answer two. - #[test] - fn a_directory_is_one_number_and_two_are_two() { - let mut draw = counter(); - let mut dirs = Directories::<()>::new(); - let a = dirs.of("/tmp/a", &mut draw, free).unwrap(); - let b = dirs.of("/tmp/b", &mut draw, free).unwrap(); - assert_ne!(a, b); - assert_eq!(dirs.of("/tmp/a", never, free), Ok(a)); - } - - /// A removal that leaves the directory standing keeps its number; one that - /// ends it forgets it, and the directory made there next is a new one. - #[test] - fn a_number_lasts_exactly_as_long_as_its_directory() { - let mut draw = counter(); - let mut dirs = Directories::<()>::new(); - let a = dirs.of("/tmp/a", &mut draw, free).unwrap(); - - dirs.removed("/tmp/a/f", |_| true); - assert_eq!(dirs.of("/tmp/a", never, free), Ok(a)); - - dirs.removed("/tmp/a/f", |dir| dir != "/tmp/a"); - let again = dirs.of("/tmp/a", &mut draw, free).unwrap(); - assert_ne!(again, a, "a directory made again where one was removed is another directory"); - } - - /// Every directory above the removal is judged, not only its parent, and - /// one that only shares a prefix with it is not. - #[test] - fn a_removal_forgets_every_directory_it_ended_and_no_other() { - let mut draw = counter(); - let mut dirs = Directories::<()>::new(); - let tmp = dirs.of("/tmp", &mut draw, free).unwrap(); - let a = dirs.of("/tmp/a", &mut draw, free).unwrap(); - let b = dirs.of("/tmp/a/b", &mut draw, free).unwrap(); - let sibling = dirs.of("/tmp/ab", &mut draw, free).unwrap(); - - dirs.removed("/tmp/a/b/f", |dir| dir == "/tmp" || dir == "/tmp/ab"); - assert_eq!(dirs.of("/tmp", never, free), Ok(tmp)); - assert_eq!(dirs.of("/tmp/ab", never, free), Ok(sibling)); - let (a2, b2) = (dirs.of("/tmp/a", &mut draw, free).unwrap(), dirs.of("/tmp/a/b", &mut draw, free).unwrap()); - assert!(![tmp, a, b, sibling].contains(&a2) && ![tmp, a, b, sibling, a2].contains(&b2)); - } - - /// The mount's directories are asked about only where a number is held: - /// a removal costs nothing for what nobody numbered. - #[test] - fn only_a_held_number_is_judged() { - let mut dirs = Directories::<()>::new(); - dirs.of("/tmp/a", || 7, free).unwrap(); - let asked = Cell::new(Vec::new()); - dirs.removed("/tmp/a/b/c/f", |dir| { - let mut seen = asked.take(); - seen.push(String::from(dir)); - asked.set(seen); - true - }); - assert_eq!(asked.take(), [String::from("/tmp/a")]); - } - - /// A path of `MAX_PATH` bytes is numbered; one byte more is refused by - /// name, and nothing is drawn or held for it. - #[test] - fn a_path_past_max_path_is_refused_invalid() { - let mut draw = counter(); - let mut dirs = Directories::<()>::new(); - let longest = format!("/{}", "a".repeat(MAX_PATH - 1)); - assert!(dirs.of(&longest, &mut draw, free).is_ok()); - let past = format!("{longest}a"); - assert_eq!(dirs.of(&past, never, free), Err(SyscallError::InvalidArgument)); - assert_eq!(dirs.of.len(), 1); - } - - /// A new directory whose charge is refused is refused with it, draws no - /// number and holds nothing; one already numbered still answers. - #[test] - fn a_refused_charge_numbers_nothing() { - let mut draw = counter(); - let mut dirs = Directories::<()>::new(); - let a = dirs.of("/tmp/a", &mut draw, free).unwrap(); - assert_eq!(dirs.of("/tmp/b", never, |_| Err(SyscallError::ResourceExhausted)), Err(SyscallError::ResourceExhausted)); - assert_eq!(dirs.of.len(), 1); - assert_eq!(dirs.of("/tmp/a", never, |_| Err(SyscallError::ResourceExhausted)), Ok(a)); - } - - /// A removal beneath a path longer than `MAX_PATH` still judges every - /// numbered directory above it. - #[test] - fn a_removal_past_max_path_judges_the_numbered_above_it() { - let mut draw = counter(); - let mut dirs = Directories::<()>::new(); - let near = format!("/tmp/{}", "a".repeat(MAX_PATH - 5)); - assert_eq!(near.len(), MAX_PATH); - let top = dirs.of("/tmp", &mut draw, free).unwrap(); - dirs.of(&near, &mut draw, free).unwrap(); - let deep = format!("{near}{}/f", "/b".repeat(MAX_PATH)); - dirs.removed(&deep, |dir| dir == "/tmp"); - assert_eq!(dirs.of("/tmp", never, free), Ok(top)); - assert_ne!(dirs.of(&near, &mut draw, free), Ok(2), "the directory MAX_PATH long ended and kept its number"); - } -} diff --git a/kernel/pure/lib.rs b/kernel/pure/lib.rs index 23ebcd9b04d..4834e491320 100644 --- a/kernel/pure/lib.rs +++ b/kernel/pure/lib.rs @@ -13,7 +13,6 @@ pub mod bootwrite; pub mod carrier; pub mod clock; pub mod cpu; -pub mod dirnodes; pub mod dmaguard; pub mod event; pub mod frames; diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index d3194de8fe9..169299b2813 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -207,6 +207,7 @@ struct Slot { /// list of its slots. pub struct Object { id: u32, + anonymous: bool, size: u64, gone: bool, head: u32, @@ -307,7 +308,13 @@ impl Pages { /// [`release`](Self::release) takes every page it held. pub fn object(&mut self, id: u32, size: u64) -> Object { assert!(size <= LAST_END, "an object of {size} bytes past the last page"); - Object { id, size, gone: false, head: NONE, pages: 0 } + Object { id, anonymous: false, size, gone: false, head: NONE, pages: 0 } + } + + /// An anonymous object owns its pages: no backing can supply them again, + /// so writeback, failure and withdrawal cannot discard present bytes. + pub fn anonymous(&mut self, id: u32, size: u64) -> Object { + Object { anonymous: true, ..self.object(id, size) } } fn hash(&self, object: u32, page: u64) -> usize { @@ -447,10 +454,11 @@ impl Pages { } None => {} } - if dirty && self.dirty_held >= self.dirty_ceiling { + let tracked = dirty && !o.anonymous; + if tracked && self.dirty_held >= self.dirty_ceiling { return Err(Refused::DirtyFull); } - let state = State { dirty, ..State::CLEAN }; + let state = State { dirty: tracked, ..State::CLEAN }; self.insert(o, page, Some(frame), state)?; if dirty { o.size = o.size.max(end); @@ -470,12 +478,12 @@ impl Pages { let i = self.find(o.id, page).ok_or(Refused::InvalidArgument)?; let s = self.slot(i); let frame = s.frame.ok_or(Refused::InvalidArgument)?; - let needs_slot = !s.state.counts_dirty(); + let needs_slot = !o.anonymous && !s.state.counts_dirty(); if needs_slot && self.dirty_held >= self.dirty_ceiling { return Err(Refused::DirtyFull); } let s = self.slot_mut(i); - s.state.dirty = true; + s.state.dirty = !o.anonymous; s.state.write_pins = s.state.write_pins.checked_add(1).expect("a page's write pins overflowed"); if s.state.writeback { s.state.pinned_since_begin = true; @@ -518,7 +526,7 @@ impl Pages { /// cleaned and keeps its slot until the end. The pages begun, as a bitmap. pub fn writeback_begin(&mut self, o: &mut Object, first: u64, count: u64) -> Result<[u64; 8], Refused> { let overlapping = self.open.iter().flatten().any(|&(id, a, n)| id == o.id && Self::overlaps((a, n), (first, count))); - if !Self::range_ok(o, first, count) || overlapping { + if o.anonymous || !Self::range_ok(o, first, count) || overlapping { return Err(Refused::InvalidArgument); } let at = self.open.iter().position(Option::is_none).ok_or(Refused::InvalidArgument)?; @@ -566,6 +574,7 @@ impl Pages { /// holds: their frames, for the caller to free at their last pin. /// A range longer than the object holds pages walks its pages, not the range. pub fn withdraw(&mut self, o: &mut Object, first: u64, count: u64, out: &mut impl FnMut(F)) -> usize { + if o.anonymous { return 0; } let end = first.saturating_add(count); let mut removed = 0; let mut take = |pages: &mut Self, o: &mut Object, i: u32| { @@ -604,6 +613,7 @@ impl Pages { let end = first.saturating_add(count).min(o.page_end()); for page in first..end { if let Some(i) = self.find(o.id, page) { + if o.anonymous && self.slot(i).frame.is_some() { continue; } if let Some(frame) = self.remove(o, i) { out(frame); } @@ -677,6 +687,28 @@ mod tests { |f| freed.push(f) } + #[test] + fn anonymous_bytes_survive_cache_operations_and_release_with_the_object() { + let mut pages = Pages::new(2, 0).unwrap(); + let mut object = pages.anonymous(1, 0); + assert_eq!(pages.supply(&mut object, 0, 42, true, PAGE), Ok(Supplied::Installed)); + assert_eq!(pages.dirty_held(), 0); + assert_eq!(pages.write_pin(&mut object, 0), Ok(42)); + pages.write_unpin(&mut object, 0, 42, PAGE); + assert_eq!(pages.withdraw(&mut object, 0, 1, &mut none), 0); + assert_eq!(pages.writeback_begin(&mut object, 0, 1), Err(Refused::InvalidArgument)); + assert_eq!(pages.fail(&mut object, 0, 1, &mut none), Ok(())); + assert_eq!(pages.lookup(&object, 0), Lookup::Present(42)); + assert_eq!(pages.supply(&mut object, 1, 43, true, 2 * PAGE), Ok(Supplied::Installed)); + assert_eq!(pages.supply(&mut object, 2, 44, true, 3 * PAGE), Err(Refused::NoRoom)); + let mut freed = Vec::new(); + pages.resize(&mut object, PAGE, &mut sink(&mut freed)); + assert_eq!(freed, [43]); + pages.release(object, &mut sink(&mut freed)); + assert_eq!(freed, [43, 42]); + assert_eq!(pages.used(), 0); + } + #[test] fn a_supplied_page_is_found_and_a_second_supply_changes_nothing() { let mut p = Pages::new(8, 4).unwrap(); diff --git a/kernel/src/actuator.rs b/kernel/src/actuator.rs index 6d1ad9a691f..d050b3cec1c 100644 --- a/kernel/src/actuator.rs +++ b/kernel/src/actuator.rs @@ -197,8 +197,6 @@ actuators! { diag_tick = "diag-tick"; - /// Run the revoked-backing controls after mount. - revoked_backing_selftest = "revoked-backing-selftest"; /// Shorten the panicked kernel's own reboot bound from a minute to seconds, /// so a guest reaches the reset. Judged by `screen_fatal_behind_a_painter`. diff --git a/kernel/src/file_cache.rs b/kernel/src/file_cache.rs deleted file mode 100644 index 1242219c64e..00000000000 --- a/kernel/src/file_cache.rs +++ /dev/null @@ -1,572 +0,0 @@ -use alloc::boxed::Box; -use alloc::collections::btree_map::Entry; -use alloc::collections::BTreeMap; -use alloc::sync::Arc; - -use alloc::vec::Vec; - -use toyos_abi::syscall::SyscallError; - -use crate::file_backing::{BlockError, FileBacking}; -use crate::mm::charge::{self, Charge, Exhausted}; -use crate::mm::space::CopySource; -use crate::sync::Lock; -use crate::user_ptr::{ByteSource, UserBytesMut}; - -pub type FileId = u64; - -/// `mm::PAGE_SIZE`, in the width this file's arrays and slices index by. -const PAGE_SIZE: usize = crate::mm::PAGE_SIZE as usize; - -/// The one ceiling on a file position or size: the page index is a `u32`, so past -/// this it would wrap and alias a low page — seek, write and truncate refuse it. -pub const MAX_FILE_SIZE: u64 = (u32::MAX as u64 + 1) * PAGE_SIZE as u64; - -/// A page's bytes, shared with each [`Snapshot`] taken while it was the -/// file's: a write to a shared page writes a copy of it. -type PageData = Arc; - -/// A page's bytes, charged to whoever made them for as long as anything holds -/// them: the file, or a snapshot of it a spawn pages from. -pub struct Page { - bytes: Box<[u8; PAGE_SIZE]>, - _charge: Charge, -} - -impl core::ops::Deref for Page { - type Target = [u8; PAGE_SIZE]; - - fn deref(&self) -> &Self::Target { - &self.bytes - } -} - -impl Page { - /// A page of zeros, or `Exhausted`, charged before it is allocated with - /// its place in its file. - fn blank() -> Result { - let charge = Charge::current(charge::heap(core::alloc::Layout::new::<[u8; PAGE_SIZE]>()) + charge::arc::() + charge::map_entry::())?; - Ok(Self { bytes: blank_page(), _charge: charge }) - } - - /// Make `data` this file's own to write, copying it if a snapshot shares - /// it; refused, it is left shared and unwritten. - fn own(data: &mut PageData) -> Result<&mut [u8; PAGE_SIZE], Exhausted> { - if Arc::get_mut(data).is_none() { - let mut copy = Self::blank()?; - *copy.bytes = ***data; - *data = Arc::new(copy); - } - Ok(&mut Arc::get_mut(data).expect("a page this file holds alone").bytes) - } -} - -struct CachedPage { - data: PageData, - /// CLOCK's second-chance bit: set on every hit, cleared when the sweep passes it over. - referenced: bool, -} - -struct CachedFile { - /// Its place in the cache, charged to whoever made it. - _charge: Charge, - pages: BTreeMap, - size: u64, - evictable: bool, - /// Where an evicted page comes back from; `None` means a dropped page cannot be re-read. - backing: Option>, - ref_count: u32, - deleted: bool, - /// When it was last written, for a mount whose pages are the file. - mtime: u64, -} - -impl CachedFile { - /// Whether this file's pages are a copy of disk data — only those count toward the budget or are evictable. - fn is_cache(&self) -> bool { - self.evictable && self.backing.is_some() - } -} - -struct FileCache { - files: BTreeMap, - next_id: u64, - /// Resident pages belonging to files that satisfy `is_cache`. - cached_pages: usize, - max_pages: usize, - evictions: u64, - /// CLOCK hand, in (file, page) key order; kept across calls so eviction costs one step, not a full scan. - hand: (FileId, u32), -} - -static FILE_CACHE: Lock = Lock::new(FileCache { - files: BTreeMap::new(), - next_id: 1, - cached_pages: 0, - // Zero, not `usize::MAX`: an uninstalled budget must fail loudly, not silently allow everything. - max_pages: 0, - evictions: 0, - hand: (0, 0), -}); - -/// Install the memory budget; must run after the PMM sizes RAM and before any file is opened. -pub fn init() { - let (total, _) = crate::mm::pmm::stats(); - let max_pages = (((total / 64) / PAGE_SIZE as u64) as usize).clamp(2048, 65536); - FILE_CACHE.lock().max_pages = max_pages; - log!("file cache: budget {} pages ({} MiB)", max_pages, max_pages * PAGE_SIZE / (1024 * 1024)); -} - -/// Allocate a new FileId. The file cache is the sole allocator. -pub fn create_file(evictable: bool, mtime: u64) -> Result { - let charge = Charge::current(charge::map_entry::())?; - let mut cache = FILE_CACHE.lock(); - let id = cache.next_id; - cache.next_id += 1; - cache.files.insert(id, CachedFile { - _charge: charge, - pages: BTreeMap::new(), - size: 0, - evictable, - backing: None, - ref_count: 1, - deleted: false, - mtime, - }); - Ok(id) -} - -/// A number from the count `FileId`s are drawn from, which no file will ever -/// hold: a kernel volume's node for a name the cache keeps no file for, so no -/// node on a kernel volume is another's (`vfs::FileSystem::node`). -pub fn draw_node() -> u64 { - let mut cache = FILE_CACHE.lock(); - let node = cache.next_id; - cache.next_id += 1; - node -} - -/// Point a file at the store its evicted pages come back from; idempotent across opens of the same file. -pub fn set_backing(file_id: FileId, backing: Arc) { - let mut cache = FILE_CACHE.lock(); - let now_governed; - { - let Some(file) = cache.files.get_mut(&file_id) else { return }; - let was_cache = file.is_cache(); - file.backing = Some(backing); - now_governed = if !was_cache && file.is_cache() { file.pages.len() } else { 0 }; - } - cache.cached_pages += now_governed; - evict_if_needed(&mut cache); -} - -/// Increment ref_count for one more open, returning a guard that undoes the -/// increment on drop unless committed: a re-open whose backing lookup fails -/// after this must not pin the file. -#[must_use = "commit() once the re-open cannot fail, or the reference is released"] -pub fn open(file_id: FileId) -> crate::rollback::Rollback { - { - let mut cache = FILE_CACHE.lock(); - if let Some(file) = cache.files.get_mut(&file_id) { - file.ref_count += 1; - } - } - crate::rollback::Rollback::new(move || undo_open(file_id)) -} - -/// Undo one [`open`]: a re-open decrements a count another handle keeps, never orphaning the file. -fn undo_open(file_id: FileId) { - let mut cache = FILE_CACHE.lock(); - if let Some(file) = cache.files.get_mut(&file_id) { - file.ref_count = file.ref_count.saturating_sub(1); - } -} - -/// Drop one open reference. The last of a deleted file takes its pages with -/// it; any other file stays, its pages clean and CLOCK's to evict, so a -/// re-open finds the id its mount keeps for the name. -pub fn release(file_id: FileId) { - let mut cache = FILE_CACHE.lock(); - let Some(file) = cache.files.get_mut(&file_id) else { return }; - file.ref_count = file.ref_count.saturating_sub(1); - if file.ref_count == 0 && file.deleted { - drop_file(&mut cache, file_id); - } -} - -/// When the file was last written, as [`create_file`] and [`touch`] said. -pub fn mtime(file_id: FileId) -> u64 { - FILE_CACHE.lock().files.get(&file_id).map_or(0, |f| f.mtime) -} - -/// Record a write at `mtime`. -pub fn touch(file_id: FileId, mtime: u64) { - if let Some(file) = FILE_CACHE.lock().files.get_mut(&file_id) { - file.mtime = mtime; - } -} - -/// Read a file page into `buf`; on `Err` the fetch failed and `buf` holds zeros, not the file's bytes. -pub fn read_page( - file_id: FileId, - page_idx: u32, - offset: usize, - buf: &mut UserBytesMut, -) -> Result<(), BlockError> { - let backing; - { - let mut cache = FILE_CACHE.lock(); - let Some(file) = cache.files.get_mut(&file_id) else { return Ok(()) }; - let file_size = file.size; - - // Beyond file size: zero-fill, no cache insert. - if (page_idx as u64) * PAGE_SIZE as u64 >= file_size { - buf.fill_zero(0, buf.len()); - return Ok(()); - } - - if let Some(page) = file.pages.get_mut(&page_idx) { - page.referenced = true; - let avail = valid_bytes_in_page(page_idx, file_size); - copy_page_region_to_buf(&page.data[..], offset, buf, avail); - return Ok(()); - } - backing = file.backing.clone(); - } - // Cache miss: unlock, fetch from backing, re-lock, insert if still absent. - - let mut fetched = match Page::blank() { - Ok(page) => page, - // A hole is zeros whether or not a page is kept for it. - Err(Exhausted) if backing.is_none() => { - buf.fill_zero(0, buf.len()); - return Ok(()); - } - Err(Exhausted) => { - buf.fill_zero(0, buf.len()); - return Err(BlockError::Exhausted); - } - }; - if let Some(backing) = &backing { - // A failed fetch must not become a resident page: a later partial write would merge into cached zeros and flush them over the file. - if let Err(e) = backing.read_page(page_idx as u64 * PAGE_SIZE as u64, &mut fetched.bytes) { - buf.fill_zero(0, buf.len()); - return Err(e); - } - } - // else: tmpfs miss → zero-filled page (fetched is already zeroed) - - let mut cache = FILE_CACHE.lock(); - let mut added = 0; - { - let Some(file) = cache.files.get_mut(&file_id) else { return Ok(()) }; - let is_cache = file.is_cache(); - if let Entry::Vacant(slot) = file.pages.entry(page_idx) { - slot.insert(CachedPage::new(fetched)); - added = is_cache as usize; - } - let file_size = file.size; - let page = file.pages.get_mut(&page_idx).unwrap(); - page.referenced = true; - let avail = valid_bytes_in_page(page_idx, file_size); - copy_page_region_to_buf(&page.data[..], offset, buf, avail); - } - cache.cached_pages += added; - evict_if_needed(&mut cache); - Ok(()) -} - -/// Write data into a file page; the lock is not held during disk I/O on a cache miss. -/// `Err` means the page could not be re-read and nothing was written — merging into zeros would destroy 4 KiB of a file that was fine — -/// or that no memory is left to keep it in. -pub fn write_page( - file_id: FileId, - page_idx: u32, - offset: usize, - data: &S, -) -> Result<(), BlockError> { - // A resident page is written under the same lock acquisition that found it; the fetch path below drops the lock, and a sibling's eviction in that window would otherwise merge the write into a blank page. - let backing; - { - let mut cache = FILE_CACHE.lock(); - { - let Some(file) = cache.files.get_mut(&file_id) else { return Ok(()) }; - if file.pages.contains_key(&page_idx) { - apply_write(file, page_idx, offset, data).map_err(|Exhausted| BlockError::Exhausted)?; - backing = None; - } else { - backing = Some(file.backing.clone()); - } - } - if backing.is_none() { - evict_if_needed(&mut cache); - return Ok(()); - } - } - let backing = backing.unwrap(); - - let mut fetched = Page::blank().map_err(|Exhausted| BlockError::Exhausted)?; - if let Some(backing) = &backing { - let page_start = page_idx as u64 * PAGE_SIZE as u64; - // Past the end there is nothing to preserve, so no fetch and no way for one to fail. - if page_start < backing.file_size() { - backing.read_page(page_start, &mut fetched.bytes)?; - } - } - - // Re-fetching after a sibling's eviction is always correct: a page written is never evicted. - let mut cache = FILE_CACHE.lock(); - let mut added = 0; - { - let Some(file) = cache.files.get_mut(&file_id) else { return Ok(()) }; - let is_cache = file.is_cache(); - if let Entry::Vacant(slot) = file.pages.entry(page_idx) { - slot.insert(CachedPage::new(fetched)); - added = is_cache as usize; - } - // A page this write just made is the file's alone, so this charges nothing. - apply_write(file, page_idx, offset, data).map_err(|Exhausted| BlockError::Exhausted)?; - } - cache.cached_pages += added; - evict_if_needed(&mut cache); - Ok(()) -} - -/// Refused, the page is unchanged: a snapshot shares it and no copy can be -/// charged. -fn apply_write( - file: &mut CachedFile, - page_idx: u32, - offset: usize, - data: &S, -) -> Result<(), Exhausted> { - let page = file.pages.get_mut(&page_idx).expect("write_page: page not resident"); - let end = (offset + data.len()).min(PAGE_SIZE); - data.read_at(0, &mut Page::own(&mut page.data)?[offset..end]); - page.referenced = true; - - let write_end = page_idx as u64 * PAGE_SIZE as u64 + end as u64; - if write_end > file.size { - file.size = write_end; - } - Ok(()) -} - -/// Copy a resident page out; `false` leaves `buf` untouched — an absent page is not zeros. -pub fn copy_page_out(file_id: FileId, page_idx: u32, buf: &mut [u8; PAGE_SIZE]) -> bool { - let cache = FILE_CACHE.lock(); - let Some(page) = cache.files.get(&file_id).and_then(|file| file.pages.get(&page_idx)) else { - return false; - }; - *buf = **page.data; - true -} - -/// A file's bytes as they were when it was taken, which no later write, -/// truncation or delete changes: a spawn's program, read without the cache's -/// lock. -pub struct Snapshot { - size: u64, - /// The resident pages by index; an absent one is a hole and reads zeros. - pages: Vec<(u32, PageData)>, - /// The list, and the snapshot itself. - _charge: Charge, -} - -/// `file_id`'s bytes now: `NotFound` for a file the cache no longer holds, -/// `ResourceExhausted` for one whose list of pages is past one heap -/// allocation or cannot be charged. Only a file with no backing: an absent -/// page of one with a backing was evicted, not a hole. -pub fn snapshot(file_id: FileId) -> Result { - let cache = FILE_CACHE.lock(); - let file = cache.files.get(&file_id).ok_or(SyscallError::NotFound)?; - assert!(file.backing.is_none(), "file cache: a snapshot of file {file_id}, whose absent pages are on its backing"); - if file.pages.len() > crate::mm::MAX_HEAP_ALLOC / core::mem::size_of::<(u32, PageData)>() { - return Err(SyscallError::ResourceExhausted); - } - let charge = Charge::current(charge::arc::() + charge::array::<(u32, PageData)>(file.pages.len()))?; - let pages = file.pages.iter().map(|(&idx, page)| (idx, Arc::clone(&page.data))).collect(); - Ok(Snapshot { size: file.size, pages, _charge: charge }) -} - -impl CopySource for Snapshot { - fn size(&self) -> u64 { - self.size - } - - fn read(&self, offset: u64, out: &mut [u8]) { - assert!(offset.checked_add(out.len() as u64).is_some_and(|end| end <= self.size), "file cache: a snapshot read past its end"); - let mut done = 0; - while done < out.len() { - let at = offset + done as u64; - let (idx, within) = ((at / PAGE_SIZE as u64) as u32, (at % PAGE_SIZE as u64) as usize); - let n = (PAGE_SIZE - within).min(out.len() - done); - match self.pages.binary_search_by_key(&idx, |&(i, _)| i) { - Ok(i) => out[done..done + n].copy_from_slice(&self.pages[i].1[within..within + n]), - Err(_) => out[done..done + n].fill(0), - } - done += n; - } - } -} - -/// Get the authoritative file size. -pub fn size(file_id: FileId) -> u64 { - FILE_CACHE.lock().files.get(&file_id).map_or(0, |f| f.size) -} - -/// Set file size and drop pages past it. Refused, nothing changes: the page -/// the new end falls inside is shared with a snapshot, and no copy of it to -/// zero can be charged. -pub fn set_size(file_id: FileId, new_size: u64) -> Result<(), Exhausted> { - let mut cache = FILE_CACHE.lock(); - let dropped; - { - let Some(file) = cache.files.get_mut(&file_id) else { return Ok(()) }; - dropped = if new_size < file.size { - // The page the new end falls inside is kept; zero its bytes past the - // new end in the one step that sets the size, so a later grow reads - // the hole as zeros rather than the discarded tail. - let tail = (new_size % PAGE_SIZE as u64) as usize; - if tail != 0 { - let straddled = (new_size / PAGE_SIZE as u64) as u32; - if let Some(page) = file.pages.get_mut(&straddled) { - Page::own(&mut page.data)?[tail..].fill(0); - } - } - // Split off whole, so that no list of the pages is allocated. - let first_removed = (new_size as usize).div_ceil(PAGE_SIZE) as u32; - let removed = file.pages.split_off(&first_removed); - if file.is_cache() { removed.len() } else { 0 } - } else { - 0 - }; - file.size = new_size; - } - cache.cached_pages -= dropped; - Ok(()) -} - -/// Mark a file as deleted (unlink). If no handles hold it, free immediately; -/// otherwise the last [`release`] does. -pub fn mark_deleted(file_id: FileId) { - let mut cache = FILE_CACHE.lock(); - let Some(file) = cache.files.get_mut(&file_id) else { return }; - file.deleted = true; - if file.ref_count == 0 { - drop_file(&mut cache, file_id); - } -} - -impl CachedPage { - fn new(data: Page) -> Self { - Self { data: Arc::new(data), referenced: false } - } -} - -/// A blank page, allocated directly on the heap: never construct via a stack-sized array. -fn blank_page() -> Box<[u8; PAGE_SIZE]> { - match alloc::vec![0u8; PAGE_SIZE].into_boxed_slice().try_into() { - Ok(page) => page, - Err(_) => unreachable!("a PAGE_SIZE slice is a [u8; PAGE_SIZE]"), - } -} - -fn drop_file(cache: &mut FileCache, file_id: FileId) { - let Some(removed) = cache.files.remove(&file_id) else { return }; - if removed.is_cache() { - cache.cached_pages -= removed.pages.len(); - } -} - -fn valid_bytes_in_page(page_idx: u32, file_size: u64) -> usize { - let page_start = page_idx as u64 * PAGE_SIZE as u64; - if page_start >= file_size { - 0 - } else { - ((file_size - page_start) as usize).min(PAGE_SIZE) - } -} - -fn copy_page_region_to_buf(page: &[u8], offset: usize, buf: &mut UserBytesMut, valid: usize) { - let start = offset.min(valid); - let end = (offset + buf.len()).min(valid); - let count = end.saturating_sub(start); - if count > 0 { - buf.write_at(0, &page[start..start + count]); - } - // Zero-fill remainder (past valid data or past file end). - if count < buf.len() { - buf.fill_zero(count, buf.len() - count); - } -} - -fn evict_if_needed(cache: &mut FileCache) { - assert!(cache.max_pages != 0, "file cache used before init installed a budget"); - let before = cache.evictions; - while cache.cached_pages > cache.max_pages { - // A governed page is a read-only mount's and never written, so one - // revolution always finds one to take. - assert!( - evict_one(cache), - "file cache: {}/{} pages resident and none evictable", - cache.cached_pages, - cache.max_pages - ); - } - // Once per full turnover, so the rate scales with the budget. - let turnover = cache.max_pages as u64; - if cache.evictions != before && (before == 0 || before / turnover != cache.evictions / turnover) { - log!("file cache: {} evictions, {}/{} pages resident", cache.evictions, cache.cached_pages, cache.max_pages); - } -} - -/// One CLOCK step-and-evict; returns false when a full revolution found no page it was allowed to take. -fn evict_one(cache: &mut FileCache) -> bool { - // Two full passes: the first may only clear reference bits, so the second must be able to evict; `+2` covers each wrap. - let steps = cache.cached_pages * 2 + 2; - for _ in 0..steps { - let Some((fid, idx)) = seek_hand(cache) else { return false }; - cache.hand = match idx.checked_add(1) { - Some(next) => (fid, next), - None => (fid + 1, 0), - }; - - { - let Some(file) = cache.files.get_mut(&fid) else { continue }; - let Some(page) = file.pages.get_mut(&idx) else { continue }; - if page.referenced { - page.referenced = false; - continue; - } - file.pages.remove(&idx); - } - cache.cached_pages -= 1; - cache.evictions += 1; - return true; - } - false -} - -/// The first resident page at or after the hand, wrapping once. -fn seek_hand(cache: &mut FileCache) -> Option<(FileId, u32)> { - if let Some(found) = page_at_or_after(cache, cache.hand) { - return Some(found); - } - cache.hand = (0, 0); - page_at_or_after(cache, cache.hand) -} - -fn page_at_or_after(cache: &FileCache, from: (FileId, u32)) -> Option<(FileId, u32)> { - for (&fid, file) in cache.files.range(from.0..) { - // Skip whole files, not page by page: a tmpfs file's pages can never be evicted, and stepping through one would exhaust the sweep's budget. - if !file.is_cache() { - continue; - } - let start = if fid == from.0 { from.1 } else { 0 }; - if let Some((&idx, _)) = file.pages.range(start..).next() { - return Some((fid, idx)); - } - } - None -} diff --git a/kernel/src/fs_rename.rs b/kernel/src/fs_rename.rs deleted file mode 100644 index 07dc7d5bdc2..00000000000 --- a/kernel/src/fs_rename.rs +++ /dev/null @@ -1,66 +0,0 @@ -//! The one order a replacing rename runs in, shared by every mount adapter: the -//! source is validated before the destination is disturbed, a rename onto the -//! same backend object (by identity, not string equality) is a no-op, and the -//! displaced destination is freed only once the move commits. [`replace_rename`] -//! is the sole sequencer, and [`Committed`] — which only `commit` mints — is what -//! `release` consumes, so freeing before commit fails to compile. - -use toyos_abi::syscall::SyscallError; - -/// Proof the move committed, carrying whatever the displaced destination left -/// for [`ReplaceRename::release`] to free. Only a successful `commit` mints one. -#[must_use] -pub(crate) struct Committed(T); - -impl Committed { - pub(crate) fn new(displaced: T) -> Self { - Committed(displaced) - } - - pub(crate) fn into_displaced(self) -> T { - self.0 - } -} - -/// A mount adapter's three rename phases, run in order by [`replace_rename`]. -pub(crate) trait ReplaceRename { - /// What a displaced destination leaves for `release` to free. - type Displaced; - - /// Is the source present? The move fails here, disturbing nothing, when not. - fn source_present(&mut self, old: &str) -> Result; - - /// Do `old` and `new` name the same backend object? The move is then a - /// no-op, never a delete of the source's own entry. - fn same_object(&mut self, old: &str, new: &str) -> Result; - - /// Commit the move on the backend, displacing whatever `new` named; on `Err` - /// nothing durable moved. - fn commit(&mut self, old: &str, new: &str) - -> Result, SyscallError>; - - /// Free the displaced destination and re-key the source, the move committed. - fn release( - &mut self, - old: &str, - new: &str, - committed: Committed, - ) -> Result<(), SyscallError>; -} - -/// Validate the source, treat a rename onto the same object as POSIX's no-op -/// success, then commit and only then release the destination. -pub(crate) fn replace_rename( - adapter: &mut R, - old: &str, - new: &str, -) -> Result<(), SyscallError> { - if !adapter.source_present(old)? { - return Err(SyscallError::NotFound); - } - if adapter.same_object(old, new)? { - return Ok(()); - } - let committed = adapter.commit(old, new)?; - adapter.release(old, new, committed) -} diff --git a/kernel/src/loader/mod.rs b/kernel/src/loader/mod.rs index 974c78e2f4a..875d0d8e59c 100644 --- a/kernel/src/loader/mod.rs +++ b/kernel/src/loader/mod.rs @@ -37,7 +37,7 @@ use crate::process::{ ProcessEntry, ThreadData, ThreadEntry, UserImage, PROCESS_TABLE, }; use crate::sync::Lock; -use crate::{scheduler, vfs, UserAddr}; +use crate::{scheduler, UserAddr}; use toyos_abi::handle::Rights; use toyos_abi::syscall::SyscallError; use toyos_elf::{Layout, TlsSegment}; @@ -49,7 +49,7 @@ const USER_STACK_SIZE: usize = 8 * 1024 * 1024; /// The reserved range below the main stack. const STACK_GUARD: u64 = 1024 * 1024; -/// Read a byte range from a file through the page cache. +/// Read a byte range from the supplied executable image. /// /// Returns only the part of the request the file actually holds — callers /// treat a short result as truncated and length-check before indexing; `len` @@ -183,7 +183,7 @@ pub fn spawn( commit: C, cwd: String, env: Vec, - image: Option>, + backing: Arc, parent: Parent, ) -> Result<(Pid, C::Held), crate::object::Refusal> { let job = Arc::clone(commit.job()); @@ -197,21 +197,6 @@ pub fn spawn( let mut charge = ProcessData::charge(&job, path, &env).map_err(SyscallError::from)?; let t0 = crate::clock::nanos_since_boot(); - let backing: Arc = match image { - Some(image) => image, - None => { - // Scoped, not held across the match: dropping `commit` on any `return` here takes the VFS lock. - let opened = vfs::lock().open_backing(path); - match opened { - Ok(b) => b, - Err(e) => { - log!("spawn: {}: {e}", path); - return Err(e.into()); - } - } - } - }; - let header_size = 4096.min(backing.file_size() as usize); let header_data = read_file_range(backing.as_ref(), 0, header_size); let layout = match elf::parse_layout(&header_data) { @@ -522,7 +507,7 @@ pub fn spawn_supervisor(images: crate::bootimage::Objects) -> Pid { ); // Built by the kernel and owing nobody anything: no table but the supervisor's own holds it. let table = SupervisorTable { job, handles, entries, labels }; - match spawn(SUPERVISOR, &[SUPERVISOR], table, String::from("/"), Vec::new(), Some(image), Parent::Root) { + match spawn(SUPERVISOR, &[SUPERVISOR], table, String::from("/"), Vec::new(), image, Parent::Root) { Ok((pid, ())) => pid, Err(crate::object::Refusal::Error(e)) => panic!("spawn_supervisor: failed to spawn: {e:?}"), Err(crate::object::Refusal::Handle(e)) => panic!("spawn_supervisor: {e}"), diff --git a/kernel/src/main.rs b/kernel/src/main.rs index cd06fc43ac0..06381666251 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -47,14 +47,7 @@ mod inventory; mod loaded; mod rollback; mod bootimage; -mod file_cache; -#[cfg(feature = "boot-actuators")] -#[cfg(feature = "boot-actuators")] -mod revoke_selftest; -mod tmpfs; mod file_backing; -mod fs_rename; -mod vfs; mod elf; mod symbols; mod process; @@ -100,7 +93,6 @@ mod late_panic { } } -use alloc::boxed::Box; use arch::{cpu, percpu}; use drivers::{acpi, gop, pci, serial}; use toyos_abi::boot::{KernelArgs, MemoryMapEntry}; @@ -445,7 +437,6 @@ pub(crate) unsafe extern "C" fn kernel_main(loader_args: &mut KernelArgs) -> ! { // Before storage and everything under it: what it covers is the rest of this // boot, and a wedge down there is the reason to have one. arch::watchdog::init(&pci_devices); - file_cache::init(); loaded::init(kernel_args); boot_phase!("peripherals ready", t_periph); @@ -453,12 +444,9 @@ pub(crate) unsafe extern "C" fn kernel_main(loader_args: &mut KernelArgs) -> ! { let t_subsys = clock::nanos_since_boot(); arch::boot::start_other_cpus(&platform, kernel_args); - vfs::init(); process::init(); scheduler::init(); - vfs::lock().mount(&["tmp"], Box::new(crate::tmpfs::TmpFs::new())); - boot_phase!("subsystems ready", t_subsys); // Off the loader's images, so nothing from here to the supervisor's @@ -467,11 +455,6 @@ pub(crate) unsafe extern "C" fn kernel_main(loader_args: &mut KernelArgs) -> ! { let pid = process::spawn_supervisor(bootimage::objects(boot_images)); log!("spawned the supervisor pid={pid}"); - #[cfg(feature = "boot-actuators")] - if actuator::revoked_backing_selftest() { - revoke_selftest::run(); - } - let t_devices = clock::nanos_since_boot(); // First in the device phase, after storage: its lines are the diagnostic diff --git a/kernel/src/object/file.rs b/kernel/src/object/file.rs deleted file mode 100644 index eb445b47a53..00000000000 --- a/kernel/src/object/file.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! An open file: two handles to one `FileObject` share the cursor; an independent cursor opens the path again. - -use alloc::sync::Arc; - -use crate::file_cache::{self, FileId}; -use crate::sync::Lock; - -use super::ObjectCore; - -pub struct OpenFileState { - pub file_id: FileId, - /// The volume it was opened on (`vfs::Vfs::open_target`). - pub volume: u64, - pub position: usize, - pub mtime: u64, -} - -// Drop runs under `Lock`, so it takes the file cache's lock and no other. -impl Drop for OpenFileState { - fn drop(&mut self) { - file_cache::release(self.file_id); - } -} - -pub struct FileObject { - pub(super) core: ObjectCore, - state: Lock, -} - -impl FileObject { - /// Over `core`, which the open charged before it opened the file. - pub fn new(core: ObjectCore, state: OpenFileState) -> Arc { - Arc::new(Self { core, state: Lock::new(state) }) - } - - pub fn with(&self, f: impl FnOnce(&mut OpenFileState) -> R) -> R { - f(&mut self.state.lock()) - } -} diff --git a/kernel/src/object/mod.rs b/kernel/src/object/mod.rs index 3cde593eaff..9d7390a4335 100644 --- a/kernel/src/object/mod.rs +++ b/kernel/src/object/mod.rs @@ -19,7 +19,6 @@ use crate::sync::Lock; pub mod device; pub mod event; -pub mod file; pub mod handle; pub mod inbox; pub mod job; @@ -273,8 +272,6 @@ kobject! { immediate Connector => port::Connector, // Immutable once built: its `Arc`s go with the last reference and nothing observes it. immediate Namespace => namespace::Namespace, - // A file's flush and cache reference ride the last `Arc`; `read`/`write` on a file never park. - immediate File => file::FileObject, // Its last handle lets go of the log's carrier, which a stranded `Arc` must not hold. deferred Console => device::ConsoleObject, // The authority is the rights on the handle; a handle going away *is* the whole event. diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index 5198dbffdb9..60095382d07 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -7,10 +7,9 @@ use alloc::sync::Arc; use toyos_abi::handle::{RawHandle, Rights}; -use toyos_abi::syscall::{FileType, OpenFlags, SeekFrom, SyscallError, MAX_WHOLE_PIPE_WRITE}; +use toyos_abi::syscall::{FileType, SyscallError, MAX_WHOLE_PIPE_WRITE}; use crate::drivers::serial; -use crate::file_cache; use crate::pipe::{self, PipeRef}; use kernel::pipe::End; use crate::user_ptr::{UserBytes, UserBytesMut}; @@ -19,7 +18,6 @@ use crate::watch::Watch; use crate::device as device_registry; use super::device::DeviceClaim; -use super::file::{FileObject, OpenFileState}; use super::handle::{HandleEntry, HandleTable}; use super::{KObjectRef, KObjectVariant}; @@ -32,7 +30,6 @@ pub fn initial_rights(object: &KObjectRef) -> Rights { KObjectRef::Connection(_) => { BASE.union(Rights::READ).union(Rights::WRITE).union(Rights::MAP) } - KObjectRef::File(_) => BASE.union(Rights::READ).union(Rights::WRITE), // No `DUP`: a claim admits exactly one handle, exclusivity by type rather than a check in `dup`. KObjectRef::Device(_) => { Rights::TRANSFER.union(Rights::WAIT).union(Rights::READ).union(Rights::WRITE) @@ -75,108 +72,6 @@ pub fn install(table: &mut HandleTable, object: KObjectRef) -> Result u64 { - // First, so the answer is the bit and not the path's own refusal. - if flags.0 & !OpenFlags::KNOWN.0 != 0 { - return SyscallError::InvalidArgument.to_u64(); - } - let writable = flags.contains(OpenFlags::WRITE); - let create = flags.contains(OpenFlags::CREATE); - let truncate = flags.contains(OpenFlags::TRUNCATE); - let append = flags.contains(OpenFlags::APPEND); - let exclusive = flags.contains(OpenFlags::EXCLUSIVE); - if exclusive && !create { - return SyscallError::InvalidArgument.to_u64(); - } - let modifies = writable || create || truncate || append; - // Charged before the file is opened or made, so a refusal makes nothing. - let core = match FileObject::new_core(0) { - Ok(core) => core, - Err(e) => return SyscallError::from(e).to_u64(), - }; - - let opened = { - let mut vfs = crate::vfs::lock(); - // Scoped to this block: dropped before the object exists, since `OpenFileState::Drop` re-takes it. - - let intent = if modifies { - crate::vfs::ResolveIntent::UserModify - } else { - crate::vfs::ResolveIntent::KernelOrRead - }; - // An exclusive create decides on the name itself, its link not - // followed, so a link (dangling or not) or a directory there is - // `AlreadyExists`, as POSIX's `O_EXCL` has it. - if exclusive { - match vfs.stat(path, false) { - Ok(_) => return SyscallError::AlreadyExists.to_u64(), - Err(SyscallError::NotFound) => {} - Err(e) => return e.to_u64(), - } - } - let target = match vfs.resolve_for_open(path, intent) { - Ok(target) => target, - Err(e) => return e.to_u64(), - }; - - if create { - let (_, file) = vfs.resolve_path("/", target.as_str()); - if file.is_empty() { - return SyscallError::InvalidArgument.to_u64(); - } - } - - // Under the same VFS lock as the check above, so two exclusive - // creates of one name cannot both find it missing; nothing is there, - // so the target is the name itself. - let built = if exclusive { - let mtime = crate::clock::mtime_now(); - vfs.create_file(target.as_str(), mtime).map(|(file_id, volume)| (file_id, volume, mtime, 0)) - } else if truncate && create { - let mtime = crate::clock::mtime_now(); - // `NotFound` is not a failure: truncating past a name that was not there is fine. - // Any `vfs.delete_file` error other than `NotFound` is propagated, not swallowed: truncating past it could silently create a file over one the mount could not confirm was missing. - match vfs.delete_file(target.as_str()) { - Ok(()) | Err(SyscallError::NotFound) => {} - Err(e) => return e.to_u64(), - } - vfs.create_file(target.as_str(), mtime).map(|(file_id, volume)| (file_id, volume, mtime, 0)) - } else { - // `mtime_target` takes no reference, so it runs first and the reference-taking - // `open_target` runs last — a refusal cannot strand a reference. `CREATE` acts on `NotFound` only. - match vfs.mtime_target(&target) { - Ok(mtime) => vfs.open_target(&target).map(|(file_id, volume)| { - let position = - if append { file_cache::size(file_id) as usize } else { 0 }; - (file_id, volume, mtime, position) - }), - Err(SyscallError::NotFound) if create => { - let mtime = crate::clock::mtime_now(); - vfs.create_file(target.as_str(), mtime).map(|(file_id, volume)| (file_id, volume, mtime, 0)) - } - Err(e) => Err(e), - } - }; - built - }; - - let (file_id, volume, mtime, position) = match opened { - Ok(v) => v, - Err(e) => return e.to_u64(), - }; - let object = KObjectRef::File(FileObject::new(core, OpenFileState { file_id, volume, position, mtime })); - // `writable` is a right, not a field: a read-only write fails for lacking `WRITE`. - let mut rights = initial_rights(&object); - if !writable { - rights = rights.without(Rights::WRITE); - } - match table.install(HandleEntry::new(object, rights)) { - Ok(h) => h.0 as u64, - Err(_) => SyscallError::ResourceExhausted.to_u64(), - } -} - /// Release one handle. /// /// What the object holds is released by its own zero-handle hook; `close` releases only what is the process's, not the object's: polls on a watch it ends. @@ -217,7 +112,7 @@ pub fn written_pipe(object: &KObjectRef) -> Option { pub fn pipe_read(object: &KObjectRef) -> Option<&PipeRef> { match object { KObjectRef::PipeRead(r) => Some(r.pipe()), - KObjectRef::Connection(_) | KObjectRef::PipeWrite(_) | KObjectRef::File(_) | KObjectRef::Device(_) + KObjectRef::Connection(_) | KObjectRef::PipeWrite(_) | KObjectRef::Device(_) | KObjectRef::Console(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) @@ -230,7 +125,7 @@ pub fn pipe_read(object: &KObjectRef) -> Option<&PipeRef> { pub fn pipe_write(object: &KObjectRef) -> Option<&PipeRef> { match object { KObjectRef::PipeWrite(w) => Some(w.pipe()), - KObjectRef::Connection(_) | KObjectRef::PipeRead(_) | KObjectRef::File(_) | KObjectRef::Device(_) + KObjectRef::Connection(_) | KObjectRef::PipeRead(_) | KObjectRef::Device(_) | KObjectRef::Console(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) @@ -295,7 +190,7 @@ pub fn read_watch(object: &KObjectRef) -> Option { }, // Named unconditionally: the watch alone cannot enforce rights. KObjectRef::SysCap(_) => Some(WatchRef::Static(&crate::log::user::WATCH)), - KObjectRef::PipeWrite(_) | KObjectRef::File(_) | KObjectRef::Inbox(_) + KObjectRef::PipeWrite(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Null(_) => None, } @@ -308,7 +203,7 @@ pub fn watched_readable(object: &KObjectRef) { KObjectRef::Notice(n) => n.listen(), KObjectRef::PipeRead(_) | KObjectRef::PipeWrite(_) | KObjectRef::Connection(_) | KObjectRef::Device(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SharedMem(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) - | KObjectRef::Namespace(_) | KObjectRef::File(_) | KObjectRef::Console(_) | KObjectRef::SysCap(_) + | KObjectRef::Namespace(_) | KObjectRef::Console(_) | KObjectRef::SysCap(_) | KObjectRef::Process(_) | KObjectRef::Null(_) | KObjectRef::Job(_) => {} } } @@ -319,7 +214,7 @@ pub fn write_watch(object: &KObjectRef) -> Option { KObjectRef::PipeWrite(w) => Some(WatchRef::Shared(w.pipe().write_watch().clone())), KObjectRef::Connection(c) => Some(WatchRef::Shared(c.tx().write_watch().clone())), KObjectRef::Console(_) => Some(WatchRef::Static(&crate::log::console::SPACE)), - KObjectRef::PipeRead(_) | KObjectRef::File(_) | KObjectRef::Device(_) + KObjectRef::PipeRead(_) | KObjectRef::Device(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) @@ -337,7 +232,7 @@ pub fn pipe_end_watch(object: &KObjectRef) -> Option { KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Notice(_) | KObjectRef::Console(_) | KObjectRef::Device(_) | KObjectRef::SysCap(_) - | KObjectRef::File(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) + | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Null(_) => None, } } @@ -351,7 +246,7 @@ pub fn other_end_gone(object: &KObjectRef) -> bool { KObjectRef::Connection(_) | KObjectRef::Acceptor(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Notice(_) | KObjectRef::Console(_) | KObjectRef::Device(_) | KObjectRef::SysCap(_) - | KObjectRef::File(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) + | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Null(_) => false, } } @@ -375,98 +270,12 @@ fn close_ends_polls(object: &KObjectRef) -> bool { device_registry::DeviceType::Framebuffer => true, }, KObjectRef::PipeRead(_) | KObjectRef::PipeWrite(_) | KObjectRef::Connection(_) - | KObjectRef::Acceptor(_) | KObjectRef::File(_) | KObjectRef::Inbox(_) + | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Null(_) | KObjectRef::Notice(_) => true, } } -/// What a file's page that could not be read or kept answers its call. -fn block_refusal(e: crate::file_backing::BlockError) -> SyscallError { - match e { - crate::file_backing::BlockError::Device => SyscallError::Io, - crate::file_backing::BlockError::Exhausted => SyscallError::ResourceExhausted, - } -} - -fn read_file(file: &FileObject, buf: &mut UserBytesMut) -> Option { - file.with(|state| { - let (answer, read) = read_file_at(state.file_id, state.position, buf); - state.position += read; - Some(answer) - }) -} - -/// Read file `file_id` at `position` into `buf`: the syscall's answer, and how -/// many bytes it read. -fn read_file_at(file_id: file_cache::FileId, position: usize, buf: &mut UserBytesMut) -> (u64, usize) { - let size = file_cache::size(file_id) as usize; - let count = buf.len().min(size.saturating_sub(position)); - let mut read = 0; - let mut refused = None; - while read < count { - let abs_pos = position + read; - let page_idx = (abs_pos / 4096) as u32; - let offset_in_page = abs_pos % 4096; - let to_read = (4096 - offset_in_page).min(count - read); - // A refused page is not a page of zeros: stop short rather than fake a hole under a success. - if let Err(e) = file_cache::read_page(file_id, page_idx, offset_in_page, &mut buf.sub(read, to_read)) { - refused = Some(e); - break; - } - read += to_read; - } - if let (0, Some(e)) = (read, refused) { - return (block_refusal(e).to_u64(), 0); - } - (read as u64, read) -} - -/// Write `buf` to file `file_id` at `position`, stamping it: the syscall's -/// answer, and how many bytes it wrote. -fn write_file_at(state: &mut OpenFileState, position: usize, buf: &UserBytes) -> (u64, usize) { - // Refused, never wrapped: past this the `u32` page index would alias a low page. - if (position as u64).saturating_add(buf.len() as u64) > file_cache::MAX_FILE_SIZE { - return (SyscallError::InvalidArgument.to_u64(), 0); - } - let mut written = 0; - let mut refused = None; - while written < buf.len() { - let abs_pos = position + written; - let page_idx = (abs_pos / 4096) as u32; - let offset_in_page = abs_pos % 4096; - let to_write = (4096 - offset_in_page).min(buf.len() - written); - // A page that cannot be re-read off the device is refused, not merged into zeros. - if let Err(e) = file_cache::write_page(state.file_id, page_idx, offset_in_page, &buf.sub(written, to_write)) { - refused = Some(e); - break; - } - written += to_write; - } - if let (0, Some(e)) = (written, refused) { - return (block_refusal(e).to_u64(), 0); - } - state.mtime = crate::clock::mtime_now(); - file_cache::touch(state.file_id, state.mtime); - (written as u64, written) -} - -/// Read a file at `offset`, leaving its position: any other object is -/// refused `NotSupported`, as a pipe has no offset to read at. -pub fn pread(object: &KObjectRef, offset: u64, buf: &mut UserBytesMut) -> u64 { - let KObjectRef::File(file) = object else { return SyscallError::NotSupported.to_u64() }; - let Ok(position) = usize::try_from(offset) else { return 0 }; - file.with(|state| read_file_at(state.file_id, position, buf).0) -} - -/// Write a file at `offset`, leaving its position: any other object is -/// refused `NotSupported`. -pub fn pwrite(object: &KObjectRef, offset: u64, buf: &UserBytes) -> u64 { - let KObjectRef::File(file) = object else { return SyscallError::NotSupported.to_u64() }; - let Ok(position) = usize::try_from(offset) else { return SyscallError::InvalidArgument.to_u64() }; - file.with(|state| write_file_at(state, position, buf).0) -} - /// Read a device claim; takes `table` because describing a device installs handles into it. pub fn read_device( claim: &DeviceClaim, @@ -545,7 +354,6 @@ fn record_bytes(record: &toyos_abi::pci::DeviceIrqRecord) -> &[u8] { /// Read whatever a handle names except a device claim; [`read_device`] needs the table mutably. pub fn try_read(object: &KObjectRef, buf: &mut UserBytesMut) -> Option { match object { - KObjectRef::File(f) => read_file(f, buf), KObjectRef::PipeRead(r) => r.pipe().try_read(buf).map(|n| n as u64), KObjectRef::Device(_) => unreachable!("a device claim is read by `read_device`"), KObjectRef::Console(_) => { @@ -596,11 +404,6 @@ fn write_pipe(object: &KObjectRef, buf: &UserBytes) -> Option { pub fn try_write(object: &KObjectRef, buf: &UserBytes) -> Option { match object { - KObjectRef::File(f) => f.with(|state| { - let (answer, written) = write_file_at(state, state.position, buf); - state.position += written; - Some(answer) - }), KObjectRef::PipeWrite(_) => write_pipe(object, buf), // The whole lines `klogd`'s queue has room for, and a trailing partial one: // a short count is a writer ahead of the console, told so. @@ -617,26 +420,6 @@ pub fn try_write(object: &KObjectRef, buf: &UserBytes) -> Option { } } -pub fn seek(object: &KObjectRef, pos: SeekFrom) -> u64 { - let KObjectRef::File(file) = object else { - return SyscallError::PermissionDenied.to_u64(); - }; - file.with(|state| { - let size = file_cache::size(state.file_id) as usize; - let new_pos = match pos { - SeekFrom::Start(n) => n as i64, - SeekFrom::Current(n) => (state.position as i64).checked_add(n).unwrap_or(-1), - SeekFrom::End(n) => (size as i64).checked_add(n).unwrap_or(-1), - }; - if new_pos < 0 || new_pos as u64 > file_cache::MAX_FILE_SIZE { - return SyscallError::InvalidArgument.to_u64(); - } - // Past EOF is a position, not an error (POSIX lseek): a later write extends the file. - state.position = new_pos as usize; - state.position as u64 - }) -} - toyos_abi::user_safe! { #[derive(Clone, Copy)] pub struct Stat { @@ -648,18 +431,10 @@ toyos_abi::user_safe! { } } -/// What kind of thing this is, how big, and which file: a file's node is its -/// `FileId`, one per file for as long as it exists (`vfs::FileSystem::open_file`). +/// The kind and size of a kernel object. pub fn fstat(object: &KObjectRef) -> Stat { let plain = |t: FileType| Stat { file_type: t as u64, size: 0, mtime: 0, volume: 0, node: 0 }; match object { - KObjectRef::File(f) => f.with(|state| Stat { - file_type: FileType::File as u64, - size: file_cache::size(state.file_id), - mtime: state.mtime, - volume: state.volume, - node: state.file_id, - }), KObjectRef::PipeRead(r) => { plain(if r.is_tty() { FileType::Tty } else { FileType::Pipe }) } @@ -696,33 +471,6 @@ pub fn fstat(object: &KObjectRef) -> Stat { } } -/// `SYS_FSYNC`: a kernel file's pages are the file (`/tmp`) or never written -/// (ROOT), so it owes nothing; nothing else here holds writes. -pub fn fsync(object: &KObjectRef) -> u64 { - match object { - KObjectRef::File(_) => 0, - _ => SyscallError::PermissionDenied.to_u64(), - } -} - -pub fn ftruncate(object: &KObjectRef, size: u64) -> u64 { - let KObjectRef::File(file) = object else { - return SyscallError::PermissionDenied.to_u64(); - }; - if size > file_cache::MAX_FILE_SIZE { - return SyscallError::InvalidArgument.to_u64(); - } - // The seek pointer is not touched (POSIX ftruncate): a shrink leaves it past EOF. - file.with(|state| { - if let Err(e) = file_cache::set_size(state.file_id, size) { - return SyscallError::from(e).to_u64(); - } - state.mtime = crate::clock::mtime_now(); - file_cache::touch(state.file_id, state.mtime); - 0 - }) -} - pub fn has_data(object: &KObjectRef) -> bool { match object { KObjectRef::PipeRead(r) => r.pipe().has_data(), @@ -734,7 +482,7 @@ pub fn has_data(object: &KObjectRef) -> bool { KObjectRef::Notice(n) => n.readable(), KObjectRef::EventWait(e) => e.ready(), KObjectRef::Pager(p) => p.has_notices(), - KObjectRef::File(_) | KObjectRef::Null(_) => true, + KObjectRef::Null(_) => true, KObjectRef::Device(d) => match d.class() { // A claim its last handle has let go has nothing to read: a poll // on it is ended where it registers (`DeviceClaim::add_poll`). @@ -759,7 +507,7 @@ pub fn read_posts_are_readiness(object: &KObjectRef) -> bool { match object { KObjectRef::SysCap(_) => true, KObjectRef::Console(_) | KObjectRef::PipeRead(_) | KObjectRef::PipeWrite(_) | KObjectRef::Connection(_) - | KObjectRef::Acceptor(_) | KObjectRef::File(_) | KObjectRef::Device(_) + | KObjectRef::Acceptor(_) | KObjectRef::Device(_) | KObjectRef::Inbox(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) | KObjectRef::SharedMem(_) | KObjectRef::Process(_) | KObjectRef::Job(_) | KObjectRef::Null(_) | KObjectRef::Notice(_) => false, @@ -772,7 +520,7 @@ pub fn has_space(object: &KObjectRef) -> bool { // one refused, retried on readiness, lands. KObjectRef::PipeWrite(w) => w.pipe().has_space(MAX_WHOLE_PIPE_WRITE), KObjectRef::Connection(c) => c.tx().writable(), - KObjectRef::File(_) | KObjectRef::Null(_) => true, + KObjectRef::Null(_) => true, KObjectRef::Console(_) => crate::log::console::has_room(), KObjectRef::PipeRead(_) | KObjectRef::Device(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) @@ -792,7 +540,7 @@ pub fn mark_tty(object: &KObjectRef) -> u64 { w.mark_tty(); 0 } - KObjectRef::Connection(_) | KObjectRef::File(_) | KObjectRef::Device(_) + KObjectRef::Connection(_) | KObjectRef::Device(_) | KObjectRef::Console(_) | KObjectRef::Acceptor(_) | KObjectRef::Inbox(_) | KObjectRef::SysCap(_) | KObjectRef::SharedMem(_) | KObjectRef::Connector(_) | KObjectRef::WakeLatency(_) | KObjectRef::EventSignal(_) | KObjectRef::EventWait(_) | KObjectRef::Pager(_) | KObjectRef::Paged(_) | KObjectRef::Namespace(_) diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index a7b74d6b77d..a054cd44713 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -268,7 +268,7 @@ impl PagerObject { /// A new object of `size` bytes: its id. `ResourceExhausted` where every /// slot holds one. - pub fn add(&self, size: u64) -> u64 { + pub fn add(&self, size: u64, anonymous: bool) -> u64 { if size > kernel::paged::LAST_END { return SyscallError::InvalidArgument.to_u64(); } @@ -277,7 +277,7 @@ impl PagerObject { return SyscallError::Gone.to_u64(); } let Some(slot) = state.free.pop() else { return SyscallError::ResourceExhausted.to_u64() }; - let object = state.pages.object(slot, size); + let object = if anonymous { state.pages.anonymous(slot, size) } else { state.pages.object(slot, size) }; state.entries[slot as usize] = Some(Entry { object, refs: 1, holders: 0, owned: true, no_holders: false }); u64::from(slot) } diff --git a/kernel/src/revoke_selftest.rs b/kernel/src/revoke_selftest.rs deleted file mode 100644 index bc7485cbfc1..00000000000 --- a/kernel/src/revoke_selftest.rs +++ /dev/null @@ -1,55 +0,0 @@ -//! The revoked-backing control, behind `revoked-backing-selftest`: on the -//! kernel's one writable mount, `/tmp`, a `FileBacking` read after the file's -//! deletion must fail rather than fault in zeros. - -use crate::file_cache; -use crate::mm::PAGE_BYTES; -use crate::vfs; - -const FILL: u8 = 0xA7; - -pub fn run() { - probe("/tmp/revoke_probe"); -} - -/// FAIL names the step so the verdict line carries the mechanism, not just the arm. -fn fail(path: &str, step: &str) { - crate::log!("revoke-selftest: {path} FAIL ({step})"); -} - -fn probe(path: &str) { - let mtime = crate::clock::mtime_now(); - let id = match vfs::lock().create_file(path, mtime) { - Ok((id, _)) => id, - Err(e) => return fail(path, &alloc::format!("create: {e:?}")), - }; - if file_cache::write_page(id, 0, 0, &[FILL; 64][..]).is_err() { - return fail(path, "write"); - } - // Released the way `OpenFileState::drop` does. - file_cache::release(id); - let backing = match vfs::lock().open_backing(path) { - Ok(b) => b, - Err(e) => return fail(path, &alloc::format!("open_backing: {e:?}")), - }; - - let mut heap = alloc::vec![0u8; PAGE_BYTES].into_boxed_slice(); - let buf: &mut [u8; PAGE_BYTES] = (&mut heap[..]).try_into().expect("PAGE_BYTES bytes"); - match backing.read_page(0, buf) { - Ok(()) if buf[0] == FILL => {} - Ok(()) => return fail(path, &alloc::format!("pre-delete read served {:#04x}", buf[0])), - Err(_) => return fail(path, "pre-delete read refused"), - } - - if let Err(e) = vfs::lock().delete_file(path) { - return fail(path, &alloc::format!("delete: {e:?}")); - } - - match backing.read_page(0, buf) { - Err(_) => crate::log!("revoke-selftest: {path} PASS (pre={FILL:#04x}, post refused)"), - Ok(()) => fail( - path, - &alloc::format!("post-delete read succeeded, byte 0 = {:#04x}", buf[0]), - ), - } -} diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 575f26ee549..370384775b7 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -27,10 +27,7 @@ use super::device::{ sys_device_reg_read, }; use super::event::{sys_event_create, sys_event_signal, sys_event_take, sys_handle_wait}; -use super::fs::{ - spawn_cwd, sys_chdir, sys_delete, sys_getcwd, sys_mkdir, sys_open, sys_readdir, sys_readlink, - sys_rename, sys_rmdir, sys_stat, sys_symlink, -}; +use super::proc::sys_getcwd; use super::handles::{sys_close, sys_handle_dup, with_object, with_object_ref}; use super::io::{ sys_console_carried, sys_null, sys_random, sys_read, sys_read_nonblock, sys_write, sys_write_nonblock, @@ -95,21 +92,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> let Some(mut buf) = ctx.user_bytes_mut(UserAddr::new(a1), a2) else { return bad_addr }; sys_random(&mut buf) } - SYS_OPEN => { - let path = match ctx.user_str(UserAddr::new(a1), a2) { Ok(s) => s, Err(e) => return e.to_u64() }; - sys_open(&path, OpenFlags(a3)) - } SYS_CLOSE => sys_close(RawHandle(a1 as u32)), - SYS_SEEK => { - let pos = match a3 { - 0 => SeekFrom::Start(a2), - 1 => SeekFrom::Current(a2 as i64), - 2 => SeekFrom::End(a2 as i64), - _ => return SyscallError::InvalidArgument.to_u64(), - }; - with_object(RawHandle(a1 as u32), Rights::READ, |o| ops::seek(o, pos)) - } - // No right: fstat moves no content, so gating on READ would fail isatty() on write-only handles. SYS_FSTAT => { let stat = match with_object_ref(RawHandle(a1 as u32), Rights::NONE, ops::fstat) { Ok(stat) => stat, @@ -120,47 +103,9 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> Err(e) => e.to_u64(), } } - // Cloned and run outside the process-data lock: fsync parks between flush - // attempts, and a park under that lock is what the baseline tripwire refuses. - // The clone outlives a concurrent close, the only other way this object is revoked. - SYS_FSYNC => { - match with_object_ref(RawHandle(a1 as u32), Rights::WRITE, KObjectRef::clone) { - Ok(object) => ops::fsync(&object), - Err(e) => e.refuse(), - } - } - SYS_STAT => { - let path = match ctx.user_str(UserAddr::new(a1), a2) { Ok(s) => s, Err(e) => return e.to_u64() }; - let follow = match a4 { - 0 => false, - 1 => true, - _ => return SyscallError::InvalidArgument.to_u64(), - }; - let stat = match sys_stat(&path, follow) { - Ok(stat) => stat, - Err(e) => return e.to_u64(), - }; - match ctx.copy_out(UserAddr::new(a3), &stat) { - Ok(()) => 0, - Err(e) => e.to_u64(), - } - } - SYS_READDIR => { - let path = match ctx.user_str(UserAddr::new(a1), a2) { Ok(s) => s, Err(e) => return e.to_u64() }; - let Some(mut buf) = ctx.user_bytes_mut(UserAddr::new(a3), a4) else { return bad_addr }; - sys_readdir(&path, &mut buf) - } - SYS_DELETE => { - let path = match ctx.user_str(UserAddr::new(a1), a2) { Ok(s) => s, Err(e) => return e.to_u64() }; - sys_delete(&path) - } SYS_SHUTDOWN => sys_shutdown(RawHandle(a1 as u32)), SYS_REBOOT => sys_reboot(RawHandle(a1 as u32)), SYS_MACHINE_STOPS => sys_machine_stops(RawHandle(a1 as u32), a2), - SYS_CHDIR => { - let path = match ctx.user_str(UserAddr::new(a1), a2) { Ok(s) => s, Err(e) => return e.to_u64() }; - sys_chdir(&path) - } SYS_GETCWD => { let Some(mut buf) = ctx.user_bytes_mut(UserAddr::new(a1), a2) else { return bad_addr }; sys_getcwd(&mut buf) @@ -171,9 +116,9 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> // First, and nothing copied: the image is the caller's object, and // an endowment below may move the caller's own handle to it. let image = match args.image_len { - 0 => None, + 0 => return SyscallError::InvalidArgument.to_u64(), len => match shared_image(args.image, len) { - Ok(image) => Some(alloc::sync::Arc::new(image) as alloc::sync::Arc), + Ok(image) => alloc::sync::Arc::new(image) as alloc::sync::Arc, Err(refused) => return refused, }, }; @@ -201,7 +146,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> }; let path = match ctx.user_str(UserAddr::new(args.path_ptr), args.path_len) { Ok(s) => s, Err(e) => return e.to_u64() }; let text = match ctx.user_str(UserAddr::new(args.argv_ptr), args.argv_len) { Ok(s) => s, Err(e) => return e.to_u64() }; - let cwd = match ctx.user_str(UserAddr::new(args.cwd_ptr), args.cwd_len).and_then(|p| spawn_cwd(&p)) { + let cwd = match ctx.user_str(UserAddr::new(args.cwd_ptr), args.cwd_len) { Ok(cwd) => cwd, Err(e) => return e.to_u64(), }; @@ -301,19 +246,6 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> SYS_HANDLE_DUP => sys_handle_dup(RawHandle(a1 as u32), a2), SYS_NULL => sys_null(a1), SYS_GETPID => process::current_process().raw() as u64, - SYS_RENAME => { - let old = match ctx.user_str(UserAddr::new(a1), a2) { Ok(s) => s, Err(e) => return e.to_u64() }; - let new = match ctx.user_str(UserAddr::new(a3), a4) { Ok(s) => s, Err(e) => return e.to_u64() }; - sys_rename(&old, &new) - } - SYS_MKDIR => { - let path = match ctx.user_str(UserAddr::new(a1), a2) { Ok(s) => s, Err(e) => return e.to_u64() }; - sys_mkdir(&path) - } - SYS_RMDIR => { - let path = match ctx.user_str(UserAddr::new(a1), a2) { Ok(s) => s, Err(e) => return e.to_u64() }; - sys_rmdir(&path) - } SYS_DLOPEN => { let Ok(args) = ctx.copy_in::(UserAddr::new(a1)) else { return bad_addr }; let name = match ctx.user_str(UserAddr::new(args.name_ptr), args.name_len) { Ok(s) => s, Err(e) => return e.to_u64() }; @@ -340,9 +272,6 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> // A no-op by contract: a dlopen'd module is shared and process-lived, and // a reload returns the same handle (`sys_dlopen`), so nothing to reclaim. SYS_DLCLOSE => 0, - SYS_FTRUNCATE => { - with_object(RawHandle(a1 as u32), Rights::WRITE, |o| ops::ftruncate(o, a2)) - } SYS_STACK_INFO => { let stack = process::with_current_data(|data| { (data.user_stack_base.raw() > 0) @@ -372,14 +301,6 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> SYS_VM_DISCARD => sys_vm_discard(a1, a2), SYS_VM_DECOMMIT => sys_vm_decommit(a1, a2), SYS_VM_REMAP => sys_vm_remap(a1, a2, a3, a4), - SYS_PREAD => { - let Some(mut buf) = ctx.user_bytes_mut(UserAddr::new(a2), a3) else { return bad_addr }; - with_object(RawHandle(a1 as u32), Rights::READ, |o| ops::pread(o, a4, &mut buf)) - } - SYS_PWRITE => { - let Some(buf) = ctx.user_bytes(UserAddr::new(a2), a3) else { return bad_addr }; - with_object(RawHandle(a1 as u32), Rights::WRITE, |o| ops::pwrite(o, a4, &buf)) - } SYS_READ_NONBLOCK => { let Some(mut buf) = ctx.user_bytes_mut(UserAddr::new(a2), a3) else { return bad_addr }; sys_read_nonblock(RawHandle(a1 as u32), &mut buf) @@ -417,16 +338,6 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> SYS_DEVICE_DMA_UNMAP => sys_device_dma_unmap(RawHandle(a1 as u32), a2), SYS_DEVICE_DMA_LIMIT => sys_device_dma_limit(RawHandle(a1 as u32), a2), SYS_DEVICE_DMA_START => sys_device_dma_start(RawHandle(a1 as u32)), - SYS_SYMLINK => { - let target = match ctx.user_str(UserAddr::new(a1), a2) { Ok(s) => s, Err(e) => return e.to_u64() }; - let link = match ctx.user_str(UserAddr::new(a3), a4) { Ok(s) => s, Err(e) => return e.to_u64() }; - sys_symlink(&target, &link) - } - SYS_READLINK => { - let path = match ctx.user_str(UserAddr::new(a1), a2) { Ok(s) => s, Err(e) => return e.to_u64() }; - let Some(mut buf) = ctx.user_bytes_mut(UserAddr::new(a3), a4) else { return bad_addr }; - sys_readlink(&path, &mut buf) - } SYS_ENDOWMENTS => { let Some(mut buf) = ctx.user_bytes_mut(UserAddr::new(a1), a2) else { return bad_addr }; sys_endowments(&mut buf) @@ -688,7 +599,10 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> SYS_ACPI => sys_acpi(&ctx, RawHandle(a1 as u32), a2, a3), SYS_DEVICE_REG_READ => sys_device_reg_read(RawHandle(a1 as u32), a2, a3), SYS_PAGER_CREATE => sys_pager_create(a1, a2, a3, a4), - SYS_PAGED_CREATE => sys_paged_create(RawHandle(a1 as u32), a2), + SYS_PAGED_CREATE => match a3 { + 0 | 1 => sys_paged_create(RawHandle(a1 as u32), a2, a3 != 0), + _ => SyscallError::InvalidArgument.to_u64(), + }, SYS_PAGED_HOLDER => sys_paged_holder(RawHandle(a1 as u32), a2), SYS_PAGED_READ => { let Some(mut buf) = ctx.user_bytes_mut(UserAddr::new(a2), a3) else { return bad_addr }; diff --git a/kernel/src/syscall/fs.rs b/kernel/src/syscall/fs.rs deleted file mode 100644 index a114e4f6dc0..00000000000 --- a/kernel/src/syscall/fs.rs +++ /dev/null @@ -1,194 +0,0 @@ -//! Filesystem syscalls: resolve a path against the caller's cwd, then read or -//! change the volume. -//! -//! Every path is a `&str` `super::dispatch` already copied out of user memory, -//! so no arm here holds a user pointer. - -use crate::object::ops; -use crate::user_ptr::UserBytesMut; -use crate::{process, vfs}; - -use toyos_abi::syscall::*; - -// Entry ops act on the name without following it; the check rides the same `vfs` -// guard the mutation will run on, so a moved mount can't separate check from act. -fn resolve_and_check( - vfs: &vfs::Vfs, - cwd: &str, - path: &str, -) -> Result { - let resolved = vfs.resolve_absolute(cwd, path); - if !vfs.user_may_modify(&resolved) { - return Err(SyscallError::PermissionDenied.to_u64()); - } - Ok(resolved) -} - -// Returns the guard held, so the mutation runs under the same lock the check ran on. -fn resolve_for_modify(path: &str) -> Result<(vfs::VfsGuard, alloc::string::String), u64> { - let cwd = process::with_process_data(|d| d.cwd.clone()); - let vfs = vfs::lock(); - let resolved = resolve_and_check(&vfs, &cwd, path)?; - Ok((vfs, resolved)) -} - -pub(super) fn sys_open(path: &str, flags: OpenFlags) -> u64 { - let cwd = process::with_process_data(|d| d.cwd.clone()); - let resolved = vfs::lock().resolve_absolute(&cwd, path); - process::with_process_data(|data| ops::open(&mut data.handles, &resolved, flags)) -} - -/// What `path` names, a final symlink followed when `follow` says so. -pub(super) fn sys_stat(path: &str, follow: bool) -> Result { - let cwd = process::with_process_data(|d| d.cwd.clone()); - let mut vfs = vfs::lock(); - let resolved = vfs.resolve_absolute(&cwd, path); - vfs.stat(&resolved, follow) -} - -/// Returns the length needed; writes the listing only when it fits (`n <= out.len()`). -pub(super) fn sys_readdir(path: &str, out: &mut UserBytesMut) -> u64 { - let cwd = process::with_process_data(|d| d.cwd.clone()); - // Held until the listing is let go, as this call returns. - let mut listing = crate::mm::charge::Charge::none(&crate::mm::charge::current()); - let entries = match vfs::lock().list(&cwd, path, &mut listing) { - Ok(e) => e, - Err(e) => return e.to_u64(), - }; - - // A directory name is stored with its trailing slash and encoded without. - let encoded = |name: &alloc::string::String| 1 + name.trim_end_matches('/').len() + 1 + 8; - let needed: usize = entries.iter().map(|(name, _)| encoded(name)).sum(); - if needed > out.len() { - return needed as u64; - } - - let mut pos = 0; - for (name, node) in &entries { - let is_dir = name.ends_with('/'); - let clean_name = if is_dir { &name[..name.len() - 1] } else { name.as_str() }; - out.write_at(pos, &[if is_dir { DIRENT_DIR } else { DIRENT_FILE }]); - pos += 1; - out.write_at(pos, clean_name.as_bytes()); - pos += clean_name.len(); - out.write_at(pos, &[0]); - pos += 1; - out.write_at(pos, &node.to_le_bytes()); - pos += 8; - } - debug_assert_eq!(pos, needed); - pos as u64 -} - -pub(super) fn sys_delete(path: &str) -> u64 { - let (mut vfs, resolved) = match resolve_for_modify(path) { - Ok(pair) => pair, - Err(refusal) => return refusal, - }; - match vfs.delete(&resolved) { - Ok(()) => 0, - Err(e) => e.to_u64(), - } -} - -pub(super) fn sys_chdir(path: &str) -> u64 { - let cwd = process::with_process_data(|d| d.cwd.clone()); - // The VFS guard ends with this statement: `sys_open` takes VFS under process-data. - let resolved = vfs::lock().cd(&cwd, path); - match resolved { - Ok(new_cwd) => { - process::with_process_data(|d| d.cwd = new_cwd); - 0 - } - Err(e) => e.to_u64(), - } -} - -/// A spawned child's working directory, judged as `SYS_CHDIR` judges a path but -/// against nobody's cwd: a relative one would be the caller's by another road. -pub(super) fn spawn_cwd(path: &str) -> Result { - if !path.starts_with('/') { - return Err(SyscallError::InvalidArgument); - } - vfs::lock().cd("/", path) -} - -/// Returns the length needed; writes the cwd only when it fits (`n <= out.len()`). -pub(super) fn sys_getcwd(out: &mut UserBytesMut) -> u64 { - process::with_process_data(|data| { - let cwd = data.cwd.as_bytes(); - if cwd.len() <= out.len() { - out.write_at(0, cwd); - } - cwd.len() as u64 - }) -} - -pub(super) fn sys_rename(old: &str, new: &str) -> u64 { - let cwd = process::with_process_data(|d| d.cwd.clone()); - let mut vfs = vfs::lock(); - let old_abs = match resolve_and_check(&vfs, &cwd, old) { - Ok(resolved) => resolved, - Err(refusal) => return refusal, - }; - let new_abs = match resolve_and_check(&vfs, &cwd, new) { - Ok(resolved) => resolved, - Err(refusal) => return refusal, - }; - match vfs.rename(&old_abs, &new_abs) { - Ok(()) => 0, - Err(e) => e.to_u64(), - } -} - -pub(super) fn sys_mkdir(path: &str) -> u64 { - let (mut vfs, resolved) = match resolve_for_modify(path) { - Ok(pair) => pair, - Err(refusal) => return refusal, - }; - match vfs.create_dir(&resolved) { - Ok(()) => 0, - Err(e) => e.to_u64(), - } -} - -pub(super) fn sys_rmdir(path: &str) -> u64 { - let (mut vfs, resolved) = match resolve_for_modify(path) { - Ok(pair) => pair, - Err(refusal) => return refusal, - }; - match vfs.remove_dir(&resolved) { - Ok(()) => 0, - Err(e) => e.to_u64(), - } -} - -pub(super) fn sys_symlink(target: &str, link: &str) -> u64 { - let (mut vfs, resolved) = match resolve_for_modify(link) { - Ok(pair) => pair, - Err(refusal) => return refusal, - }; - match vfs.create_symlink(&resolved, target) { - Ok(()) => 0, - Err(e) => { - log!("symlink({target} -> {link}): {e}"); - e.to_u64() - } - } -} - -pub(super) fn sys_readlink(path: &str, out: &mut UserBytesMut) -> u64 { - let cwd = process::with_process_data(|d| d.cwd.clone()); - let mut vfs = vfs::lock(); - let resolved = vfs.resolve_absolute(&cwd, path); - match vfs.read_link(&resolved) { - Ok(Some(target)) => { - let bytes = target.as_bytes(); - let len = bytes.len().min(out.len()); - out.write_at(0, &bytes[..len]); - len as u64 - } - Ok(None) => SyscallError::NotFound.to_u64(), - Err(e) => e.to_u64(), - } -} diff --git a/kernel/src/syscall/mod.rs b/kernel/src/syscall/mod.rs index 6000ccfd59d..aa55ef2c404 100644 --- a/kernel/src/syscall/mod.rs +++ b/kernel/src/syscall/mod.rs @@ -7,7 +7,6 @@ mod debug; mod device; pub(crate) mod dispatch; mod event; -mod fs; mod handles; mod io; mod ipc; diff --git a/kernel/src/syscall/paged.rs b/kernel/src/syscall/paged.rs index cd7f1e5f78d..6e1bc1bc5cc 100644 --- a/kernel/src/syscall/paged.rs +++ b/kernel/src/syscall/paged.rs @@ -48,9 +48,9 @@ fn pager(h: RawHandle, rights: Rights) -> Result, u64> { process::with_process_data(|data| data.handles.get::(h, rights)).map_err(|e| e.refuse()) } -pub(super) fn sys_paged_create(h: RawHandle, size: u64) -> u64 { +pub(super) fn sys_paged_create(h: RawHandle, size: u64, anonymous: bool) -> u64 { match pager(h, Rights::WRITE) { - Ok(pager) => pager.add(size), + Ok(pager) => pager.add(size, anonymous), Err(refused) => refused, } } diff --git a/kernel/src/syscall/proc.rs b/kernel/src/syscall/proc.rs index 12dfd91da5d..0dd9db1f18f 100644 --- a/kernel/src/syscall/proc.rs +++ b/kernel/src/syscall/proc.rs @@ -52,7 +52,7 @@ pub(super) fn sys_spawn( pending: crate::loader::PendingHandles, cwd: alloc::string::String, env: Vec, - image: Option>, + image: alloc::sync::Arc, parent: process::Parent, ) -> u64 { // Nothing to clean up: spawn's frame owns the child's resources on error. @@ -342,3 +342,12 @@ pub(super) fn sys_account_info(ctx: &crate::user_ptr::SyscallContext, job: RawHa Err(e) => e.to_u64(), } } + +/// The initial working-directory string the parent supplied at spawn. +pub(super) fn sys_getcwd(out: &mut crate::user_ptr::UserBytesMut) -> u64 { + process::with_process_data(|data| { + let cwd = data.cwd.as_bytes(); + if cwd.len() <= out.len() { out.write_at(0, cwd); } + cwd.len() as u64 + }) +} diff --git a/kernel/src/tmpfs.rs b/kernel/src/tmpfs.rs deleted file mode 100644 index aded31712d5..00000000000 --- a/kernel/src/tmpfs.rs +++ /dev/null @@ -1,247 +0,0 @@ -use alloc::collections::BTreeMap; -use alloc::string::String; -use alloc::sync::Arc; -use alloc::vec::Vec; -use core::sync::atomic::{AtomicBool, Ordering}; - -use crate::file_backing::FileBacking; -use crate::file_cache::{self, FileId}; -use crate::fs_rename::{self, Committed, ReplaceRename}; -use crate::mm::charge::{self, Charge}; -use crate::mm::space::CopySource; -use crate::mm::PAGE_BYTES; -use toyos_abi::syscall::SyscallError; - -use crate::vfs::FileSystem; - -struct TmpfsBacking { - file_id: FileId, - /// Shared by every backing for the entry; cleared by [`retire`]. - alive: Arc, -} - -impl FileBacking for TmpfsBacking { - /// `Err` once the file is deleted: a gone file is not a file of zeros. - fn read_page(&self, file_offset: u64, buf: &mut [u8; PAGE_BYTES]) -> crate::file_backing::BlockResult { - // copy_page_out, not file_cache::read_page: reading through the miss path here would recurse. - // A hole below the file size, left by a seek-and-write, reads as zero. - if file_offset >= file_cache::size(self.file_id) - || !file_cache::copy_page_out(self.file_id, (file_offset / PAGE_BYTES as u64) as u32, buf) - { - // After the miss: `retire` clears the flag before the pages drop. - if !self.alive.load(Ordering::Acquire) { - log!("tmpfs: read through a backing whose file was deleted"); - return Err(crate::file_backing::BlockError::Device); - } - buf.fill(0); - } - Ok(()) - } - - fn file_size(&self) -> u64 { - file_cache::size(self.file_id) - } - - fn source(&self) -> Result, SyscallError> { - if !self.alive.load(Ordering::Acquire) { - return Err(SyscallError::NotFound); - } - Ok(Arc::new(file_cache::snapshot(self.file_id)?)) - } -} - -/// Ends a file entry's life: cleared first (so every backing fails), pages after. -fn retire(id: FileId, alive: &AtomicBool) { - alive.store(false, Ordering::Release); - file_cache::mark_deleted(id); -} - -/// One name's one entry — a file or a symlink, never both. A single map keys -/// every name once, so `create`, `create_symlink`, `rename` and `delete` cannot -/// each see a different namespace. -pub(crate) enum Entry { - File { id: FileId, alive: Arc }, - /// `node` is drawn at the link's making and moves with its entry. - Symlink { target: String, node: u64 }, -} - -/// In-memory filesystem: file data lives in the file cache. -pub struct TmpFs { - /// Each name with the charge of what it keeps here: its place, its bytes - /// and its entry's. - entries: BTreeMap, -} - -/// The charge of `name` holding an entry that keeps `kept` heap bytes besides. -fn name_charge(name: &str, kept: usize) -> Result { - Ok(Charge::current(charge::map_entry::() + charge::array::(name.len()) + kept)?) -} - -/// The heap a file's entry keeps besides its name. -fn file_heap() -> usize { - charge::arc::() -} - -impl TmpFs { - pub fn new() -> Self { - Self { entries: BTreeMap::new() } - } -} - -impl ReplaceRename for TmpFs { - type Displaced = Option; - - fn source_present(&mut self, old: &str) -> Result { - Ok(self.entries.contains_key(old)) - } - - fn same_object(&mut self, old: &str, new: &str) -> Result { - // tmpfs keys each name exactly; equal strings are the one entry. - Ok(old == new) - } - - fn commit(&mut self, old: &str, new: &str) -> Result>, SyscallError> { - let kept = match self.entries.get(old) { - Some((Entry::File { .. }, _)) => file_heap(), - Some((Entry::Symlink { target, .. }, _)) => charge::array::(target.len()), - None => return Err(SyscallError::NotFound), - }; - // Charged under its new name before anything moves. - let charge = name_charge(new, kept)?; - let (moved, _) = self.entries.remove(old).expect("the name just found"); - let displaced = self.entries.remove(new).map(|(entry, _)| entry); - self.entries.insert(String::from(new), (moved, charge)); - Ok(Committed::new(displaced)) - } - - fn release( - &mut self, - _old: &str, - _new: &str, - committed: Committed>, - ) -> Result<(), SyscallError> { - if let Some(Entry::File { id, alive, .. }) = committed.into_displaced() { - retire(id, &alive); - } - Ok(()) - } -} - -impl FileSystem for TmpFs { - // Nothing else caps file count here. - fn list(&mut self, dir: &str, limit: usize, charge: &mut Charge) -> Result, SyscallError> { - let mut out = Vec::new(); - for (name, (entry, _)) in &self.entries { - if let Entry::File { .. } = entry { - if !crate::vfs::under_directory(name, dir) { - continue; - } - if out.len() == limit { - return Err(SyscallError::ResourceExhausted); - } - charge.grow(crate::vfs::listed_heap::(name))?; - out.push(name.clone()); - } - } - Ok(out) - } - - fn node(&mut self, name: &str) -> Result { - match self.entries.get(name) { - Some((Entry::File { id, .. }, _)) => Ok(*id), - Some((Entry::Symlink { node, .. }, _)) => Ok(*node), - None => Err(SyscallError::NotFound), - } - } - - /// The first name at or after `dir/` in key order is the only one that can - /// lie beneath it. - fn is_dir(&mut self, dir: &str) -> Result { - if dir.is_empty() { - return Ok(true); - } - let prefix = alloc::format!("{dir}/"); - Ok(self.entries.range::((core::ops::Bound::Included(prefix.as_str()), core::ops::Bound::Unbounded)) - .next() - .is_some_and(|(name, _)| name.starts_with(&prefix))) - } - - fn file_mtime(&mut self, name: &str) -> Result { - match self.entries.get(name) { - Some((Entry::File { id, .. }, _)) => Ok(file_cache::mtime(*id)), - _ => Err(SyscallError::NotFound), - } - } - - fn read_link(&mut self, name: &str) -> Result, SyscallError> { - Ok(match self.entries.get(name) { - Some((Entry::Symlink { target, .. }, _)) => Some(target.clone()), - _ => None, - }) - } - - fn open_file(&mut self, name: &str) -> Result<(FileId, Option>), SyscallError> { - match self.entries.get(name) { - Some((Entry::File { id, .. }, _)) => { - file_cache::open(*id).commit(); - Ok((*id, None)) // tmpfs: no backing, data is in the file cache - } - _ => Err(SyscallError::NotFound), - } - } - - fn create(&mut self, name: &str, mtime: u64) -> Result { - if let Some((Entry::File { id, .. }, _)) = self.entries.get(name) { - return Ok(*id); - } - let charge = name_charge(name, file_heap())?; - // A dangling symlink of this name is displaced: one name, one entry. - let id = file_cache::create_file(false, mtime)?; // non-evictable - self.entries.insert(String::from(name), (Entry::File { id, alive: Arc::new(AtomicBool::new(true)) }, charge)); - Ok(id) - } - - fn delete(&mut self, name: &str) -> Result<(), SyscallError> { - match self.entries.remove(name) { - Some((Entry::File { id, alive, .. }, _)) => { - retire(id, &alive); - Ok(()) - } - Some((Entry::Symlink { .. }, _)) => Ok(()), - None => Err(SyscallError::NotFound), - } - } - - fn rename(&mut self, old: &str, new: &str) -> Result<(), SyscallError> { - fs_rename::replace_rename(self, old, new) - } - - // `NotSupported`, not a refusal: no directory representation here, so the - // VFS carries created directories itself. - fn create_dir(&mut self, _name: &str) -> Result<(), SyscallError> { - Err(SyscallError::NotSupported) - } - - fn remove_dir(&mut self, _name: &str) -> Result<(), SyscallError> { - Err(SyscallError::NotSupported) - } - - fn create_symlink(&mut self, name: &str, target: &str) -> Result<(), SyscallError> { - let charge = name_charge(name, charge::array::(target.len()))?; - // Displaces whatever answered to this name: one name, one entry. - let link = Entry::Symlink { target: String::from(target), node: file_cache::draw_node() }; - if let Some((Entry::File { id, alive, .. }, _)) = self.entries.insert(String::from(name), (link, charge)) { - retire(id, &alive); - } - Ok(()) - } - - fn open_backing(&mut self, name: &str) -> Result, SyscallError> { - match self.entries.get(name) { - Some((Entry::File { id, alive, .. }, _)) => { - Ok(Arc::new(TmpfsBacking { file_id: *id, alive: Arc::clone(alive) })) - } - _ => Err(SyscallError::NotFound), - } - } -} diff --git a/kernel/src/user_ptr.rs b/kernel/src/user_ptr.rs index 34db1fbb9e5..7cd16cf426f 100644 --- a/kernel/src/user_ptr.rs +++ b/kernel/src/user_ptr.rs @@ -300,28 +300,3 @@ impl UserBytesMut<'_> { } } -/// Bytes the kernel copies *from*, wherever they live — lets `file_cache::write_page` name the capability it needs instead of a concrete window type. -pub trait ByteSource { - fn len(&self) -> usize; - fn read_at(&self, off: usize, dst: &mut [u8]); -} - -impl ByteSource for [u8] { - fn len(&self) -> usize { - <[u8]>::len(self) - } - - fn read_at(&self, off: usize, dst: &mut [u8]) { - dst.copy_from_slice(&self[off..off + dst.len()]); - } -} - -impl ByteSource for UserBytes<'_> { - fn len(&self) -> usize { - UserBytes::len(self) - } - - fn read_at(&self, off: usize, dst: &mut [u8]) { - UserBytes::read_at(self, off, dst); - } -} diff --git a/kernel/src/vfs.rs b/kernel/src/vfs.rs deleted file mode 100644 index 8f98cf5c2db..00000000000 --- a/kernel/src/vfs.rs +++ /dev/null @@ -1,699 +0,0 @@ -use alloc::boxed::Box; -use alloc::format; -use alloc::string::String; -use alloc::collections::{BTreeMap, BTreeSet}; -use alloc::vec::Vec; - -use core::ops::{Deref, DerefMut}; -use toyos_abi::syscall::{FileType, SyscallError, MAX_LINKS, MAX_PATH}; -use crate::file_cache::{self, FileId}; -use crate::mm::charge::{self, Charge}; -use crate::object::ops::Stat; -use crate::sync::{Lock, LockGuard}; -use kernel::dirnodes::Directories; - -static VFS: Lock> = Lock::new(None); - -pub fn init() { - *VFS.lock() = Some(Vfs::new()); -} - -pub struct VfsGuard(LockGuard<'static, Option>); - -impl Deref for VfsGuard { - type Target = Vfs; - fn deref(&self) -> &Vfs { self.0.as_ref().expect("VFS not initialized") } -} - -impl DerefMut for VfsGuard { - fn deref_mut(&mut self) -> &mut Vfs { self.0.as_mut().expect("VFS not initialized") } -} - -pub fn lock() -> VfsGuard { - VfsGuard(VFS.lock()) -} - -/// A device that would not answer is `Io`; `NotFound` means only that the name is absent. -pub trait FileSystem: Send { - /// Every name [`under_directory`] puts at or under `dir` (`""` is the mount - /// root), or `ResourceExhausted` above `limit`, which counts only those, - /// or where `charge` cannot grow by what [`listed_heap`] says each takes. - fn list(&mut self, dir: &str, limit: usize, charge: &mut Charge) -> Result, SyscallError>; - - /// The number of the file or symlink `name`, not followed, as - /// `toyos_abi::syscall::Stat::node` says: a file's is the `FileId` - /// [`open_file`](Self::open_file) answers, a symlink's one drawn by - /// `file_cache::draw_node`; `NotFound` for anything else. - fn node(&mut self, name: &str) -> Result; - - /// Whether `dir` (`""` is the mount root) is a directory: a working - /// directory is judged by this, on every spawn and `SYS_CHDIR`, under the - /// VFS lock. Nothing beneath it is materialised and nothing caps it; what it costs is - /// the mount's. - fn is_dir(&mut self, dir: &str) -> Result; - - /// When `name` was last written, as `toyos_abi::syscall::Stat::mtime` says, - /// to the precision the mount stores. - fn file_mtime(&mut self, name: &str) -> Result; - - /// What `name` points at; `Ok(None)` for a non-link or an absent name, never for a device that would not answer. - fn read_link(&mut self, name: &str) -> Result, SyscallError>; - - /// Open `name`: the same `FileId` on every open of the same file, plus a backing for cache misses. - fn open_file(&mut self, name: &str) -> Result<(FileId, Option>), SyscallError>; - /// Create an empty file, registered under `name`. - fn create(&mut self, name: &str, mtime: u64) -> Result; - - /// Unlink `name`, or `NotFound` if there was nothing by that name. - fn delete(&mut self, name: &str) -> Result<(), SyscallError>; - fn rename(&mut self, old: &str, new: &str) -> Result<(), SyscallError>; - - /// Create the directory `name` on the volume; `NotSupported` from a mount - /// with no directory representation makes the VFS carry it instead. - fn create_dir(&mut self, name: &str) -> Result<(), SyscallError>; - /// Remove the empty directory `name`, refusing a file, a missing name and - /// a non-empty directory each by its own error. - fn remove_dir(&mut self, name: &str) -> Result<(), SyscallError>; - - fn create_symlink(&mut self, name: &str, target: &str) -> Result<(), SyscallError>; - - /// `open_backing` has no default body: an unimplemented one would silently report every file on that mount as missing (the sentinel this trait exists to remove). - fn open_backing(&mut self, name: &str) -> Result, SyscallError>; -} - - -/// The entries `/` has, in listing order. `/` is synthesized rather than -/// mounted: it is no filesystem, and nothing outside this set can be mounted. -pub const ROOT_ENTRIES: [&str; 9] = - ["apps", "boot", "config", "home", "log", "media", "state", "system", "tmp"]; - -struct Mount { - fs: Box, -} - -/// A name at `/`, and where in its filesystem that name begins. -#[derive(Clone, Copy)] -struct MountPoint { - fs: usize, - /// Empty for a filesystem one name reaches; the name itself for one several - /// reach, which is how DATA is `/apps` and `/home` at once. - prefix: &'static str, -} - -impl MountPoint { - fn path(&self, file: &str) -> String { - match (self.prefix.is_empty(), file.is_empty()) { - (true, _) => String::from(file), - (false, true) => String::from(self.prefix), - (false, false) => format!("{}/{}", self.prefix, file), - } - } - - /// The volume its files are on, as `toyos_abi::syscall::Stat::volume` - /// says: its filesystem's index plus one, so 0 stays "no volume" and every - /// kernel mount is below a file server's bit 63. - fn volume(&self) -> u64 { - self.fs as u64 + 1 - } -} - -/// The volume `/` and every [`ROOT_ENTRIES`] name nothing is mounted at are -/// on: the directories the VFS makes itself, below a file server's bit 63 and -/// above every [`MountPoint::volume`]. -const NAMESPACE_VOLUME: u64 = (1 << 63) - 1; - -/// The node of `/` on [`NAMESPACE_VOLUME`]. -const ROOT_NODE: u64 = 1; - -/// The node on [`NAMESPACE_VOLUME`] of the [`ROOT_ENTRIES`] name at `slot`. -fn namespace_node(slot: usize) -> u64 { - ROOT_NODE + 1 + slot as u64 -} - -/// A path with every symlink followed, minted only by [`Vfs::resolve_for_open`]: -/// the mount an `OpenTarget` names is the mount opened, never one a link aimed away from. -pub struct OpenTarget(String); - -impl OpenTarget { - pub fn as_str(&self) -> &str { &self.0 } -} - -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum ResolveIntent { - KernelOrRead, - UserModify, -} - -/// Virtual filesystem that dispatches to the mount points `/` synthesizes. -pub struct Vfs { - /// Indexed by position in [`ROOT_ENTRIES`], so a name outside that set has - /// no slot to be mounted in. - at: [Option; ROOT_ENTRIES.len()], - /// Each mounted filesystem once, however many names reach it. - mounts: Vec, - /// What `mkdir` left on a mount that keeps no directories, each charged - /// to whoever made it. - created_dirs: BTreeMap, - /// Every mounted directory's node, by absolute path. - dirs: Directories, -} - -/// The most entries one `FileSystem::list` may materialise for one directory. -pub const MAX_LIST_ENTRIES: usize = 16_384; - -/// Whether `name` is the directory `dir` or lies beneath it; `dir` empty is the mount root. -pub fn under_directory(name: &str, dir: &str) -> bool { - dir.is_empty() - || name == dir - || (name.starts_with(dir) && name.as_bytes().get(dir.len()) == Some(&b'/')) -} - -/// The heap a listing's entry of `name` takes, held while the call that -/// listed it runs: its bytes, and its place in a list that doubles. -pub fn listed_heap(name: &str) -> usize { - charge::array::(name.len()) + 2 * core::mem::size_of::() -} - -/// The heap a listing's set of the directories it has seen takes for one. -fn seen_heap(name: &str) -> usize { - charge::array::(name.len()) + charge::map_entry::() -} - -/// The charge of a directory a map of `V` beside its path holds. -fn dir_charge(path: &str) -> Result { - Ok(Charge::current(charge::map_entry::() + charge::array::(path.len()))?) -} - -const _: () = assert!(core::mem::size_of::<(String, u64)>() == 32); - -/// The `created_dirs` key for a directory — the one construction its writer and readers share. -fn directory(mount: &str, subdir: &str) -> String { - if subdir.is_empty() { - format!("/{mount}") - } else { - format!("/{mount}/{subdir}") - } -} - -fn normalize(path: &str) -> String { - // `parts` is the allocation `MAX_PATH` is sized against. - let mut parts: Vec<&str> = Vec::new(); - for part in path.split('/') { - match part { - "" | "." => {} - ".." => { parts.pop(); } - other => parts.push(other), - } - } - if parts.is_empty() { - String::from("/") - } else { - format!("/{}", parts.join("/")) - } -} - -impl Vfs { - fn new() -> Self { - Self { - at: [const { None }; ROOT_ENTRIES.len()], - mounts: Vec::new(), - created_dirs: BTreeMap::new(), - dirs: Directories::new(), - } - } - - /// Mount one filesystem under `names`, each a [`ROOT_ENTRIES`] name. Several - /// names put each under its own directory of it. - pub fn mount(&mut self, names: &[&'static str], fs: Box) { - let index = self.mounts.len(); - self.mounts.push(Mount { fs }); - for name in names { - let prefix = if names.len() == 1 { "" } else { *name }; - let slot = Self::entry(name).expect("a mount name is one of the root's entries"); - self.at[slot] = Some(MountPoint { fs: index, prefix }); - } - } - - fn entry(name: &str) -> Option { - ROOT_ENTRIES.iter().position(|e| *e == name) - } - - /// What is mounted at `name`; `None` for a bare mount point and a name `/` has not. - fn point(&self, name: &str) -> Option { - self.at[Self::entry(name)?] - } - - /// May a syscall acting for userland change what is at `path`? `/` itself, - /// and a bare mount point, answer no: the read-only refusal, not a missing name. - pub fn user_may_modify(&self, path: &str) -> bool { - let (mount, _) = self.resolve_path("/", path); - self.point(&mount).is_some() - } - - fn resolve_fs(&mut self, mount: &str, file: &str) -> Option<(&mut dyn FileSystem, String)> { - let point = self.point(mount)?; - Some((self.mounts[point.fs].fs.as_mut(), point.path(file))) - } - - pub fn resolve_absolute(&self, cwd: &str, path: &str) -> String { - if path.starts_with('/') { - normalize(path) - } else if cwd == "/" { - normalize(&format!("/{}", path)) - } else { - normalize(&format!("{}/{}", cwd, path)) - } - } - - pub fn resolve_path(&self, cwd: &str, arg: &str) -> (String, String) { - let full = if arg.starts_with('/') { - normalize(arg) - } else if cwd == "/" { - normalize(&format!("/{}", arg)) - } else { - normalize(&format!("{}/{}", cwd, arg)) - }; - - if full == "/" { - return (String::new(), String::new()); - } - - let without_leading = &full[1..]; - if let Some(pos) = without_leading.find('/') { - let mount = &without_leading[..pos]; - let file = &without_leading[pos + 1..]; - (String::from(mount), String::from(file)) - } else { - (String::from(without_leading), String::new()) - } - } - - pub fn cd(&mut self, cwd: &str, target: &str) -> Result { - let (mount, subdir) = self.resolve_path(cwd, target); - - if mount.is_empty() { - return Ok(String::from("/")); - } - - let abs = directory(&mount, &subdir); - - // Refused rather than truncated: a shortened path names a different directory. - if abs.len() > MAX_PATH { - return Err(SyscallError::InvalidArgument); - } - - // A mount point exists whether or not anything is mounted on it; and - // under one this kernel mounts nothing at, the directory is a file - // server's, which the caller's client asked and this kernel cannot. - if Self::entry(&mount).is_some() && self.point(&mount).is_none() { - return Ok(abs); - } - - if self.is_mounted_dir(&mount, &subdir)? { Ok(abs) } else { Err(SyscallError::NotFound) } - } - - /// Whether `/mount/subdir` is a directory of what is mounted at `mount`: - /// its root, one `mkdir` left in `created_dirs`, or one the filesystem holds. - fn is_mounted_dir(&mut self, mount: &str, subdir: &str) -> Result { - let Some(point) = self.point(mount) else { return Ok(false) }; - Ok(subdir.is_empty() - || self.created_dirs.contains_key(&directory(mount, subdir)) - || self.mounts[point.fs].fs.is_dir(&point.path(subdir))?) - } - - /// The node of the mounted directory at the absolute `path` - /// ([`Directories`]), which the caller found is one; refused past - /// [`Directories`]'s bounds or where the caller cannot be charged for it. - fn dir_node(&mut self, path: &str) -> Result { - self.dirs.of(path, file_cache::draw_node, dir_charge::<(u64, Charge)>) - } - - /// A name at the absolute `path` was removed or moved away: every - /// directory that ended with it loses its node. A mount that cannot answer - /// keeps the node, since forgetting a directory that stands renumbers it. - fn forget_ended_dirs(&mut self, path: &str) { - let mut dirs = core::mem::take(&mut self.dirs); - dirs.removed(path, |dir| { - let (mount, subdir) = self.resolve_path("/", dir); - self.is_mounted_dir(&mount, &subdir).unwrap_or(true) - }); - self.dirs = dirs; - } - - /// Each entry of the directory, a directory's name ending in `/`, and its - /// node, as [`Self::stat`] answers it. - /// - /// `list` refuses above [`MAX_LIST_ENTRIES`] rather than truncating because a short listing is a confidently wrong answer to a caller checking existence or deleting a tree. - /// What the listing takes of the heap grows `charge`, which the caller - /// holds until it has let the listing go: refused, `ResourceExhausted`. - pub fn list(&mut self, cwd: &str, path: &str, charge: &mut Charge) -> Result, SyscallError> { - let (mount, subdir) = if path.is_empty() { - self.resolve_path(cwd, "") - } else { - self.resolve_path(cwd, path) - }; - - if mount.is_empty() { - let mut entries = Vec::with_capacity(ROOT_ENTRIES.len()); - for (slot, name) in ROOT_ENTRIES.iter().enumerate() { - let node = match self.point(name) { - Some(_) => self.dir_node(&format!("/{name}"))?, - None => namespace_node(slot), - }; - entries.push((format!("{name}/"), node)); - } - return Ok(entries); - } - - let Some(point) = self.point(&mount) else { - // A mount point with nothing mounted on it is an empty directory; - // anything under it, and any other name, is not there at all. - return match Self::entry(&mount) { - Some(_) if subdir.is_empty() => Ok(Vec::new()), - _ => Err(SyscallError::NotFound), - }; - }; - let fs_path = point.path(&subdir); - let all_files = self.mounts[point.fs].fs.list(&fs_path, MAX_LIST_ENTRIES, charge)?; - let listed = directory(&mount, &subdir); - - let prefix = if fs_path.is_empty() { - String::new() - } else { - format!("{}/", fs_path) - }; - - fn under_prefix<'a>(name: &'a str, prefix: &str) -> Option<&'a str> { - if prefix.is_empty() { Some(name) } else { name.strip_prefix(prefix) } - } - - // Dedup only removes entries, so `matching` is a true upper bound on the final result count. - let matching = all_files.iter().filter(|n| under_prefix(n, &prefix).is_some()).count(); - let mut result = Vec::with_capacity(matching); - let mut seen_dirs = BTreeSet::new(); - let mut saw_self = false; - - for name in &all_files { - let Some(rest) = under_prefix(name, &prefix) else { continue }; - - // The listed directory's own entry: proof it exists, not a child. - if rest.is_empty() { - saw_self = true; - continue; - } - if let Some(slash_pos) = rest.find('/') { - let child = &rest[..slash_pos]; - let dir_name = format!("{child}/"); - if !seen_dirs.contains(&dir_name) { - charge.grow(listed_heap::<(String, u64)>(&dir_name) + seen_heap(&dir_name))?; - seen_dirs.insert(dir_name.clone()); - let node = self.dir_node(&format!("{listed}/{child}"))?; - result.push((dir_name, node)); - } - } else { - charge.grow(listed_heap::<(String, u64)>(rest))?; - result.push((String::from(rest), self.mounts[point.fs].fs.node(name)?)); - } - } - - // A directory the VFS carries is its parent's entry whether or not a - // file is under it yet. - let parent = format!("{listed}/"); - for dir in self.created_dirs.range(parent.clone()..).map(|(dir, _)| dir) { - let Some(rest) = dir.strip_prefix(parent.as_str()) else { break }; - let child = rest.split('/').next().unwrap_or(rest); - if !child.is_empty() { - let dir_name = format!("{child}/"); - if !seen_dirs.contains(&dir_name) { - if result.len() == MAX_LIST_ENTRIES { - return Err(SyscallError::ResourceExhausted); - } - charge.grow(listed_heap::<(String, u64)>(&dir_name) + seen_heap(&dir_name))?; - seen_dirs.insert(dir_name.clone()); - result.push((dir_name, self.dirs.of(&format!("{parent}{child}"), file_cache::draw_node, dir_charge::<(u64, Charge)>)?)); - } - } - } - - // An empty directory's witnesses: its own listing entry on a mount - // that represents directories, `created_dirs` on one the VFS carries. - if result.is_empty() - && !saw_self - && !subdir.is_empty() - && !self.created_dirs.contains_key(&directory(&mount, &subdir)) - { - return Err(SyscallError::NotFound); - } - Ok(result) - } - - pub fn resolve_for_open(&mut self, path: &str, intent: ResolveIntent) -> Result { - // Before the resolution as well as after: a name at `/` resolves to - // nothing, and "no such file" is the wrong answer to "may I make one". - if intent == ResolveIntent::UserModify && !self.user_may_modify(path) { - return Err(SyscallError::PermissionDenied); - } - let (mount, file) = self.followed(path)?; - // A mount point is a directory, never a file to open. - if file.is_empty() || self.point(&mount).is_none() { - return Err(SyscallError::NotFound); - } - let target = OpenTarget(format!("/{mount}/{file}")); - if intent == ResolveIntent::UserModify && !self.user_may_modify(target.as_str()) { - return Err(SyscallError::PermissionDenied); - } - Ok(target) - } - - /// The mount and the path under it that the absolute `path` names once - /// every symlink it ends in is followed, [`MAX_LINKS`] at most. - fn followed(&mut self, path: &str) -> Result<(String, String), SyscallError> { - let (mut mount, mut file) = self.resolve_path("/", path); - for _ in 0..=MAX_LINKS { - let link = match self.resolve_fs(&mount, &file) { - Some((fs, fs_path)) if !file.is_empty() => fs.read_link(&fs_path)?, - _ => None, - }; - let Some(target) = link else { return Ok((mount, file)) }; - // An absolute target names the whole hierarchy; a relative one is - // read against the mount the link is on. - let next = if target.starts_with('/') { target } else { format!("/{mount}/{target}") }; - (mount, file) = self.resolve_path("/", &next); - } - Err(SyscallError::InvalidArgument) - } - - /// What the absolute `path` names, a final symlink followed when `follow` - /// says so, as `toyos_abi::syscall::Stat` describes it. - pub fn stat(&mut self, path: &str, follow: bool) -> Result { - let (mount, file) = if follow { self.followed(path)? } else { self.resolve_path("/", path) }; - let named = |file_type: FileType, size, mtime, volume, node| { - Stat { file_type: file_type as u64, size, mtime, volume, node } - }; - if mount.is_empty() { - return Ok(named(FileType::Directory, 0, 0, NAMESPACE_VOLUME, ROOT_NODE)); - } - let Some(point) = self.point(&mount) else { - return match Self::entry(&mount) { - Some(slot) if file.is_empty() => { - Ok(named(FileType::Directory, 0, 0, NAMESPACE_VOLUME, namespace_node(slot))) - } - _ => Err(SyscallError::NotFound), - }; - }; - let volume = point.volume(); - if !file.is_empty() { - let fs_path = point.path(&file); - let fs = &mut self.mounts[point.fs].fs; - if let Some(target) = fs.read_link(&fs_path)? { - return Ok(named(FileType::Symlink, target.len() as u64, 0, volume, fs.node(&fs_path)?)); - } - match fs.file_mtime(&fs_path) { - Ok(mtime) => { - // A file's node is its `FileId`. - let node = fs.node(&fs_path)?; - return Ok(named(FileType::File, file_cache::size(node), mtime, volume, node)); - } - Err(SyscallError::NotFound) => {} - Err(e) => return Err(e), - } - } - if !self.is_mounted_dir(&mount, &file)? { - return Err(SyscallError::NotFound); - } - Ok(named(FileType::Directory, 0, 0, volume, self.dir_node(&directory(&mount, &file))?)) - } - - /// `target`'s filesystem, its path there, and its volume ([`MountPoint::volume`]). - fn fs_for_target(&mut self, target: &OpenTarget) -> Result<(&mut dyn FileSystem, String, u64), SyscallError> { - let (mount, file) = self.resolve_path("/", target.as_str()); - if file.is_empty() { return Err(SyscallError::NotFound); } - let point = self.point(&mount).ok_or(SyscallError::NotFound)?; - Ok((self.mounts[point.fs].fs.as_mut(), point.path(&file), point.volume())) - } - - /// The file `target` names, and the volume it is on. - pub fn open_target(&mut self, target: &OpenTarget) -> Result<(FileId, u64), SyscallError> { - let (fs, fs_path, volume) = self.fs_for_target(target)?; - let (file_id, backing) = fs.open_file(&fs_path)?; - if let Some(backing) = backing { - crate::file_cache::set_backing(file_id, backing); - } - Ok((file_id, volume)) - } - - pub fn mtime_target(&mut self, target: &OpenTarget) -> Result { - let (fs, fs_path, _) = self.fs_for_target(target)?; - fs.file_mtime(&fs_path) - } - - /// Create a new empty file: it, and the volume it is on. - pub fn create_file(&mut self, path: &str, mtime: u64) -> Result<(FileId, u64), SyscallError> { - let (mount, file) = self.resolve_path("/", path); - if file.is_empty() { return Err(SyscallError::InvalidArgument); } - let point = self.point(&mount).ok_or(SyscallError::NotFound)?; - let file_id = self.mounts[point.fs].fs.create(&point.path(&file), mtime)?; - Ok((file_id, point.volume())) - } - - /// Unlink `path` on its mount, every directory keeping its node: for a - /// caller that puts a file back at once (`ops::open`'s truncate). - pub fn delete_file(&mut self, path: &str) -> Result<(), SyscallError> { - let (mount, file) = self.resolve_path("/", path); - if file.is_empty() { return Err(SyscallError::InvalidArgument); } - let (fs, fs_path) = self.resolve_fs(&mount, &file).ok_or(SyscallError::NotFound)?; - fs.delete(&fs_path) - } - - pub fn rename(&mut self, old_path: &str, new_path: &str) -> Result<(), SyscallError> { - let (old_mount, old_file) = self.resolve_path("/", old_path); - let (new_mount, new_file) = self.resolve_path("/", new_path); - if old_file.is_empty() || new_file.is_empty() { return Err(SyscallError::InvalidArgument); } - if old_mount != new_mount { return Err(SyscallError::NotSupported); } - let point = self.point(&old_mount).ok_or(SyscallError::NotFound)?; - let (old_fs_path, new_fs_path) = (point.path(&old_file), point.path(&new_file)); - self.mounts[point.fs].fs.rename(&old_fs_path, &new_fs_path)?; - self.forget_ended_dirs(&directory(&old_mount, &old_file)); - Ok(()) - } - - /// The `Result` is as much the point as the bound: a caller that discards it and reports success anyway turns the bound into a silent failure. - pub fn create_dir(&mut self, path: &str) -> Result<(), SyscallError> { - if path.len() > MAX_PATH { - return Err(SyscallError::InvalidArgument); - } - let (mount, subdir) = self.resolve_path("/", path); - // `/` and a mount root already exist. - if subdir.is_empty() { - return Err(SyscallError::AlreadyExists); - } - if let Some((fs, fs_path)) = self.resolve_fs(&mount, &subdir) { - match fs.create_dir(&fs_path) { - // No directory representation on this mount; carried below. - Err(SyscallError::NotSupported) => {} - outcome => return outcome, - } - } - // A new key is charged to its maker; a repeat of one already held costs nothing. - if !self.created_dirs.contains_key(path) { - let charge = dir_charge::(path)?; - self.created_dirs.insert(String::from(path), charge); - } - Ok(()) - } - - /// Remove an empty directory, reporting the real outcome — the `Result` is - /// the point, as in [`Self::create_dir`]. A mount root, a missing name, a - /// file, and a non-empty directory are each refused, not reported as removed. - pub fn remove_dir(&mut self, path: &str) -> Result<(), SyscallError> { - self.remove_empty_dir(path)?; - self.forget_ended_dirs(&normalize(path)); - Ok(()) - } - - fn remove_empty_dir(&mut self, path: &str) -> Result<(), SyscallError> { - let (mount, subdir) = self.resolve_path("/", path); - // A mount point (and `/`) is not a directory a caller may remove. - if subdir.is_empty() { - return Err(SyscallError::InvalidArgument); - } - let dir = directory(&mount, &subdir); - - let forwarded = { - let (fs, fs_path) = self.resolve_fs(&mount, &subdir).ok_or(SyscallError::NotFound)?; - fs.remove_dir(&fs_path) - }; - match forwarded { - // No directory representation on this mount; judged below from - // the listing and `created_dirs`, as `create_dir` carried it. - Err(SyscallError::NotSupported) => {} - Ok(()) => { - self.created_dirs.remove(&dir); - return Ok(()); - } - outcome => return outcome, - } - - let (fs, fs_path) = self.resolve_fs(&mount, &subdir).ok_or(SyscallError::NotFound)?; - let mut listing = Charge::none(&charge::current()); - let names = fs.list(&fs_path, MAX_LIST_ENTRIES, &mut listing)?; - let child_prefix = format!("{fs_path}/"); - let is_file = names.contains(&fs_path); - // A listing mount's own `name/` self-entry is not a child, or every empty directory reads non-empty. - let has_file_child = - names.iter().any(|n| n.starts_with(&child_prefix) && *n != child_prefix); - - // The name resolves to a file, not a directory. - if is_file { - return Err(SyscallError::InvalidArgument); - } - - let created_prefix = format!("{dir}/"); - // The first carried directory at or after `dir/` in key order is the - // only one that can lie beneath it. - let has_created_child = self - .created_dirs - .range::((core::ops::Bound::Included(created_prefix.as_str()), core::ops::Bound::Unbounded)) - .next() - .is_some_and(|(d, _)| d.starts_with(&created_prefix)); - if !(self.created_dirs.contains_key(&dir) || has_file_child || has_created_child) { - return Err(SyscallError::NotFound); - } - if has_file_child || has_created_child { - return Err(SyscallError::InvalidArgument); - } - self.created_dirs.remove(&dir); - Ok(()) - } - - pub fn create_symlink(&mut self, path: &str, target: &str) -> Result<(), SyscallError> { - let (mount, file) = self.resolve_path("/", path); - if file.is_empty() { - return Err(SyscallError::InvalidArgument); - } - let (fs, fs_path) = self.resolve_fs(&mount, &file).ok_or(SyscallError::NotFound)?; - fs.create_symlink(&fs_path, target) - } - - pub fn read_link(&mut self, path: &str) -> Result, SyscallError> { - let (mount, file) = self.resolve_path("/", path); - if file.is_empty() { - return Ok(None); - } - let Some((fs, fs_path)) = self.resolve_fs(&mount, &file) else { return Ok(None) }; - fs.read_link(&fs_path) - } - - /// Unlink `path`, and forget the node of every directory that ended with it. - pub fn delete(&mut self, path: &str) -> Result<(), SyscallError> { - self.delete_file(path)?; - self.forget_ended_dirs(&normalize(path)); - Ok(()) - } - - pub fn open_backing(&mut self, path: &str) -> Result, SyscallError> { - let target = self.resolve_for_open(path, ResolveIntent::KernelOrRead)?; - let (fs, fs_path, _) = self.fs_for_target(&target)?; - fs.open_backing(&fs_path) - } -} diff --git a/lib/manifest/src/lib.rs b/lib/manifest/src/lib.rs index 1922794704e..b5ebbeffbb9 100644 --- a/lib/manifest/src/lib.rs +++ b/lib/manifest/src/lib.rs @@ -124,11 +124,12 @@ pub const SYSTEM: &str = "system"; /// The file-server roles, and the directories each serves: one port per role, /// and one grant on it per directory. -pub const ROLES: [(&str, &[RoleDir]); 4] = [ +pub const ROLES: [(&str, &[RoleDir]); 5] = [ (SYSTEM, &[RoleDir { dir: "/system", root: "", write: false }]), ("data", &[dir("/apps", "apps"), dir("/config", "config"), dir("/home", "home"), dir("/state", "state")]), ("log", &[dir("/log", "")]), ("boot", &[dir("/boot", "")]), + ("memory", &[dir("/tmp", "")]), ]; /// The directories `role` serves, or `None` for a name that is no role. @@ -999,10 +1000,10 @@ mod tests { let whole_but_state = |own: Option<&str>| -> Vec<_> { std::iter::once(system()) .chain( - ["/apps", "/config", "/home", "/log", "/boot"] + ["/apps", "/config", "/home", "/log", "/boot", "/tmp"] .iter() - .zip(["apps", "config", "home", "", ""]) - .zip(["data", "data", "data", "log", "boot"]) + .zip(["apps", "config", "home", "", "", ""]) + .zip(["data", "data", "data", "log", "boot", "memory"]) .map(|((dir, root), role)| (role, dir.to_string(), root.to_string(), true)), ) .chain(own.map(|name| ("data", format!("/state/{name}"), format!("state/{name}"), true))) diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 390a8bca26b..992452cb17d 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -3435,6 +3435,12 @@ pub fn paged_create(pager: RawHandle, size: u64) -> Result { check(syscall(SYS_PAGED_CREATE, pager.0 as u64, size, 0, 0)) } +/// A resizable anonymous object: its present pages are authoritative, never +/// withdrawn or discarded by a failure, and writes need no writeback slot. +pub fn paged_anonymous(pager: RawHandle, size: u64) -> Result { + check(syscall(SYS_PAGED_CREATE, pager.0 as u64, size, 1, 0)) +} + /// A holder handle on `pager`'s object `object`, carrying `READ`, which may /// be narrowed, duplicated and sent. pub fn paged_holder(pager: RawHandle, object: u64) -> Result { diff --git a/sdk/toyos/src/paged.rs b/sdk/toyos/src/paged.rs index dfca567c87c..5ab1b3236ff 100644 --- a/sdk/toyos/src/paged.rs +++ b/sdk/toyos/src/paged.rs @@ -86,6 +86,12 @@ impl Pager { syscall::paged_create(self.0.raw(), size).map_err(|e| Refused::of(e, None)) } + /// Anonymous bytes, retained until shrunk or released. No backing can + /// refill a present page, so withdrawal and writeback cannot discard it. + pub fn anonymous(&self, size: u64) -> Result { + syscall::paged_anonymous(self.0.raw(), size).map_err(|e| Refused::of(e, None)) + } + /// A holder handle on object `id`, to send. pub fn holder(&self, id: u64) -> Result { let raw = syscall::paged_holder(self.0.raw(), id)?; diff --git a/system/fileserver/src/lib.rs b/system/fileserver/src/lib.rs index 34a77418c0e..87b50878f05 100644 --- a/system/fileserver/src/lib.rs +++ b/system/fileserver/src/lib.rs @@ -27,6 +27,7 @@ pub mod data; pub mod descriptions; pub mod fat; pub mod listing; +pub mod memory; pub mod pages; pub mod ram; pub mod resolve; diff --git a/system/fileserver/src/main.rs b/system/fileserver/src/main.rs index 92133ef9669..767ec130917 100644 --- a/system/fileserver/src/main.rs +++ b/system/fileserver/src/main.rs @@ -57,7 +57,7 @@ use fileserver::descriptions::{self, Description, Held, Ids, Shared}; use fileserver::listing; use diskserver::disk::Served; use fileserver::cache::Disk; -use fileserver::ram::Ram; +use fileserver::memory::MemoryVolume; use fileserver::fat::FatVolume; use fileserver::resolve::{self, Found, Refusal as Escape, Resolved}; use fileserver::rights; @@ -103,10 +103,6 @@ const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(2); /// whatever its grant. const NO_SUCH_OPERATION: &str = "it asked for an operation this protocol does not have"; -/// The volume memory stands in for when DATA has no partition: 1 GiB of -/// blocks, of which only what is written costs anything. -const RAM_BLOCKS: u64 = 1 << 18; - /// What a volume's cache keeps clean: its tree, or its FAT and directories. /// A file's pages are its paged object's ([`pager`]). const CLEAN_BLOCKS: usize = (16 << 20) / toyos_blockdev::BLOCK; @@ -127,6 +123,7 @@ const REPORT: Duration = Duration::from_secs(5); enum Role { System, Data, + Memory, Log, Boot, } @@ -136,6 +133,7 @@ impl Role { match text { "system" => Some(Self::System), "data" => Some(Self::Data), + "memory" => Some(Self::Memory), "log" => Some(Self::Log), "boot" => Some(Self::Boot), _ => None, @@ -146,6 +144,7 @@ impl Role { match self { Self::System => "system", Self::Data => "data", + Self::Memory => "memory", Self::Log => "log", Self::Boot => "boot", } @@ -291,7 +290,7 @@ const DATA_DIRTY: u64 = 8 * 1024; fn pager(role: Role) -> Option { let (capacity, dirty, objects) = match role { Role::System => return None, - Role::Data => { + Role::Data | Role::Memory => { let share = toyos::system::memory_total() / PAGES_SHARE / toyos::paged::PAGE; let capacity = share.saturating_sub(2 * SMALL_PAGES).max(SMALL_PAGES); // A file held for every 16 pages, the table's own cost a small @@ -340,8 +339,9 @@ fn open_volume(role: Role, guid: Option<&str>, roots: &[&str]) -> (Box Box::new(MemoryVolume::new(roots, fileserver::volume::now_nanos)) as Box, Role::Data => match fileserver::data::find(data_partitions().as_deref().map_err(String::clone)) { - Located::Nowhere => ram(roots, "this machine has no DATA partition"), + Located::Nowhere => memory(roots, "this machine has no DATA partition"), Located::Served(guid) => match served(guid) { Some(disk) => data_on(disk, roots), None => absent(roots, "the DATA partition would not open".into()), @@ -401,12 +401,9 @@ fn data_on(disk: D, roots: &[&str]) -> Box { } } -fn ram(roots: &[&str], why: &str) -> Box { +fn memory(roots: &[&str], why: &str) -> Box { println!("fileserver: {why}; /apps, /config, /home and /state are in memory and will not survive a reboot"); - match DataVolume::format(Ram::new(RAM_BLOCKS), CLEAN_BLOCKS, roots, fileserver::volume::now_nanos) { - Ok(volume) => Box::new(volume), - Err(why) => panic!("fileserver: a volume in memory would not format: {why}"), - } + Box::new(MemoryVolume::new(roots, fileserver::volume::now_nanos)) } fn fat_on(disk: D, writable: bool) -> Result, String> { @@ -1027,7 +1024,7 @@ impl Server { let gone = self.volume.identity(&path).ok(); self.volume.unlink(&path)?; if let Some(identity) = gone { - self.files.forget(&mut *self.volume, identity); + self.files.unlinked(&mut *self.volume, identity)?; } } } @@ -1053,7 +1050,7 @@ impl Server { }; self.volume.rename(&from, &to)?; if let Some(identity) = over { - self.files.forget(&mut *self.volume, identity); + self.files.unlinked(&mut *self.volume, identity)?; } self.each_dir(|held, _| descriptions::moved(held, &from, &to)); self.dirtied(); diff --git a/system/fileserver/src/memory.rs b/system/fileserver/src/memory.rs new file mode 100644 index 00000000000..7864795ea96 --- /dev/null +++ b/system/fileserver/src/memory.rs @@ -0,0 +1,252 @@ +//! An in-memory namespace. File bytes live only in their retained paged +//! objects; a page absent from an object is a zero-filled hole. + +use std::collections::BTreeMap; + +use toyos::fs::{Error, Room, MAX_FILE_BYTES}; + +use crate::volume::{parent, Kind, Listed, Meta, Node, OpenHow, Out, Reserved, Volume}; + +struct Inode { + meta: Meta, + target: Option, + opens: usize, + linked: bool, +} + +pub struct MemoryVolume { + names: BTreeMap, + nodes: BTreeMap, + next: Node, + clock: fn() -> u64, +} + +impl MemoryVolume { + pub fn new(roots: &[&str], clock: fn() -> u64) -> Self { + let mut volume = Self { names: BTreeMap::new(), nodes: BTreeMap::new(), next: 1, clock }; + volume.insert("", Kind::Dir, None); + for root in roots.iter().filter(|r| !r.is_empty()) { + volume.mkdir(root).expect("memory volume roots are distinct direct children"); + } + volume + } + + fn insert(&mut self, path: &str, kind: Kind, target: Option) -> Node { + let node = self.next; + self.next = self.next.checked_add(1).expect("memory inode numbers exhausted"); + let size = target.as_ref().map_or(0, |t| t.len() as u64); + self.nodes.insert(node, Inode { meta: Meta { kind, size, mtime: (self.clock)() }, target, opens: 0, linked: true }); + assert!(self.names.insert(path.into(), node).is_none(), "a new memory inode has a vacant name"); + node + } + + fn node(&self, path: &str) -> Result { + self.names.get(path).copied().ok_or(Error::NotFound) + } + + fn inode(&mut self, node: Node) -> Result<&mut Inode, Error> { + self.nodes.get_mut(&node).ok_or(Error::Stale) + } + + fn directory(&self, path: &str) -> Result<(), Error> { + match self.nodes[&self.node(path)?].meta.kind { + Kind::Dir => Ok(()), + _ => Err(Error::NotADirectory), + } + } + + fn vacant(&self, path: &str) -> Result<(), Error> { + if self.names.contains_key(path) { + return Err(Error::AlreadyExists); + } + self.directory(parent(path)) + } + + fn detach(&mut self, path: &str) { + let node = self.names.remove(path).expect("detaching an existing name"); + let inode = self.nodes.get_mut(&node).expect("a name has an inode"); + inode.linked = false; + if inode.opens == 0 { + self.nodes.remove(&node); + } + } +} + +impl Volume for MemoryVolume { + fn memory(&self) -> bool { true } + fn writable(&self) -> bool { true } + + fn linked(&mut self, node: Node) -> Result { + Ok(self.inode(node)?.linked) + } + + fn lstat(&mut self, path: &str) -> Result { + self.node_meta(self.node(path)?) + } + + fn read_link(&mut self, path: &str) -> Result { + self.inode(self.node(path)?)?.target.clone().ok_or(Error::InvalidRequest) + } + + fn list(&mut self, dir: &str) -> Result, Error> { + self.directory(dir)?; + Ok(self.names.iter().filter(|(path, _)| !path.is_empty() && parent(path) == dir).map(|(path, node)| Listed { + name: path.rsplit('/').next().expect("a path has a name").into(), + meta: self.nodes[node].meta, + identity: *node, + }).collect()) + } + + fn open(&mut self, path: &str, how: OpenHow) -> Result { + let node = match self.node(path) { + Ok(_) if how.create_new => return Err(Error::AlreadyExists), + Ok(node) => node, + Err(Error::NotFound) if how.create || how.create_new => { + self.vacant(path)?; + self.insert(path, Kind::File, None) + } + Err(e) => return Err(e), + }; + match self.nodes[&node].meta.kind { + Kind::File => {} + Kind::Dir => return Err(Error::IsADirectory), + Kind::Symlink => return Err(Error::InvalidRequest), + } + if how.truncate { self.truncate(node, 0)?; } + self.inode(node)?.opens += 1; + Ok(node) + } + + fn close(&mut self, node: Node) -> Result<(), Error> { + let inode = self.inode(node)?; + inode.opens = inode.opens.checked_sub(1).expect("an open memory inode is closed once"); + if inode.opens == 0 && !inode.linked { self.nodes.remove(&node); } + Ok(()) + } + + fn node_meta(&mut self, node: Node) -> Result { Ok(self.inode(node)?.meta) } + fn identity(&mut self, path: &str) -> Result { self.node(path) } + + fn read(&mut self, node: Node, offset: u64, out: &mut dyn Out) -> Result { + let size = self.inode(node)?.meta.size; + let len = out.len().min(size.saturating_sub(offset) as usize); + out.zero(0, len); + Ok(len) + } + + fn reserve(&mut self, node: Node, offset: u64, len: u64) -> Result { + let end = offset.checked_add(len).filter(|&n| n <= MAX_FILE_BYTES).ok_or(Error::FileTooLarge)?; + let size = self.inode(node)?.meta.size; + self.truncate(node, size.max(end))?; + Ok(Reserved { size, held: 0 }) + } + + fn write_back(&mut self, _: Node, _: u64, _: &[u8]) -> Result<(), Error> { + unreachable!("memory file bytes stay in their paged object") + } + + fn truncate(&mut self, node: Node, size: u64) -> Result<(), Error> { + if size > MAX_FILE_BYTES { return Err(Error::FileTooLarge); } + let now = (self.clock)(); + let inode = self.inode(node)?; + inode.meta.size = size; + inode.meta.mtime = now; + Ok(()) + } + + fn set_mtime(&mut self, node: Node, mtime: u64) -> Result<(), Error> { + self.inode(node)?.meta.mtime = mtime; + Ok(()) + } + + fn mkdir(&mut self, path: &str) -> Result<(), Error> { + self.vacant(path)?; + self.insert(path, Kind::Dir, None); + Ok(()) + } + + fn rmdir(&mut self, path: &str) -> Result<(), Error> { + if path.is_empty() { return Err(Error::Root); } + self.directory(path)?; + if self.names.keys().any(|name| parent(name) == path) { return Err(Error::NotEmpty); } + self.detach(path); + Ok(()) + } + + fn unlink(&mut self, path: &str) -> Result<(), Error> { + if self.lstat(path)?.kind == Kind::Dir { return Err(Error::IsADirectory); } + self.detach(path); + Ok(()) + } + + fn rename(&mut self, from: &str, to: &str) -> Result<(), Error> { + if from.is_empty() || to.is_empty() { return Err(Error::Root); } + let node = self.node(from)?; + if from == to { return Ok(()); } + self.directory(parent(to))?; + let kind = self.nodes[&node].meta.kind; + if to.strip_prefix(from).is_some_and(|rest| rest.starts_with('/')) { return Err(Error::IntoItself); } + if let Ok(over) = self.node(to) { + if self.nodes[&over].meta.kind == Kind::Dir { return Err(Error::IsADirectory); } + if kind == Kind::Dir { return Err(Error::NotADirectory); } + self.detach(to); + } + let moved: Vec<_> = self.names.keys().filter(|path| path.as_str() == from || path.strip_prefix(from).is_some_and(|s| s.starts_with('/'))).cloned().collect(); + for path in moved { + let node = self.names.remove(&path).expect("a moved name exists"); + self.names.insert(format!("{to}{}", &path[from.len()..]), node); + } + Ok(()) + } + + fn symlink(&mut self, path: &str, target: &str) -> Result<(), Error> { + self.vacant(path)?; + self.insert(path, Kind::Symlink, Some(target.into())); + Ok(()) + } + + fn sync(&mut self) -> Result, Error> { Ok(Vec::new()) } + fn room(&mut self) -> Result { Err(Error::Unsupported) } + fn describe(&self) -> String { format!("memory: {} named inodes, {} retained", self.names.len(), self.nodes.len()) } +} + +#[cfg(test)] +mod tests { + use super::*; + + const CREATE: OpenHow = OpenHow { create: true, create_new: false, truncate: false }; + fn clock() -> u64 { 123 } + + #[test] + fn unlink_and_replacement_keep_open_inode_identity() { + let mut volume = MemoryVolume::new(&[], clock); + let old = volume.open("old", CREATE).unwrap(); + let new = volume.open("new", CREATE).unwrap(); + volume.truncate(old, 12).unwrap(); + volume.rename("new", "old").unwrap(); + assert_eq!(volume.identity("old"), Ok(new)); + assert_eq!(volume.node_meta(old).unwrap().size, 12); + assert!(!volume.linked(old).unwrap()); + volume.close(old).unwrap(); + assert_eq!(volume.node_meta(old), Err(Error::Stale)); + volume.unlink("old").unwrap(); + assert_eq!(volume.node_meta(new).unwrap().size, 0); + let fresh = volume.open("old", CREATE).unwrap(); + assert!(fresh > old && fresh > new); + volume.close(new).unwrap(); + assert_eq!(volume.node_meta(new), Err(Error::Stale)); + } + + #[test] + fn directory_rename_carries_inodes_without_matching_sibling_prefixes() { + let mut volume = MemoryVolume::new(&["home"], clock); + volume.mkdir("home/a").unwrap(); + let node = volume.open("home/a/file", CREATE).unwrap(); + let sibling = volume.open("home/a-file", CREATE).unwrap(); + volume.rename("home/a", "home/b").unwrap(); + assert_eq!(volume.identity("home/b/file"), Ok(node)); + assert_eq!(volume.identity("home/a-file"), Ok(sibling)); + assert_eq!(volume.rmdir("home/b"), Err(Error::NotEmpty)); + assert_eq!(volume.rename("home/b", "home/b/inner"), Err(Error::IntoItself)); + } +} diff --git a/system/fileserver/src/pages.rs b/system/fileserver/src/pages.rs index b93579abb6f..b6378db0cc0 100644 --- a/system/fileserver/src/pages.rs +++ b/system/fileserver/src/pages.rs @@ -33,6 +33,7 @@ const RANGE: u64 = PAGED_RANGE_PAGES; /// A pager's calls on its objects, by id, as `toyos::paged::Pager` makes them. pub trait Pager { fn object(&self, size: u64) -> Result; + fn anonymous(&self, size: u64) -> Result; fn read(&self, id: u64, out: Bytes<'_>, offset: u64) -> Result; fn supply(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result; fn write(&self, id: u64, offset: u64, bytes: &[u8]) -> Result; @@ -48,6 +49,9 @@ impl Pager for toyos::paged::Pager { fn object(&self, size: u64) -> Result { toyos::paged::Pager::object(self, size) } + fn anonymous(&self, size: u64) -> Result { + toyos::paged::Pager::anonymous(self, size) + } fn read(&self, id: u64, out: Bytes<'_>, offset: u64) -> Result { toyos::paged::Pager::read(self, id, out, offset) } @@ -98,10 +102,12 @@ struct File { dirty: Option<(u64, u64)>, /// When it was last used, by [`Files::tick`]. used: u64, + anonymous: bool, } impl File { fn dirtied(&mut self, first: u64, end: u64) { + if self.anonymous { return; } let (lo, hi) = self.dirty.unwrap_or((first, end)); self.dirty = Some((lo.min(first), hi.max(end))); } @@ -176,6 +182,7 @@ impl Files

{ /// more for it. The object's file holds it once for good, so a second /// hold goes back at once. pub fn opened(&mut self, volume: &mut dyn Volume, identity: u64, node: Node) -> Result<(), Error> { + assert!(!volume.memory() || self.pager.is_some(), "memory files require a pager"); if self.pager.is_none() { return Ok(()); } @@ -192,13 +199,14 @@ impl Files

{ // Objects are bounded by count as pages are: a file nobody holds goes // to make room for another's. let object = loop { - match self.pg().object(size) { + let created = if volume.memory() { self.pg().anonymous(size) } else { self.pg().object(size) }; + match created { Ok(object) => break object, Err(Refused::NoRoom) if self.drop_oldest(volume, identity) => {} Err(why) => return Err(word(why)), } }; - self.files.insert(identity, File { object, node, opens: 1, held: false, dirty: None, used: 0 }); + self.files.insert(identity, File { object, node, opens: 1, held: false, dirty: None, used: 0, anonymous: volume.memory() }); self.by_object.insert(object, identity); self.touch(identity); Ok(()) @@ -209,6 +217,7 @@ impl Files

{ match self.files.get_mut(&identity) { Some(f) => { f.opens = f.opens.saturating_sub(1); + self.collect_unlinked(volume, identity)?; Ok(()) } None if self.pager.is_none() => volume.close(node), @@ -216,6 +225,25 @@ impl Files

{ } } + /// A name was removed or replaced. Memory inodes remain usable through + /// every open description and paged holder until the last one ends. + pub fn unlinked(&mut self, volume: &mut dyn Volume, identity: u64) -> Result<(), Error> { + if volume.memory() { + self.collect_unlinked(volume, identity) + } else { + self.forget(volume, identity); + Ok(()) + } + } + + fn collect_unlinked(&mut self, volume: &mut dyn Volume, identity: u64) -> Result<(), Error> { + let Some(f) = self.files.get(&identity) else { return Ok(()) }; + if volume.memory() && f.opens == 0 && !f.held && !volume.linked(f.node)? { + self.forget(volume, identity); + } + Ok(()) + } + /// `identity`'s file is gone from the volume: its object goes, its /// holders answer `Gone`, and its node is let go. pub fn forget(&mut self, volume: &mut dyn Volume, identity: u64) { @@ -312,6 +340,7 @@ impl Files

{ /// nobody holds dropped, else another's clean pages withdrawn, else this /// one's outside the range, else the dirty written back. Whether any went. fn make_room(&mut self, volume: &mut dyn Volume, identity: u64, first: u64, count: u64) -> Result { + if volume.memory() { return Ok(false); } if self.drop_oldest(volume, identity) { return Ok(true); } @@ -339,6 +368,7 @@ impl Files

{ /// The oldest file nobody holds, open or dirties, but `identity`, /// forgotten: whether there was one. fn drop_oldest(&mut self, volume: &mut dyn Volume, identity: u64) -> bool { + if volume.memory() { return false; } match self.oldest(identity, &File::droppable).first() { Some(&id) => { self.forget(volume, id); @@ -550,8 +580,11 @@ impl Files

{ } } Notice::NoHolders { object } => { - if let Some(f) = self.by_object.get(&object).and_then(|identity| self.files.get_mut(identity)) { - f.held = false; + if let Some(&identity) = self.by_object.get(&object) { + self.files.get_mut(&identity).expect("an indexed object has a file").held = false; + if let Err(e) = self.collect_unlinked(volume, identity) { + println!("fileserver: unlinked file {identity} would not be collected ({e:?})"); + } } } } diff --git a/system/fileserver/src/volume.rs b/system/fileserver/src/volume.rs index 25faa80d37a..7745ed558b2 100644 --- a/system/fileserver/src/volume.rs +++ b/system/fileserver/src/volume.rs @@ -105,6 +105,13 @@ impl Out for Window<'_> { } pub trait Volume { + /// Its paged objects are the only backing for file contents. Their pages + /// cannot be evicted while the inode remains linked or held open. + fn memory(&self) -> bool { false } + + /// Whether an open inode still has a name. + fn linked(&mut self, _node: Node) -> Result { Ok(true) } + /// Whether anything here may be changed. fn writable(&self) -> bool; diff --git a/system/supervisor/src/main.rs b/system/supervisor/src/main.rs index 911400e832f..aee9d39d8a9 100644 --- a/system/supervisor/src/main.rs +++ b/system/supervisor/src/main.rs @@ -3592,6 +3592,7 @@ fn storage_endowment(role: Option<&str>, syscap: &SysCap, grants: &Grants<'_>, b return Ok(storage); }; storage.args.push(role.to_string()); + if role == "memory" { return Ok(storage); } if role == toyos_manifest::SYSTEM { let root = boot.root.try_clone() .map_err(|e| std::io::Error::other(format!("ROOT's image did not duplicate: {e:?}")))?; From 9ceeb34d10b005e9a0d663963363013a133f4467 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 17:01:24 +0100 Subject: [PATCH 09/93] Route filesystem paths through process capabilities Remove the kernel pathname and file-offset syscall declarations and SDK fallback. Keep generic object reads, writes and metadata queries. File metadata comes from the server so unlink and link counts stay authoritative; paged reads retain their atomic offset publication. Share runtime cwd through the descriptor owner's SDK ABI, initialized only once from spawn metadata. Derive synthetic read-only root and ancestor directories from the names actually held by the immutable namespace, exposed by a bounded generic namespace inspection operation. Host validation: SDK fd-table check and clippy, 5 files tests, 49 ABI tests, and all 106 libc module tests pass. Combined guest compilation follows the kernel filesystem removal and shared Rust reexport update. --- kernel/src/syscall/dispatch.rs | 7 +- kernel/src/syscall/ipc.rs | 22 ++ sdk/abi/src/handle.rs | 26 +- sdk/abi/src/syscall.rs | 259 ++---------------- sdk/libc/arch/src/path_resolution.rs | 334 +++++------------------- sdk/libc/src/posix_io.rs | 89 ++----- sdk/libc/src/resolve.rs | 165 +----------- sdk/libc/src/spawn.rs | 11 +- sdk/libc/src/stdio.rs | 2 +- sdk/std/sys/fs.rs | 27 +- sdk/std/sys/pal/os.rs | 6 +- sdk/std/sys/process.rs | 16 +- sdk/toyos/src/fd.rs | 3 - sdk/toyos/src/files.rs | 83 +++--- sdk/toyos/src/files/namespace.rs | 58 +++++ sdk/toyos/src/files/owner.rs | 377 ++++++++++----------------- sdk/toyos/src/namespace.rs | 18 ++ 17 files changed, 418 insertions(+), 1085 deletions(-) create mode 100644 sdk/toyos/src/files/namespace.rs diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index b993c1c37b1..686cc32c41e 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -38,7 +38,7 @@ use super::io::{ use super::ipc::{ Incoming, Outgoing, sys_accept, sys_inbox_setup, sys_message_recv, sys_message_send, - sys_inbox_submit, sys_namespace_build, sys_namespace_open, sys_pipe, + sys_inbox_submit, sys_namespace_build, sys_namespace_names, sys_namespace_open, sys_pipe, sys_port_badge, sys_port_create, sys_port_mint, sys_shm_create, sys_shm_map, sys_shm_sub, }; use super::machine::{ @@ -500,6 +500,11 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> // ctx lets sys_namespace_build bound both counts before it reads either vector. sys_namespace_build(&ctx, &args) } + SYS_NAMESPACE_NAMES => { + let len = a3.min((MAX_NAMESPACE_ENTRIES * (2 + MAX_SERVICE_NAME)) as u64); + let Some(mut out) = ctx.user_bytes_mut(UserAddr::new(a2), len) else { return bad_addr }; + sys_namespace_names(RawHandle(a1 as u32), &mut out) + } SYS_NAMESPACE_OPEN => { let name = match ctx.user_str(UserAddr::new(a2), a3) { Ok(s) => s, Err(e) => return e.to_u64() }; sys_namespace_open(RawHandle(a1 as u32), &name) diff --git a/kernel/src/syscall/ipc.rs b/kernel/src/syscall/ipc.rs index 90dfe083089..ba025a1ab10 100644 --- a/kernel/src/syscall/ipc.rs +++ b/kernel/src/syscall/ipc.rs @@ -193,6 +193,28 @@ pub(super) fn sys_namespace_open(ns_h: RawHandle, name: &str) -> u64 { connect_through(&connector) } +/// Inspect only the names this namespace already holds. The Arc keeps the +/// immutable entries alive after the handle-table lock is released; copying +/// into user memory therefore holds no object or process lock. +pub(super) fn sys_namespace_names(ns_h: RawHandle, out: &mut crate::user_ptr::UserBytesMut) -> u64 { + let namespace = match process::with_process_data(|data| { + data.handles.get::(ns_h, Rights::READ) + }) { + Ok(namespace) => namespace, + Err(e) => return e.refuse(), + }; + let required: usize = namespace.entries().iter().map(|(name, _)| 2 + name.len()).sum(); + if out.len() >= required { + let mut at = 0; + for (name, _) in namespace.entries() { + out.write_at(at, &(name.len() as u16).to_le_bytes()); + out.write_at(at + 2, name.as_bytes()); + at += 2 + name.len(); + } + } + required as u64 +} + /// The client half of a connection, and the server half queued on the port. fn connect_through(connector: &port::Connector) -> u64 { if connector.closed() { diff --git a/sdk/abi/src/handle.rs b/sdk/abi/src/handle.rs index 4a8bdd230cb..8c95df9795c 100644 --- a/sdk/abi/src/handle.rs +++ b/sdk/abi/src/handle.rs @@ -82,22 +82,21 @@ pub enum ObjectKind { SharedMem = 6, Connector = 7, Namespace = 8, - File = 9, - Console = 10, - SysCap = 11, - Process = 12, - Null = 13, - Job = 14, - Notice = 15, - WakeLatency = 16, - EventSignal = 17, - EventWait = 18, - Pager = 19, - Paged = 20, + Console = 9, + SysCap = 10, + Process = 11, + Null = 12, + Job = 13, + Notice = 14, + WakeLatency = 15, + EventSignal = 16, + EventWait = 17, + Pager = 18, + Paged = 19, } impl ObjectKind { - pub const ALL: [ObjectKind; 21] = [ + pub const ALL: [ObjectKind; 20] = [ Self::PipeRead, Self::PipeWrite, Self::Connection, @@ -107,7 +106,6 @@ impl ObjectKind { Self::SharedMem, Self::Connector, Self::Namespace, - Self::File, Self::Console, Self::SysCap, Self::Process, diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 217d66d9a92..b3e6ab07ae7 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -3,17 +3,10 @@ pub const SYS_WRITE: u64 = 0; pub const SYS_READ: u64 = 1; pub const SYS_THREAD_EXIT: u64 = 5; pub const SYS_RANDOM: u64 = 6; -pub const SYS_OPEN: u64 = 9; pub const SYS_CLOSE: u64 = 10; -pub const SYS_SEEK: u64 = 13; pub const SYS_FSTAT: u64 = 14; -pub const SYS_FSYNC: u64 = 15; /// What a path names, as a [`Stat`]: [`stat`]. -pub const SYS_STAT: u64 = 16; -pub const SYS_READDIR: u64 = 17; -pub const SYS_DELETE: u64 = 18; pub const SYS_SHUTDOWN: u64 = 19; -pub const SYS_CHDIR: u64 = 20; pub const SYS_GETCWD: u64 = 21; pub const SYS_PIPE: u64 = 24; /// Start a program, endowing it exactly what the caller names. Answers a @@ -85,15 +78,11 @@ pub const SYS_NANOSLEEP: u64 = 49; /// [`dup`] and [`dup_narrowed`]. pub const SYS_HANDLE_DUP: u64 = 50; pub const SYS_GETPID: u64 = 51; -pub const SYS_RENAME: u64 = 52; -pub const SYS_MKDIR: u64 = 53; -pub const SYS_RMDIR: u64 = 54; pub const SYS_DLOPEN: u64 = 55; pub const SYS_DLSYM: u64 = 56; pub const SYS_DLCLOSE: u64 = 57; pub const SYS_FUTEX_WAIT: u64 = 58; pub const SYS_FUTEX_WAKE: u64 = 59; -pub const SYS_FTRUNCATE: u64 = 60; pub const SYS_STACK_INFO: u64 = 61; pub const SYS_CPU_COUNT: u64 = 62; pub const SYS_VM_RESERVE: u64 = 63; @@ -103,8 +92,6 @@ pub const SYS_WRITE_NONBLOCK: u64 = 67; pub const SYS_EXIT: u64 = 72; pub const SYS_GET_ENV: u64 = 73; pub const SYS_CLOCK_EPOCH: u64 = 75; -pub const SYS_SYMLINK: u64 = 81; -pub const SYS_READLINK: u64 = 82; /// Accept a queued connection from an [`Acceptor`] handle. /// /// [`Acceptor`]: crate::handle::RawHandle @@ -312,12 +299,6 @@ pub const SYS_MACHINE_STOPS: u64 = 136; /// /// [`Rights::INVENTORY`]: crate::handle::Rights::INVENTORY pub const SYS_BOOT_SOURCES: u64 = 137; -/// Read a kernel file at an offset, leaving its handle's offset where it is. -/// See [`pread`]. -pub const SYS_PREAD: u64 = 138; -/// Write a kernel file at an offset, whether or not its handle appends, and -/// leaving the handle's offset where it is. See [`pwrite`]. -pub const SYS_PWRITE: u64 = 139; pub const SYS_VM_PROTECT: u64 = 140; pub const SYS_VM_DISCARD: u64 = 141; pub const SYS_VM_DECOMMIT: u64 = 142; @@ -375,6 +356,8 @@ pub const SYS_CURRENT_CPU: u64 = 163; pub const SYS_DEVICE_FIRMWARE_WINDOW: u64 = 164; /// What a holder's paged object publishes. See [`paged_stat`]. pub const SYS_PAGED_STAT: u64 = 165; +/// List the names held by a namespace, without opening any connector. +pub const SYS_NAMESPACE_NAMES: u64 = 166; /// Bins in the per-process syscall profile — one for every number this ABI /// issues, and one at the end for every number it does not. @@ -382,14 +365,14 @@ pub const SYS_PAGED_STAT: u64 = 165; /// **The profile's parts sum to its total, and that is the whole requirement.** /// A bin array narrower than the ABI reaches drops calls out of the line while /// the total goes on counting them. -pub const SYSCALL_PROFILE_BINS: usize = 167; +pub const SYSCALL_PROFILE_BINS: usize = 168; /// Where a number this ABI does not issue is counted. Merging is a degradation /// a reader can see in the line; dropping is one nobody can. pub const SYSCALL_PROFILE_OTHER: usize = SYSCALL_PROFILE_BINS - 1; const _: () = assert!(SYS_DEVICE_FIRMWARE_WINDOW < SYSCALL_PROFILE_OTHER as u64); -const _: () = assert!(SYS_PAGED_STAT < SYSCALL_PROFILE_OTHER as u64); +const _: () = assert!(SYS_NAMESPACE_NAMES < SYSCALL_PROFILE_OTHER as u64); pub const WNOHANG: u64 = 1; /// Every bit `SYS_PROCESS_WAIT`'s flag word defines; a wait carrying another @@ -723,44 +706,6 @@ impl FileType { } } -/// Seek position for [`seek`]. -pub enum SeekFrom { - Start(u64), - Current(i64), - End(i64), -} - -/// Flags for [`open`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct OpenFlags(pub u64); - -impl OpenFlags { - pub const READ: Self = Self(1); - pub const WRITE: Self = Self(2); - pub const CREATE: Self = Self(4); - pub const TRUNCATE: Self = Self(8); - pub const APPEND: Self = Self(16); - /// With [`Self::CREATE`] only: refused `AlreadyExists` when the name is - /// there, a link or a directory included, decided in the one call that - /// creates it. - pub const EXCLUSIVE: Self = Self(32); - /// Every bit defined above. An open carrying another is refused - /// `InvalidArgument` before its path is resolved. - pub const KNOWN: Self = - Self(Self::READ.0 | Self::WRITE.0 | Self::CREATE.0 | Self::TRUNCATE.0 | Self::APPEND.0 | Self::EXCLUSIVE.0); - - pub const fn contains(self, flag: Self) -> bool { self.0 & flag.0 != 0 } -} - -impl core::ops::BitOr for OpenFlags { - type Output = Self; - fn bitor(self, rhs: Self) -> Self { Self(self.0 | rhs.0) } -} - -impl core::ops::BitOrAssign for OpenFlags { - fn bitor_assign(&mut self, rhs: Self) { self.0 |= rhs.0; } -} - /// What pages of a process's own memory may be used for. No value is both /// writable and executable. `None` pages fault on any access. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -896,18 +841,6 @@ pub fn read(handle: RawHandle, buf: &mut [u8]) -> Result { check(syscall(SYS_READ, handle.0 as u64, buf.as_mut_ptr() as u64, buf.len() as u64, 0)).map(|n| n as usize) } -/// Read a file at `offset`: how many bytes, 0 at or past its end. Any handle -/// but a file's is [`SyscallError::NotSupported`]. -pub fn pread(handle: RawHandle, buf: &mut [u8], offset: u64) -> Result { - check(syscall(SYS_PREAD, handle.0 as u64, buf.as_mut_ptr() as u64, buf.len() as u64, offset)).map(|n| n as usize) -} - -/// Write a file at `offset`: how many bytes. Any handle but a file's is -/// [`SyscallError::NotSupported`]. -pub fn pwrite(handle: RawHandle, buf: &[u8], offset: u64) -> Result { - check(syscall(SYS_PWRITE, handle.0 as u64, buf.as_ptr() as u64, buf.len() as u64, offset)).map(|n| n as usize) -} - /// Exit the current thread only. Does not return. /// Use `exit()` to exit the entire process (all threads). pub fn thread_exit(code: i32) -> ! { @@ -1293,10 +1226,6 @@ pub fn set_thread_name(name: &[u8]) { syscall(SYS_SET_THREAD_NAME, name.as_ptr() as u64, name.len() as u64, 0, 0); } -/// The most links `SYS_OPEN` follows from the name it is given: the link a -/// chain reaches past them is refused `InvalidArgument`. -pub const MAX_LINKS: u32 = 10; - /// The longest path a file server's wire carries (`sdk/toyos/src/fs.rs`), and /// libc's `PATH_MAX` (`sdk/libc/include/limits.h`). pub const MAX_PATH: usize = 4096; @@ -1315,11 +1244,6 @@ pub const MAX_WHOLE_PIPE_WRITE: usize = 4096; /// for memory. pub const PIPE_CAPACITY: usize = 2 << 20; -/// Open a file. -pub fn open(path: &[u8], flags: OpenFlags) -> Result { - check(syscall(SYS_OPEN, path.as_ptr() as u64, path.len() as u64, flags.0, 0)).map(|v| RawHandle(v as u32)) -} - /// Close a handle. /// /// # Safety @@ -1329,16 +1253,6 @@ pub unsafe fn close(handle: RawHandle) { syscall(SYS_CLOSE, handle.0 as u64, 0, 0, 0); } -/// Seek within a file handle. Returns new offset. -pub fn seek(handle: RawHandle, pos: SeekFrom) -> Result { - let (offset, whence) = match pos { - SeekFrom::Start(n) => (n as i64, 0u64), - SeekFrom::Current(n) => (n, 1u64), - SeekFrom::End(n) => (n, 2u64), - }; - check(syscall(SYS_SEEK, handle.0 as u64, offset as u64, whence, 0)) -} - /// Get file metadata for a file handle. pub fn fstat(handle: RawHandle) -> Result { let mut stat = Stat::default(); @@ -1346,81 +1260,8 @@ pub fn fstat(handle: RawHandle) -> Result { Ok(stat) } -/// What the kernel's `path` names: a file, a directory, or, unless `follow` -/// follows it, a symlink, each with the identity [`fstat`] answers a file. -pub fn stat(path: &[u8], follow: bool) -> Result { - let mut stat = Stat::default(); - let out = &mut stat as *mut Stat as u64; - check_unit(syscall(SYS_STAT, path.as_ptr() as u64, path.len() as u64, out, follow as u64))?; - Ok(stat) -} - -/// Flush a file handle to disk, or a partition claim's writes out of its -/// device's cache ([`crate::part`]): `Ok` is the device's own word that every -/// write the claim returned from before this call is durable. -pub fn fsync(handle: RawHandle) -> Result<(), SyscallError> { - check_unit(syscall(SYS_FSYNC, handle.0 as u64, 0, 0, 0)) -} - -/// Read directory entries. Returns the number of bytes the listing *needs*. -/// -/// `Ok(n)` with `n <= buf.len()` means the entries are in `buf`; `n > -/// buf.len()` means nothing was written and `n` is the size to retry with. -/// The kernel never writes a partial listing — see `sys_readdir`. -/// -/// The error is returned rather than folded into `0`: "the directory is too -/// large to list" and "the directory is empty" are different answers and a -/// caller has to be able to tell them apart. -pub fn readdir(path: &[u8], buf: &mut [u8]) -> Result { - let n = syscall(SYS_READDIR, path.as_ptr() as u64, path.len() as u64, buf.as_mut_ptr() as u64, buf.len() as u64); - match SyscallError::from_u64(n) { - Some(e) => Err(e), - None => Ok(n as usize), - } -} - -/// The kind byte a [`readdir`] entry opens with: a file. -pub const DIRENT_FILE: u8 = 1; -/// The kind byte a [`readdir`] entry opens with: a directory. -pub const DIRENT_DIR: u8 = 2; - -/// One entry of a [`readdir`] listing: its kind byte, its name, a NUL, and its -/// node as eight bytes, little-endian: the [`Stat::node`] [`stat`] answers it. -pub struct Dirent<'a> { - pub is_dir: bool, - pub name: &'a [u8], - pub node: u64, -} - -/// The entry at `*at` in a whole [`readdir`] listing, with `*at` moved past -/// it; `None` once `*at` is at its end. A listing of any other shape is a -/// kernel that broke this ABI, and panics. -pub fn dirent<'a>(listing: &'a [u8], at: &mut usize) -> Option> { - let rest = listing.get(*at..).filter(|r| !r.is_empty())?; - let is_dir = match rest[0] { - DIRENT_FILE => false, - DIRENT_DIR => true, - kind => panic!("SYS_READDIR answered an entry of kind {kind}"), - }; - let len = rest[1..].iter().position(|&b| b == 0).expect("SYS_READDIR answered a name with no NUL"); - let node = rest - .get(len + 2..len + 10) - .expect("SYS_READDIR answered an entry with no node"); - *at += len + 10; - Some(Dirent { is_dir, name: &rest[1..1 + len], node: u64::from_le_bytes(node.try_into().unwrap()) }) -} - -/// Delete a file or directory. -pub fn delete(path: &[u8]) -> Result<(), SyscallError> { - check_unit(syscall(SYS_DELETE, path.as_ptr() as u64, path.len() as u64, 0, 0)) -} - -/// Change current working directory. -pub fn chdir(path: &[u8]) -> Result<(), SyscallError> { - check_unit(syscall(SYS_CHDIR, path.as_ptr() as u64, path.len() as u64, 0, 0)) -} - -/// Get the current working directory. +/// Get the initial working directory supplied by spawn. Runtime changes +/// belong to the SDK process state (`toyos::files::getcwd`). /// /// Returns the length the path *needs*, not the number of bytes written: /// `n <= buf.len()` means the path is in `buf[..n]`, and `n > buf.len()` means @@ -2097,6 +1938,15 @@ pub fn namespace_open(ns: RawHandle, name: &str) -> Result Result { + check(syscall(SYS_NAMESPACE_NAMES, namespace.0 as u64, out.as_mut_ptr() as u64, out.len() as u64, 0)).map(|n| n as usize) +} + + /// Bytes one badge may carry. Policy on the primitive: a badge is kernel /// memory no process is charged for, so the bound is what its one reader, the /// supervisor's launch authority, needs. @@ -2562,31 +2412,6 @@ pub fn getpid() -> Pid { Pid(syscall(SYS_GETPID, 0, 0, 0, 0) as u32) } -/// Rename a file. -pub fn rename(old: &[u8], new: &[u8]) -> Result<(), SyscallError> { - check_unit(syscall(SYS_RENAME, old.as_ptr() as u64, old.len() as u64, new.as_ptr() as u64, new.len() as u64)) -} - -/// Create a directory. -pub fn mkdir(path: &[u8]) -> Result<(), SyscallError> { - check_unit(syscall(SYS_MKDIR, path.as_ptr() as u64, path.len() as u64, 0, 0)) -} - -/// Remove a directory. -pub fn rmdir(path: &[u8]) -> Result<(), SyscallError> { - check_unit(syscall(SYS_RMDIR, path.as_ptr() as u64, path.len() as u64, 0, 0)) -} - -/// Create a symbolic link at `link` pointing to `target`. -pub fn symlink(target: &[u8], link: &[u8]) -> Result<(), SyscallError> { - check_unit(syscall(SYS_SYMLINK, target.as_ptr() as u64, target.len() as u64, link.as_ptr() as u64, link.len() as u64)) -} - -/// Read the target of a symbolic link. Returns the number of bytes written to `buf`. -pub fn readlink(path: &[u8], buf: &mut [u8]) -> Result { - check(syscall(SYS_READLINK, path.as_ptr() as u64, path.len() as u64, buf.as_mut_ptr() as u64, buf.len() as u64)).map(|n| n as usize) -} - /// Libraries one process holds, `DT_NEEDED` and loaded alike; [`dl_open`] of a /// name it does not hold past it is `ResourceExhausted`. pub const MAX_LIBRARIES: usize = 1024; @@ -2661,11 +2486,6 @@ pub unsafe fn futex_wake(addr: *const u32, count: u32) -> u64 { syscall(SYS_FUTEX_WAKE, addr as u64, count as u64, 0, 0) } -/// Truncate a file handle to `size` bytes. -pub fn ftruncate(handle: RawHandle, size: u64) -> Result<(), SyscallError> { - check_unit(syscall(SYS_FTRUNCATE, handle.0 as u64, size, 0, 0)) -} - /// Get the current thread's stack base address and size. pub fn stack_info() -> Option<(u64, u64)> { let mut base: u64 = 0; @@ -3276,56 +3096,7 @@ mod tests { assert_eq!(PciId::from_wire(1 << 32), None); } - /// A listing of `entries`, each a name, whether it is a directory and a - /// node, encoded as `sys_readdir` (`kernel/src/syscall/fs.rs`) encodes it. - fn listing(entries: &[(&str, bool, u64)]) -> std::vec::Vec { - let mut out = std::vec::Vec::new(); - for (name, is_dir, node) in entries { - out.push(if *is_dir { 2 } else { 1 }); - out.extend_from_slice(name.as_bytes()); - out.push(0); - out.extend_from_slice(&node.to_le_bytes()); - } - out - } - - fn read_listing(bytes: &[u8]) -> std::vec::Vec<(std::string::String, bool, u64)> { - let mut at = 0; - let mut out = std::vec::Vec::new(); - while let Some(entry) = dirent(bytes, &mut at) { - out.push((std::string::String::from_utf8(entry.name.to_vec()).unwrap(), entry.is_dir, entry.node)); - } - assert_eq!(at, bytes.len(), "the reader stopped short of the listing's end"); - out - } - - #[test] - fn every_dirent_is_read_back_whatever_its_node_holds() { - // Nodes whose bytes are NULs, kind bytes and letters: a reader that - // does not step over all eight takes them for names. - let entries = [ - ("a", false, 0), - ("dir", true, 0x0201_0000_0000_0000), - ("b.txt", false, 0x6162_6300_0102_0304), - ("", false, u64::MAX), - ("last", true, 17), - ]; - let want: std::vec::Vec<_> = entries.iter().map(|(n, d, s)| ((*n).into(), *d, *s)).collect(); - assert_eq!(read_listing(&listing(&entries)), want); - assert!(read_listing(&[]).is_empty()); - } - #[test] - #[should_panic(expected = "of kind 3")] - fn a_dirent_of_another_kind_is_a_broken_kernel() { - read_listing(&[3, b'x', 0, 0, 0, 0, 0, 0, 0, 0, 0]); - } - - #[test] - #[should_panic(expected = "no node")] - fn a_short_dirent_is_a_broken_kernel() { - read_listing(&[1, b'x', 0, 0, 0]); - } } diff --git a/sdk/libc/arch/src/path_resolution.rs b/sdk/libc/arch/src/path_resolution.rs index 2f80369e363..bbaae27c30c 100644 --- a/sdk/libc/arch/src/path_resolution.rs +++ b/sdk/libc/arch/src/path_resolution.rs @@ -1,309 +1,93 @@ -//! `realpath`'s resolution (`resolve.rs`) over a tree laid out here and asked -//! as the kernel answers it: `..` read off the text, only the last name's link -//! followed, a relative link read against its mount, a chain of links past -//! `MAX_LINKS` refused `ELOOP` as a loop is, and a refusal's errno. - +//! Canonicalization through a process's capability namespace, including its +//! synthetic ancestors. No second resolution policy handles missing names. use std::collections::BTreeMap; - -use toyos_abi::syscall::{SyscallError, MAX_LINKS, MAX_PATH}; - +use toyos_abi::syscall::MAX_PATH; +use toyos::files::MAX_LINKS; use crate::errcode::{EIO, ELOOP, ENAMETOOLONG, ENOENT, ENOTDIR}; use crate::header::{self, LIMITS_H}; -use crate::resolve::{resolve, resolve_path, Kernel, Name, Served}; +use crate::resolve::{resolve_path, Name, Served}; #[derive(Clone, Copy)] -enum Node { - File, - Dir, - Link(&'static str), - /// A name every call about refuses `Io`, as a device that would not answer. - Broken, -} - -/// A tree keyed by absolute path, answering as `kernel/src/vfs.rs` does: -/// `open` follows a last-name link, `readdir` and `readlink` follow none. +enum Node { File, Dir, Link(&'static str), Broken } struct Tree(BTreeMap); - impl Tree { fn new(nodes: &[(&str, Node)]) -> Self { - Tree(nodes.iter().map(|&(path, node)| (String::from(path), node)).collect()) - } - - fn at(&self, at: &[u8]) -> Option<&Node> { - self.0.get(std::str::from_utf8(at).unwrap()) + Self(nodes.iter().map(|&(path, node)| (String::from(path), node)).collect()) } } - -impl Kernel for Tree { - fn read_link(&mut self, at: &[u8], target: &mut [u8]) -> Result { - match self.at(at) { - Some(&Node::Link(t)) => { - let len = t.len().min(target.len()); - target[..len].copy_from_slice(&t.as_bytes()[..len]); - Ok(len) - } - Some(Node::Broken) => Err(SyscallError::Io), - _ => Err(SyscallError::NotFound), - } - } - - fn open(&mut self, at: &[u8]) -> Result<(), SyscallError> { - match self.at(at) { - Some(Node::File) => Ok(()), - Some(&Node::Link(t)) if t.starts_with('/') => self.open(t.as_bytes()), - Some(Node::Broken) => Err(SyscallError::Io), - _ => Err(SyscallError::NotFound), +impl Served for Tree { + fn name(&mut self, at: &[u8]) -> Result { + match self.0.get(std::str::from_utf8(at).unwrap()) { + Some(Node::File) => Ok(Name::Other), + Some(Node::Dir) => Ok(Name::Directory), + Some(Node::Link(target)) => Ok(Name::Link(target.as_bytes().to_vec())), + Some(Node::Broken) => Err(EIO), + None => Err(ENOENT), } } - - fn is_dir(&mut self, at: &[u8]) -> bool { - at == b"/" || matches!(self.at(at), Some(Node::Dir)) - } } - -/// The tree every case reads: `/tmp` a mount, `/tmp/d` a directory in it. fn tree() -> Tree { Tree::new(&[ - ("/tmp", Node::Dir), - ("/tmp/d", Node::Dir), - ("/tmp/d/f", Node::File), - ("/tmp/x", Node::Dir), - ("/tmp/x/f", Node::File), - ("/tmp/d/abs", Node::Link("/tmp/x/f")), - ("/tmp/d/rel", Node::Link("x/f")), - ("/tmp/d/up", Node::Link("../system/bin/toybox")), - ("/tmp/d/chain", Node::Link("/tmp/d/abs")), - ("/tmp/d/dir", Node::Link("/tmp/x")), - ("/tmp/d/dangling", Node::Link("/tmp/none")), - ("/tmp/d/self", Node::Link("/tmp/d/self")), - ("/tmp/d/ping", Node::Link("/tmp/d/pong")), - ("/tmp/d/pong", Node::Link("d/ping")), - ("/tmp/d/broken", Node::Broken), - ("/system", Node::Dir), - ("/system/bin", Node::Dir), - ("/system/bin/toybox", Node::File), + ("/home", Node::Dir), ("/home/u", Node::Dir), + ("/home/u/a", Node::Dir), ("/home/u/a/b", Node::Dir), + ("/home/u/a/f", Node::File), ("/home/u/f", Node::File), + ("/home/u/rel", Node::Link("a/f")), + ("/home/u/abs", Node::Link("/home/u/a/f")), + ("/home/u/d", Node::Link("a")), ("/home/u/deep", Node::Link("a/b")), + ("/home/u/out", Node::Link("/tmp/f")), ("/home/u/denied", Node::Link("/system/private")), + ("/home/u/x", Node::Link("y")), ("/home/u/y", Node::Link("x")), + ("/home/u/gone", Node::Link("absent")), ("/home/u/broken", Node::Broken), + ("/tmp", Node::Dir), ("/tmp/f", Node::File), ]) } - fn answer(cwd: &str, path: &str) -> Result { - resolve(cwd.as_bytes(), path.as_bytes(), &mut tree()).map(|at| String::from_utf8(at).unwrap()) -} - -#[test] -fn a_relative_path_is_read_against_the_working_directory() { - assert_eq!(answer("/tmp/d", "f"), Ok("/tmp/d/f".into())); - assert_eq!(answer("/tmp", "./d//f"), Ok("/tmp/d/f".into())); - assert_eq!(answer("/", "tmp/d"), Ok("/tmp/d".into())); + resolve_path(cwd.as_bytes(), path.as_bytes(), &mut tree()).map(|at| String::from_utf8(at).unwrap()) } - #[test] -fn dot_dot_is_read_off_the_text_before_any_name_is_looked_up() { - assert_eq!(answer("/tmp/d", "../x/f"), Ok("/tmp/x/f".into())); - // `dir` is a link to `/tmp/x`, whose parent is not `/tmp/d`'s. - assert_eq!(answer("/", "/tmp/d/dir/../f"), Ok("/tmp/d/f".into())); - assert_eq!(answer("/tmp", "../../../.."), Ok("/".into())); - // `..` takes off a name that does not exist without asking. - assert_eq!(answer("/", "/tmp/d/none/../f"), Ok("/tmp/d/f".into())); +fn links_are_followed_at_each_component_and_relative_to_their_directory() { + assert_eq!(answer("/", "/home/u/rel"), Ok("/home/u/a/f".into())); + assert_eq!(answer("/", "/home/u/abs"), Ok("/home/u/a/f".into())); + assert_eq!(answer("/home/u", "./a//f"), Ok("/home/u/a/f".into())); + assert_eq!(answer("/", "/home/u/d/f"), Ok("/home/u/a/f".into())); + assert_eq!(answer("/", "/home/u/deep/../f"), Ok("/home/u/a/f".into())); + assert_eq!(answer("/", "/home/u/a/../f"), Ok("/home/u/f".into())); } - #[test] -fn the_last_names_links_are_followed_to_the_file() { - assert_eq!(answer("/", "/tmp/d/abs"), Ok("/tmp/x/f".into())); - assert_eq!(answer("/tmp/d", "chain"), Ok("/tmp/x/f".into())); - assert_eq!(answer("/", "/tmp/d/dir"), Ok("/tmp/x".into())); +fn absolute_links_reenter_the_same_capability_namespace() { + assert_eq!(answer("/", "/home/u/out"), Ok("/tmp/f".into())); + assert_eq!(answer("/", "/home/u/denied"), Err(ENOENT)); + assert_eq!(answer("/home/u", "../../tmp/f"), Ok("/tmp/f".into())); } - #[test] -fn a_relative_link_is_read_against_its_mount_not_its_directory() { - // `/tmp/d/x/f` is not there; `/tmp/x/f` is. - assert_eq!(answer("/", "/tmp/d/rel"), Ok("/tmp/x/f".into())); - // Its `..` leaves the mount, as the kernel's does. - assert_eq!(answer("/", "/tmp/d/up"), Ok("/system/bin/toybox".into())); +fn missing_names_files_as_parents_and_refusals_are_preserved() { + assert_eq!(answer("/", "/home/u/gone"), Err(ENOENT)); + assert_eq!(answer("/", "/home/u/a/f/g"), Err(ENOTDIR)); + assert_eq!(answer("/", "/home/u/a/f/.."), Err(ENOTDIR)); + assert_eq!(answer("/", "/home/u/missing/../f"), Err(ENOENT)); + assert_eq!(answer("/", "/home/u/broken"), Err(EIO)); + assert_eq!(answer("/", ""), Err(ENOENT)); + assert_eq!(answer("/", "/"), Ok("/".into())); } - #[test] -fn a_link_before_the_last_name_is_not_followed() { - // The kernel looks `/tmp/d/dir/f` up as that name, and there is none. - assert_eq!(answer("/", "/tmp/d/dir/f"), Err(ENOENT)); -} - -#[test] -fn a_loop_of_links_is_eloop() { - assert_eq!(answer("/", "/tmp/d/self"), Err(ELOOP)); - assert_eq!(answer("/", "/tmp/d/ping"), Err(ELOOP)); -} - -/// `/tmp/l0`, a chain of `links` links, each relative, ending in `/tmp/end`. -fn chain(links: u32) -> Tree { - let mut nodes: Vec<(&'static str, Node)> = vec![("/tmp", Node::Dir), ("/tmp/end", Node::File)]; - for i in 0..links { - let target = if i + 1 == links { "/tmp/end".to_string() } else { format!("l{}", i + 1) }; - nodes.push((leak(format!("/tmp/l{i}")), Node::Link(leak(target)))); +fn links_are_bounded_even_across_directories() { + assert_eq!(answer("/", "/home/u/x"), Err(ELOOP)); + for (links, expected) in [(MAX_LINKS, Ok(b"/tmp/f".to_vec())), (MAX_LINKS + 1, Err(ELOOP))] { + let mut t = tree(); + for i in 0..links { + let target = if i + 1 == links { "/tmp/f".into() } else { format!("l{}", i + 1) }; + t.0.insert(format!("/tmp/l{i}"), Node::Link(Box::leak(target.into_boxed_str()))); + } + assert_eq!(resolve_path(b"/", b"/tmp/l0", &mut t), expected); } - Tree::new(&nodes) } - -fn leak(s: String) -> &'static str { - Box::leak(s.into_boxed_str()) -} - -#[test] -fn a_chain_of_max_links_is_followed_and_one_more_is_eloop() { - let at = resolve(b"/", b"/tmp/l0", &mut chain(MAX_LINKS)).map(|at| String::from_utf8(at).unwrap()); - assert_eq!(at, Ok("/tmp/end".into())); - assert_eq!(resolve(b"/", b"/tmp/l0", &mut chain(MAX_LINKS + 1)), Err(ELOOP)); -} - #[test] -fn a_name_that_is_not_there_is_enoent() { - assert_eq!(answer("/", "/tmp/d/none"), Err(ENOENT)); - assert_eq!(answer("/", "/tmp/none/f"), Err(ENOENT)); - assert_eq!(answer("/", "/tmp/d/dangling"), Err(ENOENT)); - assert_eq!(answer("/tmp", ""), Err(ENOENT)); -} - -#[test] -fn a_file_passed_through_as_a_directory_is_enotdir() { - assert_eq!(answer("/", "/tmp/d/f/g"), Err(ENOTDIR)); - // A link to a file, as POSIX has it. - assert_eq!(answer("/", "/tmp/d/abs/g/h"), Err(ENOTDIR)); -} - -#[test] -fn a_path_or_an_answer_past_path_max_is_enametoolong() { +fn paths_and_link_targets_are_bounded() { let long = format!("/tmp/{}", "a".repeat(MAX_PATH)); assert_eq!(answer("/", &long), Err(ENAMETOOLONG)); - // Short enough as written, too long once read against the directory. let name = "b".repeat(MAX_PATH - 6); - assert_eq!(answer("/tmp/d", &name), Err(ENAMETOOLONG)); - // A working directory and a path joined past it, which `..` shortens. - let deep = format!("/tmp/{}", "c".repeat(MAX_PATH - 10)); - assert_eq!(answer(&deep, "../d/f"), Ok("/tmp/d/f".into())); -} - -#[test] -fn a_link_target_that_fills_the_buffer_is_enametoolong() { - let target: &'static str = Box::leak(format!("/{}", "t".repeat(MAX_PATH)).into_boxed_str()); - let mut tree = Tree::new(&[("/tmp", Node::Dir), ("/tmp/long", Node::Link(target))]); - assert_eq!(resolve(b"/", b"/tmp/long", &mut tree), Err(ENAMETOOLONG)); -} - -#[test] -fn a_kernel_refusal_is_its_errno() { - assert_eq!(answer("/", "/tmp/d/broken"), Err(EIO)); -} - -#[test] -fn path_max_is_limits_h_s() { + assert_eq!(answer("/home/u", &name), Err(ENAMETOOLONG)); + let mut t = tree(); + t.0.insert("/tmp/long".into(), Node::Link(Box::leak(long.into_boxed_str()))); + assert_eq!(resolve_path(b"/", b"/tmp/long", &mut t), Err(ENAMETOOLONG)); assert_eq!(MAX_PATH, header::int(LIMITS_H, "PATH_MAX") as usize); } - -/// A file server holding the capability `grant`, its names in the tree, -/// beside the kernel's [`tree`]: as `toyos::files` answers, a name is its -/// link's target or nothing, one that is not there `ENOENT`, and one through -/// a file `ENOTDIR`. -struct Home(Tree, &'static str); - -impl Served for Home { - fn holds(&mut self, at: &[u8]) -> bool { - let grant = self.1.as_bytes(); - at == grant || (at.starts_with(grant) && at.get(grant.len()) == Some(&b'/')) - } - - fn name(&mut self, at: &[u8]) -> Result { - let parent = &at[..at.iter().rposition(|&b| b == b'/').unwrap()]; - if parent.len() > b"/home".len() && matches!(self.0.at(parent), Some(Node::File)) { - return Err(ENOTDIR); - } - match self.0.at(at) { - Some(Node::Link(target)) => Ok(Name::Link(target.as_bytes().to_vec())), - Some(_) => Ok(Name::Other), - None if at == self.1.as_bytes() => Ok(Name::Other), - None => Err(ENOENT), - } - } -} - -fn home() -> Home { - Home(Tree::new(&[ - ("/home/u", Node::Dir), - ("/home/u/a", Node::Dir), - ("/home/u/a/b", Node::Dir), - ("/home/u/a/file", Node::File), - ("/home/u/a/f", Node::File), - ("/home/u/rel", Node::Link("a/file")), - ("/home/u/abs", Node::Link("/home/u/a/file")), - ("/home/u/d", Node::Link("a")), - ("/home/u/deep", Node::Link("a/b")), - ("/home/u/out", Node::Link("/tmp/d/f")), - ("/home/u/outdir", Node::Link("/tmp/x")), - ("/home/u/x", Node::Link("y")), - ("/home/u/y", Node::Link("x")), - ("/home/u/dangling", Node::Link("gone")), - ]), "/home") -} - -fn mixed(cwd: &str, path: &str) -> Result { - resolve_path(cwd.as_bytes(), path.as_bytes(), &mut home(), &mut tree()).map(|at| String::from_utf8(at).unwrap()) -} - -/// A served link is read against its own directory, as POSIX has it, and -/// not against a mount as the kernel's is; an absolute one from the root. -#[test] -fn a_served_link_is_read_against_its_own_directory() { - assert_eq!(mixed("/", "/home/u/rel"), Ok("/home/u/a/file".into())); - assert_eq!(mixed("/", "/home/u/abs"), Ok("/home/u/a/file".into())); - assert_eq!(mixed("/home/u", "./a//file"), Ok("/home/u/a/file".into()), "relative and lexically clean"); -} - -/// Every name's link is followed where it is met, so `..` after one is the -/// parent of where it led, and two spellings of one file answer one path. -#[test] -fn a_served_link_before_the_last_name_is_followed_and_dot_dot_after_it_is_its_parent() { - assert_eq!(mixed("/", "/home/u/d/file"), Ok("/home/u/a/file".into())); - assert_eq!(mixed("/", "/home/u/deep/../f"), Ok("/home/u/a/f".into()), "the parent of a/b, not of deep"); - assert_eq!(mixed("/", "/home/u/d/../d/file"), mixed("/", "/home/u/a/file")); -} - -/// A served link to a path the kernel holds is resolved there, by the -/// kernel's rules, whether it is the last name or one before it. -#[test] -fn a_served_link_out_to_the_kernel_is_resolved_by_the_kernel() { - assert_eq!(mixed("/", "/home/u/out"), Ok("/tmp/d/f".into())); - assert_eq!(mixed("/", "/home/u/outdir/f"), Ok("/tmp/x/f".into())); - assert_eq!(mixed("/", "/tmp/d/abs"), Ok("/tmp/x/f".into()), "a kernel path is the kernel's from its first name"); - assert_eq!(mixed("/home/u", "../../tmp/d/f"), Ok("/tmp/d/f".into())); -} - -#[test] -fn a_served_loop_is_eloop_and_a_missing_name_enoent_or_enotdir() { - assert_eq!(mixed("/", "/home/u/x"), Err(ELOOP)); - assert_eq!(mixed("/", "/home/u/dangling"), Err(ENOENT)); - assert_eq!(mixed("/", "/home/u/a/file/g"), Err(ENOTDIR)); - assert_eq!(mixed("/", "/"), Ok("/".into())); -} - -/// A capability three names deep, `fs:/home/toy/Games` without `fs:/home`, as -/// a folder grant gives one: the names above it are passed through as the -/// client routes past them, and a path beside it is the kernel's. -#[test] -fn a_path_under_a_deeper_capability_is_walked_and_one_beside_it_is_the_kernels() { - let games = || { - Home( - Tree::new(&[ - ("/home/toy/Games/save", Node::File), - ("/home/toy/Games/rom", Node::Link("roms/a.gba")), - ("/home/toy/Games/roms", Node::Dir), - ("/home/toy/Games/roms/a.gba", Node::File), - ]), - "/home/toy/Games", - ) - }; - let answer = |path: &str| { - resolve_path(b"/", path.as_bytes(), &mut games(), &mut tree()).map(|at| String::from_utf8(at).unwrap()) - }; - assert_eq!(answer("/home/toy/Games/save"), Ok("/home/toy/Games/save".into())); - assert_eq!(answer("/home/toy/Games/rom"), Ok("/home/toy/Games/roms/a.gba".into())); - assert_eq!(answer("/home/toy/./Games/../Games/save"), Ok("/home/toy/Games/save".into())); - assert_eq!(answer("/home/toy/Other"), Err(ENOENT), "beside the capability: the kernel's, which holds nothing there"); -} diff --git a/sdk/libc/src/posix_io.rs b/sdk/libc/src/posix_io.rs index 227a3bd710f..261fe7e187b 100644 --- a/sdk/libc/src/posix_io.rs +++ b/sdk/libc/src/posix_io.rs @@ -1,6 +1,5 @@ // POSIX file I/O: a path or a descriptor is `toyos::files`', which routes it to -// the kernel or to the file server behind one of the process's directory -// capabilities. +// the file server behind one of the process's directory capabilities. #![allow(non_camel_case_types)] @@ -9,16 +8,16 @@ use alloc::vec::Vec; use core::ptr; use toyos_abi::{RawHandle, Rights}; -use toyos_abi::syscall::{self, OpenFlags, SeekFrom}; +use toyos_abi::syscall; -use toyos::files::{self, At}; +use toyos::files::{self, At, SeekFrom}; use crate::errcode::{EBADF, EINTR, EINVAL}; use crate::signal::Call; use crate::time::Timespec; -pub(crate) use crate::atreq::{O_APPEND, O_CREAT, O_EXCL, O_RDONLY, O_RDWR, O_TRUNC, O_WRONLY}; -use crate::atreq::{read_write, AT_EMPTY_PATH, AT_FDCWD, AT_REMOVEDIR, AT_SYMLINK_FOLLOW, AT_SYMLINK_NOFOLLOW}; +pub(crate) use crate::atreq::{O_APPEND, O_CREAT, O_RDONLY, O_RDWR, O_TRUNC, O_WRONLY}; +use crate::atreq::{AT_EMPTY_PATH, AT_FDCWD, AT_REMOVEDIR, AT_SYMLINK_FOLLOW, AT_SYMLINK_NOFOLLOW}; const SEEK_SET: i32 = 0; const SEEK_CUR: i32 = 1; @@ -113,25 +112,17 @@ pub unsafe extern "C" fn openat(dirfd: i32, path: *const u8, flags: i32, _mode: } } -/// A spawn's `addopen` file, as a handle of this process's for the child. -/// A file a file server holds is refused: no kernel handle reaches it. +/// A spawn's `addopen` file exported as a server stream for the child. pub(crate) fn open_handle(path: &[u8], flags: i32, _mode: u32) -> Result { - let (readable, writable) = read_write(flags); + let how = crate::atreq::how(flags)?; if path == DEV_NULL { - return syscall::null(null_rights(readable, writable)).map_err(crate::errcode::of); + return syscall::null(null_rights(how.read, how.write)).map_err(crate::errcode::of); } - if files::is_served(path) { - return Err(crate::errcode::EOPNOTSUPP); - } - - let mut oflags = OpenFlags(0); - if readable { oflags |= OpenFlags::READ; } - if writable { oflags |= OpenFlags::WRITE; } - if flags & O_CREAT != 0 { oflags |= OpenFlags::CREATE; } - if flags & O_CREAT != 0 && flags & O_EXCL != 0 { oflags |= OpenFlags::EXCLUSIVE; } - if flags & O_TRUNC != 0 { oflags |= OpenFlags::TRUNCATE; } - if flags & O_APPEND != 0 { oflags |= OpenFlags::APPEND; } - syscall::open(path, oflags).map_err(crate::errcode::of) + let fd = files::open(At::Cwd, path, how).map_err(crate::errcode::of_files)?; + let stream = files::stream(fd).map(|h| h.into_raw()).map_err(crate::errcode::of_files); + // SAFETY: this function's descriptor, after the server exported the stream. + let _ = unsafe { toyos::fd::close(fd) }; + stream } /// POSIX's null device, which libc answers with a kernel null handle: the @@ -436,7 +427,7 @@ pub unsafe extern "C" fn linkat(from_dirfd: i32, from: *const u8, to_dirfd: i32, pub unsafe extern "C" fn getcwd(buf: *mut u8, size: usize) -> *mut u8 { if buf.is_null() || size == 0 { return ptr::null_mut(); } let slice = core::slice::from_raw_parts_mut(buf, size); - let n = syscall::getcwd(slice); + let n = files::getcwd(slice); if n == 0 || n >= size { return ptr::null_mut(); } @@ -478,9 +469,6 @@ pub unsafe extern "C" fn fstatat(dirfd: i32, path: *const u8, buf: *mut Stat, fl } } -/// `SYS_READLINK` answers `NotFound` for a path that is no link as for one -/// that names nothing, and POSIX tells them apart: `EINVAL` for the first. A -/// file server tells them apart itself. #[no_mangle] pub unsafe extern "C" fn readlink(path: *const u8, buf: *mut u8, size: usize) -> isize { unsafe { readlinkat(AT_FDCWD, path, buf, size) } @@ -503,20 +491,10 @@ pub unsafe extern "C" fn readlinkat(dirfd: i32, path: *const u8, buf: *mut u8, s unsafe { ptr::copy_nonoverlapping(target.as_ptr(), buf, n) }; n as isize } - Err(files::Error::Kernel(syscall::SyscallError::NotFound)) => { - // `stat` sets `errno` for a path that names nothing. - if unsafe { fstatat(dirfd, path, ptr::null_mut(), AT_SYMLINK_NOFOLLOW) } == 0 { - crate::errno::set(EINVAL); - } - -1 - } Err(e) => refuse_files(e) as isize, } } -/// A file server refuses a name that exists, as POSIX has it. The kernel's -/// `SYS_SYMLINK` displaces whatever holds the name, and a question asked first -/// races its answer, so a link the kernel would hold is refused `ENOSYS`. #[no_mangle] pub unsafe extern "C" fn symlink(target: *const u8, link: *const u8) -> i32 { unsafe { symlinkat(target, AT_FDCWD, link) } @@ -529,52 +507,25 @@ pub unsafe extern "C" fn symlinkat(target: *const u8, dirfd: i32, link: *const u Ok(at) => at, Err(e) => return refuse(e), }; - match files::is_served_at(at, link) { - Ok(true) => files::symlink(target, (at, link)).map_or_else(refuse_files, |()| 0), - Ok(false) => refuse(crate::errcode::ENOSYS), - Err(e) => refuse_files(e), - } + files::symlink(target, (at, link)).map_or_else(refuse_files, |()| 0) } struct TheServers; impl crate::resolve::Served for TheServers { - fn holds(&mut self, at: &[u8]) -> bool { - files::is_served(at) - } - fn name(&mut self, at: &[u8]) -> Result { let st = files::stat(At::Cwd, at, false).map_err(crate::errcode::of_files)?; match st.file_type { syscall::FileType::Symlink => { files::read_link(At::Cwd, at).map(crate::resolve::Name::Link).map_err(crate::errcode::of_files) } + syscall::FileType::Directory => Ok(crate::resolve::Name::Directory), _ => Ok(crate::resolve::Name::Other), } } } -struct TheKernel; - -impl crate::resolve::Kernel for TheKernel { - fn read_link(&mut self, at: &[u8], target: &mut [u8]) -> Result { - syscall::readlink(at, target) - } - - fn open(&mut self, at: &[u8]) -> Result<(), syscall::SyscallError> { - // SAFETY: a handle just opened, closed once. - syscall::open(at, OpenFlags::READ).map(|h| unsafe { syscall::close(h) }) - } - - fn is_dir(&mut self, at: &[u8]) -> bool { - syscall::stat(at, true).is_ok_and(|st| st.file_type == syscall::FileType::Directory) - } -} - -/// POSIX's `realpath` (`resolve::resolve_path`): POSIX's resolution where a -/// file server holds the names, the kernel's rules where it does, so the -/// answer is the file `open` of `path` reaches, in a block of `malloc`'s when -/// `resolved` is null. +/// POSIX path resolution through this process's filesystem capabilities. #[no_mangle] pub unsafe extern "C" fn realpath(path: *const u8, resolved: *mut u8) -> *mut u8 { use syscall::MAX_PATH; @@ -583,9 +534,9 @@ pub unsafe extern "C" fn realpath(path: *const u8, resolved: *mut u8) -> *mut u8 return ptr::null_mut(); } let mut cwd = [0u8; MAX_PATH]; - let n = syscall::getcwd(&mut cwd); - assert!(n != 0 && n <= MAX_PATH, "SYS_GETCWD: a working directory past MAX_PATH"); - let resolved_at = crate::resolve::resolve_path(&cwd[..n], c_str_to_bytes(path), &mut TheServers, &mut TheKernel); + let n = files::getcwd(&mut cwd); + assert!(n != 0 && n <= MAX_PATH, "SDK working directory past MAX_PATH"); + let resolved_at = crate::resolve::resolve_path(&cwd[..n], c_str_to_bytes(path), &mut TheServers); let at = match resolved_at { Ok(at) => at, Err(code) => { diff --git a/sdk/libc/src/resolve.rs b/sdk/libc/src/resolve.rs index 38dd011637b..d95b5152461 100644 --- a/sdk/libc/src/resolve.rs +++ b/sdk/libc/src/resolve.rs @@ -1,158 +1,31 @@ -//! Where a path leads, as `realpath` answers it: by the kernel's rules of -//! resolution (`kernel/src/vfs.rs`, `resolve_absolute` and -//! `resolve_for_open`), which every path libc hands the kernel is read by. It -//! asks the kernel only through [`Kernel`], so the host tests it -//! (`toyos-libc-copies`). -//! -//! A path a file server holds is resolved POSIX's way by [`resolve_path`], -//! which hands the kernel what it holds. -//! -//! The kernel's rules are not POSIX's (`issues/the-kernel-resolves-a-path-by-rules-of-its-own.md`): -//! `..` is read off the text before any name is looked up, only the last -//! name's link is followed, at most [`MAX_LINKS`] of them, and a relative link -//! is read against the mount it is on, the first name of its path. The kernel -//! follows no other name as a link, so a path through one names nothing, and -//! an answer holds none. - +//! POSIX canonical path resolution through the process's filesystem namespace. use alloc::vec::Vec; - -use toyos_abi::syscall::{SyscallError, MAX_LINKS, MAX_PATH}; - -use crate::errcode::{self, ELOOP, ENAMETOOLONG, ENOENT, ENOTDIR}; - -/// The three calls resolution asks the kernel, each of an absolute path. -pub(crate) trait Kernel { - /// `SYS_READLINK` into `target`, [`MAX_PATH`] bytes: the length it wrote. - fn read_link(&mut self, at: &[u8], target: &mut [u8]) -> Result; - /// Whether `SYS_OPEN` opens `at` to read; what it opened is closed. - fn open(&mut self, at: &[u8]) -> Result<(), SyscallError>; - /// Whether `SYS_READDIR` lists `at`. - fn is_dir(&mut self, at: &[u8]) -> bool; -} - -/// `path` made absolute against `cwd`, its last name's links followed, or the -/// errno it is refused with. -pub(crate) fn resolve(cwd: &[u8], path: &[u8], kernel: &mut impl Kernel) -> Result, i32> { - if path.is_empty() { - return Err(ENOENT); - } - let mut at = joined(cwd, path)?; - let mut followed = 0; - let mut target = alloc::vec![0u8; MAX_PATH]; - loop { - let len = match kernel.read_link(&at, &mut target) { - Ok(len) => len, - // A name that is no link, or none at all: `kind` tells them apart. - Err(SyscallError::NotFound) => break, - Err(e) => return Err(errcode::of(e)), - }; - if followed == MAX_LINKS { - return Err(ELOOP); - } - followed += 1; - let mount = at.split(|&b| b == b'/').nth(1).expect("an absolute path's first name"); - let mut base = Vec::with_capacity(1 + mount.len()); - base.push(b'/'); - base.extend_from_slice(mount); - at = joined(&base, &target[..len])?; - } - match kind(&at, kernel)? { - Kind::File | Kind::Dir => Ok(at), - Kind::Absent => Err(absent(&at, kernel)?), - } -} - -/// What a path no link is followed in names. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] -enum Kind { - File, - Dir, - Absent, -} - -/// `open` refuses every directory, so a refused name is asked `readdir`. -fn kind(at: &[u8], kernel: &mut impl Kernel) -> Result { - match kernel.open(at) { - Ok(()) => Ok(Kind::File), - Err(_) if kernel.is_dir(at) => Ok(Kind::Dir), - Err(SyscallError::NotFound) => Ok(Kind::Absent), - Err(e) => Err(errcode::of(e)), - } -} - -/// `rel` read against the absolute `base`, `.` and `..` taken off the text, -/// or `ENAMETOOLONG` where it or the answer reaches [`MAX_PATH`]: a link -/// target that fills the buffer may have been cut short. -fn joined(base: &[u8], rel: &[u8]) -> Result, i32> { - if rel.len() >= MAX_PATH { - return Err(ENAMETOOLONG); - } - let text = if rel.first() == Some(&b'/') { &[][..] } else { base }; - let mut names: Vec<&[u8]> = Vec::new(); - for name in text.split(|&b| b == b'/').chain(rel.split(|&b| b == b'/')) { - match name { - b"" | b"." => {} - b".." => { - names.pop(); - } - name => names.push(name), - } - } - let mut out = Vec::new(); - for name in &names { - out.push(b'/'); - out.extend_from_slice(name); - } - if out.is_empty() { - out.push(b'/'); - } - if out.len() >= MAX_PATH { - return Err(ENAMETOOLONG); - } - Ok(out) -} - -/// Why `at`, which names nothing, does: `ENOTDIR` where a directory it passes -/// through is a file, `ENOENT` otherwise. -fn absent(at: &[u8], kernel: &mut impl Kernel) -> Result { - for end in (1..at.len()).filter(|&i| at[i] == b'/') { - match kind(&at[..end], kernel)? { - Kind::Dir => {} - Kind::File => return Ok(ENOTDIR), - Kind::Absent => return Ok(ENOENT), - } - } - Ok(ENOENT) -} +use toyos_abi::syscall::MAX_PATH; +use toyos::files::MAX_LINKS; +use crate::errcode::{ELOOP, ENAMETOOLONG, ENOENT, ENOTDIR}; /// What a name a file server holds is, as resolution asks it, its own link /// not followed. pub(crate) enum Name { Link(Vec), + Directory, Other, } /// The two calls [`resolve_path`] asks of the file servers, each of an /// absolute path without `.` or `..`. pub(crate) trait Served { - /// Whether a file server holds `at`, and not the kernel: by the deepest - /// capability `at` is under, which may be several names below its first. - fn holds(&mut self, at: &[u8]) -> bool; /// `at` itself, or the errno it is refused with: one that names nothing, /// or passes through a file, is refused. fn name(&mut self, at: &[u8]) -> Result; } -/// `path` made absolute against `cwd` and resolved as POSIX resolves it, a -/// name at a time, while a file server holds the names: each link is -/// followed where it is met, a relative one against its own directory, and -/// `..` after it is the parent of where it led. Once the path as it stands -/// is the kernel's, it is [`resolve`]d by the kernel's rules, which its -/// `open` reads it by. The answer holds no link a file server keeps. -pub(crate) fn resolve_path(cwd: &[u8], path: &[u8], served: &mut impl Served, kernel: &mut impl Kernel) -> Result, i32> { +/// Follow each name's links where encountered, including before `..`. +pub(crate) fn resolve_path(cwd: &[u8], path: &[u8], served: &mut impl Served) -> Result, i32> { if path.is_empty() { return Err(ENOENT); } + if path.len() >= MAX_PATH { return Err(ENAMETOOLONG); } let names = |text: &[u8]| -> Vec> { text.split(|&b| b == b'/').map(<[u8]>::to_vec).collect() }; let mut pending: alloc::collections::VecDeque> = alloc::collections::VecDeque::new(); if path.first() != Some(&b'/') { @@ -177,25 +50,13 @@ pub(crate) fn resolve_path(cwd: &[u8], path: &[u8], served: &mut impl Served, ke if next.len() >= MAX_PATH { return Err(ENAMETOOLONG); } - let mut full = next.clone(); - for rest in &pending { - full.push(b'/'); - full.extend_from_slice(rest); - } - // Asked of the path as it reads, `.` and `..` taken off its text: a - // link met further on is asked again where it leads. - if !served.holds(&joined(b"/", &full)?) { - return resolve(b"/", &full, kernel); - } - // A name above the capability that holds the path, which the client - // routes past and no server holds: a directory, as its routing reads it. - if !served.holds(&next) { - at = next; - continue; - } match served.name(&next)? { - Name::Other => at = next, + Name::Directory => at = next, + Name::Other if pending.is_empty() => at = next, + Name::Other => return Err(ENOTDIR), Name::Link(target) => { + if target.len() >= MAX_PATH { return Err(ENAMETOOLONG); } + if target.is_empty() { return Err(ENOENT); } if followed == MAX_LINKS { return Err(ELOOP); } diff --git a/sdk/libc/src/spawn.rs b/sdk/libc/src/spawn.rs index ea006edcfb1..d78aa9b4197 100644 --- a/sdk/libc/src/spawn.rs +++ b/sdk/libc/src/spawn.rs @@ -329,7 +329,7 @@ fn spawn_counted(path: &str, argv: &[u8], env: &[u8], actions: &[Action, Kind)]) -> Result { let mut cwd = alloc::vec![0u8; syscall::MAX_PATH]; - let n = syscall::getcwd(&mut cwd); + let n = toyos::files::getcwd(&mut cwd); let cwd = core::str::from_utf8(&cwd[..n]).map_err(|_| EINVAL)?; let request = Spawn { program: path, @@ -349,13 +349,8 @@ fn start(path: &str, argv: &[u8], env: &[u8], fds: &[(u32, toyos::BorrowedHandle let Start::Helper(helper) = request.start().map_err(errno_of)? else { unreachable!("a spawn that is not independent is a helper or refused") }; - // A program on a file server is read here into an image the kernel pages - // the child from: the kernel opens only what it serves itself. - let image = match toyos::files::is_served(path.as_bytes()) { - true => Some(toyos::files::read_image(path.as_bytes()).map_err(crate::errcode::of_image)?), - false => None, - }; - helper.spawn(image.as_ref()).map_err(errno_of) + let image = toyos::files::read_image(path.as_bytes()).map_err(crate::errcode::of_image)?; + helper.spawn(Some(&image)).map_err(errno_of) } fn errno_of(refused: Refused) -> i32 { diff --git a/sdk/libc/src/stdio.rs b/sdk/libc/src/stdio.rs index 22512178f20..fd19d74e79a 100644 --- a/sdk/libc/src/stdio.rs +++ b/sdk/libc/src/stdio.rs @@ -1,6 +1,6 @@ use core::ptr; use toyos::files; -use toyos_abi::syscall::SeekFrom; +use toyos::files::SeekFrom; // Platform fd operations diff --git a/sdk/std/sys/fs.rs b/sdk/std/sys/fs.rs index 3ed16801fea..589f80e1d54 100644 --- a/sdk/std/sys/fs.rs +++ b/sdk/std/sys/fs.rs @@ -1,10 +1,9 @@ -//! Files on ToyOS: a path is served by the kernel (`/tmp`) or by a file -//! server behind one of the process's directory capabilities, and -//! `toyos::files` routes it, follows its links and keeps what it opened. A -//! `File` is one descriptor of the process's table either way. +//! Files through the process's directory capabilities and namespace ancestors. +//! `toyos::files` owns routing, connections and descriptors shared with libc. use toyos::files::{self, At, Open}; -use toyos_abi::syscall::{self, SeekFrom as AbiSeek}; +use toyos_abi::syscall; +use toyos::files::SeekFrom as AbiSeek; use crate::ffi::OsString; use crate::fmt; @@ -239,7 +238,7 @@ impl DirEntry { } /// What the listing said, or where it said only that the entry is no - /// directory, as the kernel's does, what the entry is asked. + /// directory, what the entry is asked. pub fn file_type(&self) -> io::Result { match self.file_type { FileType { is_file: false, is_dir: false, is_symlink: false } => lstat(&self.path()).map(|a| a.file_type), @@ -301,11 +300,6 @@ fn bytes(path: &Path) -> &[u8] { path.as_os_str().as_encoded_bytes() } -/// Whether `path` is on a file server rather than the kernel. -pub fn is_served(path: &Path) -> bool { - files::is_served(bytes(path)) -} - impl File { /// The file descriptor `fd` names, owned from here on. pub fn from_fd(fd: i32) -> Self { @@ -333,9 +327,8 @@ impl File { crate::sys::on_fd(self.0, |h| Ok(f(h.raw()))) } - /// A kernel handle a child may use this file through: a copy of a kernel - /// file's own, or a pipe its server writes to the file at this file's - /// offset. The caller holds it until the child has its own. + /// A child's output handle: a copied generic object or a pipe the server + /// writes at this open description's offset. pub fn as_child_stdio(&self) -> io::Result { files::stream(self.0).map_err(files_error) } @@ -592,8 +585,7 @@ pub fn set_times_nofollow(p: &Path, times: FileTimes) -> io::Result<()> { files::set_times(At::Cwd, bytes(p), false, ×.change()?).map_err(files_error) } -/// The whole of the file at `path` as a program image: what a spawn of a -/// program the kernel cannot open itself hands the kernel. +/// Read a program image for the kernel to map into a child. pub fn read_image(path: &Path, job: Option>) -> io::Result { files::read_image_for(bytes(path), job).map_err(|refused| match refused { toyos::process::ImageRefused::Empty => { @@ -607,8 +599,7 @@ pub fn read_image(path: &Path, job: Option>) -> io::Re }) } -/// Make the working directory `p`, which a file server judges when it serves -/// it and the kernel judges when it does. +/// Validate `p` and store it in the shared SDK working-directory state. pub fn chdir(p: &Path) -> io::Result<()> { files::chdir(bytes(p)).map_err(files_error) } diff --git a/sdk/std/sys/pal/os.rs b/sdk/std/sys/pal/os.rs index 4c820bcaea0..acda8b2bc93 100644 --- a/sdk/std/sys/pal/os.rs +++ b/sdk/std/sys/pal/os.rs @@ -10,7 +10,7 @@ pub fn getcwd() -> io::Result { // bytes came back as a shorter, valid-looking path to a *different* // directory, and every path derived from it pointed somewhere wrong. let mut buf = [0u8; 256]; - let n = toyos_abi::syscall::getcwd(&mut buf); + let n = toyos::files::getcwd(&mut buf); if n == 0 { return Err(io::Error::new(io::ErrorKind::Other, "getcwd failed")); } @@ -19,10 +19,10 @@ pub fn getcwd() -> io::Result { } // Too long for the stack buffer: allocate exactly what it asked for. The - // kernel bounds cwd, so this retries once rather than looping. + // SDK bounds cwd, so this retries once rather than looping. let mut heap: crate::vec::Vec = crate::vec::Vec::new(); heap.resize(n, 0); - let got = toyos_abi::syscall::getcwd(&mut heap); + let got = toyos::files::getcwd(&mut heap); if got == 0 || got > heap.len() { return Err(io::Error::new(io::ErrorKind::Other, "getcwd failed")); } diff --git a/sdk/std/sys/process.rs b/sdk/std/sys/process.rs index a6ab2b6f2d8..96d32767b5c 100644 --- a/sdk/std/sys/process.rs +++ b/sdk/std/sys/process.rs @@ -174,7 +174,7 @@ impl Command { let kept = self.image.as_ref().is_some_and(|(read, _)| *read == source); if !kept { self.image = None; - if self.boot_image.is_none() && crate::sys::fs::is_served(Path::new(&source)) { + if self.boot_image.is_none() && !self.independent { let image = crate::sys::fs::read_image(Path::new(&source), self.endowed_job())?; self.image = Some((source, image)); } @@ -184,8 +184,7 @@ impl Command { } /// The program a prepared spawn starts, as [`Self::prepare`] read it off a - /// file server: `None` before a prepare, and for a program the kernel - /// opens itself. + /// file server: `None` before a prepare and for a boot image or launch. pub fn prepared_image(&self) -> Option<&[u8]> { self.prepared.as_ref()?; self.image.as_ref().map(|(_, image)| image.bytes()) @@ -321,9 +320,9 @@ impl Command { Some(dir) => crate::env::current_dir()?.join(dir), None => crate::env::current_dir()?, }; - // The kernel judges a directory it serves itself; one on a file server - // is judged here, by that server, since the kernel cannot. - if crate::sys::fs::is_served(&dir) && !crate::fs::metadata(&dir)?.is_dir() { + // An explicitly requested directory is validated by its server. An + // inherited cwd may be bootstrap metadata without root authority. + if self.cwd.is_some() && !crate::fs::metadata(&dir)?.is_dir() { return Err(io::const_error!( io::ErrorKind::NotFound, "working directory is not a directory", @@ -429,7 +428,7 @@ impl Command { // A helper's program on a file server is taken here as the memory // object its server holds it in, or read into one counted to the job // the child is endowed to run in or else to this process's, which the - // kernel pages the child from: it opens only what it serves itself. A + // kernel pages the child from. A // launch runs its row's own program. let source = Path::new(image_path.unwrap_or(program)); let image = match (prepared_image, boot_image) { @@ -444,8 +443,7 @@ impl Command { })?; Some(toyos::process::Image::immutable(region)) } - (None, None) if crate::sys::fs::is_served(source) => Some(crate::sys::fs::read_image(source, job_of(&endowments))?), - (None, None) => None, + (None, None) => Some(crate::sys::fs::read_image(source, job_of(&endowments))?), }; let (labels, endowed): (Vec, Vec) = endowments.into_iter().unzip(); let (names, provided): (Vec, Vec) = provided.into_iter().unzip(); diff --git a/sdk/toyos/src/fd.rs b/sdk/toyos/src/fd.rs index 42e6de3a77a..abce3576bcc 100644 --- a/sdk/toyos/src/fd.rs +++ b/sdk/toyos/src/fd.rs @@ -55,9 +55,6 @@ pub const SERVED_FILE: u32 = 1; /// let go by the copy of this crate that holds the stream's end, which only /// that copy knows. pub const STREAM: u32 = 2; -/// The tag of a directory the kernel serves, which has no handle -/// (`crate::files`). -pub const KERNEL_DIR: u32 = 3; /// The tag of a listener's or a datagram socket's descriptor /// (`crate::net::socket`): a copy of its bell, which a poller watches and no /// file call reaches, let go by the copy of this crate that holds the diff --git a/sdk/toyos/src/files.rs b/sdk/toyos/src/files.rs index 6aabdd02d30..bd43c34a005 100644 --- a/sdk/toyos/src/files.rs +++ b/sdk/toyos/src/files.rs @@ -1,36 +1,21 @@ -//! Files by path, and the descriptors that name open ones, wherever they are -//! served: the kernel serves `/tmp`, and every other directory a -//! process may reach is a capability in its namespace, `fs:`, -//! served by a file server ([`crate::fs`]). A program or a library is read -//! here into an image the kernel pages it from ([`read_image`]). +//! Files and directories through the capabilities a process holds, named +//! `fs:` in its namespace. Absolute paths and paths from the +//! working directory use the longest matching prefix, at most +//! [`MAX_CAPABILITY_DEPTH`] components deep. A relative path from a +//! descriptor stays on that directory's server. Absolute symlinks return +//! here for routing through the same capabilities. //! -//! **An absolute path, or one from the working directory, goes to the -//! capability with the longest matching prefix**, at most -//! [`MAX_CAPABILITY_DEPTH`] components deep, relative to it, made absolute and -//! its `.` and `..` taken lexically; a path under no capability the process -//! holds is the kernel's. **A relative path from a descriptor ([`At::Fd`]) -//! goes to whoever serves that directory**: its server resolves the path, -//! `..` by the directory itself. An absolute link a server meets is followed -//! here, in this process's own table, at most [`MAX_LINKS`] times. A file -//! server that restarts is survived: its capability is connected again -//! ([`crate::fs::Dir`]), and what a descriptor held on it answers `Stale`. -//! -//! **A directory the kernel serves has no handle**: a descriptor of one names -//! its absolute path, and a path from it is that path's lexically -//! ([`crate::fd::KERNEL_DIR`]). The kernel keeps no hard link, unnamed file or -//! times a caller sets, and refuses each `NotSupported`. -//! -//! **One copy of this crate keeps this state**, the copy that owns the -//! descriptor table ([`crate::fd`]), and every copy reaches it through -//! `toyos_fs_*`: std and libc see one set of connections and one table. A -//! file or directory a server holds open is a descriptor naming an object of -//! this module's ([`crate::fd::SERVED_FILE`]); a kernel file is one naming its -//! handle. +//! One SDK copy owns the descriptor table, connections and working directory; +//! every copy, including std and libc, reaches it through `toyos_fs_*`. +//! Kernel objects still support generic reads, writes and metadata queries. + +#[cfg(any(feature = "fd-table", test))] +mod namespace; use alloc::string::String; use alloc::vec::Vec; -use toyos_abi::syscall::{FileType, SeekFrom, SyscallError}; +use toyos_abi::syscall::{FileType, SyscallError}; use crate::fd::{self, Fd}; use crate::fs; @@ -44,6 +29,14 @@ pub const MAX_CAPABILITY_DEPTH: usize = 4; /// The most absolute links one path's resolution follows. pub const MAX_LINKS: usize = 40; +/// Where a file seek starts. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SeekFrom { + Start(u64), + Current(i64), + End(i64), +} + /// What an open asks for. #[repr(C)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -134,7 +127,7 @@ impl Stat { /// Why a call was not answered. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Error { - /// The kernel's refusal, of a path or a handle it serves. + /// A kernel object or transport refused the operation. Kernel(SyscallError), /// A file server's refusal, or its client end's ([`crate::fs::Error`]). Served(fs::Error), @@ -144,8 +137,7 @@ pub enum Error { NotUtf8, /// More absolute links on the way than are followed. Loop, - /// A rename or a link between two servers, or between a server and the - /// kernel. + /// A rename or a link between different filesystem authorities. CrossesServers, /// A descriptor that names no file, but an object of a library's own. NotAFile, @@ -244,7 +236,7 @@ extern "C" { fn toyos_fs_readlink(at: Fd, path: *const u8, len: usize, out: *mut u8, cap: usize) -> i64; fn toyos_fs_set_times(at: Fd, path: *const u8, len: usize, follow: bool, flags: u64, atime: u64, mtime: u64) -> i64; fn toyos_fs_readdir(fd: Fd, cursor: u64, out: *mut u8, cap: usize, next: *mut u64) -> i64; - fn toyos_fs_is_served(at: Fd, path: *const u8, len: usize) -> i64; + fn toyos_fs_getcwd(out: *mut u8, len: usize) -> usize; fn toyos_fs_chdir(path: *const u8, len: usize) -> i64; } @@ -311,9 +303,9 @@ pub fn fsync(fd: Fd) -> Result<(), Error> { decode(unsafe { toyos_fs_fsync(fd) }).map(drop) } -/// A kernel handle a child may write the file through as its stdout: a copy -/// of a kernel file's own, or a pipe the file's server writes to it at its -/// offset. The caller holds it until the child has its own. +/// A child's output handle: a copy of a generic object handle, or a pipe +/// the filesystem server writes at the open description's offset. The caller +/// holds it until the child has its own. pub fn stream(fd: Fd) -> Result { // SAFETY: plain values; a handle answered is this process's alone. decode(unsafe { toyos_fs_stream(fd) }).map(|h| OwnedHandle(toyos_abi::RawHandle(h as u32))) @@ -331,8 +323,7 @@ pub fn unlink(at: At, path: &[u8], dir: bool) -> Result<(), Error> { decode(unsafe { toyos_fs_unlink(at.word(), path.as_ptr(), path.len(), dir) }).map(drop) } -/// Make `path` the working directory, which a file server judges when it -/// serves it and the kernel when it does. +/// Validate `path` as a directory and make it the process's working directory. pub fn chdir(path: &[u8]) -> Result<(), Error> { // SAFETY: `path` is live for the call. decode(unsafe { toyos_fs_chdir(path.as_ptr(), path.len()) }).map(drop) @@ -418,7 +409,7 @@ pub fn read_image_for(path: &[u8], job: Option>) -> Re NotImmutable::Kernel(e) => ImageRefused::Memory(e), NotImmutable::Writable => ImageRefused::Read(Error::Served(fs::Error::Malformed)), }), - Err(Error::Served(fs::Error::Unsupported) | Error::Kernel(SyscallError::NotSupported)) => match fstat(fd) { + Err(Error::Served(fs::Error::Unsupported)) => match fstat(fd) { Ok(stat) => Image::read_for(job, stat.size, |buf| read(fd, buf)), Err(e) => Err(ImageRefused::Read(e)), }, @@ -445,7 +436,7 @@ pub enum DlOpenRefused { pub fn dl_open(path: &[u8]) -> Result { let text = core::str::from_utf8(path).map_err(|_| DlOpenRefused::Image(ImageRefused::Read(Error::NotUtf8)))?; let mut cwd = alloc::vec![0u8; toyos_abi::syscall::MAX_PATH]; - let n = toyos_abi::syscall::getcwd(&mut cwd); + let n = getcwd(&mut cwd); let name = absolute(&String::from_utf8_lossy(&cwd[..n.min(cwd.len())]), text); let image = read_image(name.as_bytes()).map_err(DlOpenRefused::Image)?; let (handle, len) = image.spawn_words(); @@ -465,15 +456,11 @@ fn named(path: &[u8]) -> Result<&str, Error> { } } -/// Whether `path` is on a file server rather than the kernel. -pub fn is_served(path: &[u8]) -> bool { - is_served_at(At::Cwd, path) == Ok(true) -} - -/// Whether `path` from `at` is on a file server rather than the kernel. -pub fn is_served_at(at: At, path: &[u8]) -> Result { - // SAFETY: `path` is live for the call. - decode(unsafe { toyos_fs_is_served(at.word(), path.as_ptr(), path.len()) }).map(|n| n == 1) +/// The process's working directory. Returns the required length, writing +/// the complete path only when it fits in `out`. +pub fn getcwd(out: &mut [u8]) -> usize { + // SAFETY: `out` lends its writable bytes for this call. + unsafe { toyos_fs_getcwd(out.as_mut_ptr(), out.len()) } } /// `path`, absolute against `cwd` and without `.` or `..`, which are read off diff --git a/sdk/toyos/src/files/namespace.rs b/sdk/toyos/src/files/namespace.rs new file mode 100644 index 00000000000..406860295ac --- /dev/null +++ b/sdk/toyos/src/files/namespace.rs @@ -0,0 +1,58 @@ +//! Read-only directory ancestors of the filesystem capabilities actually held. +use alloc::collections::BTreeSet; +use alloc::string::String; +use alloc::vec::Vec; + +pub(super) struct Directories(Vec); + +impl Directories { + pub(super) fn new(names: impl IntoIterator) -> Self { + let mut paths = BTreeSet::from([String::from("/")]); + for name in names { + let Some(path) = name.strip_prefix(crate::fs::CAPABILITY_PREFIX) else { continue }; + if !path.starts_with('/') || super::absolute("/", path) != path { + continue; + } + let parts: Vec<_> = path.split('/').filter(|p| !p.is_empty()).collect(); + if parts.len() > super::MAX_CAPABILITY_DEPTH { continue; } + for end in 1..=parts.len() { + paths.insert(alloc::format!("/{}", parts[..end].join("/"))); + } + } + Self(paths.into_iter().collect()) + } + + pub(super) fn node(&self, path: &str) -> Option { + self.0.binary_search_by(|name| name.as_str().cmp(path)).ok().map(|i| i as u64 + 1) + } + + pub(super) fn children<'a>(&'a self, path: &'a str) -> impl Iterator { + self.0.iter().enumerate().filter_map(move |(i, child)| { + let rest = child.strip_prefix(path)?; + let rest = if path == "/" { rest } else { rest.strip_prefix('/')? }; + (!rest.is_empty() && !rest.contains('/')).then_some((rest, i as u64 + 1)) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_held_filesystem_names_create_directories() { + let dirs = Directories::new(["fs:/home/jan", "fs:/tmp", "fs:/home/jan/Games", "net", "fs:relative", "fs:/bad/../name", "fs:/a/b/c/d/e"].map(String::from)); + assert_eq!(dirs.children("/").map(|(s, _)| s).collect::>(), ["home", "tmp"]); + assert_eq!(dirs.children("/home").map(|(s, _)| s).collect::>(), ["jan"]); + assert!(dirs.node("/home/jane").is_none()); + assert!(dirs.node("/system").is_none()); + assert_eq!(dirs.node("/"), Some(1)); + } + + #[test] + fn an_empty_namespace_has_an_empty_root() { + let dirs = Directories::new([]); + assert_eq!(dirs.node("/"), Some(1)); + assert_eq!(dirs.children("/").count(), 0); + } +} diff --git a/sdk/toyos/src/files/owner.rs b/sdk/toyos/src/files/owner.rs index 980cf660891..3cc04cfb94d 100644 --- a/sdk/toyos/src/files/owner.rs +++ b/sdk/toyos/src/files/owner.rs @@ -6,11 +6,10 @@ use alloc::string::String; use alloc::sync::Arc; use alloc::vec::Vec; -use toyos_abi::syscall::{self, FileType, OpenFlags, SeekFrom, SyscallError}; -use toyos_abi::RawHandle; +use toyos_abi::syscall::{self, FileType, SyscallError}; use super::{encode, At, Error, Open, Stat, MAX_CAPABILITY_DEPTH, MAX_LINKS, SEEK_CURRENT, SEEK_END, SEEK_START}; -use crate::fd::{self, Fd, Object, KERNEL_DIR, SERVED_FILE}; +use crate::fd::{self, Fd, Object, SERVED_FILE}; use crate::fs::{self, Base, Change, Dir, Fid, Refused}; use crate::sync::{Lock, Turnstile}; @@ -23,46 +22,53 @@ struct Capability { /// Every capability this process has looked for, found or not. static CAPABILITIES: Lock>>> = Lock::new(BTreeMap::new()); -/// Where a path is resolved: by the kernel, from its absolute path; or by a -/// server, from a directory of its own. -enum Place { - Kernel(String), - Served(Arc, Base, String), +/// A path's capability, starting directory and relative name. +enum Place { Served(Arc, Base, String), Directory(String) } + +/// A file or directory held open by its server, with a paged holder where supplied. +struct Served { + cap: Arc, + fid: Fid, + position: Position, + holder: Option, } -/// What a descriptor of this module's names. enum Held { - /// A file or directory a file server holds open, and the holder of its - /// paged object where the server sent one. - Served { cap: Arc, fid: Fid, position: Position, holder: Option }, - /// A directory the kernel serves, by its absolute path. - KernelDir(String), + Served(Served), + Directory(String), } -impl Drop for Held { +impl Drop for Served { fn drop(&mut self) { - if let Held::Served { cap, fid, .. } = self { - cap.dir.lock().close(*fid); - } + self.cap.dir.lock().close(self.fid); } } -/// A served file's paged object, and what its open answered of it: an -/// `fstat` is the object's size and mtime over the rest of that, which no -/// change to the file moves. -struct Holder { - paged: crate::paged::Paged, - opened: fs::Stat, +/// Namespace inspection is bounded and immutable; no service is opened to +/// discover its names. Failed inspection is kept as an error, not an empty view. +static DIRECTORIES: crate::endow::Once> = crate::endow::Once::new(); + +fn directories() -> Result<&'static super::namespace::Directories, Error> { + DIRECTORIES.get_or_init(|| { + let names = match crate::endow::namespace() { + Some(namespace) => namespace.names().map_err(Error::Kernel)?, + None => Vec::new(), + }; + Ok(super::namespace::Directories::new(names)) + }).as_ref().map_err(|e| *e) } -impl Holder { - /// `None` while a write is under way, and the server is asked. - fn stat(&self) -> Option { - let published = self.paged.stat().ok()?; - Some(fs::Stat { size: published.size, mtime: published.mtime, ..self.opened }) - } +fn directory_stat(path: &str) -> Result { + Ok(Stat { + file_type: FileType::Directory, size: 0, nlink: 1, + atime: 0, mtime: 0, ctime: 0, volume: 0, + node: directories()?.node(path).expect("a routed namespace directory"), + }) } +fn read_only(_: &str) -> Result { Err(Error::Served(fs::Error::ReadOnly)) } +fn is_directory(_: &str) -> Result { Err(Error::Served(fs::Error::IsADirectory)) } + /// A served file's offset: kept here while this process is the /// description's only holder, so a seek costs no request and a read or a /// write goes at it as a positioned one; the server's from the moment the @@ -177,11 +183,28 @@ fn held(word: u64) -> Option> { OPEN.lock().get(usize::try_from(word).ok()?)?.clone() } -/// The working directory, as the kernel keeps it. +/// Process-owned working directory, shared by every SDK copy through this owner. +/// The kernel supplies only the spawn's initial value. +static CWD: Lock> = Lock::new(None); + fn cwd() -> String { - let mut buf = alloc::vec![0u8; syscall::MAX_PATH]; - let n = syscall::getcwd(&mut buf); - String::from_utf8_lossy(&buf[..n.min(buf.len())]).into_owned() + let mut cwd = CWD.lock(); + cwd.get_or_insert_with(|| { + let mut buf = alloc::vec![0u8; syscall::MAX_PATH]; + let n = syscall::getcwd(&mut buf); + assert!(n > 0 && n <= buf.len(), "spawn supplied an invalid working directory"); + String::from_utf8(buf[..n].to_vec()).expect("spawn supplied a non-UTF-8 working directory") + }).clone() +} + +#[no_mangle] +unsafe extern "C" fn toyos_fs_getcwd(out: *mut u8, len: usize) -> usize { + let cwd = cwd(); + if cwd.len() <= len { + // SAFETY: the caller lends `len` writable bytes for the call. + unsafe { core::ptr::copy_nonoverlapping(cwd.as_ptr(), out, cwd.len()) }; + } + cwd.len() } fn utf8(path: &[u8]) -> Result<&str, Error> { @@ -214,7 +237,11 @@ fn route(abs: &str) -> Result { return Ok(Place::Served(cap, Base::Root, parts[depth..].join("/"))); } } - Ok(Place::Kernel(String::from(abs))) + match capability("/")? { + Some(cap) => Ok(Place::Served(cap, Base::Root, parts.join("/"))), + None if directories()?.node(abs).is_some() => Ok(Place::Directory(String::from(abs))), + None => Err(Error::Served(fs::Error::NotHeld)), + } } /// Where `path` from `at` is resolved: the empty path names nothing @@ -232,15 +259,15 @@ fn place(at: At, path: &[u8]) -> Result { /// `path` from the directory `fd` names; the empty path is that directory. fn from_dir(fd: Fd, path: &str) -> Result { let named = fd::with(fd, |o| match o.tag() { - SERVED_FILE | KERNEL_DIR if o.kernel_handle().is_none() => held(o.word()), + SERVED_FILE if o.kernel_handle().is_none() => held(o.word()), _ => None, }); match named.map_err(Error::Fd)?.as_deref() { - Some(Held::Served { cap, fid, .. }) => { + Some(Held::Served(Served { cap, fid, .. })) => { let rel = fs::wire_path(path).ok_or(Error::Served(fs::Error::NameTooLong))?; Ok(Place::Served(Arc::clone(cap), Base::Fid(*fid), rel)) } - Some(Held::KernelDir(dir)) => route(&super::absolute(dir, path)), + Some(Held::Directory(dir)) => route(&super::absolute(dir, path)), None => Err(Error::NotADirectory), } } @@ -252,14 +279,14 @@ fn from_dir(fd: Fd, path: &str) -> Result { fn on_path( at: At, path: &[u8], - mut kernel: impl FnMut(&str) -> Result, + mut synthetic: impl FnMut(&str) -> Result, mut served: impl FnMut(&Arc, &mut Dir, Base, &str) -> Result, ) -> Result { let mut place = place(at, path)?; for _ in 0..MAX_LINKS { let (cap, base, rel) = match place { - Place::Kernel(abs) => return kernel(&abs).map_err(Error::Kernel), Place::Served(cap, base, rel) => (cap, base, rel), + Place::Directory(path) => return synthetic(&path), }; let mut dir = cap.dir.lock(); match served(&cap, &mut dir, base, &rel) { @@ -277,22 +304,15 @@ fn on_path( Err(Error::Loop) } -/// What `fd` names, held for the call: a kernel handle, or an object of this -/// module's. -enum Named { - Handle(RawHandle), - Held(Arc), -} - /// `kernel` on `fd`'s handle, held for the call, or `served` on what a server -/// holds open for it. A kernel directory answers `kernel_dir`. +/// holds open for it. fn on_fd( fd: Fd, kernel: impl FnOnce(crate::BorrowedHandle<'_>) -> Result, - kernel_dir: impl FnOnce(&str) -> Result, + synthetic: impl FnOnce(&str) -> Result, served: impl FnOnce(&mut Dir, Fid, &Position) -> Result, ) -> Result { - on_fd_held(fd, kernel, kernel_dir, |_, _| None, served) + on_fd_held(fd, kernel, synthetic, |_, _| None, served) } /// [`on_fd`], trying `holding` first on a served file whose paged object @@ -302,7 +322,7 @@ fn on_fd( fn on_fd_held( fd: Fd, kernel: impl FnOnce(crate::BorrowedHandle<'_>) -> Result, - kernel_dir: impl FnOnce(&str) -> Result, + synthetic: impl FnOnce(&str) -> Result, holding: impl FnOnce(&crate::paged::Paged, &Position) -> Option, served: impl FnOnce(&mut Dir, Fid, &Position) -> Result, ) -> Result { @@ -310,7 +330,7 @@ fn on_fd_held( // of this module's is taken out of it, counted, and asked after. let named = fd::with(fd, |o| match (o.kernel_handle(), o.tag()) { (Some(handle), _) => Ok(Err(kernel(handle).map_err(Error::Kernel))), - (None, SERVED_FILE | KERNEL_DIR) => { + (None, SERVED_FILE) => { Ok(Ok(held(o.word()).expect("toyos::files: a held descriptor names an object of this module's"))) } (None, _) => Err(Error::NotAFile), @@ -318,34 +338,18 @@ fn on_fd_held( match named.map_err(Error::Fd)?? { Err(answer) => answer, Ok(held) => match &*held { - Held::Served { cap, fid, position, holder } => match holder.as_ref().and_then(|h| holding(&h.paged, position)) { + Held::Directory(path) => synthetic(path), + Held::Served(held) => match held.holder.as_ref().and_then(|h| holding(h, &held.position)) { Some(answer) => Ok(answer), - None => served(&mut cap.dir.lock(), *fid, position).map_err(Error::Served), + None => served(&mut held.cap.dir.lock(), held.fid, &held.position).map_err(Error::Served), }, - Held::KernelDir(path) => kernel_dir(path), }, } } -/// What `fd` names, for a call that asks about it rather than through it. -fn named(fd: Fd) -> Result { - let named = fd::with(fd, |o| match (o.kernel_handle(), o.tag()) { - (Some(handle), _) => Ok(Named::Handle(handle.raw())), - (None, SERVED_FILE | KERNEL_DIR) => { - Ok(Named::Held(held(o.word()).expect("toyos::files: a held descriptor names an object of this module's"))) - } - (None, _) => Err(Error::NotAFile), - }); - named.map_err(Error::Fd)? -} - /// A descriptor for `held`; on a full table it is let go again, which takes /// its capability's turn, so no turn may be held here. fn install(held: Held, cloexec: bool) -> Result { - let tag = match held { - Held::Served { .. } => SERVED_FILE, - Held::KernelDir(_) => KERNEL_DIR, - }; let held = Arc::new(held); let word = { let mut open = OPEN.lock(); @@ -360,7 +364,7 @@ fn install(held: Held, cloexec: bool) -> Result { } } }; - fd::install(Object::tagged(tag, word as u64, release_held), !cloexec, 0) + fd::install(Object::tagged(SERVED_FILE, word as u64, release_held), !cloexec, 0) } /// What a server answered, as this module says it. @@ -399,42 +403,6 @@ unsafe fn bytes<'a>(ptr: *const u8, len: usize) -> &'a [u8] { unsafe { core::slice::from_raw_parts(ptr, len) } } -/// What an open opened, before a descriptor names it. -enum Opened { - Kernel(crate::OwnedHandle), - Held(Held), -} - -/// The kernel's open of `abs`; a directory, which the kernel opens no handle -/// on, is its path. The kernel's own refusal is the outer error; the inner -/// one is `O_DIRECTORY`'s, for a directory opened to write or anything else -/// opened as one. -fn kernel_open(abs: &str, how: Open) -> Result, SyscallError> { - if how.unnamed { - return Err(SyscallError::NotSupported); - } - let writes = how.write || how.append || how.create || how.create_new || how.truncate; - let is_dir = || syscall::stat(abs.as_bytes(), true).map(|st| st.file_type == FileType::Directory); - if how.directory { - return Ok(match (is_dir()?, writes) { - (true, false) => Ok(Opened::Held(Held::KernelDir(String::from(abs)))), - (true, true) => Err(Error::Served(fs::Error::IsADirectory)), - (false, _) => Err(Error::NotADirectory), - }); - } - if !writes && is_dir().unwrap_or(false) { - return Ok(Ok(Opened::Held(Held::KernelDir(String::from(abs))))); - } - let mut flags = OpenFlags(0); - if how.read { flags |= OpenFlags::READ; } - if how.write || how.append { flags |= OpenFlags::WRITE; } - if how.append { flags |= OpenFlags::APPEND; } - if how.create || how.create_new { flags |= OpenFlags::CREATE; } - if how.create_new { flags |= OpenFlags::EXCLUSIVE; } - if how.truncate { flags |= OpenFlags::TRUNCATE; } - syscall::open(abs.as_bytes(), flags).map(|handle| Ok(Opened::Kernel(crate::OwnedHandle(handle)))) -} - #[no_mangle] unsafe extern "C" fn toyos_fs_open(at: Fd, path: *const u8, len: usize, how: Open) -> i64 { // SAFETY: the caller's. @@ -442,7 +410,12 @@ unsafe extern "C" fn toyos_fs_open(at: Fd, path: *const u8, len: usize, how: Ope let opened = on_path( At::of(at), path, - |abs| kernel_open(abs, how), + |path| { + if how.write || how.append || how.create || how.create_new || how.truncate || how.unnamed { + return Err(Error::Served(fs::Error::ReadOnly)); + } + Ok(Held::Directory(String::from(path))) + }, |cap, dir, base, rel| { let mut flags = 0; if how.read { flags |= fs::O_READ; } @@ -461,25 +434,17 @@ unsafe extern "C" fn toyos_fs_open(at: Fd, path: *const u8, len: usize, how: Ope // it; a pager's process reads through the server. let holder = match opened.holder.map(crate::paged::Paged::received) { None | Some(Err(crate::paged::NotAHolder::Refused | crate::paged::NotAHolder::Ended)) => None, - Some(Ok(paged)) => Some(Holder { paged, opened: opened.stat }), + Some(Ok(paged)) => Some(paged), Some(Err(crate::paged::NotAHolder::Other)) => { dir.close(opened.fid); return Err(Refused::Error(fs::Error::Malformed)); } }; - Ok(Ok(Opened::Held(Held::Served { cap: Arc::clone(cap), fid: opened.fid, position, holder }))) + Ok(Held::Served(Served { cap: Arc::clone(cap), fid: opened.fid, position, holder })) }, - ) - .and_then(|opened| opened); - // Installed once the turn is let go: on a full table what was opened is - // closed again (`install`), and a kernel handle by the table. - let installed = opened.and_then(|opened| { - match opened { - Opened::Kernel(handle) => fd::install(Object::handle(handle), !how.cloexec, 0), - Opened::Held(held) => install(held, how.cloexec), - } - .map_err(Error::Fd) - }); + ); + // Install outside the capability turn: refusal closes the opened fid. + let installed = opened.and_then(|held| install(held, how.cloexec).map_err(Error::Fd)); answer(installed.map(|fd| fd as u64)) } @@ -489,21 +454,16 @@ unsafe extern "C" fn toyos_fs_stat(at: Fd, path: *const u8, len: usize, follow: let path = unsafe { bytes(path, len) }; let stat = match (At::of(at), path.is_empty()) { // What the descriptor names itself. - (At::Fd(fd), true) => match named(fd) { - Ok(Named::Handle(handle)) => syscall::fstat(handle).map(Stat::of_kernel).map_err(Error::Kernel), - Ok(Named::Held(held)) => match &*held { - Held::Served { cap, fid, holder, .. } => match holder.as_ref().and_then(Holder::stat) { - Some(stat) => Ok(stat_of(stat)), - None => cap.dir.lock().fstat(*fid).map(stat_of).map_err(Error::Served), - }, - Held::KernelDir(dir) => syscall::stat(dir.as_bytes(), true).map(Stat::of_kernel).map_err(Error::Kernel), - }, - Err(e) => Err(e), - }, + (At::Fd(fd), true) => on_fd( + fd, + |h| syscall::fstat(h.raw()).map(Stat::of_kernel), + directory_stat, + |dir, fid, _| dir.fstat(fid).map(stat_of), + ), (at, _) => on_path( at, path, - |abs| syscall::stat(abs.as_bytes(), follow).map(Stat::of_kernel), + directory_stat, |_, dir, base, rel| dir.stat(base, rel, follow).map(stat_of), ), }; @@ -524,11 +484,6 @@ fn on_stream(fd: Fd, f: impl FnOnce(crate::BorrowedHandle<'_>) -> Result(_: &str) -> Result { - Err(Error::Served(fs::Error::IsADirectory)) -} - #[no_mangle] unsafe extern "C" fn toyos_fs_read(fd: Fd, buf: *mut u8, len: usize) -> i64 { // SAFETY: the caller lends `len` writable bytes for the call. The @@ -542,7 +497,7 @@ unsafe extern "C" fn toyos_fs_read(fd: Fd, buf: *mut u8, len: usize) -> i64 { let n = on_fd_held( fd, |h| syscall::read(h.raw(), out()), - is_a_directory, + is_directory, |holder, position| position.read_held(holder, out()), |dir, fid, position| position.read(dir, fid, out()), ); @@ -557,7 +512,7 @@ unsafe extern "C" fn toyos_fs_write(fd: Fd, buf: *const u8, len: usize) -> i64 { if let Some(n) = on_stream(fd, |h| crate::net::stream::write(h, buf, blocking)) { return answer(n.map(|n| n as u64)); } - let n = on_fd(fd, |h| syscall::write(h.raw(), buf), is_a_directory, |dir, fid, position| position.write(dir, fid, buf)); + let n = on_fd(fd, |h| syscall::write(h.raw(), buf), is_directory, |dir, fid, position| position.write(dir, fid, buf)); answer(n.map(|n| n as u64)) } @@ -569,8 +524,8 @@ unsafe extern "C" fn toyos_fs_pread(fd: Fd, buf: *mut u8, len: usize, offset: u6 let out = || unsafe { core::slice::from_raw_parts_mut(buf, len) }; let n = on_fd_held( fd, - |h| syscall::pread(h.raw(), out(), offset), - is_a_directory, + |_| Err(SyscallError::NotSupported), + is_directory, |holder, _| holder.read(out(), offset).ok(), |dir, fid, _| dir.pread(fid, offset, out()), ); @@ -581,46 +536,42 @@ unsafe extern "C" fn toyos_fs_pread(fd: Fd, buf: *mut u8, len: usize, offset: u6 unsafe extern "C" fn toyos_fs_pwrite(fd: Fd, buf: *const u8, len: usize, offset: u64) -> i64 { // SAFETY: the caller's. let buf = unsafe { bytes(buf, len) }; - let n = on_fd(fd, |h| syscall::pwrite(h.raw(), buf, offset), is_a_directory, |dir, fid, _| dir.pwrite(fid, offset, buf)); + let n = on_fd(fd, |_| Err(SyscallError::NotSupported), is_directory, |dir, fid, _| dir.pwrite(fid, offset, buf)); answer(n.map(|n| n as u64)) } #[no_mangle] extern "C" fn toyos_fs_seek(fd: Fd, whence: u32, delta: i64) -> i64 { - let (kernel, served) = match whence { - SEEK_START => (SeekFrom::Start(delta as u64), fs::SEEK_START), - SEEK_CURRENT => (SeekFrom::Current(delta), fs::SEEK_CURRENT), - SEEK_END => (SeekFrom::End(delta), fs::SEEK_END), + let served = match whence { + SEEK_START => fs::SEEK_START, + SEEK_CURRENT => fs::SEEK_CURRENT, + SEEK_END => fs::SEEK_END, _ => return encode(Error::Kernel(SyscallError::InvalidArgument)), }; - // A kernel directory has no offset to move. - let kernel_dir = |_: &str| Err(Error::Kernel(SyscallError::InvalidArgument)); - answer(on_fd(fd, |h| syscall::seek(h.raw(), kernel), kernel_dir, |dir, fid, position| position.seek(dir, fid, served, delta))) + answer(on_fd(fd, |_| Err(SyscallError::NotSupported), is_directory, |dir, fid, position| position.seek(dir, fid, served, delta))) } #[no_mangle] extern "C" fn toyos_fs_truncate(fd: Fd, size: u64) -> i64 { - answer(on_fd(fd, |h| syscall::ftruncate(h.raw(), size), is_a_directory, |dir, fid, _| dir.truncate(fid, size)).map(|()| 0)) + answer(on_fd(fd, |_| Err(SyscallError::NotSupported), is_directory, |dir, fid, _| dir.truncate(fid, size)).map(|()| 0)) } #[no_mangle] extern "C" fn toyos_fs_fsync(fd: Fd) -> i64 { - // A kernel directory holds nothing its files' syncs do not write. - answer(on_fd(fd, |h| syscall::fsync(h.raw()), |_| Ok(()), |dir, fid, _| dir.fsync(fid)).map(|()| 0)) + answer(on_fd(fd, |_| Err(SyscallError::NotSupported), is_directory, |dir, fid, _| dir.fsync(fid)).map(|()| 0)) } #[no_mangle] extern "C" fn toyos_fs_stream(fd: Fd) -> i64 { - let handle = on_fd(fd, |h| h.try_clone_to_owned().map(crate::OwnedHandle::into_raw), is_a_directory, |dir, fid, position| { + let handle = on_fd(fd, |h| h.try_clone_to_owned().map(crate::OwnedHandle::into_raw), is_directory, |dir, fid, position| { position.stream(dir, fid).map(|pipe| pipe.into_raw()) }); answer(handle.map(|h| u64::from(h.0))) } -/// A kernel file is `NotSupported`: the kernel opens its own files for a spawn. #[no_mangle] extern "C" fn toyos_fs_image(fd: Fd) -> i64 { - let image = on_fd(fd, |_| Err(SyscallError::NotSupported), is_a_directory, |dir, fid, _| dir.image(fid).map(|h| h.into_raw())); + let image = on_fd(fd, |_| Err(SyscallError::NotSupported), is_directory, |dir, fid, _| dir.image(fid).map(|h| h.into_raw())); answer(image.map(|h| u64::from(h.0))) } @@ -628,7 +579,7 @@ extern "C" fn toyos_fs_image(fd: Fd) -> i64 { unsafe extern "C" fn toyos_fs_mkdir(at: Fd, path: *const u8, len: usize) -> i64 { // SAFETY: the caller's. let path = unsafe { bytes(path, len) }; - let made = on_path(At::of(at), path, |abs| syscall::mkdir(abs.as_bytes()), |_, dir, base, rel| dir.mkdir(base, rel)); + let made = on_path(At::of(at), path, read_only, |_, dir, base, rel| dir.mkdir(base, rel)); answer(made.map(|()| 0)) } @@ -639,7 +590,7 @@ unsafe extern "C" fn toyos_fs_unlink(at: Fd, path: *const u8, len: usize, is_dir let removed = on_path( At::of(at), path, - |abs| if is_dir { syscall::rmdir(abs.as_bytes()) } else { syscall::delete(abs.as_bytes()) }, + read_only, |_, dir, base, rel| dir.unlink(base, rel, is_dir), ); answer(removed.map(|()| 0)) @@ -649,30 +600,20 @@ unsafe extern "C" fn toyos_fs_unlink(at: Fd, path: *const u8, len: usize, is_dir unsafe extern "C" fn toyos_fs_chdir(path: *const u8, len: usize) -> i64 { // SAFETY: the caller's. let path = unsafe { bytes(path, len) }; - let changed = utf8(path).and_then(|path| { + let changed = super::named(path).and_then(|path| { let abs = super::absolute(&cwd(), path); - if let Place::Served(..) = route(&abs)? { - let stat = on_path( - At::Cwd, - abs.as_bytes(), - |abs| syscall::stat(abs.as_bytes(), true).map(Stat::of_kernel), - |_, dir, base, rel| dir.stat(base, rel, true).map(stat_of), - )?; - if stat.file_type != FileType::Directory { - return Err(Error::Served(fs::Error::NotADirectory)); - } + let stat = on_path(At::Cwd, abs.as_bytes(), directory_stat, |_, dir, base, rel| dir.stat(base, rel, true).map(stat_of))?; + if stat.file_type != FileType::Directory { + return Err(Error::NotADirectory); } - syscall::chdir(abs.as_bytes()).map_err(Error::Kernel) + *CWD.lock() = Some(abs); + Ok(()) }); answer(changed.map(|()| 0)) } -/// Two paths one call names, on one server: both where the kernel serves -/// them, or the capability and both bases. -enum Pair { - Kernel(String, String), - Served(Arc, (Base, String), (Base, String)), -} +/// Two paths on one capability, with their starting directories. +struct Pair(Arc, (Base, String), (Base, String)); fn pair(from: (At, &[u8]), to: (At, &[u8])) -> Result { pair_of(place(from.0, from.1)?, place(to.0, to.1)?) @@ -680,8 +621,7 @@ fn pair(from: (At, &[u8]), to: (At, &[u8])) -> Result { fn pair_of(from: Place, to: Place) -> Result { match (from, to) { - (Place::Kernel(a), Place::Kernel(b)) => Ok(Pair::Kernel(a, b)), - (Place::Served(ca, ba, a), Place::Served(cb, bb, b)) if Arc::ptr_eq(&ca, &cb) => Ok(Pair::Served(ca, (ba, a), (bb, b))), + (Place::Served(ca, ba, a), Place::Served(cb, bb, b)) if Arc::ptr_eq(&ca, &cb) => Ok(Pair(ca, (ba, a), (bb, b))), _ => Err(Error::CrossesServers), } } @@ -700,13 +640,7 @@ unsafe extern "C" fn toyos_fs_rename(from_at: Fd, from: *const u8, from_len: usi // SAFETY: the caller's. let (from, to) = unsafe { (bytes(from, from_len), bytes(to, to_len)) }; let renamed = pair((At::of(from_at), from), (At::of(to_at), to)).and_then(|pair| match pair { - // The kernel refuses a rename between its own mounts as - // unsupported, which is a rename across servers as POSIX reads it. - Pair::Kernel(a, b) => syscall::rename(a.as_bytes(), b.as_bytes()).map_err(|e| match e { - SyscallError::NotSupported => Error::CrossesServers, - e => Error::Kernel(e), - }), - Pair::Served(cap, (ba, a), (bb, b)) => cap.dir.lock().rename((ba, &a), (bb, &b)).map_err(crossing), + Pair(cap, (ba, a), (bb, b)) => cap.dir.lock().rename((ba, &a), (bb, &b)).map_err(crossing), }); answer(renamed.map(|()| 0)) } @@ -729,8 +663,7 @@ unsafe extern "C" fn toyos_fs_link( (at, _) => place(at, from), }; let linked = from_place.and_then(|from| pair_of(from, place(At::of(to_at), to)?)).and_then(|pair| match pair { - Pair::Kernel(..) => Err(Error::Kernel(SyscallError::NotSupported)), - Pair::Served(cap, (ba, a), (bb, b)) => cap.dir.lock().link((ba, &a), follow, (bb, &b), None).map_err(crossing), + Pair(cap, (ba, a), (bb, b)) => cap.dir.lock().link((ba, &a), follow, (bb, &b), None).map_err(crossing), }); answer(linked.map(|()| 0)) } @@ -740,7 +673,7 @@ unsafe extern "C" fn toyos_fs_symlink(target: *const u8, target_len: usize, at: // SAFETY: the caller's. let (target, link) = unsafe { (bytes(target, target_len), bytes(link, link_len)) }; let made = utf8(target).and_then(|target| { - on_path(At::of(at), link, |abs| syscall::symlink(target.as_bytes(), abs.as_bytes()), |_, dir, base, rel| { + on_path(At::of(at), link, read_only, |_, dir, base, rel| { dir.symlink(target, (base, rel)) }) }); @@ -754,7 +687,7 @@ unsafe extern "C" fn toyos_fs_readlink(at: Fd, path: *const u8, len: usize, out: // SAFETY: the caller lends `cap` writable bytes for the call, and each // closure makes the slice only for its own run. let target = || unsafe { core::slice::from_raw_parts_mut(out, cap) }; - let n = on_path(At::of(at), path, |abs| syscall::readlink(abs.as_bytes(), target()), |_, dir, base, rel| { + let n = on_path(At::of(at), path, |_| Err(Error::Served(fs::Error::NotALink)), |_, dir, base, rel| { let mut buf = [0u8; fs::MAX_PATH]; let n = dir.read_link(base, rel, &mut buf)?; let out = target(); @@ -773,58 +706,30 @@ unsafe extern "C" fn toyos_fs_set_times(at: Fd, path: *const u8, len: usize, fol let Some(change) = Change::decode(&request).filter(|c| c.size.is_none()) else { return encode(Error::Served(fs::Error::InvalidRequest)); }; - let unsupported = || Err(Error::Kernel(SyscallError::NotSupported)); let set = match (At::of(at), path.is_empty()) { - // What the descriptor names itself. - (At::Fd(fd), true) => match named(fd) { - Ok(Named::Held(held)) => match &*held { - Held::Served { cap, fid, .. } => cap.dir.lock().set_attr(Base::Fid(*fid), "", follow, &change).map_err(crossing), - Held::KernelDir(_) => unsupported(), - }, - Ok(Named::Handle(_)) => unsupported(), - Err(e) => Err(e), - }, - (at, _) => on_path(at, path, |_| Err(SyscallError::NotSupported), |_, dir, base, rel| dir.set_attr(base, rel, follow, &change)), + (At::Fd(fd), true) => from_dir(fd, "").and_then(|place| match place { + Place::Served(cap, base, rel) => cap.dir.lock().set_attr(base, &rel, follow, &change).map_err(crossing), + Place::Directory(_) => Err(Error::Served(fs::Error::ReadOnly)), + }), + (at, _) => on_path(at, path, read_only, |_, dir, base, rel| dir.set_attr(base, rel, follow, &change)), }; answer(set.map(|()| 0)) } -/// A kernel listing too large for the first buffer is asked again at the size -/// it named; a directory that outgrows its own listing this many times is -/// refused rather than waited on. -const LIST_ATTEMPTS: usize = 4; - -/// The kernel's listing of `abs`, in the served one's encoding. -fn kernel_listing(abs: &str) -> Result, Error> { - let mut buf = alloc::vec![0u8; 64 * 1024]; - for _ in 0..LIST_ATTEMPTS { - let n = syscall::readdir(abs.as_bytes(), &mut buf).map_err(Error::Kernel)?; - if n <= buf.len() { - let (mut out, mut at) = (Vec::new(), 0); - while let Some(d) = syscall::dirent(&buf[..n], &mut at) { - // The kernel tells a directory from anything else, and no more. - let kind = if d.is_dir { fs::KIND_DIR } else { 0 }; - let name = core::str::from_utf8(d.name).unwrap_or(""); - let entry = fs::Entry { kind, size: 0, node: d.node, name }; - fs::encode_entry(&mut out, &entry).ok_or(Error::Kernel(SyscallError::InvalidArgument))?; - } - return Ok(out); - } - buf.resize(n, 0); - } - Err(Error::Kernel(SyscallError::ResourceExhausted)) -} - #[no_mangle] unsafe extern "C" fn toyos_fs_readdir(fd: Fd, cursor: u64, out: *mut u8, cap: usize, next: *mut u64) -> i64 { let listing = on_fd( fd, - // A kernel handle is a file, never a directory. |_| Err(SyscallError::InvalidArgument), - // The kernel answers a directory whole, as one batch. - |abs| match cursor { - 0 => kernel_listing(abs).map(|listing| (listing, None)), - _ => Ok((Vec::new(), None)), + |path| { + let mut out = Vec::new(); + if cursor == 0 { + for (name, node) in directories()?.children(path) { + fs::encode_entry(&mut out, &fs::Entry { kind: fs::KIND_DIR, size: 0, node, name }) + .expect("namespace service names fit directory entries"); + } + } + Ok((out, None)) }, |dir, fid, _| { let mut batch = Vec::new(); @@ -843,11 +748,3 @@ unsafe extern "C" fn toyos_fs_readdir(fd: Fd, cursor: u64, out: *mut u8, cap: us listing.len() as u64 })) } - -#[no_mangle] -unsafe extern "C" fn toyos_fs_is_served(at: Fd, path: *const u8, len: usize) -> i64 { - // SAFETY: the caller's. - let path = unsafe { bytes(path, len) }; - let served = place(At::of(at), path); - answer(served.map(|place| u64::from(matches!(place, Place::Served(..))))) -} diff --git a/sdk/toyos/src/namespace.rs b/sdk/toyos/src/namespace.rs index 6c2a67468d2..bf63297a90e 100644 --- a/sdk/toyos/src/namespace.rs +++ b/sdk/toyos/src/namespace.rs @@ -26,6 +26,24 @@ impl Namespace { syscall::namespace_open(self.0.raw(), name).map(|h| Connection(OwnedHandle(h))) } + /// The names this namespace holds, sorted. Inspection opens no connector + /// and conveys no authority beyond the namespace itself. + pub fn names(&self) -> Result, SyscallError> { + let required = syscall::namespace_names(self.0.raw(), &mut [])?; + assert!(required <= MAX_NAMESPACE_ENTRIES * (2 + syscall::MAX_SERVICE_NAME), "namespace names exceed the ABI bound"); + let mut bytes = alloc::vec![0; required]; + let read = syscall::namespace_names(self.0.raw(), &mut bytes)?; + assert_eq!(required, read, "an immutable namespace changed names"); + let mut names = alloc::vec::Vec::new(); + let mut rest = &bytes[..]; + while !rest.is_empty() { + let len = u16::from_le_bytes(rest[..2].try_into().expect("namespace name length")) as usize; + names.push(alloc::string::String::from_utf8(rest[2..2 + len].to_vec()).expect("namespace name is UTF-8")); + rest = &rest[2 + len..]; + } + Ok(names) + } + pub fn into_raw(self) -> RawHandle { self.0.into_raw() } From d5e69ee464190cd764cfa4eac3aecebf71e057c0 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 17:05:53 +0100 Subject: [PATCH 10/93] Retain memory files in anonymous objects and remove kernel filesystems Serve /tmp and diskless DATA roots through the fileserver namespace with one retained anonymous paged object per regular inode. Keep names and open descriptions independent, release unlinked bytes only after the last description and holder, and read sparse holes without allocating frames. Delete kernel VFS, tmpfs, pathname syscalls, file cache and file objects. Executable loading now takes its supplied immutable SharedImage directly. Keep bootstrap cwd transport; SDK routing removal is coordinated separately. Add owner-only advisory pager quota headroom, explicit memory grants, and host plus guest coverage for retention, sparse reads and quota refusal. --- images/system.toml | 2 +- kernel/CLAUDE.md | 2 +- kernel/pure/paged.rs | 12 +++- kernel/src/clock.rs | 6 -- kernel/src/elf/mod.rs | 17 +++--- kernel/src/file_backing.rs | 40 ++----------- kernel/src/loader/mod.rs | 34 +++-------- kernel/src/object/ops.rs | 2 +- kernel/src/object/paged.rs | 30 +++++++--- kernel/src/syscall/dispatch.rs | 4 +- kernel/src/syscall/paged.rs | 1 + kernel/src/syscall/proc.rs | 2 +- kernel/src/user_ptr.rs | 16 ----- lib/manifest/src/lib.rs | 21 ++++--- sdk/abi/src/syscall.rs | 3 + sdk/toyos/src/paged.rs | 6 ++ system/fileserver/Cargo.toml | 2 +- system/fileserver/src/lib.rs | 25 ++------ system/fileserver/src/main.rs | 40 ++++++------- system/fileserver/src/memory.rs | 83 ++++++++++++++++++-------- system/fileserver/src/pages.rs | 21 +++++-- system/fileserver/src/pages/tests.rs | 89 +++++++++++++++++++++++++--- system/fileserver/src/volume.rs | 8 +-- system/supervisor/src/main.rs | 4 +- tests/guest/pagerprobe/src/main.rs | 31 +++++++++- 25 files changed, 299 insertions(+), 202 deletions(-) diff --git a/images/system.toml b/images/system.toml index 6886bbb8aa3..1cedcedec89 100644 --- a/images/system.toml +++ b/images/system.toml @@ -369,7 +369,7 @@ receives = ["disk"] drives = ["pci:0c:03:30"] receives = ["disk", "input"] -# The file servers: DATA, the log and the boot volume, one process +# The file servers: DATA, temporary files, the log and boot volumes, one process # each, serving every program the directories of its role. Started again when # one ends, on the same ports. A pager: its readers wait on the pages it # supplies, which only a row of this file may make a program do. diff --git a/kernel/CLAUDE.md b/kernel/CLAUDE.md index 6959fc22b54..d532dc2caf6 100644 --- a/kernel/CLAUDE.md +++ b/kernel/CLAUDE.md @@ -9,7 +9,7 @@ The module header at the site owns its subsystem — read it before changing a m - **The idle loop may not take a global lock unconditionally** — the crash report reads the process table through a `try_lock` it must never block on. `sched/reap_gate.rs` is the pattern: a relaxed-load gate in front of the lock. - **Every x86 Ring 0 entry clears the direction flag** — no hardware gate does it, and `memmove` sets it across `rep` operations with interrupts on; `arch::entry::ring3_naked_asm` is where it lives. - **`BackendGuard` masks interrupts for its whole life**, so anything written under it is an interrupt latency; a new holder bounds itself as the console drain and the userland `write` flush do. -- **A syscall that can block resolves its handle and clones the object out before it blocks** — a `with_object`/`with_process_data` guard held across a park is a runtime panic no compile check catches; the `SYS_FSYNC` arm in `syscall/dispatch.rs` is the pattern. +- **A syscall that can block resolves its handle and clones the object out before it blocks** — a `with_object`/`with_process_data` guard held across a park is a runtime panic no compile check catches; the paged-object reads in `syscall/paged.rs` are the pattern. - **A decision the process table makes lives in `kernel/proclife/`, never in `process.rs`** — its defects are interleavings and its host tests are the only machine that can enumerate one. - **A task holds at most one watch registration** — a standing registration across a loop must not call anything that registers again. A double registration panics only at attempt ≥ 2, so the contention depth is the coverage. - **A console is per holder, minted at spawn** — the object *is* the line buffer. diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index df72bd973da..3b56f1c6901 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -4,7 +4,8 @@ //! //! **A page is present, failed or absent.** A present page holds a frame and //! its dirty state; a failed one holds no frame and answers `EIO` until it is -//! supplied again; an absent one has no slot. Nothing here waits: a caller +//! supplied again; an absent one has no slot. Unwritten anonymous memory +//! reads zeros without a slot, and its present pages cannot be withdrawn. Nothing here waits: a caller //! that finds a page absent records its request in its own wait record. //! //! **A dirty page holds a slot until a write-back cleans it.** Slots are @@ -169,6 +170,8 @@ pub enum Refused { #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Lookup { Present(F), + /// Unwritten anonymous memory, without a resident frame. + Zero, Absent, Failed, PastEnd, @@ -497,6 +500,7 @@ impl Pages { return Lookup::PastEnd; } match self.find(o.id, page) { + None if o.anonymous => Lookup::Zero, None => Lookup::Absent, Some(i) => self.slot(i).frame.map_or(Lookup::Failed, Lookup::Present), } @@ -679,10 +683,10 @@ impl Pages { /// Mark the range failed: present pages give up their frames, and a read /// of any page of it answers `Failed` until it is supplied again. pub fn fail(&mut self, o: &mut Object, first: u64, count: u64, out: &mut impl FnMut(F)) -> Result<(), Refused> { + if o.anonymous { return Err(Refused::InvalidArgument); } let end = first.saturating_add(count).min(o.page_end()); for page in first..end { if let Some(i) = self.find(o.id, page) { - if o.anonymous && self.slot(i).frame.is_some() { continue; } if let Some(frame) = self.remove(o, i) { out(frame); } @@ -810,10 +814,12 @@ mod tests { pages.write_unpin(&mut object, 0, 42, PAGE); assert_eq!(pages.withdraw(&mut object, 0, 1, &mut none), 0); assert_eq!(pages.writeback_begin(&mut object, 0, 1), Err(Refused::InvalidArgument)); - assert_eq!(pages.fail(&mut object, 0, 1, &mut none), Ok(())); + assert_eq!(pages.fail(&mut object, 0, 1, &mut none), Err(Refused::InvalidArgument)); assert_eq!(pages.lookup(&object, 0), Lookup::Present(42)); assert_eq!(pages.supply(&mut object, 1, 43, true, 2 * PAGE), Ok(Supplied::Installed)); assert_eq!(pages.supply(&mut object, 2, 44, true, 3 * PAGE), Err(Refused::NoRoom)); + pages.resize(&mut object, 4 * PAGE, &mut none); + assert_eq!(pages.lookup(&object, 3), Lookup::Zero, "a hole reads without any free slot"); let mut freed = Vec::new(); pages.resize(&mut object, PAGE, &mut sink(&mut freed)); assert_eq!(freed, [43]); diff --git a/kernel/src/clock.rs b/kernel/src/clock.rs index 68829bad25a..aa6903bfd49 100644 --- a/kernel/src/clock.rs +++ b/kernel/src/clock.rs @@ -275,9 +275,3 @@ pub fn utc_nanos() -> Option { .then(|| BOOT_SECS.load(Relaxed).saturating_mul(NANOS_PER_SEC).saturating_add(nanos_since_boot())) } -/// What a file written now is stamped with (`toyos_abi::syscall::Stat::mtime`): -/// [`utc_nanos`], and 0 — undated — on a machine whose RTC never answered. -pub fn mtime_now() -> u64 { - utc_nanos().unwrap_or(0) -} - diff --git a/kernel/src/elf/mod.rs b/kernel/src/elf/mod.rs index aa5bbfff15a..5026380fea6 100644 --- a/kernel/src/elf/mod.rs +++ b/kernel/src/elf/mod.rs @@ -310,13 +310,12 @@ impl ModuleImage { /// Read `dst.size()` bytes from a file backing straight into `dst`. /// -/// `dst` is a `KernelSlice`, so the read is bounded by its own allocation; `Err` leaves it partially zeroed, and callers must refuse rather than continue. -#[must_use = "an image assembled from a failed read is zeros, not the program"] +/// `dst` is a `KernelSlice`, so the read is bounded by its own allocation. pub(crate) fn read_backing_into( - backing: &dyn crate::file_backing::FileBacking, + backing: &crate::file_backing::SharedImage, offset: u64, dst: KernelSlice, -) -> crate::file_backing::BlockResult { +) { let mut remaining = dst.size(); let mut file_off = offset; let mut buf_off = 0usize; @@ -324,7 +323,7 @@ pub(crate) fn read_backing_into( while remaining > 0 { let off_in_block = (file_off % 4096) as usize; let chunk = (4096 - off_in_block).min(remaining); - backing.read_page(file_off - off_in_block as u64, &mut page_buf)?; + backing.read_page(file_off - off_in_block as u64, &mut page_buf); // SAFETY: `copy_from` asserts `buf_off + chunk <= dst.size()`; `page_buf` // and `dst` cannot overlap, and nothing else can see `dst` yet. unsafe { dst.copy_from(buf_off, &page_buf[off_in_block..off_in_block + chunk]) }; @@ -332,12 +331,11 @@ pub(crate) fn read_backing_into( buf_off += chunk; remaining -= chunk; } - Ok(()) } /// Load a shared object into one contiguous allocation and apply its /// `R_X86_64_RELATIVE` relocations. -pub fn load_shared_lib(backing: &dyn crate::file_backing::FileBacking) -> Result { +pub fn load_shared_lib(backing: &crate::file_backing::SharedImage) -> Result { let header_size = 4096.min(backing.file_size() as usize); let header_data = crate::loader::read_file_range(backing, 0, header_size); let layout = parse_layout(&header_data)?; @@ -375,8 +373,7 @@ pub fn load_shared_lib(backing: &dyn crate::file_backing::FileBacking) -> Result // subslice turns a weakening of that into an assert, not an overwrite. for seg in layout.segments() { let dst = module.at(seg.image()).subslice(0, seg.filesz() as usize); - read_backing_into(backing, seg.file_offset(), dst) - .map_err(|_| "a segment could not be read off the device")?; + read_backing_into(backing, seg.file_offset(), dst); } let dyn_info = match layout.dynamic() { @@ -492,7 +489,7 @@ pub fn load_shared_lib(backing: &dyn crate::file_backing::FileBacking) -> Result /// `.dynsym`'s entry count as the section header table declares it, or `None` /// when the file does not say. fn dynsym_count_from_sections( - backing: &dyn crate::file_backing::FileBacking, + backing: &crate::file_backing::SharedImage, layout: &Layout, ) -> Option { let table = layout.section_headers()?; diff --git a/kernel/src/file_backing.rs b/kernel/src/file_backing.rs index 156170c745c..a41f93db769 100644 --- a/kernel/src/file_backing.rs +++ b/kernel/src/file_backing.rs @@ -7,33 +7,6 @@ use toyos_abi::syscall::SyscallError; /// `mm::PAGE_SIZE`: `usize` for buffer sizing, `u64` for file offsets. const BLOCK_SIZE: usize = crate::mm::PAGE_SIZE as usize; -/// Why a page a backing holds could not be read. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum BlockError { - /// The backing itself failed the read. - Device, - /// No memory to keep the page in: the kernel holds back what is left. - Exhausted, -} - -pub type BlockResult = Result<(), BlockError>; - -/// Backing store for a memory-mapped file: an image a caller handed over, or a -/// tmpfs file; callers do not know which. -pub trait FileBacking: Send + Sync { - /// Reads one page of file data at `file_offset` into `buf`, zero-filling past EOF. - #[must_use = "a failed read left the buffer zeroed; it does not hold the file's bytes"] - fn read_page(&self, file_offset: u64, buf: &mut [u8; BLOCK_SIZE]) -> BlockResult; - - /// Total file size in bytes. - fn file_size(&self) -> u64; - - /// The file's bytes as a spawn's segments copy them, each page at its - /// first touch: `NotFound` once the file is gone, `ResourceExhausted` - /// where what keeps them cannot be charged. - fn source(&self) -> Result, SyscallError>; -} - /// An executable in a shared memory object handed over by handle, since no /// file server's volume is the kernel's: a file server's object for the file /// (for ROOT, part of the boot image), or one a caller read into an object of @@ -61,25 +34,22 @@ impl SharedImage { } Ok(Self { object, size: len }) } -} -impl FileBacking for SharedImage { - fn read_page(&self, file_offset: u64, buf: &mut [u8; BLOCK_SIZE]) -> BlockResult { + pub fn read_page(&self, file_offset: u64, buf: &mut [u8; BLOCK_SIZE]) { buf.fill(0); if file_offset >= self.size { - return Ok(()); + return; } let valid = BLOCK_SIZE.min((self.size - file_offset) as usize); // The one fetch of each byte: its holders may be writing it. self.object.read(file_offset, &mut buf[..valid]); - Ok(()) } - fn file_size(&self) -> u64 { + pub fn file_size(&self) -> u64 { self.size } - fn source(&self) -> Result, SyscallError> { - Ok(self.object.clone()) + pub fn source(&self) -> Arc { + self.object.clone() } } diff --git a/kernel/src/loader/mod.rs b/kernel/src/loader/mod.rs index 875d0d8e59c..4493b8623d6 100644 --- a/kernel/src/loader/mod.rs +++ b/kernel/src/loader/mod.rs @@ -56,7 +56,7 @@ const STACK_GUARD: u64 = 1024 * 1024; /// comes from untrusted ELF fields, so the read is clamped rather than sized /// by it. pub(crate) fn read_file_range( - backing: &dyn crate::file_backing::FileBacking, + backing: &crate::file_backing::SharedImage, offset: u64, len: usize, ) -> Vec { @@ -71,10 +71,7 @@ pub(crate) fn read_file_range( let off_in_block = (file_off % 4096) as usize; let chunk = (4096 - off_in_block).min(remaining); - // A page the store refuses ends the read here rather than filling zeros. - if backing.read_page(file_off - off_in_block as u64, &mut page_buf).is_err() { - break; - } + backing.read_page(file_off - off_in_block as u64, &mut page_buf); result.extend_from_slice(&page_buf[off_in_block..off_in_block + chunk]); file_off += chunk as u64; @@ -171,8 +168,8 @@ fn segment_prot(seg: &toyos_elf::Segment) -> kernel::vm::Prot { /// the child's handle table around its handle to itself once nothing is left /// to refuse. /// -/// `path` is the program, opened here — or, with `image`, the bytes the caller -/// read from it itself, and then nothing opens it. `argv[0]` is only the name the child goes by. +/// `path` labels the supplied executable image; the kernel resolves no path. +/// `argv[0]` is the name the child goes by. /// /// `Refusal`, not `-> !`, is the error type: every failure below owns a /// partly built process (address space, stack, kernel stack), and nothing @@ -183,7 +180,7 @@ pub fn spawn( commit: C, cwd: String, env: Vec, - backing: Arc, + backing: Arc, parent: Parent, ) -> Result<(Pid, C::Held), crate::object::Refusal> { let job = Arc::clone(commit.job()); @@ -214,15 +211,7 @@ pub fn spawn( return Err(SyscallError::InvalidArgument.into()); }; - // Taken before the child's space is locked: a file's snapshot takes the - // file cache's lock, which a user copy may hold while it takes a space's. - let source = match backing.source() { - Ok(source) => source, - Err(e) => { - log!("spawn: {}: {e}", path); - return Err(e.into()); - } - }; + let source = backing.source(); let t1 = crate::clock::nanos_since_boot(); @@ -281,16 +270,11 @@ pub fn spawn( return Err(SyscallError::ResourceExhausted.into()); }; // `slice` bounds `filesz` against the `memsz` allocation, re-checking what `Layout::parse` already refused. - if elf::read_backing_into( + elf::read_backing_into( backing.as_ref(), tls_file_off, tls_buf.slice(tls.template().len() as usize), - ) - .is_err() - { - log!("spawn: {}: the TLS template could not be read off the device", path); - return Err(SyscallError::NotFound.into()); - } + ); Some(tls_buf) } None => None, @@ -501,7 +485,7 @@ pub fn spawn_supervisor(images: crate::bootimage::Objects) -> Pid { labels.extend_from_slice(label.as_bytes()); } let len = images.supervisor.immutable().expect("a boot image is immutable"); - let image: Arc = Arc::new( + let image: Arc = Arc::new( crate::file_backing::SharedImage::over(images.supervisor, len) .expect("spawn_supervisor: a boot image is ordinary memory, its length inside it"), ); diff --git a/kernel/src/object/ops.rs b/kernel/src/object/ops.rs index 60095382d07..8bb3e6994ac 100644 --- a/kernel/src/object/ops.rs +++ b/kernel/src/object/ops.rs @@ -19,7 +19,7 @@ use crate::device as device_registry; use super::device::DeviceClaim; use super::handle::{HandleEntry, HandleTable}; -use super::{KObjectRef, KObjectVariant}; +use super::KObjectRef; /// What a freshly created object's one handle carries. pub fn initial_rights(object: &KObjectRef) -> Rights { diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index fe57a5cfc2f..eba640bad6f 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -262,6 +262,11 @@ impl PagerObject { &self.watch } + pub fn room(&self) -> u64 { + let state = self.state.lock(); + (state.pages.capacity() - state.pages.used()) as u64 + } + /// Whether a notice is owed: a request no notice has named, or a last /// holder gone. pub fn has_notices(&self) -> bool { @@ -480,13 +485,18 @@ impl PagerObject { match step { Step::Copy(count) => { for piece in &pieces[..count] { - // SAFETY: `phys` is a frame of the object, pinned by - // `pin_chunk`, and `within + len` lies inside it. - unsafe { out.write_raw(done, DirectMap::from_phys(piece.phys).as_ptr::().add(piece.within), piece.len) }; + match piece.phys { + Some(phys) => { + // SAFETY: `phys` is pinned by `pin_chunk`, + // and `within + len` lies inside its frame. + unsafe { out.write_raw(done, DirectMap::from_phys(phys).as_ptr::().add(piece.within), piece.len) }; + } + None => out.fill_zero(done, piece.len), + } done += piece.len; } for piece in &pieces[..count] { - pmm::unpin_range(piece.phys, PAGE as usize); + if let Some(phys) = piece.phys { pmm::unpin_range(phys, PAGE as usize); } } let (gone, written) = { let state = self.state.lock(); @@ -565,13 +575,19 @@ impl PagerObject { Lookup::Present(phys) if pmm::pin_range(phys, PAGE as usize) => { let within = (pos % PAGE) as usize; let len = ((PAGE - pos % PAGE).min(range.end - pos)) as usize; - pieces[count] = Piece { phys, within, len }; + pieces[count] = Piece { phys: Some(phys), within, len }; count += 1; pos += len as u64; } // A pin count at its limit: copy what is pinned, and pin again. Lookup::Present(_) if count > 0 => break, Lookup::Present(_) => panic!("paged: a frame's pins overflowed"), + Lookup::Zero => { + let len = ((PAGE - pos % PAGE).min(range.end - pos)) as usize; + pieces[count] = Piece { phys: None, within: 0, len }; + count += 1; + pos += len as u64; + } _ if count > 0 => break, Lookup::Absent => { if let Some(waiting) = waiting { @@ -894,10 +910,10 @@ impl PagerObject { } } -/// Part of one pinned frame a read copies. +/// Part of one pinned frame or unwritten anonymous page a read copies. #[derive(Clone, Copy, Default)] struct Piece { - phys: u64, + phys: Option, within: usize, len: usize, } diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index 9093de416ad..370f9f9f72d 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -8,7 +8,7 @@ //! runs for a decode refusal exactly as for a handler. `syscall_total` and //! `syscall_total_ns` then count and time the same set of calls. -use crate::object::{ops, KObjectRef}; +use crate::object::ops; use crate::user_ptr::SyscallContext; use crate::UserAddr; use crate::process; @@ -118,7 +118,7 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> let image = match args.image_len { 0 => return SyscallError::InvalidArgument.to_u64(), len => match shared_image(args.image, len) { - Ok(image) => alloc::sync::Arc::new(image) as alloc::sync::Arc, + Ok(image) => alloc::sync::Arc::new(image) as alloc::sync::Arc, Err(refused) => return refused, }, }; diff --git a/kernel/src/syscall/paged.rs b/kernel/src/syscall/paged.rs index 348754f27b3..f7b49046038 100644 --- a/kernel/src/syscall/paged.rs +++ b/kernel/src/syscall/paged.rs @@ -128,6 +128,7 @@ pub(super) fn sys_paged_op(ctx: &SyscallContext<'_>, h: RawHandle, op: u64, args let range_ok = |count: u64| (1..=PAGED_RANGE_PAGES).contains(&count) && args.offset.checked_add(count).is_some(); let id = args.object; match op { + x if x == PagedOp::Room as u64 => pager.room(), x if x == PagedOp::Read as u64 => { if args.flags != 0 || !within_reach(args.offset, args.len) { return SyscallError::InvalidArgument.to_u64(); diff --git a/kernel/src/syscall/proc.rs b/kernel/src/syscall/proc.rs index 0dd9db1f18f..7416353bd5c 100644 --- a/kernel/src/syscall/proc.rs +++ b/kernel/src/syscall/proc.rs @@ -52,7 +52,7 @@ pub(super) fn sys_spawn( pending: crate::loader::PendingHandles, cwd: alloc::string::String, env: Vec, - image: alloc::sync::Arc, + image: alloc::sync::Arc, parent: process::Parent, ) -> u64 { // Nothing to clean up: spawn's frame owns the child's resources on error. diff --git a/kernel/src/user_ptr.rs b/kernel/src/user_ptr.rs index 7cd16cf426f..a34eb7a9112 100644 --- a/kernel/src/user_ptr.rs +++ b/kernel/src/user_ptr.rs @@ -181,11 +181,6 @@ impl View { } } - fn sub(&self, what: &str, off: usize, len: usize) -> View { - assert!(off.checked_add(len).is_some_and(|end| end <= self.len), "{what} {off}+{len} past a {}-byte window", self.len); - View { start: self.start + off as u64, len, access: self.access } - } - fn read_into(&self, off: usize, dst: &mut [u8]) { let into = dst.as_mut_ptr(); self.pieces("UserBytes::read_at", off, dst.len(), |run, at, n| { @@ -242,11 +237,6 @@ impl UserBytes<'_> { unsafe { core::ptr::copy_nonoverlapping(run, dst.add(at), n) } }); } - - /// The `len`-byte window at `off` inside this one. - pub fn sub(&self, off: usize, len: usize) -> UserBytes<'_> { - UserBytes(self.0.sub("UserBytes::sub", off, len), PhantomData) - } } /// A bulk buffer the kernel copies into and never reads back, so it cannot act on a value another thread substituted. @@ -293,10 +283,4 @@ impl UserBytesMut<'_> { unsafe { core::ptr::write_bytes(run, 0, n) } }); } - - /// The `len`-byte window at `off` inside this one. - pub fn sub(&mut self, off: usize, len: usize) -> UserBytesMut<'_> { - UserBytesMut(self.0.sub("UserBytesMut::sub", off, len), PhantomData) - } } - diff --git a/lib/manifest/src/lib.rs b/lib/manifest/src/lib.rs index b5ebbeffbb9..90a42dd4209 100644 --- a/lib/manifest/src/lib.rs +++ b/lib/manifest/src/lib.rs @@ -108,7 +108,7 @@ pub struct RoleDir { pub dir: &'static str, /// Where it is on the role's volume, which a grant on it names /// (`toyos::fs::Grant`): DATA keeps its directories side by side, and - /// SYSTEM, LOG and BOOT are each served whole. + /// SYSTEM, LOG, BOOT and MEMORY are each served whole. pub root: &'static str, /// Whether a grant on it may change it: SYSTEM's ROOT is the signed /// image, which nothing does. @@ -161,6 +161,11 @@ pub fn whole_tree() -> Vec { .collect() } +/// `/tmp`, read-write: an explicit capability in every view. +fn temporary() -> View { + View { role: "memory", dir: "/tmp".into(), root: String::new(), write: true } +} + /// `/system`, read-only: in every view. fn system() -> View { let [only] = role_dirs(SYSTEM).expect("SYSTEM is a role") else { panic!("SYSTEM serves one directory") }; @@ -397,7 +402,7 @@ impl Program { /// The directories this row's program is endowed. **An installed package /// sees `/system` and its own directory read-only, its own folder of the - /// home read-write and the folder its launch was granted, and nothing else + /// home read-write, `/tmp`, and the folder its launch was granted, and nothing else /// any role serves**: no other package, no other part of the home, no /// `/config`, `/state`, `/log` or `/boot`. /// **Every other row sees the whole tree but [`STATE`], and of that only its @@ -408,7 +413,7 @@ impl Program { match self.package() { Some(name) => { let granted = self.folder.iter().map(|f| data_dir(f.path.clone(), f.access == grants::Access::ReadWrite)); - [system(), data_dir(package::Package::dir(name), false), data_dir(app_home(name), true)] + [system(), temporary(), data_dir(package::Package::dir(name), false), data_dir(app_home(name), true)] .into_iter() .chain(granted) .collect() @@ -946,8 +951,8 @@ mod tests { } /// **An installed package sees `/system` and its own directory read-only - /// and its own folder read-write, and nothing else any role serves.** - /// Spelled out whole, so a fourth directory, a wider root or a writable + /// and its own folder and `/tmp` read-write, and nothing else any role serves.** + /// Spelled out whole, so an extra directory, a wider root or a writable /// package is red. #[test] fn a_package_s_view_is_its_own_directory_read_only_and_its_own_folder() { @@ -956,6 +961,7 @@ mod tests { views(&row), [ system(), + ("memory", "/tmp".into(), String::new(), true), ("data", "/apps/gbae".into(), "apps/gbae".into(), false), ("data", "/home/toy/Apps/gbae".into(), "home/toy/Apps/gbae".into(), true), ] @@ -964,7 +970,7 @@ mod tests { let longest = "n".repeat(MAX_PROGRAM_NAME); let row = sample().app_row(&longest, &format!("/apps/{longest}/{longest}")).unwrap(); assert_eq!( - views(&row).into_iter().skip(1).map(|(_, _, root, write)| (root, write)).collect::>(), + views(&row).into_iter().skip(2).map(|(_, _, root, write)| (root, write)).collect::>(), [(format!("apps/{longest}"), false), (format!("home/toy/Apps/{longest}"), true)] ); } @@ -980,6 +986,7 @@ mod tests { views(&row), [ system(), + ("memory", "/tmp".into(), String::new(), true), ("data", "/apps/gbae".into(), "apps/gbae".into(), false), ("data", "/home/toy/Apps/gbae".into(), "home/toy/Apps/gbae".into(), true), ("data", "/home/toy/Games".into(), "home/toy/Games".into(), write), @@ -1125,7 +1132,7 @@ mod tests { /// role is refused where it is written: the supervisor would start a server for /// directories nobody named. #[test] - fn a_role_is_one_of_the_three_and_its_directories_are_fixed() { + fn a_role_has_fixed_directories() { let m = sample(); let fileserver = m.program("fileserver").unwrap(); assert_eq!(fileserver.roles, ["data", "log"]); diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 648b3c04c57..cb52cb75c9d 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -3387,6 +3387,9 @@ pub enum PagedOp { /// Set the mtime to `time`, as a file's `utimens` does; the bytes and /// the version stay. Stamp = 10, + /// Unused slots in the pager's resident-page quota. This advisory + /// snapshot reserves nothing and promises no physical memory. + Room = 11, } /// [`PagedOp::Supply`]'s pages are dirty. diff --git a/sdk/toyos/src/paged.rs b/sdk/toyos/src/paged.rs index f12ad7adf34..93834fa5a06 100644 --- a/sdk/toyos/src/paged.rs +++ b/sdk/toyos/src/paged.rs @@ -93,6 +93,12 @@ impl Pager { syscall::paged_anonymous(self.0.raw(), size, mtime).map_err(|e| Refused::of(e, None)) } + /// Unused slots in this pager's page quota, without reserving them or + /// promising that physical memory will be available for them. + pub fn room(&self) -> Result { + self.op(PagedOp::Room, PagedArgs::default(), None) + } + /// A holder handle on object `id`, to send. pub fn holder(&self, id: u64) -> Result { let raw = syscall::paged_holder(self.0.raw(), id)?; diff --git a/system/fileserver/Cargo.toml b/system/fileserver/Cargo.toml index ae4a6c629e8..87279c5e829 100644 --- a/system/fileserver/Cargo.toml +++ b/system/fileserver/Cargo.toml @@ -2,7 +2,7 @@ name = "fileserver" edition = "2024" license = "MIT OR Apache-2.0" -description = "ToyOS's file server: one process per storage role — ROOT, DATA, the log or the boot volume — serves that role's paths." +description = "ToyOS's file server: one process per storage role — ROOT, DATA, temporary files, the log or the boot volume — serves that role's paths." # The library is the server's decisions — the block cache, the volumes and the # resolver — so the host can test them; the service is `src/main.rs`. diff --git a/system/fileserver/src/lib.rs b/system/fileserver/src/lib.rs index 87b50878f05..0308247cff9 100644 --- a/system/fileserver/src/lib.rs +++ b/system/fileserver/src/lib.rs @@ -1,24 +1,7 @@ -//! A file server's decisions: the blocks memory stands in for a partition -//! with ([`ram`]) — a partition's own is `diskserver::disk::Served` — the -//! cache of its volume's structures ([`cache`]), its files' pages -//! ([`pages`]), the volumes it -//! can serve ([`data`] for the bcachefs DATA role, [`root`] for the SYSTEM -//! role's ROOT in memory, [`fat`] for FAT32's LOG and BOOT, [`absent`] for a -//! role with no volume this boot), and the resolver that -//! keeps every path inside the directory a connection was given ([`resolve`]), -//! which requests a read-only one is refused ([`rights`]), what a client -//! may take of the server and name in a request ([`bounds`]), the open -//! files its connections share ([`descriptions`]), and a directory's listing -//! in batches ([`listing`]). -//! -//! **A file's bytes are in the kernel's paged objects** ([`pages`]), one per -//! file, which its holders read without asking this server; a volume's own -//! structures — a btree node, a FAT sector — are in [`cache::Cache`]. A write -//! lands in either and reaches the disk at a sync, or when either holds more -//! dirty than it keeps. -//! -//! The service around it is `src/main.rs`: its role's port and the grant on -//! each connection, its clients and their windows, and the wire (`toyos::fs`). +//! File-server volumes, namespace resolution and open-file ownership. +//! Persistent file contents use supplied paged objects; memory volumes keep +//! their bytes in anonymous objects until the last name and holder are gone. +//! The service and its capability protocol are in `main.rs`. pub mod absent; pub mod bounds; diff --git a/system/fileserver/src/main.rs b/system/fileserver/src/main.rs index 945b7283c7c..aad95ffc2ac 100644 --- a/system/fileserver/src/main.rs +++ b/system/fileserver/src/main.rs @@ -1,9 +1,9 @@ -//! fileserver: one role's file server — SYSTEM, DATA, LOG or BOOT — serving the +//! fileserver: one role's file server — SYSTEM, DATA, LOG, BOOT or MEMORY — serving the //! directories of that role as capabilities. //! //! **What it holds**: the acceptor of its role's one port, endowed by the //! supervisor under `serve:fs:`; for its volume, SYSTEM's ROOT image -//! (`ROOT_IMAGE_LABEL`), which needs no disk, or `block` in its namespace, a +//! (`ROOT_IMAGE_LABEL`), MEMORY's anonymous objects, or `block` in its namespace, a //! connector to the block service minted for its role's partition alone — //! every DATA partition, or the one the loader named, read-only for the boot //! volume; and nothing else of the machine. Two DATA partitions are refused by @@ -231,7 +231,7 @@ impl Exports { fn main() { let args: Vec = std::env::args().collect(); let role = args.get(1).and_then(|r| Role::parse(r)).unwrap_or_else(|| { - panic!("fileserver: started with {args:?}; the first argument is a role: system, data, log or boot") + panic!("fileserver: started with {args:?}; the first argument is a role: system, data, log, boot or memory") }); let mut guid = None; for arg in args.iter().skip(2) { @@ -254,9 +254,10 @@ fn main() { dirs.iter().map(|d| d.dir).collect::>().join(", "), volume.describe() ); + let files = Files::new(pager(role, volume.memory())); Server { volume, - files: Files::new(pager(role)), + files, pager_armed: false, memory, acceptor, @@ -274,7 +275,7 @@ fn main() { } /// The share of physical memory the file servers' pagers together hold in -/// pages: LOG and BOOT [`SMALL_PAGES`] each, DATA the rest. An interim +/// pages: LOG and BOOT [`SMALL_PAGES`] each, DATA and MEMORY share the rest. An interim /// bound, since nothing reclaims a pager's clean pages for another job /// (`issues/a-pager-holds-a-fixed-capacity-until-the-memory-service-reclaims-its-clean-pages.md`). const PAGES_SHARE: u64 = 8; @@ -288,12 +289,12 @@ const DATA_DIRTY: u64 = 8 * 1024; /// The role's pager: how many pages its files hold, how many of them may be /// dirty and how many files it keeps. None for SYSTEM, whose files are /// immutable and are served from ROOT's memory. -fn pager(role: Role) -> Option { +fn pager(role: Role, anonymous: bool) -> Option { let (capacity, dirty, objects) = match role { Role::System => return None, Role::Data | Role::Memory => { let share = toyos::system::memory_total() / PAGES_SHARE / toyos::paged::PAGE; - let capacity = share.saturating_sub(2 * SMALL_PAGES).max(SMALL_PAGES); + let capacity = (share.saturating_sub(2 * SMALL_PAGES) / 2).max(SMALL_PAGES); // A file held for every 16 pages, the table's own cost a small // part of the pages': at most 65,536. (capacity, DATA_DIRTY.min(capacity / 2), (capacity / 16).clamp(1024, 64 * 1024)) @@ -301,7 +302,7 @@ fn pager(role: Role) -> Option { Role::Log => (SMALL_PAGES, 512, 1024), Role::Boot => (SMALL_PAGES, 0, 1024), }; - let pager = toyos::paged::Pager::create(capacity, dirty, objects, REPORT) + let pager = toyos::paged::Pager::create(capacity, if anonymous { 0 } else { dirty }, objects, REPORT) .unwrap_or_else(|e| panic!("fileserver: the {role:?} server makes no pager ({e:?}); its row is no pager row")); Some(pager) } @@ -458,8 +459,8 @@ enum Answer { /// What `meta` and `identity` say a node is, on `volume`. These volumes keep /// one name per file and no access or change time. -fn stat_reply(meta: Meta, identity: u64, volume: u64) -> Reply { - let stat = Stat { kind: meta.kind.wire(), size: meta.size, nlink: 1, atime: 0, mtime: meta.mtime, ctime: 0, volume, node: identity }; +fn stat_reply(meta: Meta, identity: u64, volume: u64, nlink: u64) -> Reply { + let stat = Stat { kind: meta.kind.wire(), size: meta.size, nlink, atime: 0, mtime: meta.mtime, ctime: 0, volume, node: identity }; Reply::stat(&stat) } @@ -813,7 +814,7 @@ impl Server { fn stat_path(&mut self, path: &str) -> Result { let meta = self.volume.lstat(path)?; let identity = self.volume.identity(path)?; - Ok(Answer::Reply(stat_reply(meta, identity, self.volume_id))) + Ok(Answer::Reply(stat_reply(meta, identity, self.volume_id, 1))) } /// What a description holds open is. @@ -826,13 +827,12 @@ impl Server { }; (held, f.identity) }; - let meta = match held { - Ok(node) => self.volume.node_meta(node)?, - // A removed directory is what it was, with nothing in it. - Err((_, true)) => Meta { kind: Kind::Dir, size: 0, mtime: 0 }, - Err((path, false)) => self.volume.lstat(&path)?, + let (meta, linked) = match held { + Ok(node) => (self.volume.node_meta(node)?, self.volume.linked(node)?), + Err((_, true)) => (Meta { kind: Kind::Dir, size: 0, mtime: 0 }, false), + Err((path, false)) => (self.volume.lstat(&path)?, true), }; - Ok(Answer::Reply(stat_reply(meta, identity, self.volume_id))) + Ok(Answer::Reply(stat_reply(meta, identity, self.volume_id, u64::from(linked)))) } /// The open file `fid` names, a directory refused `IsADirectory`, and @@ -851,7 +851,7 @@ impl Server { } match op { OPEN => self.open(id, &r), - // These volumes keep no file without a name. + // No volume currently creates an inode without a name. CREATE_UNNAMED => Err(Error::Unsupported), CLOSE => { // Nobody waits on the answer: an id the client does not hold @@ -939,7 +939,7 @@ impl Server { } SYNC => self.sync(), ROOM => { - let room = self.volume.room()?; + let room = self.files.room(&mut *self.volume)?; Ok(Answer::Reply(Reply { value: room.free, value2: room.unit, ..Reply::ok() })) } STREAM => { @@ -1198,7 +1198,7 @@ impl Server { return Err(e); } }; - let reply = Reply { value: fid, ..stat_reply(meta, identity, self.volume_id) }; + let reply = Reply { value: fid, ..stat_reply(meta, identity, self.volume_id, 1) }; Ok(match holder { Some(handle) => { self.files.held(identity); diff --git a/system/fileserver/src/memory.rs b/system/fileserver/src/memory.rs index 7864795ea96..b92293f8d98 100644 --- a/system/fileserver/src/memory.rs +++ b/system/fileserver/src/memory.rs @@ -10,12 +10,12 @@ use crate::volume::{parent, Kind, Listed, Meta, Node, OpenHow, Out, Reserved, Vo struct Inode { meta: Meta, target: Option, + children: BTreeMap, opens: usize, linked: bool, } pub struct MemoryVolume { - names: BTreeMap, nodes: BTreeMap, next: Node, clock: fn() -> u64, @@ -23,25 +23,38 @@ pub struct MemoryVolume { impl MemoryVolume { pub fn new(roots: &[&str], clock: fn() -> u64) -> Self { - let mut volume = Self { names: BTreeMap::new(), nodes: BTreeMap::new(), next: 1, clock }; - volume.insert("", Kind::Dir, None); + let mut volume = Self { nodes: BTreeMap::new(), next: 1, clock }; + volume.allocate(Kind::Dir, None); for root in roots.iter().filter(|r| !r.is_empty()) { volume.mkdir(root).expect("memory volume roots are distinct direct children"); } volume } - fn insert(&mut self, path: &str, kind: Kind, target: Option) -> Node { + fn allocate(&mut self, kind: Kind, target: Option) -> Node { let node = self.next; self.next = self.next.checked_add(1).expect("memory inode numbers exhausted"); let size = target.as_ref().map_or(0, |t| t.len() as u64); - self.nodes.insert(node, Inode { meta: Meta { kind, size, mtime: (self.clock)() }, target, opens: 0, linked: true }); - assert!(self.names.insert(path.into(), node).is_none(), "a new memory inode has a vacant name"); + self.nodes.insert(node, Inode { meta: Meta { kind, size, mtime: (self.clock)() }, target, children: BTreeMap::new(), opens: 0, linked: true }); node } + fn insert(&mut self, path: &str, kind: Kind, target: Option) -> Result { + self.vacant(path)?; + let parent = self.node(parent(path))?; + let node = self.allocate(kind, target); + self.nodes.get_mut(&parent).expect("the parent exists").children.insert(name(path).into(), node); + Ok(node) + } + fn node(&self, path: &str) -> Result { - self.names.get(path).copied().ok_or(Error::NotFound) + let mut node = 1; + for name in path.split('/').filter(|part| !part.is_empty()) { + let inode = &self.nodes[&node]; + if inode.meta.kind != Kind::Dir { return Err(Error::NotADirectory); } + node = *inode.children.get(name).ok_or(Error::NotFound)?; + } + Ok(node) } fn inode(&mut self, node: Node) -> Result<&mut Inode, Error> { @@ -56,14 +69,17 @@ impl MemoryVolume { } fn vacant(&self, path: &str) -> Result<(), Error> { - if self.names.contains_key(path) { - return Err(Error::AlreadyExists); + match self.node(path) { + Ok(_) => return Err(Error::AlreadyExists), + Err(Error::NotFound) => {} + Err(e) => return Err(e), } self.directory(parent(path)) } fn detach(&mut self, path: &str) { - let node = self.names.remove(path).expect("detaching an existing name"); + let parent = self.node(parent(path)).expect("an existing name has a parent"); + let node = self.nodes.get_mut(&parent).expect("the parent exists").children.remove(name(path)).expect("detaching an existing name"); let inode = self.nodes.get_mut(&node).expect("a name has an inode"); inode.linked = false; if inode.opens == 0 { @@ -85,13 +101,13 @@ impl Volume for MemoryVolume { } fn read_link(&mut self, path: &str) -> Result { - self.inode(self.node(path)?)?.target.clone().ok_or(Error::InvalidRequest) + self.inode(self.node(path)?)?.target.clone().ok_or(Error::NotALink) } fn list(&mut self, dir: &str) -> Result, Error> { self.directory(dir)?; - Ok(self.names.iter().filter(|(path, _)| !path.is_empty() && parent(path) == dir).map(|(path, node)| Listed { - name: path.rsplit('/').next().expect("a path has a name").into(), + Ok(self.nodes[&self.node(dir)?].children.iter().map(|(name, node)| Listed { + name: name.clone(), meta: self.nodes[node].meta, identity: *node, }).collect()) @@ -102,8 +118,7 @@ impl Volume for MemoryVolume { Ok(_) if how.create_new => return Err(Error::AlreadyExists), Ok(node) => node, Err(Error::NotFound) if how.create || how.create_new => { - self.vacant(path)?; - self.insert(path, Kind::File, None) + self.insert(path, Kind::File, None)? } Err(e) => return Err(e), }; @@ -160,15 +175,14 @@ impl Volume for MemoryVolume { } fn mkdir(&mut self, path: &str) -> Result<(), Error> { - self.vacant(path)?; - self.insert(path, Kind::Dir, None); + self.insert(path, Kind::Dir, None)?; Ok(()) } fn rmdir(&mut self, path: &str) -> Result<(), Error> { if path.is_empty() { return Err(Error::Root); } self.directory(path)?; - if self.names.keys().any(|name| parent(name) == path) { return Err(Error::NotEmpty); } + if !self.nodes[&self.node(path)?].children.is_empty() { return Err(Error::NotEmpty); } self.detach(path); Ok(()) } @@ -191,23 +205,26 @@ impl Volume for MemoryVolume { if kind == Kind::Dir { return Err(Error::NotADirectory); } self.detach(to); } - let moved: Vec<_> = self.names.keys().filter(|path| path.as_str() == from || path.strip_prefix(from).is_some_and(|s| s.starts_with('/'))).cloned().collect(); - for path in moved { - let node = self.names.remove(&path).expect("a moved name exists"); - self.names.insert(format!("{to}{}", &path[from.len()..]), node); - } + let old_parent = self.node(parent(from))?; + let new_parent = self.node(parent(to))?; + self.nodes.get_mut(&old_parent).expect("the old parent exists").children.remove(name(from)); + self.nodes.get_mut(&new_parent).expect("the new parent exists").children.insert(name(to).into(), node); Ok(()) } fn symlink(&mut self, path: &str, target: &str) -> Result<(), Error> { - self.vacant(path)?; - self.insert(path, Kind::Symlink, Some(target.into())); + self.insert(path, Kind::Symlink, Some(target.into()))?; Ok(()) } fn sync(&mut self) -> Result, Error> { Ok(Vec::new()) } + fn commit(&mut self, _: Option) -> Result<(), Error> { Ok(()) } fn room(&mut self) -> Result { Err(Error::Unsupported) } - fn describe(&self) -> String { format!("memory: {} named inodes, {} retained", self.names.len(), self.nodes.len()) } + fn describe(&self) -> String { format!("memory: {} inodes", self.nodes.len()) } +} + +fn name(path: &str) -> &str { + path.rsplit('/').next().expect("a path has a name") } #[cfg(test)] @@ -249,4 +266,18 @@ mod tests { assert_eq!(volume.rmdir("home/b"), Err(Error::NotEmpty)); assert_eq!(volume.rename("home/b", "home/b/inner"), Err(Error::IntoItself)); } + + #[test] + fn symlinks_keep_their_names_and_never_replace_existing_entries() { + let mut volume = MemoryVolume::new(&[], clock); + let node = volume.open("file", CREATE).unwrap(); + assert_eq!(volume.symlink("file", "target"), Err(Error::AlreadyExists)); + assert_eq!(volume.identity("file"), Ok(node)); + assert_eq!(volume.read_link("file"), Err(Error::NotALink)); + volume.symlink("link", "/home/toy/target").unwrap(); + let listed = volume.list("").unwrap(); + assert_eq!(listed[1].name, "link"); + assert_eq!(listed[1].meta.kind, Kind::Symlink); + assert_eq!(volume.read_link("link"), Ok("/home/toy/target".into())); + } } diff --git a/system/fileserver/src/pages.rs b/system/fileserver/src/pages.rs index 02251f53223..288a6307633 100644 --- a/system/fileserver/src/pages.rs +++ b/system/fileserver/src/pages.rs @@ -3,20 +3,20 @@ //! which is the only copy of the file's data the server keeps. This moves //! pages between the object and the volume: a page a reader asks for is read //! off the volume and supplied, a write lands dirty in the object, and a sync -//! writes the dirty pages back before the volume's own sync. +//! writes the dirty pages back before the volume's own sync. Memory volumes +//! use anonymous objects instead: no writeback or eviction can discard them. //! //! **An object per file, by its volume identity**, which the notices name it //! by. It holds the volume's node for its life, so a dirty file stays open //! on the volume whoever closed it, and outlives its last close for its pages' //! sake until room is wanted: then a file nobody holds and nothing dirties -//! goes first, oldest first. +//! goes first, oldest first. Anonymous objects stay while named or held. //! //! **A write is one write of the object's**, however many calls it takes: //! a call that stops short leaves it under way until its end, so no holder //! reads the file half written (`toyos::paged::Pager::end_write`). //! -//! **The volume's size and mtime are the object's**, which its holders -//! `fstat` without a request: each call that moves the volume's mtime hands +//! **The volume's size and mtime are the object's**: each call that moves the volume's mtime hands //! the object the volume's own value, at its resolution. A write reserves on the volume //! first, so the volume's size runs past the object's only for the bytes the //! write then puts in before it returns; nothing reads the volume for them @@ -40,6 +40,7 @@ const RANGE: u64 = PAGED_RANGE_PAGES; pub trait Pager { fn object(&self, size: u64, mtime: u64) -> Result; fn anonymous(&self, size: u64, mtime: u64) -> Result; + fn room(&self) -> Result; fn read(&self, id: u64, out: Bytes<'_>, offset: u64) -> Result; fn supply(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result; fn write(&self, id: u64, offset: u64, bytes: &[u8], mtime: u64) -> Result; @@ -60,6 +61,9 @@ impl Pager for toyos::paged::Pager { fn anonymous(&self, size: u64, mtime: u64) -> Result { toyos::paged::Pager::anonymous(self, size, mtime) } + fn room(&self) -> Result { + toyos::paged::Pager::room(self) + } fn read(&self, id: u64, out: Bytes<'_>, offset: u64) -> Result { toyos::paged::Pager::read(self, id, out, offset) } @@ -151,6 +155,14 @@ impl Files

{ self.pager.as_ref() } + pub fn room(&self, volume: &mut dyn Volume) -> Result { + if volume.memory() { + Ok(toyos::fs::Room { free: self.pg().room().map_err(word)?.saturating_mul(PAGE), unit: PAGE }) + } else { + volume.room() + } + } + fn pg(&self) -> &P { self.pager.as_ref().expect("fileserver: a paged file on a server with no pager") } @@ -408,6 +420,7 @@ impl Files

{ /// `data` written to `node`'s file at `offset`. pub fn write(&mut self, volume: &mut dyn Volume, identity: u64, node: Node, offset: u64, data: &[u8]) -> Result<(), Error> { + if data.is_empty() { return Ok(()); } if self.pager.is_none() || !self.files.contains_key(&identity) { return volume.write(node, offset, data); } diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index 216f35ec08b..e4b92363664 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -21,6 +21,7 @@ struct Model { pages: BTreeMap<(u64, u64), Page>, sizes: BTreeMap, mtimes: BTreeMap, + anonymous: BTreeSet, supplies: u32, /// Every supply from here is refused. refusing: bool, @@ -71,6 +72,7 @@ impl Fake { fn supply_pages(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result { let mut m = self.0.borrow_mut(); + let anonymous = m.anonymous.contains(&id); m.supplies += 1; if m.refusing { return Err(Refused::Kernel(SyscallError::Io)); @@ -85,7 +87,7 @@ impl Fake { continue; } else if m.pages.len() >= m.capacity { Some(Refused::NoRoom) - } else if dirty && m.held() >= m.ceiling { + } else if dirty && !anonymous && m.held() >= m.ceiling { Some(Refused::DirtyFull) } else { None @@ -96,7 +98,7 @@ impl Fake { let mut held = vec![0u8; PAGE as usize]; let n = if dirty { chunk.len() } else { chunk.len().min((size - page * PAGE) as usize) }; held[..n].copy_from_slice(&chunk[..n]); - m.pages.insert((id, page), Page { bytes: held, dirty, writeback: false }); + m.pages.insert((id, page), Page { bytes: held, dirty: dirty && !anonymous, writeback: false }); if dirty { let s = m.sizes.get_mut(&id).unwrap(); *s = (*s).max(page * PAGE + chunk.len() as u64); @@ -108,19 +110,20 @@ impl Fake { fn write_pages(&self, id: u64, offset: u64, bytes: &[u8]) -> Result { let mut m = self.0.borrow_mut(); + let anonymous = m.anonymous.contains(&id); let mut done = 0; while done < bytes.len() { let at = offset + done as u64; let key = (id, at / PAGE); let Some(page) = m.pages.get(&key) else { break }; - if !page.dirty && !page.writeback && m.held() >= m.ceiling { + if !anonymous && !page.dirty && !page.writeback && m.held() >= m.ceiling { return if done > 0 { Ok(done) } else { Err(Refused::DirtyFull) }; } let within = (at % PAGE) as usize; let n = (PAGE as usize - within).min(bytes.len() - done); let page = m.pages.get_mut(&key).unwrap(); page.bytes[within..within + n].copy_from_slice(&bytes[done..done + n]); - page.dirty = true; + page.dirty = !anonymous; let s = m.sizes.get_mut(&id).unwrap(); *s = (*s).max(at + n as u64); done += n; @@ -130,6 +133,11 @@ impl Fake { } impl Pager for Fake { + fn room(&self) -> Result { + let model = self.0.borrow(); + Ok((model.capacity - model.pages.len()) as u64) + } + fn object(&self, size: u64, mtime: u64) -> Result { let mut m = self.0.borrow_mut(); if m.sizes.len() >= m.objects { @@ -142,6 +150,12 @@ impl Pager for Fake { Ok(id) } + fn anonymous(&self, size: u64, mtime: u64) -> Result { + let id = self.object(size, mtime)?; + self.0.borrow_mut().anonymous.insert(id); + Ok(id) + } + fn release(&self, id: u64) -> Result<(), Refused> { let mut m = self.0.borrow_mut(); m.pages.retain(|(c, _), _| *c != id); @@ -154,10 +168,13 @@ impl Pager for Fake { let end = size.min(offset + out.len() as u64); let mut at = offset; while at < end { - let Some(page) = m.pages.get(&(id, at / PAGE)) else { break }; let within = (at % PAGE) as usize; let n = ((PAGE as usize - within) as u64).min(end - at) as usize; - out.write((at - offset) as usize, &page.bytes[within..within + n]); + match m.pages.get(&(id, at / PAGE)) { + Some(page) => out.write((at - offset) as usize, &page.bytes[within..within + n]), + None if m.anonymous.contains(&id) => out.sub((at - offset) as usize, n).zero(), + None => break, + } at += n as u64; } match at - offset { @@ -190,6 +207,7 @@ impl Pager for Fake { fn withdraw(&self, id: u64, first: u64, count: u64) -> Result { let mut m = self.0.borrow_mut(); + if m.anonymous.contains(&id) { return Ok(0); } let before = m.pages.len(); let end = first.saturating_add(count); m.pages.retain(|(c, p), page| *c != id || !(first..end).contains(p) || page.dirty || page.writeback); @@ -259,14 +277,14 @@ fn volume() -> DataVolume { const CREATE: OpenHow = OpenHow { create: true, create_new: false, truncate: false }; /// `path` opened through `files`: its node and identity. -fn open(files: &mut Files, v: &mut DataVolume, path: &str) -> (Node, u64) { +fn open(files: &mut Files, v: &mut dyn Volume, path: &str) -> (Node, u64) { let node = v.open(path, CREATE).unwrap(); let identity = v.identity(path).unwrap(); files.opened(v, identity, node).unwrap(); (node, identity) } -fn read(files: &mut Files, v: &mut DataVolume, (node, id): (Node, u64), offset: u64, len: usize) -> Vec { +fn read(files: &mut Files, v: &mut dyn Volume, (node, id): (Node, u64), offset: u64, len: usize) -> Vec { let mut out = vec![0xEEu8; len]; // SAFETY: `out` is the test's, reached only through these bytes meanwhile. let bytes = unsafe { Bytes::from_raw(out.as_mut_ptr(), len) }; @@ -289,6 +307,61 @@ fn pattern(len: usize, seed: u8) -> Vec { (0..len).map(|i| (i as u8).wrapping_mul(31).wrapping_add(seed)).collect() } +#[test] +fn memory_bytes_survive_close_sync_and_namespace_replacement_until_the_last_holder() { + let mut volume = crate::memory::MemoryVolume::new(&[], clock); + let fake = Fake::new(4, 0); + let mut files = Files::new(Some(fake.clone())); + let (node, id) = open(&mut files, &mut volume, "file"); + files.write(&mut volume, id, node, 0, b"old contents").unwrap(); + files.closed(&mut volume, id, node).unwrap(); + files.sync(&mut volume).unwrap(); + let again = open(&mut files, &mut volume, "file"); + assert_eq!(again, (node, id)); + assert_eq!(read(&mut files, &mut volume, again, 0, 20), b"old contents"); + let replacement = open(&mut files, &mut volume, "replacement"); + files.write(&mut volume, replacement.1, replacement.0, 0, b"new contents").unwrap(); + volume.rename("replacement", "file").unwrap(); + files.unlinked(&mut volume, id).unwrap(); + assert_eq!(read(&mut files, &mut volume, again, 0, 20), b"old contents"); + assert_eq!(volume.identity("file"), Ok(replacement.1)); + let object = files.object(id).unwrap(); + files.held(id); + files.closed(&mut volume, id, node).unwrap(); + assert!(files.object(id).is_some(), "the paged holder outlives the description"); + files.notice(&mut volume, Notice::NoHolders { object }); + assert!(files.object(id).is_none()); + assert_eq!(volume.node_meta(node), Err(Error::Stale)); + assert_eq!(read(&mut files, &mut volume, replacement, 0, 20), b"new contents"); + assert_eq!(fake.0.borrow().held(), 0, "anonymous bytes need no writeback slots"); +} + +#[test] +fn memory_sparse_files_do_not_resurrect_truncated_bytes_or_evict_existing_data() { + let mut volume = crate::memory::MemoryVolume::new(&[], clock); + let fake = Fake::new(2, 0); + let mut files = Files::new(Some(fake.clone())); + let (node, id) = open(&mut files, &mut volume, "file"); + files.write(&mut volume, id, node, PAGE + 5, b"tail").unwrap(); + assert_eq!(fake.pages_of(files.object(id).unwrap()), 1, "the hole holds no pages"); + assert_eq!(files.room(&mut volume).unwrap().free, PAGE); + let bytes = read(&mut files, &mut volume, (node, id), 0, 2 * PAGE as usize); + assert_eq!(files.room(&mut volume).unwrap().free, PAGE, "reading a hole retains no zero page"); + assert!(bytes[..PAGE as usize + 5].iter().all(|&b| b == 0)); + assert_eq!(&bytes[PAGE as usize + 5..], b"tail"); + files.truncate(&mut volume, id, node, PAGE + 6).unwrap(); + files.truncate(&mut volume, id, node, PAGE + 9).unwrap(); + assert_eq!(read(&mut files, &mut volume, (node, id), PAGE + 5, 4), b"t\0\0\0"); + files.write(&mut volume, id, node, 0, b"head").unwrap(); + let (other, other_id) = open(&mut files, &mut volume, "other"); + assert_eq!(files.write(&mut volume, other_id, other, 0, b"full"), Err(Error::NoSpace)); + assert_eq!(volume.node_meta(other).unwrap().size, 0); + assert_eq!(read(&mut files, &mut volume, (node, id), PAGE + 5, 4), b"t\0\0\0"); + files.truncate(&mut volume, id, node, PAGE).unwrap(); + files.write(&mut volume, other_id, other, 0, b"room").unwrap(); + assert_eq!(read(&mut files, &mut volume, (other, other_id), 0, 4), b"room"); +} + #[test] fn a_read_is_supplied_off_the_volume_once_and_then_served_from_the_object() { let mut v = volume(); diff --git a/system/fileserver/src/volume.rs b/system/fileserver/src/volume.rs index 5b2d255968b..91e0312b7d2 100644 --- a/system/fileserver/src/volume.rs +++ b/system/fileserver/src/volume.rs @@ -4,8 +4,8 @@ //! //! **An open file is a [`Node`]**, one per path while any client holds it //! open, so two opens of one file write one set of pages and see one length. A -//! node whose file was unlinked or renamed over answers `Stale` from then on: -//! its blocks may be another file's. +//! disk adapter may revoke a replaced node; a memory inode outlives its name +//! while an open description or paged holder still reaches it. use std::collections::BTreeMap; use std::ops::Bound; @@ -370,8 +370,8 @@ pub fn take_anchor() { } /// What a file written now is stamped with (`toyos_abi::syscall::Stat::mtime`): -/// nanoseconds since the Unix epoch, UTC, as the kernel's `clock::mtime_now` -/// reckons them, and 0 — undated — on a machine whose RTC never answered. +/// nanoseconds since the Unix epoch, UTC, and 0 — undated — on a machine whose +/// RTC never answered. pub fn now_nanos() -> u64 { let anchor = ANCHOR.get().expect("fileserver takes the anchor at its start"); anchor.map_or(0, |secs| secs.saturating_mul(NANOS_PER_SEC).saturating_add(nanos_since_boot())) diff --git a/system/supervisor/src/main.rs b/system/supervisor/src/main.rs index aee9d39d8a9..8430ac2f27e 100644 --- a/system/supervisor/src/main.rs +++ b/system/supervisor/src/main.rs @@ -3581,8 +3581,8 @@ fn guid_text(guid: [u8; 16]) -> String { /// A file server's arguments and the volume it is endowed for one start. /// -/// A file server is told its role. SYSTEM is handed ROOT's image; every other -/// role `block` minted with its grant (`toyos_supervisor::storage`), and LOG +/// A file server is told its role. SYSTEM is handed ROOT's image; MEMORY +/// needs no block grant, and disk roles receive `block` minted with their grant (`toyos_supervisor::storage`), and LOG /// and BOOT the GUID of the partition the loader named. A log or boot role the /// loader named no partition for, and a block port closed for good, each /// refuse the start: the role is then absent, never served from memory. diff --git a/tests/guest/pagerprobe/src/main.rs b/tests/guest/pagerprobe/src/main.rs index b8a70a7515a..7e17537ee46 100644 --- a/tests/guest/pagerprobe/src/main.rs +++ b/tests/guest/pagerprobe/src/main.rs @@ -25,6 +25,7 @@ use toyos::poller::{Poller, READABLE}; use toyos_abi::syscall::SyscallError; const HOLDER: &str = "pagerprobe-holder"; +const ANONYMOUS: &str = "pagerprobe-anonymous"; /// Pages of the large object: more than 512 MiB. const LARGE: u64 = 140_000; @@ -52,6 +53,17 @@ fn main() { let left = pager.object(LARGE * PAGE, 0).expect("pagerprobe: the object the pager's end takes"); fill(&pager, left); println!("pagerprobe: more than 512 MiB in one object withdrawn, released and left for the end"); + let anonymous = pager.anonymous(32 * PAGE, 0).expect("pagerprobe: anonymous memory"); + assert_eq!(pager.supply(anonymous, 0, &vec![0x37; 16 * PAGE as usize], true), Ok(16)); + assert_eq!(pager.room(), Ok(0), "the anonymous pages consumed the remaining quota"); + assert_eq!(pager.withdraw(anonymous, 0, 32), Ok(0), "anonymous bytes cannot be withdrawn"); + assert_eq!(pager.fail(anonymous, 0, 32), Err(Refused::Kernel(SyscallError::InvalidArgument))); + let mut bytes = vec![0xff; 32 * PAGE as usize]; + // SAFETY: `bytes` is local and borrowed only by this read until it returns. + let out = unsafe { toyos::shm::Bytes::from_raw(bytes.as_mut_ptr(), bytes.len()) }; + assert_eq!(pager.read(anonymous, out, 0), Ok(bytes.len())); + anonymous_contents(&bytes); + assert_eq!(pager.room(), Ok(0), "a zero-hole read needs no quota"); let object = pager.object(4096, 0).expect("pagerprobe: an object"); let own = pager.holder(object).expect("pagerprobe: a holder for itself"); // The handle is a holder, and this process a pager: refused by the kernel. @@ -63,7 +75,13 @@ fn main() { println!("pagerprobe: a pager is refused a holder's read"); let theirs = pager.holder(object).expect("pagerprobe: a holder to hand on"); - let mut helper = Command::new(SELF).arg("--hold").endow(HOLDER, theirs.into()).spawn().expect("pagerprobe: the helper starts"); + let memory = pager.holder(anonymous).expect("pagerprobe: an anonymous holder"); + let mut helper = Command::new(SELF) + .arg("--hold") + .endow(HOLDER, theirs.into()) + .endow(ANONYMOUS, memory.into()) + .spawn() + .expect("pagerprobe: the helper starts"); // The helper's request is the notice that it waits. let poller = Poller::new(1); @@ -97,6 +115,12 @@ fn fill(pager: &Pager, object: u64) { } fn hold() { + let memory: toyos::OwnedHandle = Endowments::get().take(ANONYMOUS).expect("pagerprobe --hold: anonymous memory"); + let memory = Paged::received(memory).expect("pagerprobe --hold: an anonymous holder"); + let mut bytes = vec![0xff; 32 * PAGE as usize]; + assert_eq!(memory.read(&mut bytes, 0), Ok(bytes.len())); + anonymous_contents(&bytes); + println!("pagerprobe --hold: anonymous contents and zero holes read at a full page quota"); // A peer that says a pipe end is a holder is answered as no holder, and // the process asking lives. let (read, _write) = toyos::pipe_pair().expect("pagerprobe --hold: a pipe"); @@ -114,3 +138,8 @@ fn hold() { other => panic!("pagerprobe --hold: a wait on an unsupplied page answered {other:?}"), } } + +fn anonymous_contents(bytes: &[u8]) { + assert!(bytes[..16 * PAGE as usize].iter().all(|&b| b == 0x37)); + assert!(bytes[16 * PAGE as usize..].iter().all(|&b| b == 0)); +} From 1046a1fb0d859e9aab5223d533d6257b18165137 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 19:18:54 +0100 Subject: [PATCH 11/93] Replace provisional DATA storage with bcachefs adapter Consolidation checkpoint: routes DATA through the on-disk engine and removes the provisional format and its synchronous adapter. Holds inode identities and per-page reservation tokens. Engine token APIs and userland memory changes are integrated in following commits; this checkpoint is not a validated landing. The partition signature test passes with actual bcachefs recognition. --- Cargo.lock | 1 + lib/bcachefs/Cargo.toml | 2 +- lib/bcachefs/src/alloc_bitmap.rs | 658 -------- lib/bcachefs/src/block_io.rs | 307 ---- lib/bcachefs/src/btree.rs | 963 ------------ lib/bcachefs/src/designation.rs | 33 + lib/bcachefs/src/engine/mod.rs | 2 +- lib/bcachefs/src/engine/superblock.rs | 8 + lib/bcachefs/src/fs.rs | 2066 ------------------------- lib/bcachefs/src/lib.rs | 75 +- lib/bcachefs/src/superblock.rs | 346 ----- lib/blockserve/src/content.rs | 6 +- system/fileserver/Cargo.toml | 4 +- system/fileserver/src/cache.rs | 41 - system/fileserver/src/data.rs | 1981 ++++-------------------- system/fileserver/src/fat.rs | 2 +- system/fileserver/src/pages/tests.rs | 16 +- system/fileserver/src/volume.rs | 12 +- 18 files changed, 399 insertions(+), 6124 deletions(-) delete mode 100644 lib/bcachefs/src/alloc_bitmap.rs delete mode 100644 lib/bcachefs/src/block_io.rs delete mode 100644 lib/bcachefs/src/btree.rs create mode 100644 lib/bcachefs/src/designation.rs delete mode 100644 lib/bcachefs/src/fs.rs delete mode 100644 lib/bcachefs/src/superblock.rs diff --git a/Cargo.lock b/Cargo.lock index 68672ad8580..01ce56ed62e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1010,6 +1010,7 @@ version = "0.0.0" dependencies = [ "bcachefs", "diskserver", + "getrandom 0.3.4", "toyos", "toyos-abi", "toyos-blockdev", diff --git a/lib/bcachefs/Cargo.toml b/lib/bcachefs/Cargo.toml index f25ba84d2b0..60b2f25e33f 100644 --- a/lib/bcachefs/Cargo.toml +++ b/lib/bcachefs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bcachefs" -description = "bcachefs: the on-disk format's reader, mkfs and check, and the interim ToyOS format they replace." +description = "bcachefs on-disk storage: read, write, format, recovery and integrity checking." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" diff --git a/lib/bcachefs/src/alloc_bitmap.rs b/lib/bcachefs/src/alloc_bitmap.rs deleted file mode 100644 index 59df38f1ec5..00000000000 --- a/lib/bcachefs/src/alloc_bitmap.rs +++ /dev/null @@ -1,658 +0,0 @@ -use alloc::collections::BTreeMap; -use alloc::vec::Vec; - -use crate::block_io::{BlockBuf, BlockNum, BlockIO, BlockIOExt, BLOCK_SIZE}; -use crate::fs::FsError; -use crate::superblock::Superblock; - -const BITS_PER_BLOCK: u64 = (BLOCK_SIZE * 8) as u64; - -/// A run of blocks the allocator actually reserved. -/// -/// `len` is what you got, never what you asked for. The pair this replaced was -/// `(BlockNum, u32)`, which reads as "here is your block, and how many" and was -/// destructured positionally at three sites; one of them then addressed a block -/// past the end of the short run it had just recorded, so a sparse write on a -/// fragmented volume landed on another file. A struct cannot be read as "all -/// of it" by accident. -#[must_use] -#[derive(Debug, Clone, Copy)] -pub struct Run { - pub start: BlockNum, - pub len: u32, -} - -/// Bitmap-based block allocator. -/// -/// The bitmap is stored on disk starting at `bitmap_start` and spanning -/// `bitmap_blocks` blocks. Each bit represents one block: 1 = used, 0 = free. -/// -/// **With `shadow` on, no node the last committed tree reaches is written -/// over, and no block it reaches is handed out again, before the next commit -/// lands** (`Mounted::sync`): a btree node is written in place only when the -/// operation in progress took its block ([`Self::in_place`]), and a block -/// given up is free at once only if no commit has named it since it was taken -/// (`fresh`); any other waits in `pending` for the commit. An operation -/// ([`Self::begin`]) gives up its blocks only when it succeeds, and refused -/// gives back every one it took. -/// mkfs runs with `shadow` off: nothing it writes is committed until it ends. -pub struct BitmapAllocator { - pub bitmap_start: BlockNum, - pub bitmap_blocks: u64, - pub total_blocks: u64, - pub free_blocks: u64, - pub next_alloc: u64, // cursor — scan starts here, wraps once - pub shadow: bool, - /// Taken since the last commit's superblock was handed to the device. - fresh: Runs, - /// Given up, and reached by a tree a mount may still find. - pending: Vec<(u64, u32)>, - op: Option, -} - -/// Blocks only an operation that [`Reserve::Draw`]s may take while `shadow` -/// is on, so a full volume still copies the nodes a delete or a shrink -/// changes, as many as sixteen between two commits. -pub const NODE_RESERVE: u64 = 16; - -/// Whether an operation may take the blocks [`NODE_RESERVE`] keeps: only one -/// that leaves the tree no larger, so a commit gives back all it drew. -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum Reserve { - Keep, - Draw, -} - -/// The operation in progress: what it took, and what it gives up if it succeeds. -struct Op { - reserve: Reserve, - taken: Runs, - given: Vec<(u64, u32)>, -} - -/// Disjoint runs of blocks, by first block, to one past the last. -#[derive(Default)] -struct Runs(BTreeMap); - -impl Runs { - fn insert(&mut self, start: u64, len: u64) { - self.0.insert(start, start + len); - } - - fn contains(&self, block: u64) -> bool { - self.0.range(..=block).next_back().is_some_and(|(_, &end)| block < end) - } - - /// Take `block` out, answering whether it was in. - fn remove(&mut self, block: u64) -> bool { - let Some((&start, &end)) = self.0.range(..=block).next_back() else { return false }; - if block >= end { - return false; - } - self.0.remove(&start); - if start < block { - self.0.insert(start, block); - } - if block + 1 < end { - self.0.insert(block + 1, end); - } - true - } -} - -impl BitmapAllocator { - /// The allocator of a mounted volume, its free count the last commit's. - pub fn open(sb: &Superblock) -> Self { - Self { - bitmap_start: sb.bitmap_start, - bitmap_blocks: sb.bitmap_blocks, - total_blocks: sb.block_count, - free_blocks: sb.free_blocks, - next_alloc: sb.next_alloc, - shadow: true, - fresh: Runs::default(), - pending: Vec::new(), - op: None, - } - } - - /// Read the free count off the bitmap: a stop after the last commit - /// leaves it holding fewer than the superblock counts. - pub fn count_free(&mut self, io: &dyn BlockIO) -> Result<(), FsError> { - let mut free_blocks = 0; - let mut buf = BlockBuf::zeroed(); - for i in 0..self.total_blocks.div_ceil(BITS_PER_BLOCK) { - io.read(BlockNum::new(self.bitmap_start.raw() + i), &mut buf)?; - let bits = (self.total_blocks - i * BITS_PER_BLOCK).min(BITS_PER_BLOCK); - let (whole, rest) = buf.0[..bits.div_ceil(8) as usize].split_at((bits / 8) as usize); - free_blocks += whole.iter().map(|b| b.count_zeros() as u64).sum::(); - free_blocks += rest.first().map_or(0, |b| (!b & ((1u8 << (bits % 8)) - 1)).count_ones() as u64); - } - self.free_blocks = free_blocks; - Ok(()) - } - - /// Begin one operation on the tree, which [`Self::succeed`] or - /// [`Self::fail`] ends. - pub fn begin(&mut self, reserve: Reserve) { - let op = Op { reserve, taken: Runs::default(), given: Vec::new() }; - assert!(self.op.replace(op).is_none(), "an operation began inside another"); - } - - /// What an allocation may take now: every free block, but for those - /// [`NODE_RESERVE`] keeps from all but an operation that draws on it. - pub(crate) fn spare(&self) -> u64 { - match !self.shadow || self.op.as_ref().is_some_and(|op| op.reserve == Reserve::Draw) { - true => self.free_blocks, - false => self.free_blocks.saturating_sub(NODE_RESERVE), - } - } - - /// Whether a node at `block` may be written where it is. - pub fn in_place(&self, block: BlockNum) -> bool { - !self.shadow || self.op.as_ref().is_some_and(|op| op.taken.contains(block.raw())) - } - - /// The operation took effect: what it gave up is given up now. A bit the - /// device would not clear is a leaked block, and no reason to call an - /// operation that took effect refused. - pub fn succeed(&mut self, io: &dyn BlockIO) { - let op = self.op.take().expect("an operation in progress"); - for (start, count) in op.given { - let _ = self.give(io, start, count); - } - } - - /// The operation was refused: every block it took is free again, and what - /// it would have given up stays its owner's. A bit the device would not - /// clear is a leaked block, and the refusal already in hand is the answer. - pub fn fail(&mut self, io: &dyn BlockIO) { - let op = self.op.take().expect("an operation in progress"); - for (start, end) in op.taken.0 { - for block in start..end { - self.fresh.remove(block); - let _ = self.set_free(io, BlockNum::new(block)); - } - } - } - - /// A superblock naming the tree as it stands may land from here on, so - /// nothing taken so far is free at once when given up. - pub fn seal(&mut self) { - self.fresh = Runs::default(); - } - - /// Blocks given up that a commit landing makes free. - pub fn pending(&self) -> u64 { - self.pending.iter().map(|&(_, count)| count as u64).sum() - } - - /// A commit landed: the blocks only an older tree reached are free. - pub fn committed(&mut self, io: &dyn BlockIO) -> Result<(), FsError> { - while let Some((start, count)) = self.pending.pop() { - for i in 0..count { - if let Err(e) = self.set_free(io, BlockNum::new(start + i as u64)) { - self.pending.push((start + i as u64, count - i)); - return Err(e); - } - } - } - Ok(()) - } - - /// Give up `count` blocks from `start`: free now if no commit has named - /// them, and otherwise once the next one lands. A block whose bit the - /// device would not clear is leaked, and the rest are given up still. - fn give(&mut self, io: &dyn BlockIO, start: u64, count: u32) -> Result<(), FsError> { - let mut refused = Ok(()); - for block in start..start + count as u64 { - if !self.shadow || self.fresh.remove(block) { - if let Err(e) = self.set_free(io, BlockNum::new(block)) { - refused = refused.and(Err(e)); - } - } else { - match self.pending.last_mut() { - Some((at, n)) if *at + *n as u64 == block => *n += 1, - _ => self.pending.push((block, 1)), - } - } - } - refused - } - - /// Where a block's bit lives, and which bit of that byte it is. - fn bit_of(&self, block: BlockNum) -> (BlockNum, usize, u8) { - let byte_idx = block.raw() / 8; - ( - BlockNum::new(self.bitmap_start.raw() + byte_idx / BLOCK_SIZE as u64), - (byte_idx % BLOCK_SIZE as u64) as usize, - (block.raw() % 8) as u8, - ) - } - - /// Mark a single block as used in the bitmap. - fn set_used(&self, io: &dyn BlockIO, block: BlockNum) -> Result<(), FsError> { - let (bitmap_block, byte_off, bit) = self.bit_of(block); - let mut buf = BlockBuf::zeroed(); - io.read(bitmap_block, &mut buf)?; - buf.0[byte_off] |= 1 << bit; - io.write(bitmap_block, &buf) - } - - /// Mark a single block as free in the bitmap. - /// - /// The in-memory counters move only once the bitmap block is on the - /// device: a free the device refused is not a free, and counting it would - /// leave the allocator handing out a block the bitmap still calls taken. - fn set_free(&mut self, io: &dyn BlockIO, block: BlockNum) -> Result<(), FsError> { - let (bitmap_block, byte_off, bit) = self.bit_of(block); - let mut buf = BlockBuf::zeroed(); - io.read(bitmap_block, &mut buf)?; - buf.0[byte_off] &= !(1 << bit); - io.write(bitmap_block, &buf)?; - - self.free_blocks += 1; - if block.raw() < self.next_alloc { - self.next_alloc = block.raw(); - } - Ok(()) - } - - /// Mark a contiguous range of blocks as used, one write per bitmap block. - /// A write the device refuses clears again what the run had marked, as far - /// as the device lets it, so a run is not left marked and named by nothing. - fn set_range_used(&self, io: &dyn BlockIO, start: BlockNum, count: u64) -> Result<(), FsError> { - let end = start.raw() + count; - let mut block = start.raw(); - while block < end { - match self.set_bits(io, block, end, true) { - Ok(next) => block = next, - Err(e) => { - let mut undo = start.raw(); - while undo < block { - match self.set_bits(io, undo, block, false) { - Ok(next) => undo = next, - Err(_) => break, - } - } - return Err(e); - } - } - } - Ok(()) - } - - /// Set or clear, in one read-modify-write, the bits of `from..end` that - /// lie in `from`'s bitmap block; the first block past them. - fn set_bits(&self, io: &dyn BlockIO, from: u64, end: u64, used: bool) -> Result { - let (bitmap_block, _, _) = self.bit_of(BlockNum::new(from)); - let mut buf = BlockBuf::zeroed(); - io.read(bitmap_block, &mut buf)?; - let mut block = from; - while block < end { - let (holding, byte_off, bit) = self.bit_of(BlockNum::new(block)); - if holding != bitmap_block { - break; - } - match used { - true => buf.0[byte_off] |= 1 << bit, - false => buf.0[byte_off] &= !(1 << bit), - } - block += 1; - } - io.write(bitmap_block, &buf)?; - Ok(block) - } - - /// Allocate a single block. - pub fn alloc_block(&mut self, io: &dyn BlockIO) -> Result { - Ok(self.alloc_exact(io, 1)?.start) - } - - /// Whether `block` is free. - pub fn is_free(&self, io: &dyn BlockIO, block: BlockNum) -> Result { - let (bitmap_block, byte_off, bit) = self.bit_of(block); - let mut buf = BlockBuf::zeroed(); - io.read(bitmap_block, &mut buf)?; - Ok(buf.0[byte_off] & (1 << bit) == 0) - } - - /// Reserve as much of `wanted` as one contiguous run can cover, scanning - /// from block `from`. - /// - /// The run is never empty and may be shorter than asked for, so every - /// caller has to loop or has to be wrong. - pub fn alloc_up_to(&mut self, io: &dyn BlockIO, from: u64, wanted: u32) -> Result { - let spare = self.spare(); - if spare == 0 { - return Err(FsError::NoSpace { requested: wanted, available: 0 }); - } - // A zero-length run would let a caller's loop spin without progress. - let wanted = wanted.max(1).min(u32::try_from(spare).unwrap_or(u32::MAX)); - let (start, len) = self.longest_free_run(io, from % self.total_blocks, wanted)?; - self.reserve(io, start, len.min(wanted)) - } - - /// Reserve a run at or after block `from`, of at most `wanted` blocks: the - /// first free run at least `min` long, or else the longest one seen once - /// the scan has gone round the volume or examined `budget` blocks. Each - /// block examined until a run is chosen comes off `budget`. `None` if no - /// free block was seen within it. - pub fn alloc_first_fit( - &mut self, - io: &dyn BlockIO, - from: u64, - wanted: u32, - min: u32, - budget: &mut u64, - ) -> Result, FsError> { - let spare = self.spare(); - if spare == 0 { - return Err(FsError::NoSpace { requested: wanted, available: 0 }); - } - let wanted = wanted.max(1).min(u32::try_from(spare).unwrap_or(u32::MAX)); - let min = min.clamp(1, wanted); - let total = self.total_blocks; - let chosen = { - let mut cached = (u64::MAX, BlockBuf::zeroed()); - let mut free = |pos: u64| -> Result { - let (bitmap_block, byte_off, bit) = self.bit_of(BlockNum::new(pos)); - if cached.0 != bitmap_block.raw() { - io.read(bitmap_block, &mut cached.1)?; - cached.0 = bitmap_block.raw(); - } - Ok(cached.1 .0[byte_off] & (1 << bit) == 0) - }; - let from = from % total; - // (start, length) of the run being walked, and of the longest seen. - let (mut run, mut best) = ((0u64, 0u32), (0u64, 0u32)); - let mut found = None; - for pos in (from..total).chain(0..from) { - // Block 0 does not follow the volume's last block. - if pos == 0 { - run.1 = 0; - } - if *budget == 0 { - break; - } - *budget -= 1; - if !free(pos)? { - run.1 = 0; - continue; - } - if run.1 == 0 { - run.0 = pos; - } - run.1 += 1; - if run.1 > best.1 { - best = run; - } - if run.1 >= min { - found = Some(run); - break; - } - } - match found { - Some((start, mut len)) => { - while len < wanted && start + u64::from(len) < total && free(start + u64::from(len))? { - len += 1; - } - Some((start, len)) - } - None => (best.1 > 0).then_some(best), - } - }; - match chosen { - Some((start, len)) => self.reserve(io, start, len).map(Some), - None => Ok(None), - } - } - - /// Reserve all of `count` or nothing, for callers that cannot place a - /// short run. Nothing is marked used unless the whole run is there. - pub fn alloc_exact(&mut self, io: &dyn BlockIO, count: u32) -> Result { - let spare = self.spare(); - if spare < count as u64 { - return Err(FsError::NoSpace { requested: count, available: spare }); - } - let (start, len) = self.longest_free_run(io, self.next_alloc, count)?; - if len < count { - return Err(FsError::NoSpace { - requested: count, - available: self.free_blocks, - }); - } - self.reserve(io, start, count) - } - - /// Mark a run used and move the cursor past it. - /// - /// The counters follow the bitmap write for the reason `set_free`'s do: a - /// reservation the device would not record is not a reservation, and - /// deducting it would lose blocks nothing ever gets back. - fn reserve(&mut self, io: &dyn BlockIO, start: u64, len: u32) -> Result { - let start_block = BlockNum::new(start); - self.set_range_used(io, start_block, len as u64)?; - self.fresh.insert(start, len as u64); - if let Some(op) = &mut self.op { - op.taken.insert(start, len as u64); - } - self.free_blocks -= len as u64; - self.next_alloc = start + len as u64; - if self.next_alloc >= self.total_blocks { - self.next_alloc = 0; - } - Ok(Run { start: start_block, len }) - } - - /// The longest free run found scanning from block `start_pos`, wrapping - /// once, stopping early once `wanted` blocks are in hand. - fn longest_free_run(&self, io: &dyn BlockIO, start_pos: u64, wanted: u32) -> Result<(u64, u32), FsError> { - if self.free_blocks == 0 { - return Err(FsError::NoSpace { - requested: wanted, - available: 0, - }); - } - - let total = self.total_blocks; - let mut best_start = None; - let mut best_count = 0u32; - - // Scan from start_pos, wrap once - let mut pos = start_pos; - let mut wrapped = false; - let mut run_start = None; - let mut run_count = 0u32; - - // Cache the current bitmap block to avoid re-reading for every bit - let mut cached_bitmap_block = u64::MAX; - let mut cached_buf = BlockBuf::zeroed(); - - loop { - if wrapped && pos >= start_pos { - break; - } - if pos >= total { - if wrapped { - break; - } - wrapped = true; - pos = 0; - run_start = None; - run_count = 0; - continue; - } - - // Read bitmap block if not cached - let byte_idx = pos / 8; - let bblock = self.bitmap_start.raw() + byte_idx / BLOCK_SIZE as u64; - if bblock != cached_bitmap_block { - io.read(BlockNum::new(bblock), &mut cached_buf)?; - cached_bitmap_block = bblock; - } - - let byte_off = (byte_idx % BLOCK_SIZE as u64) as usize; - let bit_idx = pos % 8; - let is_free = (cached_buf.0[byte_off] >> bit_idx) & 1 == 0; - - if is_free { - if run_start.is_none() { - run_start = Some(pos); - run_count = 0; - } - run_count += 1; - - if run_count >= wanted { - // Found exactly what we wanted - best_start = run_start; - best_count = run_count; - break; - } - - if run_count > best_count { - best_start = run_start; - best_count = run_count; - } - } else { - run_start = None; - run_count = 0; - } - - pos += 1; - } - - let start = best_start.ok_or(FsError::NoSpace { - requested: wanted, - available: self.free_blocks, - })?; - - Ok((start, best_count)) - } - - /// Give up a contiguous range of blocks: inside an operation, once it - /// succeeds. - pub fn free_range( - &mut self, - io: &dyn BlockIO, - start: BlockNum, - count: u32, - ) -> Result<(), FsError> { - match &mut self.op { - Some(op) => { - op.given.push((start.raw(), count)); - Ok(()) - } - None => self.give(io, start.raw(), count), - } - } - - /// Initialize bitmap on disk: zero all bitmap blocks, then mark metadata blocks as used. - pub fn format( - io: &dyn BlockIO, - bitmap_start: BlockNum, - bitmap_blocks: u64, - total_blocks: u64, - metadata_blocks: u64, - ) -> Result { - let zero = BlockBuf::zeroed(); - for i in 0..bitmap_blocks { - io.write(BlockNum::new(bitmap_start.raw() + i), &zero)?; - } - - let mut alloc = Self { - bitmap_start, - bitmap_blocks, - total_blocks, - free_blocks: total_blocks - metadata_blocks, - next_alloc: metadata_blocks, - shadow: false, - fresh: Runs::default(), - pending: Vec::new(), - op: None, - }; - - // Metadata blocks (superblock, bitmap, journal area), and the backup - // superblock at the far end. - for i in 0..metadata_blocks { - alloc.set_used(io, BlockNum::new(i))?; - } - alloc.set_used(io, BlockNum::new(total_blocks - 1))?; - alloc.free_blocks -= 1; - - Ok(alloc) - } -} - -#[cfg(all(test, feature = "std"))] -mod tests { - use super::*; - use crate::block_io::{DeviceError, VecBlockIO}; - use core::cell::{Cell, RefCell}; - - /// A bitmap device that records the blocks written to it, and refuses a - /// write to `refuse` once armed. - struct Counted { - io: VecBlockIO, - written: RefCell>, - refuse: Cell>, - } - - impl BlockIO for Counted { - fn read_block(&self, block: BlockNum, buf: &mut BlockBuf) -> Result<(), DeviceError> { - self.io.read_block(block, buf) - } - fn write_block(&self, block: BlockNum, buf: &BlockBuf) -> Result<(), DeviceError> { - if self.refuse.get() == Some(block.raw()) { - return Err(DeviceError::FAILED); - } - self.written.borrow_mut().push(block.raw()); - self.io.write_block(block, buf) - } - fn block_count(&self) -> u64 { - self.io.block_count() - } - fn sync(&self) -> Result<(), DeviceError> { - Ok(()) - } - } - - /// A volume of two bitmap blocks; the device holds only the bitmap. - fn two_bitmap_blocks() -> (Counted, BitmapAllocator) { - let io = Counted { io: VecBlockIO::new(2), written: RefCell::default(), refuse: Cell::new(None) }; - let alloc = BitmapAllocator::format(&io, BlockNum::new(0), 2, 40_000, 2).expect("format"); - io.written.borrow_mut().clear(); - (io, alloc) - } - - /// A run across the boundary of two bitmap blocks writes each once and - /// marks its blocks, and only those. - #[test] - fn a_run_across_two_bitmap_blocks_writes_each_once() { - let (io, mut alloc) = two_bitmap_blocks(); - let free = alloc.free_blocks; - let at = BITS_PER_BLOCK - 8; - let mut budget = u64::MAX; - let run = alloc.alloc_first_fit(&io, at, 16, 1, &mut budget).expect("a run").expect("a free block"); - assert_eq!((run.start.raw(), run.len), (at, 16)); - assert_eq!(*io.written.borrow(), [0, 1]); - for (block, taken) in [(at - 1, false), (at, true), (at + 15, true), (at + 16, false)] { - assert_eq!(alloc.is_free(&io, BlockNum::new(block)).unwrap(), !taken, "block {block}"); - } - alloc.count_free(&io).expect("count"); - assert_eq!(alloc.free_blocks, free - 16); - } - - /// A run whose second bitmap block the device refuses is refused, and - /// what it marked in the first is cleared again. - #[test] - fn a_run_refused_halfway_leaves_nothing_marked() { - let (io, mut alloc) = two_bitmap_blocks(); - let free = alloc.free_blocks; - io.refuse.set(Some(1)); - let mut budget = u64::MAX; - assert!(alloc.alloc_first_fit(&io, BITS_PER_BLOCK - 8, 16, 1, &mut budget).is_err()); - alloc.count_free(&io).expect("count"); - assert_eq!(alloc.free_blocks, free, "blocks stayed marked that nothing names"); - } -} diff --git a/lib/bcachefs/src/block_io.rs b/lib/bcachefs/src/block_io.rs deleted file mode 100644 index 4e62dce58d0..00000000000 --- a/lib/bcachefs/src/block_io.rs +++ /dev/null @@ -1,307 +0,0 @@ -use core::fmt; - -use crate::fs::FsError; - -pub const BLOCK_SIZE: usize = 4096; - -/// A block number on disk. Cannot be confused with a byte offset. -/// -/// It cannot be *turned into* one here either: the conversion this type used to -/// carry was `self.0 * BLOCK_SIZE`, an unchecked multiply on a number a btree -/// inside the image chose, and nothing in the tree ever called it. [`byte_range`] -/// is the one way a block becomes bytes, and it is `checked_mul` for the reason -/// its own doc gives. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct BlockNum(u64); - -impl BlockNum { - pub const fn new(n: u64) -> Self { - Self(n) - } - - pub const fn raw(self) -> u64 { - self.0 - } -} - -impl fmt::Display for BlockNum { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "block#{}", self.0) - } -} - -/// A 4096-byte aligned block buffer. Guarantees correct size at compile time. -#[repr(C, align(4096))] -pub struct BlockBuf(pub [u8; BLOCK_SIZE]); - -impl BlockBuf { - pub fn zeroed() -> Self { - Self([0u8; BLOCK_SIZE]) - } - - pub fn as_bytes(&self) -> &[u8; BLOCK_SIZE] { - &self.0 - } - - pub fn as_bytes_mut(&mut self) -> &mut [u8; BLOCK_SIZE] { - &mut self.0 - } -} - -impl Default for BlockBuf { - fn default() -> Self { - Self::zeroed() - } -} - -/// The two variants mean opposite give-up policies. Each carries a witness only -/// this crate can mint, so rustc's fix-it for the old erasure — which -/// re-collapses a still-durable `Refused` into `Failed`, the loss #327 removed -/// — no longer compiles, nor can the witness be named outside the crate: -/// -/// ```compile_fail -/// let r: Result<(), u32> = Err(7); -/// let _: Result<(), bcachefs::DeviceError> = r.map_err(|_| bcachefs::DeviceError::Failed); -/// ``` -/// ```compile_fail -/// use bcachefs::Sealed; -/// ``` -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum DeviceError { - /// The device's own word: the transfer was attempted and failed. - Failed(Sealed), - /// Refused before it was attempted, on the caller's time budget; safe to - /// ask again. - Refused(Sealed), -} - -/// The variants' witness: unmintable (private field) and unnameable (private module) outside the crate. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Sealed(()); - -/// What a foreign error answers at [`DeviceError::classify`]; the answer is the discriminant. -pub trait TransferError { - /// True when the transfer was never attempted — refused on budget, still durable. - fn refused_before_attempt(&self) -> bool; -} - -impl DeviceError { - pub(crate) const FAILED: Self = Self::Failed(Sealed(())); - pub(crate) const REFUSED: Self = Self::Refused(Sealed(())); - - /// The only constructor reachable outside the crate. - pub fn classify(err: &E) -> Self { - if err.refused_before_attempt() { - Self::REFUSED - } else { - Self::FAILED - } - } -} - -/// The byte range block `block` occupies in an image, or `None` when that -/// range cannot be expressed at all. -/// -/// **`checked_mul`, and that is not a spelling choice.** A block number reaches -/// this crate out of a btree that lives in the image it indexes, so a corrupt -/// or hostile image names any `u64` it likes — and `0x0020_0000_0000_0000 * 4096` -/// wraps. Under the kernel's build profile (`overflow-checks = true`) a wrapping -/// multiply is a *panic in the kernel* from input that crossed a trust boundary, -/// and under a release profile it is a small in-range offset into somebody -/// else's memory. Both are refusals here. -/// -/// One function rather than the same three lines at each of the four call -/// sites: the bound every image-backed reader applies is this one. -fn byte_range(block: BlockNum) -> Option<(usize, usize)> { - let off = (block.raw() as usize).checked_mul(BLOCK_SIZE)?; - let end = off.checked_add(BLOCK_SIZE)?; - Some((off, end)) -} - -/// Block-level I/O abstraction. -/// -/// `&self` with interior mutability — implementations handle their own -/// synchronization. `buf` is always exactly BLOCK_SIZE bytes via BlockBuf. -/// -/// Every method is fallible for the reason every [`BlockDevice`] method is: a -/// block the device would not give back is not a block of zeros, and an -/// implementation with nowhere to report that has to invent one. The kernel's -/// did — it logged and served zeros, so a read error reached the btree as a -/// block that fails its structural checks rather than as a failure. -/// -/// [`BlockDevice`]: ../../kernel/src/block.rs -pub trait BlockIO { - #[must_use = "a refused read left the buffer holding whatever it held before"] - fn read_block(&self, block: BlockNum, buf: &mut BlockBuf) -> Result<(), DeviceError>; - #[must_use = "a refused write did not reach the device"] - fn write_block(&self, block: BlockNum, buf: &BlockBuf) -> Result<(), DeviceError>; - fn block_count(&self) -> u64; - fn sync(&self) -> Result<(), DeviceError> { - Ok(()) - } -} - -/// The same three operations, reported as [`FsError`] with the block attached. -/// -/// Every call site inside this crate goes through these rather than through -/// [`BlockIO`] directly, so the block number in the error is the one the caller -/// passed in and there is no way for an implementation to name a different one. -pub(crate) trait BlockIOExt { - fn read(&self, block: BlockNum, buf: &mut BlockBuf) -> Result<(), FsError>; - fn write(&self, block: BlockNum, buf: &BlockBuf) -> Result<(), FsError>; - fn flush(&self) -> Result<(), FsError>; -} - -impl BlockIOExt for T { - fn read(&self, block: BlockNum, buf: &mut BlockBuf) -> Result<(), FsError> { - self.read_block(block, buf).map_err(|e| FsError::DeviceRead(block, e)) - } - - fn write(&self, block: BlockNum, buf: &BlockBuf) -> Result<(), FsError> { - self.write_block(block, buf).map_err(|e| FsError::DeviceWrite(block, e)) - } - - fn flush(&self) -> Result<(), FsError> { - self.sync().map_err(FsError::DeviceSync) - } -} - -// --- Host-side implementations --- - -/// In-memory block device backed by a Vec. Used by mkfs on the host. -#[cfg(feature = "std")] -pub struct VecBlockIO { - data: std::cell::RefCell>, -} - -#[cfg(feature = "std")] -impl VecBlockIO { - pub fn new(block_count: u64) -> Self { - let size = block_count as usize * BLOCK_SIZE; - Self { - data: std::cell::RefCell::new(vec![0u8; size]), - } - } - - pub fn from_vec(data: Vec) -> Self { - Self { - data: std::cell::RefCell::new(data), - } - } - - pub fn into_vec(self) -> Vec { - self.data.into_inner() - } -} - -#[cfg(feature = "std")] -impl BlockIO for VecBlockIO { - fn read_block(&self, block: BlockNum, buf: &mut BlockBuf) -> Result<(), DeviceError> { - let data = self.data.borrow(); - let (off, end) = byte_range(block).ok_or(DeviceError::FAILED)?; - buf.0.copy_from_slice(data.get(off..end).ok_or(DeviceError::FAILED)?); - Ok(()) - } - - fn write_block(&self, block: BlockNum, buf: &BlockBuf) -> Result<(), DeviceError> { - let mut data = self.data.borrow_mut(); - let (off, end) = byte_range(block).ok_or(DeviceError::FAILED)?; - data.get_mut(off..end).ok_or(DeviceError::FAILED)?.copy_from_slice(&buf.0); - Ok(()) - } - - fn block_count(&self) -> u64 { - (self.data.borrow().len() / BLOCK_SIZE) as u64 - } -} - - -/// The bound an image-backed reader applies to a block number the image itself -/// named, at every way of getting it wrong. -/// -/// It runs here rather than in a guest because there is nothing architectural -/// about it: a block number, a length, and the comparison between them. The -/// extent list lives *inside* the image, so a corrupt one names a block past -/// its own end and a reader without a length reads whatever follows. -#[cfg(all(test, feature = "std"))] -mod image_bounds { - use super::*; - - /// Three blocks of image, each byte stamped with its block number. - fn image(blocks: usize) -> VecBlockIO { - let mut v = vec![0u8; blocks * BLOCK_SIZE]; - for b in 0..blocks { - v[b * BLOCK_SIZE..(b + 1) * BLOCK_SIZE].fill(b as u8 + 1); - } - VecBlockIO::from_vec(v) - } - - #[test] - fn every_block_the_image_holds_is_served_whole() { - let io = image(3); - assert_eq!(io.block_count(), 3); - for b in 0..3u64 { - let mut buf = BlockBuf::zeroed(); - io.read_block(BlockNum::new(b), &mut buf).expect("block inside the image"); - assert!(buf.0.iter().all(|&x| x == b as u8 + 1), "block {b} served the wrong bytes"); - } - } - - #[test] - fn a_block_past_the_end_is_refused_rather_than_read() { - let io = image(3); - let mut buf = BlockBuf::zeroed(); - for b in [3u64, 4, u64::MAX / BLOCK_SIZE as u64] { - assert_eq!(io.read_block(BlockNum::new(b), &mut buf), Err(DeviceError::FAILED)); - } - } - - /// A block that *starts* inside the image and ends past it. `get(off..end)` - /// is what makes this a refusal; an `off < len` test would serve a short - /// read out of whatever follows. - #[test] - fn a_block_that_starts_inside_and_ends_outside_is_refused() { - let io = VecBlockIO::from_vec(vec![7u8; 2 * BLOCK_SIZE + 1]); - let mut buf = BlockBuf::zeroed(); - io.read_block(BlockNum::new(1), &mut buf).expect("the last whole block"); - assert_eq!( - io.read_block(BlockNum::new(2), &mut buf), - Err(DeviceError::FAILED), - "one byte of a block is not a block" - ); - } - - /// `block.raw() as usize * BLOCK_SIZE` wraps for a block number above - /// `usize::MAX / 4096`. The kernel builds this crate with - /// `overflow-checks = true`, so the unchecked multiply is a kernel panic - /// from a number an image chose — and without the checks it is a small - /// in-range offset into somebody else's memory. - #[test] - fn a_block_number_whose_byte_offset_wraps_is_refused_and_does_not_panic() { - let io = image(3); - let mut buf = BlockBuf::zeroed(); - for n in [ - usize::MAX as u64 / BLOCK_SIZE as u64 + 1, - u64::MAX, - 1u64 << 52, - (1u64 << 52) + 1, - ] { - assert_eq!(byte_range(BlockNum::new(n)), None, "block {n:#x} named a byte range"); - assert_eq!(io.read_block(BlockNum::new(n), &mut buf), Err(DeviceError::FAILED)); - } - // The largest block number that does express a range still names one - // no image holds. - let last = usize::MAX as u64 / BLOCK_SIZE as u64 - 1; - assert!(byte_range(BlockNum::new(last)).is_some()); - assert_eq!(io.read_block(BlockNum::new(last), &mut buf), Err(DeviceError::FAILED)); - } - - /// An empty image holds no block, including block 0 — the superblock's. - #[test] - fn an_empty_image_holds_no_block_at_all() { - let io = VecBlockIO::from_vec(Vec::new()); - let mut buf = BlockBuf::zeroed(); - assert_eq!(io.block_count(), 0); - assert_eq!(io.read_block(BlockNum::new(0), &mut buf), Err(DeviceError::FAILED)); - } -} diff --git a/lib/bcachefs/src/btree.rs b/lib/bcachefs/src/btree.rs deleted file mode 100644 index eb642d1199f..00000000000 --- a/lib/bcachefs/src/btree.rs +++ /dev/null @@ -1,963 +0,0 @@ -use alloc::vec::Vec; -use core::ops::ControlFlow; -use crate::block_io::{BlockBuf, BlockNum, BlockIO, BlockIOExt, BLOCK_SIZE}; -use crate::crc32c::crc32c; -use crate::alloc_bitmap::BitmapAllocator; -use crate::fs::FsError; - -pub const NODE_MAGIC: [u8; 4] = *b"BTND"; -const NODE_HEADER_SIZE: usize = 32; -const KEY_HEADER_SIZE: usize = 24; -pub(crate) const CRC_START: usize = 8; // CRC covers bytes [8..4096] -const MAX_PAYLOAD: usize = BLOCK_SIZE - NODE_HEADER_SIZE; - -/// A child pointer is one block number, and nothing else ever appears in an -/// interior node's value. -const CHILD_VALUE_SIZE: usize = 8; -const CHILD_DISK_SIZE: usize = (KEY_HEADER_SIZE + CHILD_VALUE_SIZE + 7) & !7; - -/// The most entries a 4096-byte block can physically hold. -/// -/// The count is a `u16` on disk with 387 times this range, and it used to size -/// a `Vec` before a single byte of the block had been looked at. -const MAX_ENTRIES: usize = MAX_PAYLOAD / KEY_HEADER_SIZE; - -/// The largest `Entry::disk_size` any node can ever hold. -/// -/// A node holds at least one entry, so an entry bigger than this cannot be -/// stored at all — and splitting does not help: a split of one oversized entry -/// leaves an empty left node and a right node that is still oversized. It has -/// to be refused at the door, before anything is written, because every caller -/// that builds a value builds it out of something userland chose (a name, or -/// one extent per discontiguous run of a file). -pub const MAX_ENTRY_SIZE: usize = MAX_PAYLOAD; - -/// Total on-disk size of a leaf holding these entries. -fn leaf_size(entries: &[Entry]) -> usize { - NODE_HEADER_SIZE + entries.iter().map(|e| e.disk_size()).sum::() -} - -/// Reject an entry no node could hold. Call before mutating anything. -pub fn check_entry_fits(entry: &Entry) -> Result<(), FsError> { - let size = entry.disk_size(); - if size > MAX_ENTRY_SIZE { - return Err(FsError::EntryTooLarge { size, max: MAX_ENTRY_SIZE }); - } - Ok(()) -} - -/// How much further a descent is allowed to go. -/// -/// The shape of the tree is on the disk, and a disk is not ours. A child -/// pointer naming its own node, or any cycle at all, is a descent that never -/// reaches a leaf; nothing in the block format forbids one, so the bound is -/// carried by the only operation that can go deeper. -#[derive(Clone, Copy)] -struct Depth(u8); - -impl Depth { - /// A B+ tree whose interior nodes hold at least two children is at most - /// `log2(blocks)` deep, and a block number is a `u64`. - const ROOT: Self = Self(64); - - fn descend(self, from: BlockNum) -> Result { - match self.0.checked_sub(1) { - Some(left) => Ok(Self(left)), - None => Err(FsError::TreeTooDeep(from)), - } - } -} - -/// On-disk key stored in B+ tree nodes. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Key { - pub name_hash: u64, - pub name_hash_hi: u64, - pub key_type: KeyType, -} - -impl Key { - pub const ZERO: Self = Self { - name_hash: 0, - name_hash_hi: 0, - key_type: KeyType::Deleted, - }; -} - -impl PartialOrd for Key { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for Key { - fn cmp(&self, other: &Self) -> core::cmp::Ordering { - self.name_hash - .cmp(&other.name_hash) - .then(self.name_hash_hi.cmp(&other.name_hash_hi)) - .then((self.key_type as u16).cmp(&(other.key_type as u16))) - } -} - -#[repr(u16)] -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum KeyType { - Deleted = 0, - File = 1, - Symlink = 2, -} - -impl TryFrom for KeyType { - type Error = FsError; - fn try_from(v: u16) -> Result { - match v { - 0 => Ok(Self::Deleted), - 1 => Ok(Self::File), - 2 => Ok(Self::Symlink), - _ => Err(FsError::CorruptedKey(v)), - } - } -} - -/// A key-value entry in a B+ tree leaf. -#[derive(Debug, Clone)] -pub struct Entry { - pub key: Key, - pub value: Vec, -} - -impl Entry { - /// Total size on disk: key header + value, padded to 8-byte alignment. - pub fn disk_size(&self) -> usize { - disk_size(self.value.len()) - } -} - -fn disk_size(value_len: usize) -> usize { - (KEY_HEADER_SIZE + value_len + 7) & !7 -} - -/// Whether an entry whose value is `value_len` bytes passes -/// [`check_entry_fits`], asked before the value is built. -pub fn value_fits(value_len: usize) -> bool { - disk_size(value_len) <= MAX_ENTRY_SIZE -} - -/// A child pointer: its separator, and the block the subtree lives in. Every -/// key of the subtree is at least the separator and below the next child's; -/// the first child's is never read ([`child_index`]). -#[derive(Debug, Clone, Copy)] -pub struct Child { - pub key: Key, - pub block: BlockNum, -} - -/// A parsed B+ tree node. -/// -/// The variant *is* the leaf/interior distinction. It is decided once, by -/// [`Node::parse`], which is also where a child pointer is turned into a -/// `BlockNum` — checked to be eight bytes long and to name a block that exists -/// on the device. Six descent sites used to do that decode themselves, from a -/// value whose length nothing had constrained. -/// -/// `level` rides along so the header round-trips; the only arithmetic on it is -/// the checked increment that gives a new root its level. -pub enum Node { - Leaf(Vec), - /// `children` is never empty: an interior node with no children is a - /// subtree with no bottom. - Interior { level: u16, children: Vec }, -} - -impl Node { - fn level(&self) -> u16 { - match self { - Node::Leaf(_) => 0, - Node::Interior { level, .. } => *level, - } - } - - fn count(&self) -> usize { - match self { - Node::Leaf(entries) => entries.len(), - Node::Interior { children, .. } => children.len(), - } - } - - /// Bytes this node's entries occupy after the header. - fn payload_size(&self) -> usize { - match self { - Node::Leaf(entries) => entries.iter().map(|e| e.disk_size()).sum(), - Node::Interior { children, .. } => children.len() * CHILD_DISK_SIZE, - } - } - - /// Read and parse a node from disk, verifying magic and CRC. - pub fn read(io: &dyn BlockIO, block: BlockNum) -> Result { - let device_blocks = io.block_count(); - let mut buf = BlockBuf::zeroed(); - io.read(block, &mut buf)?; - Self::parse(&buf, block, device_blocks) - } - - fn parse(buf: &BlockBuf, block: BlockNum, device_blocks: u64) -> Result { - let b = buf.as_bytes(); - - let magic = [b[0], b[1], b[2], b[3]]; - if magic != NODE_MAGIC { - return Err(FsError::BadMagic { - expected: NODE_MAGIC, - got: magic, - }); - } - - let stored_crc = u32::from_le_bytes([b[4], b[5], b[6], b[7]]); - let computed_crc = crc32c(&b[CRC_START..]); - if stored_crc != computed_crc { - return Err(FsError::ChecksumMismatch { - block, - stored: stored_crc, - computed: computed_crc, - }); - } - - let level = u16::from_le_bytes([b[8], b[9]]); - let entry_count = u16::from_le_bytes([b[10], b[11]]) as usize; - if entry_count > MAX_ENTRIES { - return Err(FsError::CorruptedNode(block)); - } - - // Grown, not reserved: `entry_count` is a number off the disk, and - // reserving from it asked the kernel allocator for 3,145,680 bytes - // against a 2 MiB ceiling. What the block can actually hold is what - // gets allocated, and the refusal above is the fail-fast on top. - let mut entries: Vec = Vec::new(); - let mut offset = NODE_HEADER_SIZE; - - for _ in 0..entry_count { - if offset + KEY_HEADER_SIZE > BLOCK_SIZE { - return Err(FsError::CorruptedNode(block)); - } - - let name_hash = read_u64(b, offset); - let name_hash_hi = read_u64(b, offset + 8); - let key_type_raw = u16::from_le_bytes([b[offset + 16], b[offset + 17]]); - let val_len = u32::from_le_bytes([ - b[offset + 18], - b[offset + 19], - b[offset + 20], - b[offset + 21], - ]) as usize; - - let key_type = KeyType::try_from(key_type_raw)?; - - let val_start = offset + KEY_HEADER_SIZE; - let val_end = val_start + val_len; - if val_end > BLOCK_SIZE { - return Err(FsError::CorruptedNode(block)); - } - - entries.push(Entry { - key: Key { name_hash, name_hash_hi, key_type }, - value: b[val_start..val_end].to_vec(), - }); - - offset = (val_end + 7) & !7; - } - - if level == 0 { - return Ok(Node::Leaf(entries)); - } - if entries.is_empty() { - return Err(FsError::CorruptedNode(block)); - } - - let mut children = Vec::with_capacity(entries.len()); - for entry in &entries { - let raw = entry - .value - .get(..CHILD_VALUE_SIZE) - .ok_or(FsError::CorruptedNode(block))?; - let mut bytes = [0u8; CHILD_VALUE_SIZE]; - bytes.copy_from_slice(raw); - let child = u64::from_le_bytes(bytes); - if child >= device_blocks { - return Err(FsError::BlockOffDevice { block: child, device_blocks }); - } - children.push(Child { key: entry.key, block: BlockNum::new(child) }); - } - Ok(Node::Interior { level, children }) - } - - /// Serialize this node to a block buffer, computing the CRC. - /// - /// Fallible rather than asserting: `used` is a sum over values whose size - /// userland chooses, so an overfull node is an input to reject, not a - /// kernel bug to scream about. This bound is also what keeps every slice - /// index below sound — the final offset is `NODE_HEADER_SIZE + used`. - pub fn write_to(&self, buf: &mut BlockBuf) -> Result<(), FsError> { - let used = self.payload_size(); - if used > MAX_PAYLOAD { - return Err(FsError::NodeOverfull { used, max: MAX_PAYLOAD }); - } - - let b = buf.as_bytes_mut(); - b.fill(0); - - b[0..4].copy_from_slice(&NODE_MAGIC); - // CRC at [4..8] filled last - b[8..10].copy_from_slice(&self.level().to_le_bytes()); - b[10..12].copy_from_slice(&(self.count() as u16).to_le_bytes()); - - let free_space = (MAX_PAYLOAD - used) as u32; - b[12..16].copy_from_slice(&free_space.to_le_bytes()); - - let mut offset = NODE_HEADER_SIZE; - match self { - Node::Leaf(entries) => { - for entry in entries { - offset = write_entry(b, offset, &entry.key, &entry.value); - } - } - Node::Interior { children, .. } => { - for child in children { - offset = write_entry(b, offset, &child.key, &child.block.raw().to_le_bytes()); - } - } - } - - let crc = crc32c(&b[CRC_START..]); - b[4..8].copy_from_slice(&crc.to_le_bytes()); - Ok(()) - } - - /// Write this node to disk. - pub fn write(&self, io: &dyn BlockIO, block: BlockNum) -> Result<(), FsError> { - let mut buf = BlockBuf::zeroed(); - self.write_to(&mut buf)?; - io.write(block, &buf)?; - Ok(()) - } -} - -fn read_u64(b: &[u8; BLOCK_SIZE], off: usize) -> u64 { - let mut bytes = [0u8; 8]; - bytes.copy_from_slice(&b[off..off + 8]); - u64::from_le_bytes(bytes) -} - -/// Write one key header and its value, returning the next entry's offset. -fn write_entry(b: &mut [u8; BLOCK_SIZE], offset: usize, key: &Key, value: &[u8]) -> usize { - b[offset..offset + 8].copy_from_slice(&key.name_hash.to_le_bytes()); - b[offset + 8..offset + 16].copy_from_slice(&key.name_hash_hi.to_le_bytes()); - b[offset + 16..offset + 18].copy_from_slice(&(key.key_type as u16).to_le_bytes()); - b[offset + 18..offset + 22].copy_from_slice(&(value.len() as u32).to_le_bytes()); - // [22..24] reserved = 0 - - let val_start = offset + KEY_HEADER_SIZE; - b[val_start..val_start + value.len()].copy_from_slice(value); - (val_start + value.len() + 7) & !7 -} - -// --- B+ tree operations --- - -/// The child to descend into for `key`. -/// -/// The children after the first are in order of their separators, so the -/// answer is the last of them whose separator is `<= key`, or the first child -/// for a key below them all. The first child's own key goes unread: it is -/// below every key routed to it only until an insert makes a new least key. -fn find_child(children: &[Child], key: &Key) -> Option { - children.get(child_index(children, key)).map(|c| c.block) -} - -/// [`find_child`]'s answer as an index into `children`. -fn child_index(children: &[Child], key: &Key) -> usize { - children.iter().skip(1).take_while(|c| c.key <= *key).count() -} - -/// The block a node the operation in progress changed is written to: its -/// own where [`BitmapAllocator::in_place`] allows, and otherwise a new one, -/// the old given up. -fn own(io: &dyn BlockIO, alloc: &mut BitmapAllocator, block: BlockNum) -> Result { - if alloc.in_place(block) { - return Ok(block); - } - let moved = alloc.alloc_block(io)?; - alloc.free_range(io, block, 1)?; - Ok(moved) -} - -/// Search the B+ tree for an exact key match. Returns the leaf entry's value. -pub fn search(io: &dyn BlockIO, root: BlockNum, key: &Key) -> Result>, FsError> { - let mut block = root; - let mut depth = Depth::ROOT; - - loop { - match Node::read(io, block)? { - Node::Leaf(entries) => { - return Ok(entries.into_iter().find(|e| e.key == *key).map(|e| e.value)); - } - Node::Interior { children, .. } => { - let next = find_child(&children, key).ok_or(FsError::CorruptedNode(block))?; - depth = depth.descend(block)?; - block = next; - } - } - } -} - -/// Search the B+ tree for all entries with a given `name_hash`. -/// Returns all matching entries (there may be multiple due to hash collisions -/// or multiple key_types with the same hash). -pub fn search_by_hash( - io: &dyn BlockIO, - root: BlockNum, - name_hash: u64, -) -> Result, FsError> { - // Use the MAXIMUM possible key for this name_hash so we descend to the - // rightmost child that could contain any entry with this hash. - // Then scan the leaf — entries with matching name_hash will be there - // because they sort between (name_hash, 0, Deleted) and (name_hash, MAX, MAX). - let search_key = Key { - name_hash, - name_hash_hi: u64::MAX, - key_type: KeyType::Symlink, // highest key_type value - }; - - let mut block = root; - let mut depth = Depth::ROOT; - - loop { - match Node::read(io, block)? { - Node::Leaf(entries) => { - return Ok(entries - .into_iter() - .filter(|e| e.key.name_hash == name_hash && e.key.key_type != KeyType::Deleted) - .collect()); - } - Node::Interior { children, .. } => { - let next = - find_child(&children, &search_key).ok_or(FsError::CorruptedNode(block))?; - depth = depth.descend(block)?; - block = next; - } - } - } -} - -/// Delete an exact key from the B+ tree: the root after it and the old value, -/// or `None` with nothing written when no entry has the key. -/// Does not merge underflowing nodes — just removes the entry from the leaf. -pub fn delete( - io: &dyn BlockIO, - alloc: &mut BitmapAllocator, - root: BlockNum, - key: &Key, -) -> Result)>, FsError> { - delete_recursive(io, alloc, root, Depth::ROOT, key) -} - -fn delete_recursive( - io: &dyn BlockIO, - alloc: &mut BitmapAllocator, - block: BlockNum, - depth: Depth, - key: &Key, -) -> Result)>, FsError> { - match Node::read(io, block)? { - Node::Leaf(mut entries) => { - let Some(pos) = entries.iter().position(|e| e.key == *key) else { - return Ok(None); - }; - let old = entries.remove(pos); - let at = own(io, alloc, block)?; - Node::Leaf(entries).write(io, at)?; - Ok(Some((at, old.value))) - } - Node::Interior { level, mut children } => { - let idx = child_index(&children, key); - let child = children.get(idx).ok_or(FsError::CorruptedNode(block))?.block; - let Some((moved, old)) = delete_recursive(io, alloc, child, depth.descend(block)?, key)? else { - return Ok(None); - }; - if moved == child { - return Ok(Some((block, old))); - } - children[idx].block = moved; - let at = own(io, alloc, block)?; - Node::Interior { level, children }.write(io, at)?; - Ok(Some((at, old))) - } - } -} - -/// Every live leaf entry, one at a time, in tree order. Nothing accumulates -/// here: what a caller keeps is the only allocation the tree's size can drive, -/// so a bound applied inside `visit` is applied before the entry lands. The -/// walk ends at the first `Break`, and returns it. -pub fn for_each_live( - io: &dyn BlockIO, - root: BlockNum, - visit: &mut dyn FnMut(Entry) -> Result, FsError>, -) -> Result, FsError> { - walk_recursive(io, root, Depth::ROOT, visit) -} - -fn walk_recursive( - io: &dyn BlockIO, - block: BlockNum, - depth: Depth, - visit: &mut dyn FnMut(Entry) -> Result, FsError>, -) -> Result, FsError> { - match Node::read(io, block)? { - Node::Leaf(entries) => { - for entry in entries.into_iter().filter(|e| e.key.key_type != KeyType::Deleted) { - if visit(entry)?.is_break() { - return Ok(ControlFlow::Break(())); - } - } - } - Node::Interior { children, .. } => { - let deeper = depth.descend(block)?; - for child in children { - if walk_recursive(io, child.block, deeper, visit)?.is_break() { - return Ok(ControlFlow::Break(())); - } - } - } - } - Ok(ControlFlow::Continue(())) -} - -/// Insert a key-value pair into the B+ tree. -/// -/// Returns the root block, which changes when the old root was split or -/// written somewhere new. -pub fn insert( - io: &dyn BlockIO, - alloc: &mut BitmapAllocator, - root: BlockNum, - entry: Entry, -) -> Result { - check_entry_fits(&entry)?; - - let Placed { at, split } = insert_recursive(io, alloc, root, Depth::ROOT, entry)?; - if split.is_empty() { - return Ok(at); - } - let level = Node::read(io, at)? - .level() - .checked_add(1) - .ok_or(FsError::CorruptedNode(at))?; - let new_root_block = alloc.alloc_block(io)?; - - // A first child's key is never read (`child_index`). - let mut children = alloc::vec![Child { key: Key::ZERO, block: at }]; - children.extend(split); - Node::Interior { level, children }.write(io, new_root_block)?; - - Ok(new_root_block) -} - -/// Where an insert left a node: its block, and the siblings it split off, in -/// key order. -struct Placed { - at: BlockNum, - split: Vec, -} - -fn insert_recursive( - io: &dyn BlockIO, - alloc: &mut BitmapAllocator, - block: BlockNum, - depth: Depth, - entry: Entry, -) -> Result { - match Node::read(io, block)? { - Node::Leaf(mut entries) => { - match entries.binary_search_by(|e| e.key.cmp(&entry.key)) { - Ok(i) => entries[i] = entry, - Err(i) => entries.insert(i, entry), - } - write_or_split(io, alloc, block, Node::Leaf(entries)) - } - Node::Interior { level, mut children } => { - let idx = child_index(&children, &entry.key); - let child_block = children.get(idx).ok_or(FsError::CorruptedNode(block))?.block; - let deeper = depth.descend(block)?; - - let Placed { at, split } = insert_recursive(io, alloc, child_block, deeper, entry)?; - if at == child_block && split.is_empty() { - return Ok(Placed { at: block, split }); - } - children[idx].block = at; - // The siblings hold the keys of `idx` past those it kept, in order. - children.splice(idx + 1..idx + 1, split); - write_or_split(io, alloc, block, Node::Interior { level, children }) - } - } -} - -fn write_or_split( - io: &dyn BlockIO, - alloc: &mut BitmapAllocator, - block: BlockNum, - node: Node, -) -> Result { - let at = own(io, alloc, block)?; - if NODE_HEADER_SIZE + node.payload_size() <= BLOCK_SIZE { - node.write(io, at)?; - return Ok(Placed { at, split: Vec::new() }); - } - Ok(Placed { at, split: split_node(io, alloc, at, node)? }) -} - -/// Split `node` across `block` and new siblings, answering the siblings. The -/// blocks it took are the operation's, which gives them back if it fails. -fn split_node( - io: &dyn BlockIO, - alloc: &mut BitmapAllocator, - block: BlockNum, - node: Node, -) -> Result, FsError> { - match node { - Node::Leaf(entries) => { - // One entry is not a split problem. Halving by *count* used to - // produce `mid == 0` here, which drained every entry into the right - // node and left an empty one behind — and the right node was still - // the oversized entry. - if entries.len() < 2 { - let size = entries.first().map_or(0, |e| e.disk_size()); - return Err(FsError::EntryTooLarge { size, max: MAX_ENTRY_SIZE }); - } - - let mut nodes = pack(entries); - let first = nodes.remove(0); - let mut children = Vec::with_capacity(nodes.len()); - for node in nodes { - let sibling = alloc.alloc_block(io)?; - children.push(Child { key: node[0].key, block: sibling }); - Node::Leaf(node).write(io, sibling)?; - } - Node::Leaf(first).write(io, block)?; - - Ok(children) - } - Node::Interior { level, mut children } => { - if children.len() < 2 { - return Err(FsError::CorruptedNode(block)); - } - // Every child costs the same 32 bytes, so halving by count is - // halving by bytes — the rule leaves are not allowed to use. - let mid = children.len() / 2; - let right: Vec = children.drain(mid..).collect(); - let Some(split_key) = right.first().map(|c| c.key) else { - return Err(FsError::CorruptedNode(block)); - }; - - let right_block = alloc.alloc_block(io)?; - Node::Interior { level, children }.write(io, block)?; - Node::Interior { level, children: right }.write(io, right_block)?; - - Ok(alloc::vec![Child { key: split_key, block: right_block }]) - } - } -} - -/// `entries` in order, each node filled before the next is begun. Every entry -/// fits a node alone ([`check_entry_fits`]), so a large entry that lands -/// between small ones takes a node of its own: a split in two has no point -/// that leaves both sides within a block for that shape. -fn pack(entries: Vec) -> Vec> { - let mut nodes: Vec> = Vec::new(); - let mut used = BLOCK_SIZE; - for entry in entries { - if used + entry.disk_size() > BLOCK_SIZE { - nodes.push(Vec::new()); - used = NODE_HEADER_SIZE; - } - used += entry.disk_size(); - nodes.last_mut().expect("a node was begun").push(entry); - } - nodes -} - -#[cfg(test)] -mod tests { - use super::*; - use alloc::vec; - - fn entry(value_len: usize) -> Entry { - Entry { key: Key::ZERO, value: vec![0u8; value_len] } - } - - /// A block holding a node header the caller chose and no valid entries. - /// The CRC is computed last, so every case below is a block the parser - /// accepts as authentic — which is the point: a checksum says the bytes - /// are the bytes somebody wrote. - fn crafted(level: u16, entry_count: u16, entries: &[(u32, &[u8])]) -> BlockBuf { - let mut buf = BlockBuf::zeroed(); - let b = buf.as_bytes_mut(); - b[0..4].copy_from_slice(&NODE_MAGIC); - b[8..10].copy_from_slice(&level.to_le_bytes()); - b[10..12].copy_from_slice(&entry_count.to_le_bytes()); - let mut offset = NODE_HEADER_SIZE; - for (declared_len, value) in entries { - b[offset + 18..offset + 22].copy_from_slice(&declared_len.to_le_bytes()); - let val_start = offset + KEY_HEADER_SIZE; - b[val_start..val_start + value.len()].copy_from_slice(value); - offset = (val_start + *declared_len as usize + 7) & !7; - } - let crc = crc32c(&b[CRC_START..]); - b[4..8].copy_from_slice(&crc.to_le_bytes()); - buf - } - - fn counts(nodes: &[Vec]) -> Vec { - nodes.iter().map(Vec::len).collect() - } - - #[test] - fn pack_fills_each_node_in_order_and_none_past_a_block() { - // Two entries that only fit apart. - assert_eq!(counts(&pack(vec![entry(3000), entry(3000)])), [1, 1]); - // A small entry ahead of two large ones, which halving by count - // leaves 6048 bytes in one node. - assert_eq!(counts(&pack(vec![entry(1000), entry(3000), entry(3000)])), [2, 1]); - // The largest entry between small ones, which no split in two fits: - // a leaf of names where one file's entry has grown. - let mut middle: Vec = (0..40).map(|_| entry(72)).collect(); - middle.insert(20, entry(MAX_ENTRY_SIZE - KEY_HEADER_SIZE)); - let nodes = pack(middle); - assert_eq!(counts(&nodes), [20, 1, 20]); - assert!(nodes.iter().all(|n| leaf_size(n) <= BLOCK_SIZE)); - } - - #[test] - fn an_entry_larger_than_the_payload_is_refused() { - assert!(check_entry_fits(&entry(MAX_ENTRY_SIZE - KEY_HEADER_SIZE)).is_ok()); - assert!(matches!( - check_entry_fits(&entry(MAX_ENTRY_SIZE)), - Err(FsError::EntryTooLarge { .. }), - )); - } - - #[test] - fn write_to_refuses_an_overfull_node_instead_of_underflowing() { - let node = Node::Leaf(vec![entry(3000), entry(3000)]); - let mut buf = BlockBuf::zeroed(); - assert!(matches!(node.write_to(&mut buf), Err(FsError::NodeOverfull { .. }))); - } - - #[test] - fn an_interior_child_shorter_than_a_block_number_is_refused() { - // Four bytes where a child pointer belongs. Six descent sites used to - // index `value[..8]` here. - let buf = crafted(1, 1, &[(4, &[1, 0, 0, 0])]); - assert!(matches!( - Node::parse(&buf, BlockNum::new(7), 64), - Err(FsError::CorruptedNode(_)), - )); - } - - #[test] - fn an_interior_node_with_no_children_is_refused() { - let buf = crafted(1, 0, &[]); - assert!(matches!( - Node::parse(&buf, BlockNum::new(7), 64), - Err(FsError::CorruptedNode(_)), - )); - } - - #[test] - fn a_child_pointer_off_the_device_is_refused() { - let buf = crafted(1, 1, &[(8, &u64::MAX.to_le_bytes())]); - assert!(matches!( - Node::parse(&buf, BlockNum::new(7), 64), - Err(FsError::BlockOffDevice { .. }), - )); - } - - #[test] - fn a_crafted_entry_count_never_reaches_the_allocator() { - // The declared count is a `u16`; the block has room for MAX_ENTRIES. - // Reserving from the former asked for more than the kernel's whole - // allocation ceiling, and returned the *same* error while doing it — - // so the peak allocation, not the return value, is the instrument. - assert_eq!(MAX_ENTRIES, 169); - let ceiling = 2 * 1024 * 1024 - 4096; // mm::MAX_HEAP_ALLOC - let reserved_from_disk = u16::MAX as usize * core::mem::size_of::(); - assert!( - reserved_from_disk > ceiling, - "{reserved_from_disk} bytes is under the ceiling — this test proves nothing", - ); - - let buf = crafted(0, u16::MAX, &[]); - crate::alloc_probe::take_peak(); - let parsed = Node::parse(&buf, BlockNum::new(7), 64); - let peak = crate::alloc_probe::take_peak(); - - assert!(matches!(parsed, Err(FsError::CorruptedNode(_)))); - assert!( - peak <= BLOCK_SIZE, - "parsing a block that declares {} entries asked the allocator for {peak} bytes", - u16::MAX, - ); - } - - /// A split that finds no block for a later sibling fails its operation, - /// which gives back the block it took for the earlier: nothing names it. - #[test] - fn a_split_short_of_a_sibling_gives_back_the_ones_it_took() { - let io = crate::block_io::VecBlockIO::new(16); - let mut alloc = BitmapAllocator::format(&io, BlockNum::new(0), 1, 16, 1).unwrap(); - let taken = alloc.free_blocks as u32 - 2; - let _ = alloc.alloc_exact(&io, taken).unwrap(); - let leaf = alloc.alloc_block(&io).unwrap(); - assert_eq!(alloc.free_blocks, 1); - - let mut middle: Vec = (0..40).map(|_| entry(72)).collect(); - middle.insert(20, entry(MAX_ENTRY_SIZE - KEY_HEADER_SIZE)); - assert_eq!(pack(middle.clone()).len(), 3, "two siblings, and a block for one"); - alloc.begin(crate::alloc_bitmap::Reserve::Keep); - assert!(matches!( - split_node(&io, &mut alloc, leaf, Node::Leaf(middle)), - Err(FsError::NoSpace { .. }), - )); - assert_eq!(alloc.free_blocks, 0, "the first sibling took the last block"); - alloc.fail(&io); - assert_eq!(alloc.free_blocks, 1, "the first sibling's block went back"); - } - - /// Keys below every key the tree holds go down its leftmost spine, into - /// child 0 of every node on it. When that child splits, its siblings take - /// keys below the one child 0 was split off at, and every key the tree - /// holds is still found by its own search afterwards. - #[test] - fn keys_below_the_tree_s_least_split_its_first_child_and_every_key_is_still_found() { - let io = crate::block_io::VecBlockIO::new(4096); - let mut alloc = BitmapAllocator::format(&io, BlockNum::new(0), 1, 4096, 1).unwrap(); - let mut root = alloc.alloc_block(&io).unwrap(); - Node::Leaf(Vec::new()).write(&io, root).unwrap(); - let key = |name_hash: u64| Key { name_hash, name_hash_hi: 0, key_type: KeyType::File }; - // Above, then ever further below, so the second run splits child 0. - let keys: Vec = (0..400).map(|i| 1_000_000 + i).chain((0..800).map(|i| 999_999 - i)).collect(); - for &k in &keys { - root = insert(&io, &mut alloc, root, Entry { key: key(k), value: vec![k as u8; 200] }).unwrap(); - } - assert!(Node::read(&io, root).unwrap().level() >= 1, "the tree never grew past a leaf"); - let lost: Vec = keys.iter().copied().filter(|&k| search(&io, root, &key(k)).unwrap().is_none()).collect(); - assert!(lost.is_empty(), "{} of {} keys are not found, {:?} first", lost.len(), keys.len(), lost.first()); - } - - /// xorshift64*: the model test's whole source of choice, so one seed is - /// one run. - struct Rng(u64); - - impl Rng { - fn next(&mut self) -> u64 { - self.0 ^= self.0 >> 12; - self.0 ^= self.0 << 25; - self.0 ^= self.0 >> 27; - self.0.wrapping_mul(0x2545_F491_4F6C_DD1D) - } - - fn below(&mut self, n: u64) -> u64 { - self.next() % n - } - } - - /// The tree against a plain map of what each operation made it: new keys, - /// replacements whose values change size, deletes, a run of ever lower - /// keys down the leftmost spine, and values so large a leaf packs three - /// ways. Every key the map holds is found with its value, every key it - /// deleted is not, and a walk yields exactly the map's keys, strictly - /// ascending, which holds the whole tree's order and not only the keys a - /// search probes. - #[test] - fn the_tree_agrees_with_a_map_through_inserts_replacements_and_deletes() { - let io = crate::block_io::VecBlockIO::new(1 << 15); - let mut alloc = BitmapAllocator::format(&io, BlockNum::new(0), 1, 1 << 15, 1).unwrap(); - let mut root = alloc.alloc_block(&io).unwrap(); - Node::Leaf(Vec::new()).write(&io, root).unwrap(); - let mut rng = Rng(0x9E37_79B9_7F4A_7C15); - let mut model: alloc::collections::BTreeMap> = alloc::collections::BTreeMap::new(); - let mut gone: Vec = Vec::new(); - let big = MAX_ENTRY_SIZE - KEY_HEADER_SIZE; - let value = |rng: &mut Rng| { - let len = match rng.below(50) { - 0 => big, - 1..=4 => 1000 + rng.below(1000) as usize, - _ => 8 + rng.below(200) as usize, - }; - vec![rng.next() as u8; len] - }; - let key = |name_hash: u64, hi: u64| Key { name_hash, name_hash_hi: hi, key_type: KeyType::File }; - - let check = |root: BlockNum, model: &alloc::collections::BTreeMap>, gone: &[Key], at: usize| { - for (k, v) in model { - assert_eq!(search(&io, root, k).unwrap().as_ref(), Some(v), "op {at}: {k:?} is not found as written"); - } - for k in gone.iter().filter(|k| !model.contains_key(k)) { - assert_eq!(search(&io, root, k).unwrap(), None, "op {at}: deleted {k:?} is found"); - } - let mut walked = Vec::new(); - let walk = for_each_live(&io, root, &mut |e| { - walked.push(e.key); - Ok(ControlFlow::Continue(())) - }); - assert!(walk.unwrap().is_continue(), "op {at}: the walk did not end"); - assert!(walked.windows(2).all(|w| w[0] < w[1]), "op {at}: the walk is not strictly ascending"); - assert!(walked.iter().eq(model.keys()), "op {at}: the walk names other keys than the map"); - }; - - for op in 0..6000 { - let roll = rng.below(10); - if (2000..2600).contains(&op) { - // Below the least key the tree holds, each time. - let least = model.keys().next().map_or(u64::MAX, |k| k.name_hash); - let k = key(least.saturating_sub(1 + rng.below(3)), rng.next()); - let v = value(&mut rng); - root = insert(&io, &mut alloc, root, Entry { key: k, value: v.clone() }).unwrap(); - model.insert(k, v); - } else if roll < 6 || model.is_empty() { - let k = key(rng.next(), rng.next()); - let v = value(&mut rng); - root = insert(&io, &mut alloc, root, Entry { key: k, value: v.clone() }).unwrap(); - model.insert(k, v); - } else { - let k = *model.keys().nth(rng.below(model.len() as u64) as usize).unwrap(); - if roll < 8 { - let v = value(&mut rng); - root = insert(&io, &mut alloc, root, Entry { key: k, value: v.clone() }).unwrap(); - model.insert(k, v); - } else { - let (at, old) = delete(&io, &mut alloc, root, &k) - .unwrap() - .unwrap_or_else(|| panic!("op {op}: the tree has no {k:?}, which the map holds, to delete")); - assert_eq!(Some(old), model.remove(&k)); - root = at; - gone.push(k); - } - } - if op % 500 == 499 { - check(root, &model, &gone, op); - } - } - check(root, &model, &gone, 6000); - assert!(Node::read(&io, root).unwrap().level() >= 2, "the tree never reached a second interior level"); - } - - #[test] - fn a_descent_gives_up_before_it_runs_out_of_stack() { - let mut depth = Depth::ROOT; - for _ in 0..64 { - depth = depth.descend(BlockNum::new(1)).expect("64 levels is a legal tree"); - } - assert!(matches!( - depth.descend(BlockNum::new(1)), - Err(FsError::TreeTooDeep(_)), - )); - } -} diff --git a/lib/bcachefs/src/designation.rs b/lib/bcachefs/src/designation.rs new file mode 100644 index 00000000000..d8f7f3ac7ee --- /dev/null +++ b/lib/bcachefs/src/designation.rs @@ -0,0 +1,33 @@ +//! An explicit permission to format a partition, separate from bcachefs's +//! layout. Mkfs overwrites this first block, consuming the designation. + +use toyos_blockdev::BLOCK; + +const MAGIC: [u8; 16] = *b"TOYOS-FORMAT-ME\0"; + +pub fn designation(blocks: u64) -> [u8; BLOCK] { + let mut bytes = [0; BLOCK]; + bytes[..16].copy_from_slice(&MAGIC); + bytes[16..24].copy_from_slice(&blocks.to_le_bytes()); + bytes +} + +pub fn designated(bytes: &[u8]) -> Option { + if bytes.get(..16)? != MAGIC { return None; } + Some(u64::from_le_bytes(bytes.get(16..24)?.try_into().ok()?)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_designation_names_exactly_its_partition_size() { + for blocks in [0, 1, 65536, u64::MAX] { + assert_eq!(designated(&designation(blocks)), Some(blocks)); + } + assert_eq!(designated(&designation(7)[..23]), None); + assert_eq!(designated(&[0; BLOCK]), None); + assert!(designation(7)[24..].iter().all(|&byte| byte == 0)); + } +} diff --git a/lib/bcachefs/src/engine/mod.rs b/lib/bcachefs/src/engine/mod.rs index 6100fcd4bce..67f79c4d33d 100644 --- a/lib/bcachefs/src/engine/mod.rs +++ b/lib/bcachefs/src/engine/mod.rs @@ -31,7 +31,7 @@ pub use fsck::{fsck, Finding}; pub use mkfs::{fit, mkfs, Encryption, Options}; pub use mount::{Link, Mount, MountOptions, New, Owner, Removed, Rename}; pub use read::{read_file, DirEntry, Inode, Volume}; -pub use superblock::{Kdf, Superblock}; +pub use superblock::{carries_volume, Kdf, Superblock}; pub use tree::{Meta, NodeRef, Tree}; pub use value::DirentType; diff --git a/lib/bcachefs/src/engine/superblock.rs b/lib/bcachefs/src/engine/superblock.rs index d636bf3f9d6..d9da18e3658 100644 --- a/lib/bcachefs/src/engine/superblock.rs +++ b/lib/bcachefs/src/engine/superblock.rs @@ -13,6 +13,14 @@ pub(crate) const MAGIC: [u8; 16] = [0xc6, 0x85, 0x73, 0xf6, 0x66, 0xce, 0x90, 0x /// Where the primary copy and the layout copy start, in sectors. pub(crate) const PRIMARY: u64 = 8; pub(crate) const LAYOUT: u64 = 7; + +/// A format signature, not validation: used to avoid overwriting a volume +/// even when its primary superblock is damaged. +pub fn carries_volume(front: &[u8]) -> bool { + [LAYOUT as usize * super::SECTOR, PRIMARY as usize * super::SECTOR + 24] + .into_iter() + .any(|at| front.get(at..at + MAGIC.len()) == Some(&MAGIC)) +} /// The fixed header's bytes. pub(crate) const HEADER: usize = 752; /// The most a copy this reader reads may hold. Upstream's default reserve is diff --git a/lib/bcachefs/src/fs.rs b/lib/bcachefs/src/fs.rs deleted file mode 100644 index a574893552c..00000000000 --- a/lib/bcachefs/src/fs.rs +++ /dev/null @@ -1,2066 +0,0 @@ -use alloc::string::String; -use alloc::vec; -use alloc::vec::Vec; -use core::marker::PhantomData; -use core::ops::ControlFlow; - -use crate::alloc_bitmap::{BitmapAllocator, Reserve}; -use crate::block_io::{BlockBuf, BlockNum, BlockIO, BlockIOExt, DeviceError, BLOCK_SIZE}; -use crate::btree::{self, Entry, Key, KeyType, Node}; -use crate::superblock::{FsUuid, Superblock}; - -/// Extent: a contiguous run of blocks on disk. -/// -/// **An extent that came off a disk is a range of the volume, and -/// [`decode_leaf_value`] is what makes that true of it.** The fields are bare -/// integers and they cross the crate boundary — `kernel/src/file_backing.rs` -/// turns `start_block` into a block to read on every page fault — so nothing -/// downstream has anything to compare them against. The comparison happens -/// once, at the decode. -#[repr(C)] -#[derive(Debug, Clone, Copy)] -pub struct Extent { - pub start_block: u64, - pub block_count: u32, - pub _reserved: u32, -} - -const EXTENT_SIZE: usize = 16; - -/// Append `count` blocks at `start`, extending the last extent when the run -/// continues it. -/// -/// Without this a file needed one extent per *page*: the allocator hands back -/// consecutive blocks for consecutive single-block requests, and every one of -/// them was pushed as a fresh 16-byte extent into a value that has to fit in a -/// 4 KiB btree node. That is what capped a file at ~250 pages and panicked the -/// kernel one page later. A sequentially written file of any size is now one -/// extent, and what remains bounded is the number of *discontiguous runs*. -fn push_extent(extents: &mut Vec, start: u64, count: u32) { - if let Some(last) = extents.last_mut() { - // `checked_add` on both halves. On the count, because it is a u32 and a - // run past 16 TiB has to become a second extent instead of wrapping. On - // the address, because the extent list this appends to can have come - // off the disk — the kernel's write path resolves a page against the - // list `decode_leaf_value` handed it — and `u64::MAX + 1` in a kernel - // built with `overflow-checks` is a panic from a number an image chose. - if last.start_block.checked_add(last.block_count as u64) == Some(start) { - if let Some(merged) = last.block_count.checked_add(count) { - last.block_count = merged; - return; - } - } - } - extents.push(Extent { start_block: start, block_count: count, _reserved: 0 }); -} - -/// What an entry records besides its name, as [`Mounted::list`] answers it. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Recorded { - pub symlink: bool, - pub size: u64, - pub mtime: u64, -} - -/// Filesystem error type with rich context. -#[derive(Debug)] -pub enum FsError { - BadMagic { expected: [u8; 4], got: [u8; 4] }, - UnsupportedVersion(u32), - ChecksumMismatch { block: BlockNum, stored: u32, computed: u32 }, - CorruptedKey(u16), - CorruptedNode(BlockNum), - /// A block number off the end of what it was read out of. On-disk pointers - /// are the disk's claim about itself; the arbiter is whichever bound is - /// smaller — the device for a btree child pointer, the *volume* for a - /// file's extents, because the volume is what the allocator's bitmap is - /// sized from. - BlockOffDevice { block: u64, device_blocks: u64 }, - /// A file claiming more blocks than the volume has — an extent list naming - /// them, or a declared size needing them. The disk's arithmetic about - /// itself, and it sizes what this crate then allocates. - NotEnoughBlocks { needed: u64, available: u64 }, - /// A descent that has gone deeper than any tree over this device can be, - /// which means it is following a cycle. - TreeTooDeep(BlockNum), - /// A superblock that does not describe the device it was read from. The - /// CRC only says the bytes are the bytes somebody wrote. - BadSuperblock { field: &'static str }, - /// The device refused a transfer. Distinct from every corruption variant - /// above: those say the bytes are wrong, these say there are no bytes — - /// and each carries the device's retry discriminant unchanged. - DeviceRead(BlockNum, DeviceError), - DeviceWrite(BlockNum, DeviceError), - DeviceSync(DeviceError), - NotFound, - NoSpace { requested: u32, available: u64 }, - NameTooLong { len: usize, max: usize }, - /// A value no node could hold. Reachable from ordinary userland writes: - /// the extent list lives inline in the value, one entry per discontiguous - /// run of the file. - EntryTooLarge { size: usize, max: usize }, - /// A node whose entries do not fit the block. Defence in depth behind - /// `EntryTooLarge` — nothing should reach it. - NodeOverfull { used: usize, max: usize }, - /// A symlink target declaring more bytes than the caller's ceiling — - /// refused before the one allocation the declared size drives. - TargetTooLong { size: u64, max: u64 }, - /// A tree with more live entries than the caller's ceiling — refused before - /// the over-bound entry materialises, as [`FsError::TargetTooLong`] is. - ListTooLong { limit: usize }, -} - -impl FsError { - /// Whether a mount refused for this reason means the volume was never - /// ours — the only question a caller deciding whether to offer a format - /// stamp may ask. Exhaustive on purpose: a caller outside this crate - /// cannot classify a refusal any other way, and a new variant has to be - /// placed here before it compiles. `BadMagic` alone disowns the volume — - /// neither copy of the superblock claims it. Every other refusal, - /// including a device that would not answer at all, is a volume of ours - /// that did not mount, and disowns nothing. - pub fn disowns_volume(&self) -> bool { - match self { - FsError::BadMagic { .. } => true, - FsError::UnsupportedVersion(_) - | FsError::ChecksumMismatch { .. } - | FsError::CorruptedKey(_) - | FsError::CorruptedNode(_) - | FsError::BlockOffDevice { .. } - | FsError::NotEnoughBlocks { .. } - | FsError::TreeTooDeep(_) - | FsError::BadSuperblock { .. } - | FsError::DeviceRead(_, _) - | FsError::DeviceWrite(_, _) - | FsError::DeviceSync(_) - | FsError::NotFound - | FsError::NoSpace { .. } - | FsError::NameTooLong { .. } - | FsError::EntryTooLarge { .. } - | FsError::NodeOverfull { .. } - | FsError::TargetTooLong { .. } - | FsError::ListTooLong { .. } => false, - } - } -} - -pub struct ReadOnly; -pub struct ReadWrite; - -/// A formatted but not yet mounted filesystem. Used for building images -/// (mkfs): written in place, since nothing it holds is committed before -/// [`Formatted::into_io`]. -pub struct Formatted(Mounted); - -/// A mounted filesystem. Mode is ReadOnly or ReadWrite. -pub struct Mounted { - io: IO, - sb: Superblock, - alloc: BitmapAllocator, - /// The superblock both copies hold on the medium, made durable by this - /// mount's last commit; `None` until one lands, and after a sync refused - /// before its superblock landed. - landed: Option, - _mode: PhantomData, -} - -// --- Hash --- - -fn siphash_2_4(data: &[u8], key: [u8; 8]) -> u64 { - // Simplified SipHash-2-4 - let k = u64::from_le_bytes(key); - let mut v0 = 0x736f6d6570736575u64 ^ k; - let mut v1 = 0x646f72616e646f6du64 ^ k; - let mut v2 = 0x6c7967656e657261u64 ^ k; - let mut v3 = 0x7465646279746573u64 ^ k; - - let len = data.len(); - let blocks = len / 8; - - for i in 0..blocks { - let mut word = [0u8; 8]; - word.copy_from_slice(&data[i * 8..i * 8 + 8]); - let m = u64::from_le_bytes(word); - v3 ^= m; - for _ in 0..2 { - sip_round(&mut v0, &mut v1, &mut v2, &mut v3); - } - v0 ^= m; - } - - let mut last = (len as u64) << 56; - let remainder = &data[blocks * 8..]; - for (i, &byte) in remainder.iter().enumerate() { - last |= (byte as u64) << (i * 8); - } - - v3 ^= last; - for _ in 0..2 { - sip_round(&mut v0, &mut v1, &mut v2, &mut v3); - } - v0 ^= last; - - v2 ^= 0xff; - for _ in 0..4 { - sip_round(&mut v0, &mut v1, &mut v2, &mut v3); - } - - v0 ^ v1 ^ v2 ^ v3 -} - -fn sip_round(v0: &mut u64, v1: &mut u64, v2: &mut u64, v3: &mut u64) { - *v0 = v0.wrapping_add(*v1); - *v1 = v1.rotate_left(13); - *v1 ^= *v0; - *v0 = v0.rotate_left(32); - *v2 = v2.wrapping_add(*v3); - *v3 = v3.rotate_left(16); - *v3 ^= *v2; - *v0 = v0.wrapping_add(*v3); - *v3 = v3.rotate_left(21); - *v3 ^= *v0; - *v2 = v2.wrapping_add(*v1); - *v1 = v1.rotate_left(17); - *v1 ^= *v2; - *v2 = v2.rotate_left(32); -} - -fn hash_name(seed: &[u8; 16], name: &str) -> (u64, u64) { - let mut seed1 = [0u8; 8]; - let mut seed2 = [0u8; 8]; - seed1.copy_from_slice(&seed[0..8]); - seed2.copy_from_slice(&seed[8..16]); - ( - siphash_2_4(name.as_bytes(), seed1), - siphash_2_4(name.as_bytes(), seed2), - ) -} - -fn make_key(seed: &[u8; 16], name: &str, key_type: KeyType) -> Key { - let (h, hi) = hash_name(seed, name); - Key { - name_hash: h, - name_hash_hi: hi, - key_type, - } -} - -// --- Leaf value encoding/decoding --- - -const MAX_NAME_LEN: usize = 512; - -/// A leaf value's type tag *is* its key type, so nothing carries the two -/// separately. They were written out by hand at three sites as `1` and `2`, -/// with `if key_type == File { 1 } else { 2 }` twice — one place for them to -/// drift, and no way to notice. -const _: () = assert!( - KeyType::File as u8 == 1 && KeyType::Symlink as u8 == 2, - "decode_leaf_value reads 1 as a file and 2 as a symlink", -); - -/// The length of a leaf value naming `name` and `extents` runs. -fn leaf_value_len(name: &str, extents: usize) -> usize { - // 1 (entry_type) + 2 (name_len) + 8 (size) + 8 (mtime) + name + extents - 1 + 2 + 8 + 8 + name.len() + extents * EXTENT_SIZE -} - -/// Whether the one entry that holds a file named `name` can name `extents`: -/// a writer that grows the list refuses the write this answers `false` for, -/// before anything the entry would have to record is accepted. -pub fn file_entry_fits(name: &str, extents: &[Extent]) -> bool { - btree::value_fits(leaf_value_len(name, extents.len())) -} - -/// Encode a file/symlink leaf value. -fn encode_leaf_value( - entry_type: KeyType, - name: &str, - size: u64, - mtime: u64, - extents: &[Extent], -) -> Vec { - let name_bytes = name.as_bytes(); - let name_len = name_bytes.len(); - let mut val = vec![0u8; leaf_value_len(name, extents.len())]; - - val[0] = entry_type as u8; - val[1..3].copy_from_slice(&(name_len as u16).to_le_bytes()); - val[3..11].copy_from_slice(&size.to_le_bytes()); - val[11..19].copy_from_slice(&mtime.to_le_bytes()); - val[19..19 + name_len].copy_from_slice(name_bytes); - - let mut off = 19 + name_len; - for ext in extents { - val[off..off + 8].copy_from_slice(&ext.start_block.to_le_bytes()); - val[off + 8..off + 12].copy_from_slice(&ext.block_count.to_le_bytes()); - val[off + 12..off + 16].copy_from_slice(&ext._reserved.to_le_bytes()); - off += EXTENT_SIZE; - } - - val -} - -/// Decoded leaf value with owned strings. -pub enum LeafValue { - File { - name: String, - size: u64, - mtime: u64, - extents: Vec, - }, - Symlink { - name: String, - size: u64, - mtime: u64, - extents: Vec, - }, -} - -impl LeafValue { - pub fn name(&self) -> &str { - match self { - LeafValue::File { name, .. } => name, - LeafValue::Symlink { name, .. } => name, - } - } - - pub fn size(&self) -> u64 { - match self { - LeafValue::File { size, .. } => *size, - LeafValue::Symlink { size, .. } => *size, - } - } - - pub fn mtime(&self) -> u64 { - match self { - LeafValue::File { mtime, .. } => *mtime, - LeafValue::Symlink { mtime, .. } => *mtime, - } - } - - pub fn extents(&self) -> &[Extent] { - match self { - LeafValue::File { extents, .. } => extents, - LeafValue::Symlink { extents, .. } => extents, - } - } -} - -/// Decode a file/symlink leaf value, refusing one that does not fit the volume -/// it was read out of. -/// -/// **`volume_blocks` is not decoration, and this is the only place it is -/// applied.** An extent's `start_block` and `block_count` are numbers off the -/// disk, and three things downstream take them as read: the kernel's demand -/// paging turns one into a device block on every page fault -/// (`kernel/src/file_backing.rs`), [`read_extents`] sizes an allocation from -/// the file size beside them, and every delete hands them to -/// [`BitmapAllocator::free_range`] — which turns a block number into a bitmap -/// block to *read, modify and write*. That last one is why the bound is the -/// volume's block count and not the device's: `bit_of` derives the bitmap block -/// as `bitmap_start + block / 32768`, and the superblock guarantees the bitmap -/// covers the volume and nothing beyond it. On a 128-block volume, block 32769 -/// lands on byte 0 of block 2 — the root node — and freeing it clears bit 1 of -/// `BTND`'s `B`. -/// -/// So the refusals are here, at the one place a leaf value becomes typed data, -/// rather than at each of the sites that consume one. -fn decode_leaf_value(value: &[u8], volume_blocks: u64) -> Result { - if value.len() < 19 { - return Err(FsError::CorruptedKey(0)); - } - - let entry_type = value[0]; - let name_len = u16::from_le_bytes([value[1], value[2]]) as usize; - let size = u64::from_le_bytes(value[3..11].try_into().unwrap()); - let mtime = u64::from_le_bytes(value[11..19].try_into().unwrap()); - - if 19 + name_len > value.len() { - return Err(FsError::CorruptedKey(0)); - } - - let name_str = core::str::from_utf8(&value[19..19 + name_len]) - .map_err(|_| FsError::CorruptedKey(0))?; - let name = String::from(name_str); - - let extent_data = &value[19 + name_len..]; - let extent_count = extent_data.len() / EXTENT_SIZE; - let mut extents = Vec::with_capacity(extent_count); - let mut named_blocks = 0u64; - for i in 0..extent_count { - let off = i * EXTENT_SIZE; - let start_block = u64::from_le_bytes(extent_data[off..off + 8].try_into().unwrap()); - let block_count = u32::from_le_bytes(extent_data[off + 8..off + 12].try_into().unwrap()); - - // Every block the run names has to be a block of this volume, and the - // *end* is where that is decided: a start on the last block passes on - // its own and still runs off the edge. - let end = start_block.checked_add(block_count as u64); - if end.is_none_or(|end| end > volume_blocks) { - return Err(FsError::BlockOffDevice { block: start_block, device_blocks: volume_blocks }); - } - - // And a file cannot hold more blocks than the volume has: every block - // of a file is one of the volume's, so the total is bounded even where - // no single run is out of range. Without this, 254 in-range runs of - // 4 GiB each are what `read_extents` sizes its `Vec` against. - named_blocks = named_blocks.saturating_add(block_count as u64); - if named_blocks > volume_blocks { - return Err(FsError::NotEnoughBlocks { needed: named_blocks, available: volume_blocks }); - } - - extents.push(Extent { start_block, block_count, _reserved: 0 }); - } - - match entry_type { - 1 => Ok(LeafValue::File { name, size, mtime, extents }), - 2 => Ok(LeafValue::Symlink { name, size, mtime, extents }), - _ => Err(FsError::CorruptedKey(entry_type as u16)), - } -} - -/// Allocate blocks and write `data` into them, returning the extent list. -/// -/// The allocator answers with a run that may be shorter than the request, so -/// covering `data` takes a loop. A run reserved by an earlier turn of that loop -/// is the operation's, which gives it back if a later turn fails. -fn write_data( - io: &dyn BlockIO, - alloc: &mut BitmapAllocator, - data: &[u8], -) -> Result, FsError> { - if data.is_empty() { - return Ok(Vec::new()); - } - - let blocks_needed = data.len().div_ceil(BLOCK_SIZE) as u32; - let mut extents: Vec = Vec::new(); - let mut remaining = blocks_needed; - let mut data_offset = 0usize; - - while remaining > 0 { - let run = alloc.alloc_up_to(io, alloc.next_alloc, remaining)?; - push_extent(&mut extents, run.start.raw(), run.len); - - let mut buf = BlockBuf::zeroed(); - for i in 0..run.len as u64 { - buf.0.fill(0); - let chunk_end = (data_offset + BLOCK_SIZE).min(data.len()); - if data_offset < data.len() { - let len = chunk_end - data_offset; - buf.0[..len].copy_from_slice(&data[data_offset..chunk_end]); - } - io.write(BlockNum::new(run.start.raw() + i), &buf)?; - data_offset += BLOCK_SIZE; - } - - remaining -= run.len; - } - - Ok(extents) -} - -/// Read file data from a list of extents. -/// -/// **`size` sizes the `Vec` and `size` is a number off the disk**, so it is -/// held against the volume: a file cannot be longer than the thing it is -/// stored on. Without that line a leaf value declaring `u64::MAX` asked the -/// allocator for `u64::MAX` bytes — in the kernel, from a symlink `read_link` -/// follows, where anything past `mm::MAX_HEAP_ALLOC` is not an `Err` but an -/// assertion. -/// -/// **The volume and not the blocks the file names**, which is the tighter -/// bound and the wrong one: a file is allowed to end in a hole. `set_len` past -/// the last block dirties no page, so the flush allocates nothing and records -/// a size the extents do not reach — `fs_truncate_persist` grows a one-page -/// `/home` file to 3 MiB — and the loop below is written for exactly that, -/// leaving the tail as the zeros the hole is. -fn read_extents( - io: &dyn BlockIO, - extents: &[Extent], - size: u64, - volume_blocks: u64, -) -> Result, FsError> { - let needed_blocks = size.div_ceil(BLOCK_SIZE as u64); - if needed_blocks > volume_blocks { - return Err(FsError::NotEnoughBlocks { needed: needed_blocks, available: volume_blocks }); - } - - let mut data = vec![0u8; size as usize]; - let mut offset = 0usize; - let mut buf = BlockBuf::zeroed(); - - for ext in extents { - for i in 0..ext.block_count as u64 { - if offset >= size as usize { - break; - } - io.read(BlockNum::new(ext.start_block + i), &mut buf)?; - let remaining = size as usize - offset; - let to_copy = remaining.min(BLOCK_SIZE); - data[offset..offset + to_copy].copy_from_slice(&buf.0[..to_copy]); - offset += to_copy; - } - } - - Ok(data) -} - -// --- Formatted (for mkfs / image building) --- - -impl Formatted { - /// Format a new filesystem on the given block device. - pub fn format(io: IO) -> Result { - let block_count = io.block_count(); - - // Layout: [superblock(1)] [bitmap] [journal(reserved, 0 for now)] [data...] [sb_backup(1)] - let bitmap_blocks = block_count.div_ceil(BLOCK_SIZE as u64 * 8); - let bitmap_start = BlockNum::new(1); - // Reserve journal space but don't use it in Phase 1 - let journal_start = BlockNum::new(1 + bitmap_blocks); - let journal_blocks = 0u32; // Phase 2 will set this to 64 - - let metadata_blocks = 1 + bitmap_blocks + journal_blocks as u64; - - // Create empty root leaf node - let root_block_num = metadata_blocks; // first data block is the root node - let total_metadata = metadata_blocks + 1; // +1 for root node - - let alloc = BitmapAllocator::format( - &io, - bitmap_start, - bitmap_blocks, - block_count, - total_metadata, - )?; - - // An empty node's entries occupy zero bytes, so the only failure - // `write` has left here is the device's. - Node::Leaf(Vec::new()).write(&io, BlockNum::new(root_block_num))?; - - // Generate random-ish hash seed from block count (deterministic for reproducible builds) - let mut hash_seed = [0u8; 16]; - let seed_val = block_count.wrapping_mul(0x517cc1b727220a95); - hash_seed[0..8].copy_from_slice(&seed_val.to_le_bytes()); - hash_seed[8..16].copy_from_slice(&seed_val.wrapping_mul(0x6c62272e07bb0142).to_le_bytes()); - - let sb = Superblock { - block_count, - root_node: BlockNum::new(root_block_num), - next_alloc: total_metadata, - free_blocks: alloc.free_blocks, - bitmap_start, - bitmap_blocks, - journal_start, - journal_blocks, - journal_head: 0, - flags: 0, // not clean until sync - hash_seed, - uuid: FsUuid::UNNAMED, - }; - - sb.write(&io)?; - - Ok(Self(Mounted { io, sb, alloc, landed: None, _mode: PhantomData })) - } - - /// Name this filesystem, so whoever serves it can say which one it is. - /// - /// A separate act from formatting: a volume nothing names is legal, and - /// nothing here invents a name for one. Persisted by [`Self::into_io`]. - pub fn set_uuid(&mut self, uuid: FsUuid) { - self.0.sb.uuid = uuid; - } - - /// Create a file on the formatted filesystem (used during mkfs). - pub fn create(&mut self, name: &str, data: &[u8], mtime: u64) -> Result<(), FsError> { - self.0.create(name, data, mtime) - } - - /// Create a symlink on the formatted filesystem. - pub fn create_symlink(&mut self, name: &str, target: &str, mtime: u64) -> Result<(), FsError> { - self.0.put(name, KeyType::Symlink, target.as_bytes(), mtime) - } - - /// Mount this formatted filesystem for read-write access. - pub fn mount(self) -> Mounted { - let mut fs = self.0; - fs.alloc.shadow = true; - fs - } - - /// Mount this formatted filesystem for read-only access. - pub fn mount_readonly(self) -> Mounted { - let Mounted { io, sb, alloc, .. } = self.0; - Mounted { io, sb, alloc, landed: None, _mode: PhantomData } - } - - /// Consume and return the underlying IO (for extracting the image bytes). - pub fn into_io(mut self) -> Result { - self.0.sync()?; - Ok(self.0.io) - } -} - -// --- Mounted (read operations, available for both ReadOnly and ReadWrite) --- - -impl Mounted { - /// Open an existing filesystem from disk, to read. - pub fn open(io: IO) -> Result { - Self::read_superblock(io) - } -} - -impl Mounted { - /// Open an existing filesystem from disk, to change; a mount that only - /// reads never asks how much is free. - pub fn open(io: IO) -> Result { - let mut fs = Self::read_superblock(io)?; - fs.alloc.count_free(&fs.io)?; - Ok(fs) - } - - /// The bytes a change may still take: every free block but those - /// [`crate::alloc_bitmap::NODE_RESERVE`] keeps for a delete. - pub fn free_bytes(&self) -> u64 { - self.alloc.spare() * BLOCK_SIZE as u64 - } -} - -impl Mounted { - fn read_superblock(io: IO) -> Result { - let sb = Superblock::read(&io)?; - let alloc = BitmapAllocator::open(&sb); - Ok(Mounted { io, sb, alloc, landed: None, _mode: PhantomData }) - } - - /// The device this filesystem was opened over. - pub fn io(&self) -> &IO { - &self.io - } - - /// What this filesystem is named, or [`FsUuid::UNNAMED`]. - pub fn uuid(&self) -> FsUuid { - self.sb.uuid - } - - /// Decode a leaf value against *this volume's* block count. - /// - /// The one way a leaf value is decoded inside a mount, so the bound cannot - /// be forgotten at one of the dozen sites that ask for one. `sb.block_count` - /// and not `io.block_count()`: the volume is what the bitmap is sized from, - /// and `Superblock::check` has already refused a volume larger than its - /// device. - fn decode(&self, value: &[u8]) -> Result { - decode_leaf_value(value, self.sb.block_count) - } - - /// Find a file entry by name. Tries File key first, then Symlink. - fn find_by_name(&self, name: &str) -> Result)>, FsError> { - // Try as File first (most common) - let key = make_key(&self.sb.hash_seed, name, KeyType::File); - if let Some(value) = btree::search(&self.io, self.sb.root_node, &key)? { - let leaf = self.decode(&value)?; - if leaf.name() == name { - return Ok(Some((key, value))); - } - } - - // Try as Symlink - let key = make_key(&self.sb.hash_seed, name, KeyType::Symlink); - if let Some(value) = btree::search(&self.io, self.sb.root_node, &key)? { - let leaf = self.decode(&value)?; - if leaf.name() == name { - return Ok(Some((key, value))); - } - } - - Ok(None) - } - - /// Read a file's contents by name. - pub fn read_file(&self, name: &str) -> Result, FsError> { - let (_, value) = self.find_by_name(name)?.ok_or(FsError::NotFound)?; - let leaf = self.decode(&value)?; - match leaf { - LeafValue::File { size, extents, .. } | LeafValue::Symlink { size, extents, .. } => { - read_extents(&self.io, &extents, size, self.sb.block_count) - } - } - } - - /// Read a symlink's target by name. `None` when the name is not a symlink. - /// `max_len` caps the allocation the target's declared size drives. - pub fn read_link(&self, name: &str, max_len: u64) -> Result, FsError> { - let Some((_, value)) = self.find_by_name(name)? else { return Ok(None) }; - match self.decode(&value)? { - LeafValue::Symlink { size, extents, .. } => { - if size > max_len { - return Err(FsError::TargetTooLong { size, max: max_len }); - } - let data = read_extents(&self.io, &extents, size, self.sb.block_count)?; - Ok(String::from_utf8(data).ok()) - } - _ => Ok(None), - } - } - - /// Modification time of a file. `None` when nothing answers to the name. - pub fn file_mtime(&self, name: &str) -> Result, FsError> { - Ok(self.leaf(name)?.map(|leaf| leaf.mtime())) - } - - /// The decoded entry `name` answers to, if any. - fn leaf(&self, name: &str) -> Result, FsError> { - match self.find_by_name(name)? { - Some((_, value)) => Ok(Some(self.decode(&value)?)), - None => Ok(None), - } - } - - /// Up to `limit` of the files `keep` accepts, with what each one's entry - /// records; more is refused before any materialise. `limit` counts the - /// kept names, so a caller listing one directory pays that directory's - /// bound and not the volume's. - pub fn list(&self, limit: usize, keep: &dyn Fn(&str) -> bool) -> Result, FsError> { - let mut result = Vec::new(); - let ControlFlow::Continue(()) = btree::for_each_live(&self.io, self.sb.root_node, &mut |entry| { - let Ok(leaf) = self.decode(&entry.value) else { return Ok(ControlFlow::Continue(())) }; - if !keep(leaf.name()) { - return Ok(ControlFlow::Continue(())); - } - if result.len() >= limit { - return Err(FsError::ListTooLong { limit }); - } - let symlink = matches!(leaf, LeafValue::Symlink { .. }); - result.push((String::from(leaf.name()), Recorded { symlink, size: leaf.size(), mtime: leaf.mtime() })); - Ok(ControlFlow::Continue(())) - })? else { - unreachable!("`list`'s visitor never breaks") - }; - Ok(result) - } - - /// Whether some name lies beneath `dir` (`""` is the root): the flat - /// namespace has no other kind of directory. Keys are hashes of whole - /// names, so the answer is a walk that stops at the first such name and - /// costs the whole tree when there is none; it keeps nothing, and a node - /// it cannot read refuses this question alone. - pub fn is_dir(&self, dir: &str) -> Result { - if dir.is_empty() { - return Ok(true); - } - let found = btree::for_each_live(&self.io, self.sb.root_node, &mut |entry| { - // As `list`: a leaf that does not decode answers to no name. - let Ok(leaf) = self.decode(&entry.value) else { return Ok(ControlFlow::Continue(())) }; - let beneath = leaf.name().strip_prefix(dir).is_some_and(|rest| rest.starts_with('/')); - Ok(if beneath { ControlFlow::Break(()) } else { ControlFlow::Continue(()) }) - })?; - Ok(found.is_break()) - } - - /// Convert back to Formatted state (for testing — insert more files after reading). - pub fn into_formatted(self) -> Formatted { - let Self { io, sb, mut alloc, .. } = self; - alloc.shadow = false; - Formatted(Mounted { io, sb, alloc, landed: None, _mode: PhantomData }) - } - - /// Return the extents and file size for a file. - /// Used by the kernel to construct a FileBacking for demand-paged loading. - pub fn file_extents(&self, name: &str) -> Result, u64)>, FsError> { - Ok(self.leaf(name)?.map(|leaf| (leaf.extents().to_vec(), leaf.size()))) - } - - /// Check if a name is a symlink. - pub fn is_symlink(&self, name: &str) -> Result { - Ok(self.find_by_name(name)?.is_some_and(|(key, _)| key.key_type == KeyType::Symlink)) - } - -} - -// --- ReadWrite-only operations --- - -/// **Every change is one operation and every sync one commit.** An operation -/// ([`Mounted::atomic`]) takes effect whole or, refused anywhere, leaves the -/// tree and the allocator as they were; it never writes over a node the last -/// commit's tree reaches, but to new blocks up to a new root. A commit -/// ([`Mounted::sync`]) is both superblock copies naming that root landing on -/// the device after everything the root reaches. A superblock's bytes lie in -/// its first 512-byte sector, which a device writes whole, so killed anywhere -/// each copy names the last commit's tree whole or the new one whole. -/// -/// That is true of names, and of each entry's length and extents; not of a -/// file's bytes. A page written over where the file already holds it -/// (`block_for`) is written in place, so a kill can -/// leave a file's committed blocks holding pages from both sides of a write -/// (`issues/a-write-over-a-files-committed-page-is-not-shadowed.md`). What a -/// kill costs besides is the blocks taken since the last commit and those only -/// the old tree reached: marked used, and named by no tree, until a sweep -/// (`issues/a-crash-leaks-the-blocks-of-its-last-commit.md`). -/// -/// A delete, and an update whose entry grows no longer, may take the blocks -/// [`crate::alloc_bitmap::NODE_RESERVE`] keeps from every other change, so a -/// full volume still shrinks, by as many committed nodes as the reserve holds -/// between two commits (`issues/a-full-data-volume-frees-space-only-at-a-commit.md`). -impl Mounted { - /// Create a file, replacing whatever answered to `name`. - pub fn create(&mut self, name: &str, data: &[u8], mtime: u64) -> Result<(), FsError> { - self.put(name, KeyType::File, data, mtime) - } - - /// Create a symlink, replacing whatever answered to `name`. - pub fn create_symlink(&mut self, name: &str, target: &str) -> Result<(), FsError> { - self.put(name, KeyType::Symlink, target.as_bytes(), 0) - } - - /// Run `change` as one operation: whole, or refused with the root and - /// every block as they were. - fn atomic( - &mut self, - reserve: Reserve, - change: impl FnOnce(&mut Self) -> Result, - ) -> Result { - let root = self.sb.root_node; - self.alloc.begin(reserve); - match change(self) { - Ok(done) => { - self.alloc.succeed(&self.io); - Ok(done) - } - Err(e) => { - self.sb.root_node = root; - self.alloc.fail(&self.io); - Err(e) - } - } - } - - /// Put `name` on the volume, displacing whatever answered to it. - /// - /// The new entry goes in before the old one comes out, for the reason - /// `rename` does it: freeing first is the old file destroyed when - /// `write_data` or `btree::insert` then fails, and a full volume is a - /// failure any caller can provoke. Measured before it did: on a 64-block - /// volume, a 5-block file overwritten by a 400-block one left the volume - /// empty and the 5 blocks unreachable. - /// - /// What that costs is that the replacement's blocks and the original's are - /// both allocated at once, so an overwrite on a nearly full volume can now - /// fail where it used to succeed. An error is the cheaper half of that - /// trade. - fn put( - &mut self, - name: &str, - key_type: KeyType, - data: &[u8], - mtime: u64, - ) -> Result<(), FsError> { - if name.is_empty() || name.len() > MAX_NAME_LEN { - return Err(FsError::NameTooLong { len: name.len(), max: MAX_NAME_LEN }); - } - - self.atomic(Reserve::Keep, |fs| { - let displaced = match fs.find_by_name(name)? { - Some((key, value)) => Some((key, fs.decode(&value)?.extents().to_vec())), - None => None, - }; - - let extents = write_data(&fs.io, &mut fs.alloc, data)?; - let value = encode_leaf_value(key_type, name, data.len() as u64, mtime, &extents); - let key = make_key(&fs.sb.hash_seed, name, key_type); - fs.sb.root_node = btree::insert(&fs.io, &mut fs.alloc, fs.sb.root_node, Entry { key, value })?; - - fs.retire_displaced(displaced, key) - }) - } - - /// Remove `key`'s entry, if the tree has one. - fn remove(&mut self, key: &Key) -> Result { - let Some((root, _)) = btree::delete(&self.io, &mut self.alloc, self.sb.root_node, key)? else { - return Ok(false); - }; - self.sb.root_node = root; - Ok(true) - } - - /// Remove the entry the insert of `new_key` did not replace, and free the - /// blocks of whatever answered to that name before. - /// - /// The insert replaces the destination only where the two keys agree. A - /// file written over a symlink keys differently, and the entry left behind - /// would answer to the name forever with blocks nothing could reach. - fn retire_displaced( - &mut self, - displaced: Option<(Key, Vec)>, - new_key: Key, - ) -> Result<(), FsError> { - let Some((old_key, old_extents)) = displaced else { return Ok(()) }; - if old_key != new_key { - self.remove(&old_key)?; - } - self.free_extents(&old_extents) - } - - /// Delete a file/symlink by name, freeing its data blocks. Returns true if found. - /// - /// `find_by_name` answers the "is this the entry we mean?" question, which - /// used to be asked after the removal: `btree::delete` took the entry out - /// and *then* the decoded name was compared, so a key collision destroyed - /// an unrelated file, leaked its blocks and returned `false` — telling the - /// caller nothing had happened. It also answers it once for both key - /// types, where the old shape fell through from File to Symlink after a - /// non-matching removal and could take two entries out in one call. - pub fn delete(&mut self, name: &str) -> Result { - self.atomic(Reserve::Draw, |fs| { - let Some((key, value)) = fs.find_by_name(name)? else { return Ok(false) }; - let extents = fs.decode(&value)?.extents().to_vec(); - - // `find_by_name` reached this key by the descent `btree::delete` is - // about to repeat, so an empty removal is not "no such file" — it is a - // tree that answers two ways. - if !fs.remove(&key)? { - return Err(FsError::CorruptedNode(fs.sb.root_node)); - } - fs.free_extents(&extents)?; - Ok(true) - }) - } - - /// Commit the tree as it stands: see this block's header. A commit whose - /// superblock is the one the medium already holds writes none, since no - /// node its root reaches has been written over since it landed; the - /// device is still flushed, for what is written in place. - pub fn sync(&mut self) -> Result<(), FsError> { - let landed = self.landed.take(); - self.io.flush()?; - // What the volume holds free once this commit's own frees are made. - self.sb.free_blocks = self.alloc.free_blocks + self.alloc.pending(); - self.sb.next_alloc = self.alloc.next_alloc; - self.sb.set_clean(true); - if landed.as_ref().is_some_and(|l| self.sb.same_commit(l)) { - self.landed = landed; - } else { - // From the first superblock write on, this commit may be what a - // mount finds, even where the write is refused. - self.alloc.seal(); - self.sb.write(&self.io)?; - self.io.flush()?; - self.landed = Some(self.sb.clone()); - } - self.alloc.committed(&self.io) - } - - /// Rename a file or symlink. - pub fn rename(&mut self, old_name: &str, new_name: &str) -> Result<(), FsError> { - self.rename_all(&[(old_name, new_name)]) - } - - /// Rename every `(old, new)` pair in turn, as one operation: a directory - /// is the names beneath it, and moves whole or not at all. - pub fn rename_all(&mut self, renames: &[(&str, &str)]) -> Result<(), FsError> { - self.atomic(Reserve::Keep, |fs| renames.iter().try_for_each(|&(old, new)| fs.rename_one(old, new))) - } - - /// The new entry goes in before the old one comes out. What that ordering - /// costs is that the insert *is* the removal of whatever `new_name` named — - /// same name and same type is the same key, and `btree::insert` replaces on - /// an equal key — so the displaced entry has to be read out of the tree - /// before the insert. Asking for it afterwards, by name, answers with the - /// file that was just renamed and frees its extents. - fn rename_one(&mut self, old_name: &str, new_name: &str) -> Result<(), FsError> { - // Every other name-taking entry point bounds its name; this one did - // not, and `user_ptr::MAX_USER_STR` lets 64 KiB of it through. - if new_name.is_empty() || new_name.len() > MAX_NAME_LEN { - return Err(FsError::NameTooLong { len: new_name.len(), max: MAX_NAME_LEN }); - } - - let (old_key, old_value) = self.find_by_name(old_name)? - .ok_or(FsError::NotFound)?; - let leaf = self.decode(&old_value)?; - let new_key = make_key(&self.sb.hash_seed, new_name, old_key.key_type); - - // What `new_name` names now. `find_by_name` matches on the decoded - // name, so an entry answering to both names is one entry — a rename - // onto itself, with nothing to displace and nothing to free. - let displaced = match self.find_by_name(new_name)? { - Some((key, _)) if key == old_key => None, - Some((key, value)) => Some((key, self.decode(&value)?.extents().to_vec())), - None => None, - }; - - let new_value = encode_leaf_value( - old_key.key_type, new_name, leaf.size(), leaf.mtime(), leaf.extents(), - ); - self.sb.root_node = btree::insert( - &self.io, &mut self.alloc, - self.sb.root_node, - Entry { key: new_key, value: new_value }, - )?; - - self.retire_displaced(displaced, new_key)?; - - // The source's blocks stay allocated: the new entry holds the same - // extent list. Nothing to delete when the two names share a key — the - // entry under it is the one the insert just wrote. - if new_key != old_key { - self.remove(&old_key)?; - } - - Ok(()) - } - - /// Update file metadata (size, mtime, extents) without rewriting data. - pub fn update_metadata( - &mut self, - name: &str, - new_extents: &[Extent], - size: u64, - mtime: u64, - ) -> Result<(), FsError> { - let (old_key, old_value) = self.find_by_name(name)?.ok_or(FsError::NotFound)?; - let leaf = self.decode(&old_value)?; - let new_value = encode_leaf_value(old_key.key_type, leaf.name(), size, mtime, new_extents); - // An entry no longer than the one it replaces splits no node. - let reserve = match new_value.len() <= old_value.len() { - true => Reserve::Draw, - false => Reserve::Keep, - }; - let new_entry = Entry { key: old_key, value: new_value }; - - self.atomic(reserve, |fs| { - // No delete first: the key is unchanged and `btree::insert` replaces - // on an equal key. Blocks the caller drops from the extent list are - // the caller's to free, through [`Self::free_extents`], after this - // records the shortened list. - fs.sb.root_node = btree::insert(&fs.io, &mut fs.alloc, fs.sb.root_node, new_entry)?; - Ok(()) - }) - } - - /// Give `extents`' blocks back to the allocator. Record first, free second: - /// the caller shortens the entry's list before calling this, so a failure - /// between the two leaks blocks rather than leaving an entry naming freed - /// ones. - pub fn free_extents(&mut self, extents: &[Extent]) -> Result<(), FsError> { - for ext in extents { - self.alloc.free_range(&self.io, BlockNum::new(ext.start_block), ext.block_count)?; - } - Ok(()) - } - - /// Allocate blocks until `extents` reach every page of `pages`. The caller - /// writes every page of `pages`, zero-filling what it does not cover; the - /// pages bridged before them are zeroed here. - /// - /// Each run is the first free one from the file's end at least `min_run` - /// long (or every page still missing, if fewer), so single-block holes do - /// not each become a run. That search examines at most one lap of the - /// bitmap for the whole call. Once it finds none, the rest of the call - /// takes runs as they come, sweeping forward from the last, within one - /// more lap: at most two laps a call, besides the blocks it takes. - pub fn allocate( - &mut self, - extents: &mut Vec, - pages: core::ops::RangeInclusive, - min_run: u32, - ) -> Result<(), FsError> { - let end = *pages.end() as u64 + 1; - let mut covered: u64 = extents.iter().map(|e| e.block_count as u64).sum(); - let hole = covered; - let total = self.alloc.total_blocks; - let mut budget = total; - // Where the second phase has swept to, once it has begun. - let mut sweep: Option = None; - while covered < end { - let want = (end - covered).min(u32::MAX as u64) as u32; - let next = extents.last().and_then(|e| e.start_block.checked_add(e.block_count as u64)); - let (from, min) = match sweep { - Some(at) => (at, 1), - None => { - let from = match next { - None => self.alloc.next_alloc, - Some(next) if next < total && self.alloc.is_free(&self.io, BlockNum::new(next))? => next, - Some(next) => next % total + SPREAD, - }; - (from, min_run.min(want)) - } - }; - match self.alloc.alloc_first_fit(&self.io, from, want, min, &mut budget)? { - Some(run) => { - push_extent(extents, run.start.raw(), run.len); - covered += run.len as u64; - let after = run.start.raw() + run.len as u64; - match sweep { - Some(_) => sweep = Some(after), - // Shorter than asked: the search spent its lap, and - // the sweep begins here, with a lap of its own. - None if run.len < min => { - sweep = Some(after); - budget = total; - } - None => {} - } - } - None if sweep.is_none() => { - sweep = Some(next.unwrap_or(self.alloc.next_alloc) % total); - budget = total; - } - None => return Err(FsError::NoSpace { requested: want, available: self.alloc.free_blocks }), - } - } - - // The blocks that bridge the gap are the allocator's most recently freed - // and still hold what the file that gave them up left there; the caller - // writes only `pages`, so every block before them is a hole and reads zero. - let zero = BlockBuf::zeroed(); - for page in hole..*pages.start() as u64 { - let block = block_for(extents, page as u32).ok_or(FsError::NotFound)?; - self.io.write(BlockNum::new(block), &zero)?; - } - Ok(()) - } -} - -/// How far past a file's last block its next run is looked for when another -/// file has taken that block. Two files that grow in turn then each extend -/// runs this long rather than alternate blocks, so the runs one entry can name -/// ([`file_entry_fits`]) last this many times longer. -const SPREAD: u64 = 256; - -/// The block holding `page_idx`, if the extents already reach that far. -/// -/// The one definition of where a page lives, used both to answer a resolve and -/// to answer it again after allocating — so an allocation that came up short -/// cannot produce a block the lookup would not agree with. -/// -/// Total arithmetic, for the reason [`push_extent`]'s is: the list walked here -/// can be one a disk wrote, and a `+` that wraps is a panic rather than a -/// wrong answer. `None` is already "the extents do not reach that page", which -/// is what a list whose own arithmetic does not close is. -fn block_for(extents: &[Extent], page_idx: u32) -> Option { - let mut cursor = 0u64; - for ext in extents { - let end = cursor.checked_add(ext.block_count as u64)?; - if (page_idx as u64) < end { - return ext.start_block.checked_add(page_idx as u64 - cursor); - } - cursor = end; - } - None -} - -#[cfg(all(test, feature = "std"))] -mod tests { - use super::*; - use crate::block_io::VecBlockIO; - use crate::btree::NODE_MAGIC; - use crate::crc32c::crc32c; - - /// A real volume with a real file on it, as raw bytes to be tampered with. - /// - /// Every crafted block below is resealed with the checksum the format asks - /// for, so the parser accepts all of them as authentic. That is the whole - /// point: whoever writes the image writes the CRC. - fn image(blocks: u64) -> Vec { - let mut fs = Formatted::format(VecBlockIO::new(blocks)).expect("format"); - fs.create("victim.txt", b"a file that was already here", 1).expect("create"); - fs.into_io().expect("sync").into_vec() - } - - fn seal_node(raw: &mut [u8], block: u64) { - let at = block as usize * BLOCK_SIZE; - let crc = crc32c(&raw[at + crate::btree::CRC_START..at + BLOCK_SIZE]); - raw[at + 4..at + 8].copy_from_slice(&crc.to_le_bytes()); - } - - fn seal_superblock(raw: &mut [u8], block: u64) { - let at = block as usize * BLOCK_SIZE; - let crc = crc32c(&raw[at + Superblock::CRC_START..at + BLOCK_SIZE]); - raw[at + 8..at + 12].copy_from_slice(&crc.to_le_bytes()); - } - - /// Patch a little-endian `u64` into both copies of the superblock. - fn patch_superblock(raw: &mut [u8], blocks: u64, off: usize, val: u64) { - for sb_block in [0, blocks - 1] { - let at = sb_block as usize * BLOCK_SIZE; - raw[at + off..at + off + 8].copy_from_slice(&val.to_le_bytes()); - seal_superblock(raw, sb_block); - } - } - - /// Point both superblocks at `root`, and set the depth field to its maximum - /// while we are here — that field is what three recursions used to descend - /// on, and a disk saying 65535 is the shape that killed the kernel. - fn set_root(raw: &mut [u8], blocks: u64, root: u64) { - for sb_block in [0, blocks - 1] { - let at = sb_block as usize * BLOCK_SIZE; - raw[at + 24..at + 32].copy_from_slice(&root.to_le_bytes()); - raw[at + 32..at + 34].copy_from_slice(&u16::MAX.to_le_bytes()); - seal_superblock(raw, sb_block); - } - } - - /// An interior node at `block` with one child pointer, `value_len` bytes - /// wide, naming `child`. - fn craft_interior(raw: &mut [u8], block: u64, child: u64, value_len: u32) { - let at = block as usize * BLOCK_SIZE; - raw[at..at + BLOCK_SIZE].fill(0); - raw[at..at + 4].copy_from_slice(&NODE_MAGIC); - raw[at + 8..at + 10].copy_from_slice(&1u16.to_le_bytes()); - raw[at + 10..at + 12].copy_from_slice(&1u16.to_le_bytes()); - - let entry = at + 32; - raw[entry + 18..entry + 22].copy_from_slice(&value_len.to_le_bytes()); - let value = entry + 24; - let bytes = child.to_le_bytes(); - let n = (value_len as usize).min(bytes.len()); - raw[value..value + n].copy_from_slice(&bytes[..n]); - seal_node(raw, block); - } - - fn mount(raw: Vec) -> Result, FsError> { - Mounted::<_, ReadOnly>::open(VecBlockIO::from_vec(raw)) - } - - fn mount_rw(raw: Vec) -> Result, FsError> { - Mounted::<_, ReadWrite>::open(VecBlockIO::from_vec(raw)) - } - - fn read_u64_at(raw: &[u8], off: usize) -> u64 { - u64::from_le_bytes(raw[off..off + 8].try_into().unwrap()) - } - - /// Rewrite the root leaf as one `victim.txt` entry with the type, size and - /// extents given, and reseal it. - /// - /// The key bytes are left exactly as [`image`] wrote them, so the name - /// still hashes to this entry and every lookup reaches the crafted value. - /// The extents go in raw, as `(start_block, block_count)` — that is the - /// whole point: these are the numbers a disk gets to choose. - fn craft_entry(raw: &mut [u8], root: u64, entry_type: u8, size: u64, extents: &[(u64, u32)]) { - const NAME: &[u8] = b"victim.txt"; - let at = root as usize * BLOCK_SIZE; - let entry = at + 32; - let value = entry + 24; - let val_len = 19 + NAME.len() + extents.len() * EXTENT_SIZE; - assert!( - value - at + val_len <= BLOCK_SIZE, - "a {val_len}-byte value does not fit the block this crafts", - ); - - // Level 0, one entry: a leaf. - raw[at + 8..at + 10].copy_from_slice(&0u16.to_le_bytes()); - raw[at + 10..at + 12].copy_from_slice(&1u16.to_le_bytes()); - raw[entry + 18..entry + 22].copy_from_slice(&(val_len as u32).to_le_bytes()); - raw[value..at + BLOCK_SIZE].fill(0); - - raw[value] = entry_type; - raw[value + 1..value + 3].copy_from_slice(&(NAME.len() as u16).to_le_bytes()); - raw[value + 3..value + 11].copy_from_slice(&size.to_le_bytes()); - raw[value + 11..value + 19].copy_from_slice(&1u64.to_le_bytes()); - raw[value + 19..value + 19 + NAME.len()].copy_from_slice(NAME); - - let mut off = value + 19 + NAME.len(); - for (start_block, block_count) in extents { - raw[off..off + 8].copy_from_slice(&start_block.to_le_bytes()); - raw[off + 8..off + 12].copy_from_slice(&block_count.to_le_bytes()); - off += EXTENT_SIZE; - } - seal_node(raw, root); - } - - #[test] - fn a_delete_of_a_name_that_is_not_here_destroys_nothing() { - // The stored entry keeps its value — the name inside it is still - // victim.txt — and answers to the key `ghost` hashes to. That is the - // 2^-128 collision, staged rather than waited for. - let blocks = 128; - let mut raw = image(blocks); - let mut seed = [0u8; 16]; - seed.copy_from_slice(&raw[90..106]); - let root = read_u64_at(&raw, 24); - let (h, hi) = hash_name(&seed, "ghost"); - - let entry = root as usize * BLOCK_SIZE + 32; - raw[entry..entry + 8].copy_from_slice(&h.to_le_bytes()); - raw[entry + 8..entry + 16].copy_from_slice(&hi.to_le_bytes()); - seal_node(&mut raw, root); - - let mut fs = mount_rw(raw).expect("mount"); - assert!( - fs.list(usize::MAX, &|_| true).expect("list").iter().any(|(n, _)| n == "victim.txt"), - "the craft did not leave victim.txt on the volume", - ); - - assert!(!fs.delete("ghost").expect("delete"), "nothing on this volume is named ghost"); - - assert!( - fs.list(usize::MAX, &|_| true).expect("list").iter().any(|(n, _)| n == "victim.txt"), - "deleting a name that does not exist destroyed the entry it collided with", - ); - } - - #[test] - fn a_root_that_points_at_itself_is_refused_not_followed() { - let blocks = 128; - let mut raw = image(blocks); - craft_interior(&mut raw, 3, 3, 8); - set_root(&mut raw, blocks, 3); - - let fs = mount(raw).expect("the volume still describes its device"); - match fs.list(usize::MAX, &|_| true) { - Err(FsError::TreeTooDeep(_)) => {} - other => panic!("expected TreeTooDeep, got {other:?}"), - } - } - - #[test] - fn an_interior_value_too_short_for_a_block_number_is_refused() { - let blocks = 128; - let mut raw = image(blocks); - craft_interior(&mut raw, 3, 4, 4); - set_root(&mut raw, blocks, 3); - - let fs = mount(raw).expect("mount"); - match fs.list(usize::MAX, &|_| true) { - Err(FsError::CorruptedNode(_)) => {} - other => panic!("expected CorruptedNode, got {other:?}"), - } - } - - #[test] - fn a_child_pointer_past_the_end_of_the_device_is_refused() { - let blocks = 128; - let mut raw = image(blocks); - craft_interior(&mut raw, 3, u64::MAX, 8); - set_root(&mut raw, blocks, 3); - - let fs = mount(raw).expect("mount"); - match fs.list(usize::MAX, &|_| true) { - Err(FsError::BlockOffDevice { .. }) => {} - other => panic!("expected BlockOffDevice, got {other:?}"), - } - } - - #[test] - fn a_superblock_claiming_more_blocks_than_the_device_has_is_refused() { - let blocks = 128; - let mut raw = image(blocks); - patch_superblock(&mut raw, blocks, 12, 1 << 40); - - match mount(raw) { - Err(FsError::BadSuperblock { field }) => assert_eq!(field, "block_count"), - Err(other) => panic!("expected BadSuperblock, got {other:?}"), - Ok(_) => panic!("mounted a superblock describing a device 8192 times this one"), - } - } - - #[test] - fn a_volume_copied_onto_a_larger_device_does_not_mount() { - let blocks = 128; - let mut raw = image(blocks); - mount(raw.clone()).expect("the volume mounts on the device it was formatted on"); - raw.resize(raw.len() + 8 * BLOCK_SIZE, 0); - - match mount(raw) { - Err(FsError::BadSuperblock { field }) => assert_eq!(field, "block_count"), - Err(other) => panic!("expected BadSuperblock, got {other:?}"), - Ok(_) => panic!("mounted a volume that did not come from this device"), - } - } - - #[test] - fn a_bitmap_outside_the_volume_is_refused() { - // `set_used`, `set_free` and `is_free` all compute - // `bitmap_start + byte_idx / 4096` and check it against nothing, so a - // mounted volume with this field wrong writes its bitmap over whatever - // is at those blocks. - let blocks = 128; - let mut raw = image(blocks); - patch_superblock(&mut raw, blocks, 52, 1 << 40); - - match mount(raw) { - Err(FsError::BadSuperblock { field }) => assert_eq!(field, "bitmap_start"), - Err(other) => panic!("expected BadSuperblock, got {other:?}"), - Ok(_) => panic!("mounted a volume whose bitmap is not on the device"), - } - } - - #[test] - fn a_bitmap_too_small_to_cover_the_volume_is_refused() { - // One bit per block is not negotiable: a shorter bitmap means blocks - // whose free/used state is read out of whatever follows it. - let blocks = 128; - let mut raw = image(blocks); - patch_superblock(&mut raw, blocks, 60, 0); - - match mount(raw) { - Err(FsError::BadSuperblock { field }) => assert_eq!(field, "bitmap_blocks"), - Err(other) => panic!("expected BadSuperblock, got {other:?}"), - Ok(_) => panic!("mounted a volume with no bitmap at all"), - } - } - - #[test] - fn an_untampered_volume_still_mounts_and_reads() { - let blocks = 128; - let fs = mount(image(blocks)).expect("a volume this crate wrote must mount"); - assert_eq!(fs.read_file("victim.txt").expect("read"), b"a file that was already here"); - } - - #[test] - fn an_extent_past_the_end_of_the_volume_is_refused() { - // A run that *begins* on the last block and ends one past it. The - // start alone is in range, which is why the comparison is on the end. - let blocks = 128; - let mut raw = image(blocks); - let root = read_u64_at(&raw, 24); - craft_entry(&mut raw, root, 1, 2 * BLOCK_SIZE as u64, &[(blocks - 1, 2)]); - - let fs = mount(raw).expect("mount"); - match fs.read_file("victim.txt") { - Err(FsError::BlockOffDevice { block, device_blocks }) => { - assert_eq!((block, device_blocks), (blocks - 1, blocks)); - } - // Never `{other:?}` over a read's `Ok`: an unrefused one holds - // whatever the crafted size asked for, and the panic message is - // where that lands — 2 GB of it, measured, the first time. - Err(other) => panic!("expected BlockOffDevice, got {other:?}"), - Ok(data) => panic!("a run ending past the volume was read: {} bytes", data.len()), - } - } - - #[test] - fn a_run_ending_on_the_volumes_last_block_is_accepted() { - // The other side of the same comparison: a file whose run ends exactly - // at the volume's end is a legal file, and a bound that refuses it is - // an off-by-one nothing else would catch. - let blocks = 128; - let mut raw = image(blocks); - let root = read_u64_at(&raw, 24); - craft_entry(&mut raw, root, 1, BLOCK_SIZE as u64, &[(blocks - 1, 1)]); - - let fs = mount(raw).expect("mount"); - let data = fs.read_file("victim.txt").expect("a run of the last block is on the volume"); - assert_eq!(data.len(), BLOCK_SIZE); - } - - #[test] - fn a_file_naming_more_blocks_than_the_volume_has_is_refused() { - // Every run here is inside the volume; there are simply too many of - // them. This is the bound that keeps `read_extents`' `Vec` under the - // volume's own size — without it, 254 in-range runs of 4 GiB each are - // what a declared size is measured against. - let blocks = 128; - let mut raw = image(blocks); - let root = read_u64_at(&raw, 24); - let extents: Vec<(u64, u32)> = (0..64).map(|_| (3u64, 4u32)).collect(); - craft_entry(&mut raw, root, 1, BLOCK_SIZE as u64, &extents); - - let fs = mount(raw).expect("mount"); - match fs.read_file("victim.txt") { - Err(FsError::NotEnoughBlocks { needed, available }) => { - // 33 runs of 4 is the first total past 128. - assert_eq!((needed, available), (132, blocks)); - } - Err(other) => panic!("expected NotEnoughBlocks, got {other:?}"), - Ok(data) => panic!( - "a file naming 256 blocks of a {blocks}-block volume was read: {} bytes", - data.len(), - ), - } - } - - #[test] - fn a_size_longer_than_the_volume_never_reaches_the_allocator() { - // The kernel-reachable shape: `read_link` is on the adapter, and the - // target it returns is `read_extents(size)` with a `size` that came off - // the disk. A gibibyte is small enough that a host allocator hands it - // over and 500 times what the kernel's asserts on - // (`mm::MAX_HEAP_ALLOC`, 2_093_056), so the peak allocation is the - // instrument here and the return value is not. - let blocks = 128; - let mut raw = image(blocks); - let root = read_u64_at(&raw, 24); - let size = 1u64 << 30; - craft_entry(&mut raw, root, 2, size, &[(3, 1)]); - - let fs = mount(raw).expect("mount"); - crate::alloc_probe::take_peak(); - let target = fs.read_link("victim.txt", u64::MAX); - let peak = crate::alloc_probe::take_peak(); - - // The allocation is asserted first: it is the harm, and it happened - // before there was a return value to look at. - assert!( - peak <= BLOCK_SIZE, - "a symlink declaring {size} bytes asked the allocator for {peak}", - ); - match target { - Err(FsError::NotEnoughBlocks { needed, available }) => { - assert_eq!((needed, available), (size / BLOCK_SIZE as u64, blocks)); - } - Err(other) => panic!("expected NotEnoughBlocks, got {other:?}"), - Ok(target) => panic!( - "a symlink declaring {size} bytes resolved to a {}-byte target", - target.map_or(0, |t| t.len()), - ), - } - } - - #[test] - fn a_size_inside_the_volume_but_over_the_ceiling_never_reaches_the_allocator() { - // The band the volume bound leaves open: a size the volume holds and a - // kernel heap cannot. Same instrument as above, with a smaller number. - let blocks = 2048; - let mut raw = image(blocks); - let root = read_u64_at(&raw, 24); - let size = 3_000_000u64; - let ceiling = 65_536u64; - craft_entry(&mut raw, root, 2, size, &[(3, 1)]); - - let fs = mount(raw).expect("mount"); - crate::alloc_probe::take_peak(); - let target = fs.read_link("victim.txt", ceiling); - let peak = crate::alloc_probe::take_peak(); - - assert!( - peak <= BLOCK_SIZE, - "a symlink declaring {size} bytes asked the allocator for {peak}", - ); - match target { - Err(FsError::TargetTooLong { size: got, max }) => { - assert_eq!((got, max), (size, ceiling)); - } - Err(other) => panic!("expected TargetTooLong, got {other:?}"), - Ok(target) => panic!( - "a symlink declaring {size} bytes resolved to a {}-byte target", - target.map_or(0, |t| t.len()), - ), - } - } - - #[test] - fn a_tree_past_the_ceiling_is_refused_before_it_materialises() { - // 40,000 entries is past where the unbounded walk's `Vec` doubles to - // 65,536 elements — over the kernel heap ceiling where `GlobalAlloc` - // asserts, so materialise-then-check was a `sys_readdir` panic on - // `/home`. The peak allocation is the instrument, not the refusal alone. - let ceiling = 2 * 1024 * 1024 - 4096; // mm::MAX_HEAP_ALLOC - let limit = 16_384; // vfs::MAX_LIST_ENTRIES, what the kernel passes down - let count = 40_000usize; - let doubled = 65_536 * core::mem::size_of::<(String, u64)>(); - assert!( - doubled > ceiling, - "{doubled} bytes is under the ceiling — this test proves nothing", - ); - - let mut fs = Formatted::format(VecBlockIO::new(16_384)).expect("format"); - for i in 0..count { - fs.create(&format!("f{i}"), &[], 0).expect("create"); - } - let fs = fs.mount(); - - crate::alloc_probe::take_peak(); - let listed = fs.list(limit, &|_| true); - let peak = crate::alloc_probe::take_peak(); - - // Asserted first: the allocation is the harm, before any return value. - assert!( - peak <= ceiling, - "listing a {count}-entry tree under a {limit} ceiling asked the allocator for {peak}", - ); - match listed { - Err(FsError::ListTooLong { limit: said }) => assert_eq!(said, limit), - Err(other) => panic!("expected ListTooLong, got {other:?}"), - Ok(names) => panic!("a {count}-entry tree listed {} names past the ceiling", names.len()), - } - - // The legal case the bound must not take with it. - let all = fs.list(count, &|_| true).expect("list with room"); - assert_eq!(all.len(), count, "a listing with room came back short"); - - // What `keep` bounds is the kept set, so one directory in a volume - // past the ceiling still lists. - let mut fs = fs.into_formatted(); - fs.create("d/only", &[], 0).expect("create"); - let fs = fs.mount(); - let listed = fs - .list(limit, &|name| name == "d" || name.starts_with("d/")) - .expect("one directory in an over-full volume"); - assert_eq!(listed.len(), 1, "listing d/ returned {listed:?}"); - } - - #[test] - fn a_file_ending_in_a_hole_still_reads_as_zeros() { - // The legal case the bound above must not take with it. `set_len` past - // the last block dirties no page, so a flush records a size the extents - // do not reach — `fs_truncate_persist` grows a one-page `/home` file to - // 3 MiB — and the tail of that file is a hole, which is zeros and not a - // refusal. - let blocks = 128; - let mut raw = image(blocks); - let root = read_u64_at(&raw, 24); - craft_entry(&mut raw, root, 1, 4 * BLOCK_SIZE as u64, &[(3, 1)]); - - let fs = mount(raw).expect("mount"); - let data = fs.read_file("victim.txt").expect("a file may end in a hole"); - assert_eq!(data.len(), 4 * BLOCK_SIZE); - assert!( - data[BLOCK_SIZE..].iter().all(|&b| b == 0), - "the hole past the file's one block is not zeros", - ); - } - - #[test] - fn an_extent_outside_the_volume_is_refused_before_the_bitmap_is_touched() { - // What a delete does with an extent: `free_range` → `set_free` → - // `bit_of`, which is `bitmap_start + block / 32768` and is held against - // nothing. This volume's bitmap is one block, at block 1, so every - // block from 32768 up has its "bit" in block 2 — the root node. 32769 - // picks bit 1 of byte 0, and byte 0 is the `B` of `BTND` (0x42), whose - // bit 1 is set: clearing it leaves `@TND` and the volume has no root. - let blocks = 128; - let mut raw = image(blocks); - let root = read_u64_at(&raw, 24); - assert_eq!(root, 2, "the bit arithmetic above is keyed to the root's block"); - craft_entry(&mut raw, root, 1, BLOCK_SIZE as u64, &[(32769, 1)]); - - let at = root as usize * BLOCK_SIZE; - let before = raw[at..at + BLOCK_SIZE].to_vec(); - - let mut fs = mount_rw(raw).expect("mount"); - let deleted = fs.delete("victim.txt"); - - // The bytes first: what the delete *returned* is the smaller half of - // this, and asserting it first would hide the damage behind a panic. - let after = fs.into_formatted().into_io().expect("sync").into_vec(); - assert_eq!(&after[at..at + 4], &NODE_MAGIC[..], "the root node stopped being a node"); - assert!( - after[at..at + BLOCK_SIZE] == before[..], - "a delete of an extent outside the volume wrote to the root node", - ); - - match deleted { - Err(FsError::BlockOffDevice { block, device_blocks }) => { - assert_eq!((block, device_blocks), (32769, blocks)); - } - other => panic!("expected BlockOffDevice, got {other:?}"), - } - } - - #[test] - fn a_page_resolved_against_an_extent_at_the_end_of_the_address_space_is_total() { - // Two halves of one hole. The list `allocate` extends is - // the one `decode_leaf_value` handed out, so the door is where the - // refusal belongs — and the arithmetic behind the door is total anyway, - // because `u64::MAX + 1` under the kernel's `overflow-checks` is a - // panic rather than a wrong answer. - let blocks = 128; - let mut raw = image(blocks); - let root = read_u64_at(&raw, 24); - craft_entry(&mut raw, root, 1, BLOCK_SIZE as u64, &[(u64::MAX, 1)]); - - let mut fs = mount_rw(raw).expect("mount"); - - // The arithmetic first, on a list handed in directly: past the door, - // `push_extent` adds `start_block + block_count` and `block_for` - // accumulates the counts, and neither may be a panic. - let mut extents = Vec::from([Extent { - start_block: u64::MAX, - block_count: 1, - _reserved: 0, - }]); - fs.allocate(&mut extents, 1..=1, 1).expect("a block for page 1"); - let block = block_for(&extents, 1).expect("page 1 resolves"); - assert!(block < blocks, "page 1 resolved to block {block}, which is not on this volume"); - - // And the door itself: nothing gets to hand that list to the kernel. - match fs.file_extents("victim.txt") { - Err(FsError::BlockOffDevice { block, device_blocks }) => { - assert_eq!((block, device_blocks), (u64::MAX, blocks)); - } - other => panic!("expected BlockOffDevice, got {other:?}"), - } - } - - /// The kernel's shrink: record the shortened extent list, then free the - /// dropped tail. The record answers with the short list, an emptied list - /// records as empty rather than silently keeping the old blocks, and the - /// freed tail is the allocator's again — block for block. - #[test] - fn a_shortened_record_gives_its_tail_back() { - let mut fs = Formatted::format(VecBlockIO::new(64)).expect("format").mount(); - fs.create("shrink.bin", b"", 1).expect("create"); - let mut extents = Vec::new(); - for page in 0..3 { - fs.allocate(&mut extents, page..=page, 1).expect("allocate"); - } - fs.update_metadata("shrink.bin", &extents, 3 * BLOCK_SIZE as u64, 2) - .expect("record three pages"); - let free_full = fs.alloc.free_blocks; - - let kept = vec![Extent { - start_block: extents[0].start_block, - block_count: 1, - _reserved: 0, - }]; - let dropped = vec![Extent { - start_block: extents[0].start_block + 1, - block_count: 2, - _reserved: 0, - }]; - fs.update_metadata("shrink.bin", &kept, 100, 3).expect("record the shortened list"); - fs.free_extents(&dropped).expect("free the dropped tail"); - assert_eq!( - fs.alloc.free_blocks, - free_full + 2, - "the dropped tail did not come back to the allocator", - ); - - let (back, size) = fs.file_extents("shrink.bin").expect("file_extents").expect("present"); - assert_eq!(size, 100); - assert_eq!(back.len(), 1, "the record kept the dropped tail: {back:?}"); - assert_eq!(back[0].block_count, 1); - - fs.update_metadata("shrink.bin", &[], 0, 5).expect("record an emptied list"); - let (back, size) = fs.file_extents("shrink.bin").expect("file_extents").expect("present"); - assert_eq!(size, 0); - assert!(back.is_empty(), "an emptied extent list kept blocks: {back:?}"); - } - - /// A block allocated only to bridge a gap is a hole, and a hole reads zero. - /// - /// `set_free` walks `next_alloc` down to whatever it just freed, so the - /// blocks a shrink gave back are the first ones the next allocation takes: - /// a gap bridged with them serves that file's own discarded tail. - #[test] - fn a_block_allocated_to_bridge_a_gap_reads_as_zeros() { - let mut fs = Formatted::format(VecBlockIO::new(64)).expect("format").mount(); - fs.create("gap.bin", b"", 1).expect("create"); - - let mut stamped = BlockBuf::zeroed(); - stamped.0.fill(0xA7); - let mut extents = Vec::new(); - for page in 0..3u32 { - fs.allocate(&mut extents, page..=page, 1).expect("allocate"); - let block = block_for(&extents, page).expect("allocated"); - fs.io.write_block(BlockNum::new(block), &stamped).expect("stamp"); - } - let first = extents[0].start_block; - fs.free_extents(&[Extent { start_block: first + 1, block_count: 2, _reserved: 0 }]) - .expect("give the tail back"); - - // The shrunk list, then a write that reaches page 2 again: page 1 is - // the gap, and the allocator answers with the blocks just freed. - let mut short = vec![Extent { start_block: first, block_count: 1, _reserved: 0 }]; - fs.allocate(&mut short, 2..=2, 1).expect("allocate through the gap"); - let target = block_for(&short, 2).expect("page 2 resolves"); - assert_eq!(target, first + 2, "the allocator did not reuse the freed tail"); - let gap = block_for(&short, 1).expect("page 1 resolves"); - assert_eq!(gap, first + 1, "page 1 is not the block the shrink freed"); - - let mut buf = BlockBuf::zeroed(); - fs.io.read_block(BlockNum::new(gap), &mut buf).expect("read the gap block"); - if let Some(at) = buf.0.iter().position(|&b| b != 0) { - panic!("the gap block holds {:#04x} at byte {at}, not zero", buf.0[at]); - } - } - - /// A directory is a name with something beneath it, and follows every - /// write that adds or takes one. - #[test] - fn is_dir_is_a_name_beneath() { - let mut fs = Formatted::format(VecBlockIO::new(512)).expect("format"); - fs.create("a/b/file", b"x", 1).expect("create"); - fs.create("ab", b"x", 1).expect("create"); - fs.create_symlink("s/link", "a", 1).expect("symlink"); - let mut fs = mount_rw(fs.into_io().expect("sync").into_vec()).expect("mount"); - for dir in ["", "a", "a/b", "s"] { - assert!(fs.is_dir(dir).expect("a walk of a sound tree"), "{dir:?}"); - } - // `ab` shares `a`'s bytes, so a prefix test without the separator answers it. - for not in ["a/b/file", "ab", "a/missing", "b", "a/b/file/deeper", "a/"] { - assert!(!fs.is_dir(not).expect("a walk of a sound tree"), "{not:?}"); - } - - fs.rename("a/b/file", "e/moved").expect("rename"); - assert!(!fs.is_dir("a").expect("walk") && fs.is_dir("e").expect("walk")); - assert!(fs.delete("e/moved").expect("delete")); - assert!(!fs.is_dir("e").expect("walk")); - } - - /// A node the walk cannot follow refuses the one question that met it; - /// the mount before it stands. - #[test] - fn is_dir_over_a_corrupt_node_is_refused() { - let blocks = 128; - let mut raw = image(blocks); - craft_interior(&mut raw, 3, 4, 4); - set_root(&mut raw, blocks, 3); - - let fs = mount(raw).expect("mount"); - match fs.is_dir("a") { - Err(FsError::CorruptedNode(_)) => {} - other => panic!("expected CorruptedNode, got {other:?}"), - } - assert!(fs.is_dir("").expect("the root needs no walk")); - } - - /// `is_dir_is_a_name_beneath` never grows the root past one leaf, so a - /// walk that stopped propagating `Break` through an interior node would - /// still pass it. More names than `MAX_ENTRIES` (169) fit in one leaf - /// forces at least one split, so the root here is `Interior` and every - /// `d{i}` is beneath a child the walk has to descend into and return from. - #[test] - fn is_dir_over_a_split_root_finds_every_directory() { - const COUNT: usize = 200; - let mut fs = Formatted::format(VecBlockIO::new(2048)).expect("format"); - for i in 0..COUNT { - fs.create(&format!("d{i}/f"), b"x", i as u64) - .unwrap_or_else(|e| panic!("create d{i}/f: {e:?}")); - } - let fs = mount_rw(fs.into_io().expect("sync").into_vec()).expect("mount"); - for i in 0..COUNT { - let dir = format!("d{i}"); - assert!(fs.is_dir(&dir).expect("a walk of a sound tree"), "{dir}"); - } - } - - /// A device that records the blocks it is asked to write and counts its - /// syncs, and refuses the sync `refuse` numbers. - struct Counted { - io: VecBlockIO, - written: core::cell::RefCell>, - syncs: core::cell::Cell, - refuse: core::cell::Cell>, - } - - impl Counted { - fn counts(&self) -> (usize, u32) { - (self.written.borrow().len(), self.syncs.get()) - } - } - - impl BlockIO for Counted { - fn read_block(&self, block: BlockNum, buf: &mut BlockBuf) -> Result<(), DeviceError> { - self.io.read_block(block, buf) - } - fn write_block(&self, block: BlockNum, buf: &BlockBuf) -> Result<(), DeviceError> { - self.written.borrow_mut().push(block.raw()); - self.io.write_block(block, buf) - } - fn block_count(&self) -> u64 { - self.io.block_count() - } - fn sync(&self) -> Result<(), DeviceError> { - let n = self.syncs.get(); - self.syncs.set(n + 1); - if self.refuse.get() == Some(n) { Err(DeviceError::FAILED) } else { Ok(()) } - } - } - - fn counted(blocks: u64) -> Mounted { - let io = Counted { - io: VecBlockIO::from_vec(image(blocks)), - written: core::cell::RefCell::default(), - syncs: core::cell::Cell::default(), - refuse: core::cell::Cell::default(), - }; - Mounted::<_, ReadWrite>::open(io).expect("mount") - } - - /// A write's pages are allocated as the runs the volume has, with one - /// write to each bitmap block their bits lie in, and no page the caller - /// writes is written here. Only a hole before the pages is zeroed. - #[test] - fn a_range_of_pages_is_allocated_with_one_write_per_bitmap_block() { - let mut fs = counted(4096); - let bitmap = fs.alloc.bitmap_start.raw(); - let mut extents = Vec::new(); - let written = |fs: &Mounted, from: usize| fs.io().written.borrow()[from..].to_vec(); - - let from = fs.io().counts().0; - fs.allocate(&mut extents, 0..=255, 32).expect("allocate"); - assert_eq!(extents.len(), 1, "256 pages on an empty volume took {extents:?}"); - assert_eq!(written(&fs, from), [bitmap], "allocating 256 pages wrote these blocks"); - - let from = fs.io().counts().0; - fs.allocate(&mut extents, 260..=299, 32).expect("allocate past a hole"); - let hole: Vec = (256..260).map(|page| block_for(&extents, page).expect("allocated")).collect(); - assert_eq!(written(&fs, from), [vec![bitmap], hole].concat(), "a write after a 4-page hole wrote these blocks"); - } - - /// Mark `blocks` used on the bitmap directly, to lay out a volume's free - /// space, and recount what is free. - fn mark_used(fs: &mut Mounted, blocks: impl Iterator) { - let bits = (BLOCK_SIZE * 8) as u64; - let mut buf = BlockBuf::zeroed(); - let mut held = u64::MAX; - let start = fs.alloc.bitmap_start.raw(); - for block in blocks { - if block / bits != held { - if held != u64::MAX { - fs.io.write_block(BlockNum::new(start + held), &buf).expect("write the bitmap"); - } - held = block / bits; - fs.io.read_block(BlockNum::new(start + held), &mut buf).expect("read the bitmap"); - } - buf.0[(block % bits / 8) as usize] |= 1 << (block % 8); - } - if held != u64::MAX { - fs.io.write_block(BlockNum::new(start + held), &buf).expect("write the bitmap"); - } - fs.alloc.count_free(&fs.io).expect("recount"); - } - - /// 31 single free blocks at the cursor, then a long free run: a write's - /// pages go to the long run as one extent, where a page at a time takes - /// the holes one by one. - #[test] - fn a_write_skips_single_holes_for_a_run_a_request_long() { - let volume = || { - let mut fs = Formatted::format(VecBlockIO::new(4096)).expect("format").mount(); - let at = fs.alloc.next_alloc; - mark_used(&mut fs, (at..at + 64).step_by(2)); - (fs, at) - }; - let (mut fs, at) = volume(); - let mut extents = Vec::new(); - fs.allocate(&mut extents, 0..=63, 32).expect("allocate"); - let runs: Vec<(u64, u32)> = extents.iter().map(|e| (e.start_block, e.block_count)).collect(); - assert_eq!(runs, [(at + 63, 64)], "64 pages after 31 single holes"); - - let (mut paged, at) = volume(); - let mut extents = Vec::new(); - for page in 0..4 { - paged.allocate(&mut extents, page..=page, 1).expect("allocate"); - } - assert_eq!(extents[0].start_block, at + 1, "a page at a time takes the first hole"); - } - - /// A device that keeps only the blocks written to it, the rest reading - /// zeros, and counts its reads. - struct Sparse { - blocks: u64, - held: core::cell::RefCell>, - reads: core::cell::Cell, - } - - impl BlockIO for Sparse { - fn read_block(&self, block: BlockNum, buf: &mut BlockBuf) -> Result<(), DeviceError> { - self.reads.set(self.reads.get() + 1); - buf.0 = self.held.borrow().get(&block.raw()).map_or([0; BLOCK_SIZE], |b| b.0); - Ok(()) - } - fn write_block(&self, block: BlockNum, buf: &BlockBuf) -> Result<(), DeviceError> { - self.held.borrow_mut().insert(block.raw(), BlockBuf(buf.0)); - Ok(()) - } - fn block_count(&self) -> u64 { - self.blocks - } - fn sync(&self) -> Result<(), DeviceError> { - Ok(()) - } - } - - /// Bitmap blocks of the volumes below. - const BITMAP_BLOCKS: u64 = 8; - - /// A volume of [`BITMAP_BLOCKS`] bitmap blocks, every block of it used - /// but those `free` names. - fn clustered(free: impl Fn(u64) -> bool) -> Mounted { - let blocks = BITMAP_BLOCKS * (BLOCK_SIZE * 8) as u64; - let io = Sparse { blocks, held: core::cell::RefCell::default(), reads: core::cell::Cell::default() }; - let mut fs = Formatted::format(io).expect("format").mount(); - mark_used(&mut fs, (0..blocks).filter(|&b| !free(b))); - fs - } - - /// `pages` pages written to an empty file, at most two laps of the bitmap - /// read besides each run's own: a lap passes `BITMAP_BLOCKS` and the one it - /// starts in, and each run reads its first block's bitmap block, marks it - /// and checks the block after the file. Returns the extents. - fn within_two_laps(fs: &mut Mounted, pages: u32) -> Vec { - fs.io.reads.set(0); - let mut extents = Vec::new(); - fs.allocate(&mut extents, 0..=pages - 1, 32).expect("allocate"); - let reads = fs.io.reads.get(); - let runs = extents.len() as u64; - assert!(reads <= 2 * (BITMAP_BLOCKS + 1) + 3 * runs, "{runs} runs read the bitmap {reads} times"); - extents - } - - /// No free run longer than one block anywhere: one lap finds no run a - /// request long, and the rest of the write takes single blocks forward. - #[test] - fn a_write_on_a_volume_of_single_holes_reads_two_laps_at_most() { - let mut fs = clustered(|b| b.is_multiple_of(2)); - let extents = within_two_laps(&mut fs, 16); - assert!(extents.iter().all(|e| e.block_count == 1), "a run longer than one block: {extents:?}"); - assert_eq!(extents.len(), 16); - } - - /// The only free blocks are 64 single ones in 128 blocks, with nothing - /// free beyond them: the write sweeps forward through them, not round the - /// volume once a run. - #[test] - fn a_write_into_one_cluster_of_single_holes_reads_two_laps_at_most() { - const AT: u64 = 100_000; - let mut fs = clustered(|b| (AT..AT + 128).contains(&b) && (b - AT).is_multiple_of(2)); - let extents = within_two_laps(&mut fs, 16); - let starts: Vec = extents.iter().map(|e| e.start_block).collect(); - assert_eq!(starts, (0..16).map(|k| AT + 2 * k).collect::>()); - } - - /// The only free blocks are eight runs of 40, one used block apart, and a - /// write takes seven runs' worth (the rest is the tree's reserve). The jump - /// past a file's taken next block overshoots the cluster, so the search - /// for a run a request long spends its lap, and the write sweeps forward - /// for the rest: at most two laps in all. - #[test] - fn a_write_into_one_cluster_of_long_runs_reads_two_laps_at_most() { - const AT: u64 = 100_000; - let mut fs = clustered(|b| (AT..AT + 8 * 41).contains(&b) && (b - AT) % 41 < 40); - let extents = within_two_laps(&mut fs, 7 * 40); - assert_eq!(extents.iter().map(|e| e.block_count).sum::(), 7 * 40); - } - - /// A sync with nothing changed since the last commit writes no block and - /// asks the device for one sync, for whatever was written in place; a - /// change after it is committed again. - #[test] - fn a_sync_with_nothing_uncommitted_writes_nothing() { - let mut fs = counted(128); - fs.create("a", b"one", 2).expect("create"); - fs.sync().expect("the commit"); - let (writes, syncs) = fs.io().counts(); - fs.sync().expect("a clean sync"); - assert_eq!(fs.io().counts(), (writes, syncs + 1)); - - fs.create("b", b"two", 3).expect("create"); - fs.sync().expect("the next commit"); - let (after, _) = fs.io().counts(); - assert!(after > writes + 2, "a change and its commit wrote {} blocks", after - writes); - let raw = mount_rw(fs.io.io.into_vec()).expect("remount"); - assert_eq!(raw.read_file("b").expect("read"), b"two"); - } - - /// A commit whose superblock was written and whose sync was refused is - /// made again by the next, though nothing changed between them. - #[test] - fn a_sync_after_a_refused_one_commits_again() { - let mut fs = counted(128); - fs.create("a", b"one", 2).expect("create"); - fs.sync().expect("the commit"); - fs.create("b", b"two", 3).expect("create"); - fs.io().refuse.set(Some(fs.io().syncs.get() + 1)); - assert!(fs.sync().is_err(), "the device refused the sync after the superblock"); - let (writes, _) = fs.io().counts(); - fs.sync().expect("the commit again"); - let again = fs.io().written.borrow()[writes..].to_vec(); - assert!(again.contains(&0) && again.contains(&127), "the retry wrote {again:?}, no superblock"); - } -} diff --git a/lib/bcachefs/src/lib.rs b/lib/bcachefs/src/lib.rs index ec4e1141e63..7776542525f 100644 --- a/lib/bcachefs/src/lib.rs +++ b/lib/bcachefs/src/lib.rs @@ -1,76 +1,9 @@ -//! bcachefs. [`engine`] implements its on-disk format; the modules beside it -//! are the interim ToyOS format DATA's file server still uses. -//! -//! **Every number this crate parses came off a disk it does not own, and a -//! CRC is not authentication — whoever writes the image writes the CRC.** Each -//! bound is applied where the bytes become typed data and nowhere else: a -//! superblock's fields in `Superblock::check`, a btree child pointer in -//! `Node::parse`, a file's extents in `decode_leaf_value`. A consumer of what -//! those hand back does not re-check, so a new one of them is a new refusal -//! here rather than a comparison at the call site. +//! bcachefs on-disk storage and ToyOS's explicit formatting designation. #![cfg_attr(not(feature = "std"), no_std)] -#![allow(dead_code)] - extern crate alloc; - -mod block_io; mod crc32c; -mod superblock; -mod alloc_bitmap; -mod btree; -mod fs; -#[warn(dead_code)] +mod designation; pub mod engine; - -pub use block_io::{BlockIO, BlockBuf, BlockNum, DeviceError, TransferError}; -#[cfg(feature = "std")] -pub use block_io::VecBlockIO; -pub use fs::{file_entry_fits, Formatted, Mounted, ReadOnly, ReadWrite, Recorded, FsError, Extent}; -pub use superblock::{carries_volume, designated, designation, FsUuid, Superblock}; - -/// Records the largest single allocation each test thread makes, so a test can -/// assert what parsing a crafted block asks the allocator for. -/// -/// Nothing else can see that. The defect this exists to catch — a `Vec` sized -/// from an on-disk `u16` — leaves the parse returning the same error either -/// way, so a test that only checks the return value passes with the bug in. -#[cfg(test)] -mod alloc_probe { - use core::alloc::{GlobalAlloc, Layout}; - use core::cell::Cell; - use std::alloc::System; - - std::thread_local! { - static PEAK: Cell = const { Cell::new(0) }; - } - - pub struct Probe; - - fn record(size: usize) { - // `try_with` because TLS destruction can free after the slot is gone. - let _ = PEAK.try_with(|p| p.set(p.get().max(size))); - } - - /// The largest single allocation this thread has made since the last call. - pub fn take_peak() -> usize { - PEAK.with(|p| p.replace(0)) - } - - unsafe impl GlobalAlloc for Probe { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - record(layout.size()); - unsafe { System.alloc(layout) } - } - unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - unsafe { System.dealloc(ptr, layout) } - } - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - record(new_size); - unsafe { System.realloc(ptr, layout, new_size) } - } - } - - #[global_allocator] - static PROBE: Probe = Probe; -} +pub use designation::{designated, designation}; +pub use engine::carries_volume; diff --git a/lib/bcachefs/src/superblock.rs b/lib/bcachefs/src/superblock.rs deleted file mode 100644 index 2462012247d..00000000000 --- a/lib/bcachefs/src/superblock.rs +++ /dev/null @@ -1,346 +0,0 @@ -use core::fmt; - -use crate::block_io::{BlockBuf, BlockNum, BlockIO, BlockIOExt, BLOCK_SIZE}; -use crate::crc32c::crc32c; -use crate::fs::FsError; - -pub const MAGIC: [u8; 4] = *b"BCFS"; -pub const VERSION: u32 = 2; - -/// The designation stamp: the only thing that authorises a file server to -/// format a partition that holds no volume of this format. -/// -/// Not a bcachefs concept, and it lives here anyway, because it occupies the -/// same block as the superblock and the one invariant that matters is that -/// the two can never be confused — which is checkable at a glance only if -/// they sit next to each other. The assertion below is the check. -/// -/// Bytes 0..16 are this magic and bytes 16..24 the little-endian block count -/// of the device the stamp designates, so a stamped image copied onto a -/// different disk designates nothing. The rest of the block is zero when -/// written and ignored when read. -const DESIGNATION_MAGIC: [u8; 16] = *b"TOYOS-FORMAT-ME\0"; - -const DESIGNATION_BLOCKS_OFFSET: usize = 16; - -/// Block 0 of a device of `blocks` blocks that designates it, whole. -pub fn designation(blocks: u64) -> [u8; BLOCK_SIZE] { - let mut block = [0u8; BLOCK_SIZE]; - block[..DESIGNATION_MAGIC.len()].copy_from_slice(&DESIGNATION_MAGIC); - block[DESIGNATION_BLOCKS_OFFSET..DESIGNATION_BLOCKS_OFFSET + 8].copy_from_slice(&blocks.to_le_bytes()); - block -} - -/// Whether `block0` begins as a volume of this format's does, whatever state -/// the rest of the volume is in. -pub fn carries_volume(block0: &[u8]) -> bool { - block0.get(..MAGIC.len()) == Some(&MAGIC) -} - -/// The block count a stamp at the front of `block0` names, or `None` where -/// `block0` carries none. -pub fn designated(block0: &[u8]) -> Option { - if block0.get(..DESIGNATION_MAGIC.len())? != DESIGNATION_MAGIC { - return None; - } - let count = block0.get(DESIGNATION_BLOCKS_OFFSET..DESIGNATION_BLOCKS_OFFSET + 8)?; - Some(u64::from_le_bytes(count.try_into().ok()?)) -} - -const _: () = assert!( - DESIGNATION_MAGIC[0] != MAGIC[0] - || DESIGNATION_MAGIC[1] != MAGIC[1] - || DESIGNATION_MAGIC[2] != MAGIC[2] - || DESIGNATION_MAGIC[3] != MAGIC[3], - "a designation stamp would parse as a superblock, or the reverse", -); - -/// A filesystem's name: sixteen bytes in its superblock, printed as 32 hex -/// digits. -/// -/// Not a partition GUID and never mixed with one — a partition is where a -/// filesystem's bytes are, and one filesystem may occupy several. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub struct FsUuid(pub [u8; 16]); - -impl FsUuid { - /// A filesystem no role names, which is what [`crate::Formatted::format`] - /// leaves: naming one is a deliberate act, not a side effect of making one. - pub const UNNAMED: Self = Self([0; 16]); - - /// The inverse of [`fmt::Display`]: 32 hex digits, either case, or `None`. - pub fn parse(text: &str) -> Option { - let text = text.as_bytes(); - if text.len() != 32 { - return None; - } - let mut out = [0u8; 16]; - for (byte, pair) in out.iter_mut().zip(text.as_chunks::<2>().0) { - let digit = |c: u8| match c { - b'0'..=b'9' => Some(c - b'0'), - b'a'..=b'f' => Some(c - b'a' + 10), - b'A'..=b'F' => Some(c - b'A' + 10), - _ => None, - }; - *byte = digit(pair[0])? << 4 | digit(pair[1])?; - } - Some(Self(out)) - } -} - -impl fmt::Display for FsUuid { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - for byte in self.0 { - write!(f, "{byte:02x}")?; - } - Ok(()) - } -} - -/// On-disk superblock layout. Stored at block 0 and backed up at the last block. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Superblock { - pub block_count: u64, - pub root_node: BlockNum, - pub next_alloc: u64, - pub free_blocks: u64, - pub bitmap_start: BlockNum, - pub bitmap_blocks: u64, - pub journal_start: BlockNum, - pub journal_blocks: u32, - pub journal_head: u64, - pub flags: u16, - pub hash_seed: [u8; 16], - pub uuid: FsUuid, -} - -impl Superblock { - pub(crate) const CRC_START: usize = 12; // CRC covers bytes [12..4096] - - /// Whether `self` names the same commit as `other`, which the medium - /// holds. The allocation cursor is not compared: it is only where a scan - /// begins, and a commit's own frees move it after its superblock lands. - pub fn same_commit(&self, other: &Self) -> bool { - *self == Self { next_alloc: self.next_alloc, ..other.clone() } - } - - pub fn is_clean(&self) -> bool { - self.flags & 1 != 0 - } - - pub fn set_clean(&mut self, clean: bool) { - if clean { - self.flags |= 1; - } else { - self.flags &= !1; - } - } - - /// Parse a superblock from a block buffer. Verifies magic, version, and CRC. - /// - /// The envelope only. Every field below is a number the disk chose, and a - /// CRC is not authentication — whoever writes the image writes the CRC. - /// [`Superblock::read`] is the entry point that also refuses a superblock - /// describing a device other than the one it came off, and it is the only - /// road to a mount. - pub fn parse(buf: &BlockBuf) -> Result { - let b = buf.as_bytes(); - - let magic = [b[0], b[1], b[2], b[3]]; - if magic != MAGIC { - return Err(FsError::BadMagic { expected: MAGIC, got: magic }); - } - - let version = u32::from_le_bytes([b[4], b[5], b[6], b[7]]); - if version != VERSION { - return Err(FsError::UnsupportedVersion(version)); - } - - let stored_crc = u32::from_le_bytes([b[8], b[9], b[10], b[11]]); - let computed_crc = crc32c(&b[Self::CRC_START..]); - if stored_crc != computed_crc { - return Err(FsError::ChecksumMismatch { - block: BlockNum::new(0), - stored: stored_crc, - computed: computed_crc, - }); - } - - let mut hash_seed = [0u8; 16]; - hash_seed.copy_from_slice(&b[90..106]); - let mut uuid = [0u8; 16]; - uuid.copy_from_slice(&b[106..122]); - - Ok(Self { - block_count: read_u64(b, 12), - root_node: BlockNum::new(read_u64(b, 24)), - next_alloc: read_u64(b, 36), - free_blocks: read_u64(b, 44), - bitmap_start: BlockNum::new(read_u64(b, 52)), - bitmap_blocks: read_u64(b, 60), - journal_start: BlockNum::new(read_u64(b, 68)), - journal_blocks: read_u32(b, 76), - journal_head: read_u64(b, 80), - flags: read_u16(b, 88), - hash_seed, - uuid: FsUuid(uuid), - }) - } - - /// Serialize the superblock into a block buffer, computing the CRC. - pub fn write_to(&self, buf: &mut BlockBuf) { - let b = buf.as_bytes_mut(); - b.fill(0); - - b[0..4].copy_from_slice(&MAGIC); - write_u32(b, 4, VERSION); - // CRC at [8..12] filled last - - write_u64(b, 12, self.block_count); - write_u32(b, 20, BLOCK_SIZE as u32); - write_u64(b, 24, self.root_node.raw()); - // [32..36] pad — the tree's depth used to live here, and drove three - // recursions against a 128 KiB kernel stack. The descent ends at a - // `Node::Leaf` now, so the disk has no say in how deep it goes. - write_u64(b, 36, self.next_alloc); - write_u64(b, 44, self.free_blocks); - write_u64(b, 52, self.bitmap_start.raw()); - write_u64(b, 60, self.bitmap_blocks); - write_u64(b, 68, self.journal_start.raw()); - write_u32(b, 76, self.journal_blocks); - write_u64(b, 80, self.journal_head); - write_u16(b, 88, self.flags); - b[90..106].copy_from_slice(&self.hash_seed); - b[106..122].copy_from_slice(&self.uuid.0); - - let crc = crc32c(&b[Self::CRC_START..]); - write_u32(b, 8, crc); - } - - /// Refuse a superblock that does not describe the device it was read from. - /// - /// Nine of these fields are indices into a device whose size the - /// superblock does not get to declare, and `Mounted::open` copied five of - /// them straight into the allocator. An unchecked `bitmap_start` puts - /// bitmap writes on arbitrary blocks; an unchecked `block_count` above the - /// device puts the backup superblock past the end of it. - fn check(&self, device_blocks: u64) -> Result<(), FsError> { - let bad = |field| Err(FsError::BadSuperblock { field }); - - // Exact, not a bound: `format` writes the device's own block count, so a - // volume naming any other number did not come from this device. - if self.block_count != device_blocks { - return bad("block_count"); - } - if self.root_node.raw() >= self.block_count { - return bad("root_node"); - } - // Block 0 is the superblock, so a bitmap starting there overwrites it. - if self.bitmap_start.raw() == 0 || self.bitmap_start.raw() >= self.block_count { - return bad("bitmap_start"); - } - let bitmap_needed = self.block_count.div_ceil(BLOCK_SIZE as u64 * 8); - let bitmap_end = self.bitmap_start.raw().checked_add(self.bitmap_blocks); - if self.bitmap_blocks < bitmap_needed - || bitmap_end.is_none_or(|end| end > self.block_count) - { - return bad("bitmap_blocks"); - } - if self.journal_start.raw() >= self.block_count { - return bad("journal_start"); - } - if self.free_blocks > self.block_count { - return bad("free_blocks"); - } - if self.next_alloc >= self.block_count { - return bad("next_alloc"); - } - Ok(()) - } - - /// Read superblock from disk, trying block 0 first, then backup at last block. - pub fn read(io: &dyn BlockIO) -> Result { - let device_blocks = io.block_count(); - if device_blocks == 0 { - return Err(FsError::BadSuperblock { field: "device has no blocks" }); - } - - let checked = |buf: &BlockBuf| { - Self::parse(buf).and_then(|sb| sb.check(device_blocks).map(|()| sb)) - }; - - let mut buf = BlockBuf::zeroed(); - // A block 0 the device would not read is not a block 0 that failed to - // parse: the backup is the answer to a bad *superblock*, not to a bad - // device, and reaching for it after a refused read would mount a - // volume from a device that is not answering. - io.read(BlockNum::new(0), &mut buf)?; - match checked(&buf) { - Ok(sb) => Ok(sb), - Err(primary_err) => { - io.read(BlockNum::new(device_blocks - 1), &mut buf)?; - // `BadMagic` is returned only when no copy claims to be a - // volume: a caller tells a disk that is not ours from one of - // ours that broke by it. - checked(&buf).map_err(|backup_err| match primary_err { - FsError::BadMagic { .. } => backup_err, - _ => primary_err, - }) - } - } - } - - /// Write superblock to both block 0 and the backup at the last block. - pub fn write(&self, io: &dyn BlockIO) -> Result<(), FsError> { - let mut buf = BlockBuf::zeroed(); - self.write_to(&mut buf); - io.write(BlockNum::new(0), &buf)?; - io.write(BlockNum::new(self.block_count - 1), &buf) - } -} - -// --- Little-endian helpers --- - -fn read_u16(buf: &[u8], off: usize) -> u16 { - u16::from_le_bytes([buf[off], buf[off + 1]]) -} - -fn read_u32(buf: &[u8], off: usize) -> u32 { - u32::from_le_bytes(buf[off..off + 4].try_into().unwrap()) -} - -fn read_u64(buf: &[u8], off: usize) -> u64 { - u64::from_le_bytes(buf[off..off + 8].try_into().unwrap()) -} - -fn write_u16(buf: &mut [u8], off: usize, val: u16) { - buf[off..off + 2].copy_from_slice(&val.to_le_bytes()); -} - -fn write_u32(buf: &mut [u8], off: usize, val: u32) { - buf[off..off + 4].copy_from_slice(&val.to_le_bytes()); -} - -fn write_u64(buf: &mut [u8], off: usize, val: u64) { - buf[off..off + 8].copy_from_slice(&val.to_le_bytes()); -} - -#[cfg(test)] -mod designation_tests { - use super::*; - - /// A stamp names the count it was made for, and a block that does not - /// begin with the magic names nothing, a superblock's first among them. - #[test] - fn a_stamp_names_its_block_count_and_nothing_else_is_one() { - for blocks in [1, 43_000_000, u64::MAX] { - assert_eq!(designated(&designation(blocks)), Some(blocks)); - } - let mut superblock = [0u8; BLOCK_SIZE]; - superblock[..4].copy_from_slice(&MAGIC); - assert_eq!(designated(&superblock), None); - assert_eq!(designated(&[0u8; BLOCK_SIZE]), None); - assert_eq!(designated(&designation(7)[..20]), None, "a block too short for its count"); - assert!(designation(7)[24..].iter().all(|b| *b == 0), "the rest of the stamp's block is zero"); - } -} diff --git a/lib/blockserve/src/content.rs b/lib/blockserve/src/content.rs index 0e3684aec71..6c686016f6f 100644 --- a/lib/blockserve/src/content.rs +++ b/lib/blockserve/src/content.rs @@ -29,6 +29,7 @@ fn has(front: &[u8], at: usize, bytes: &[u8]) -> bool { /// Every format recognised, by name: where its signature is, and what it is. const SIGNATURES: &[(&str, usize, &[u8])] = &[ + ("the retired ToyOS provisional file system", 0, b"BCFS"), // The ext4 Disk Layout: the superblock at 1024, `s_magic` 0xEF53 at its byte 0x38. ("an ext2/3/4 file system", 1024 + 0x38, &[0x53, 0xEF]), // XFS Algorithms & Data Structures, the superblock: `sb_magicnum` "XFSB" at 0. @@ -105,7 +106,10 @@ mod tests { assert_eq!(judge(&stamped, 100), Content::Designated(100)); assert_eq!(judge(&stamped, 99), Content::Foreign("a stamp for another size")); let mut ours = blank.clone(); - ours[..4].copy_from_slice(b"BCFS"); + ours[4096 + 24..4096 + 40].copy_from_slice(&[ + 0xc6, 0x85, 0x73, 0xf6, 0x66, 0xce, 0x90, 0xa9, + 0xd9, 0x6a, 0x60, 0xcf, 0x80, 0x3d, 0xf7, 0xef, + ]); assert_eq!(judge(&ours, 100), Content::Volume); assert_eq!(judge(&blank[..4096], 1), Content::Unrecognised, "a partition shorter than the read"); } diff --git a/system/fileserver/Cargo.toml b/system/fileserver/Cargo.toml index ae4a6c629e8..8c4f36748a3 100644 --- a/system/fileserver/Cargo.toml +++ b/system/fileserver/Cargo.toml @@ -25,13 +25,13 @@ toyos-fat32 = { path = "../../lib/fat32" } bcachefs = { path = "../../lib/bcachefs" } diskserver = { path = "../diskserver" } toyos-blockdev = { path = "../../lib/blockdev" } +toyos-crypto = { path = "../../lib/crypto" } +getrandom = "0.3" [dev-dependencies] # The volumes the FAT role leaves, judged by the checker that shares no code # with the driver it wrote them with. toyos-fat32-check = { path = "../../lib/fat32/check" } -# The draws a test ROOT's mkfs takes. -toyos-crypto = { path = "../../lib/crypto" } [package.metadata.toyos.host] exempt.owns = "ToyOS's storage roles, served from the partitions ToyOS claims for it" diff --git a/system/fileserver/src/cache.rs b/system/fileserver/src/cache.rs index 41998f8e3b7..9f87465dc81 100644 --- a/system/fileserver/src/cache.rs +++ b/system/fileserver/src/cache.rs @@ -481,47 +481,6 @@ impl Inner { } } -/// The cache as the bcachefs crate reads it: shared, since the volume reads -/// and writes files' data past the same cache its btree lives in. The crate -/// is synchronous, so each request is driven to its end here. -pub struct Shared(pub std::rc::Rc>); - -impl Clone for Shared { - fn clone(&self) -> Self { - Self(std::rc::Rc::clone(&self.0)) - } -} - -impl bcachefs::BlockIO for Shared { - fn read_block(&self, block: bcachefs::BlockNum, buf: &mut bcachefs::BlockBuf) -> Result<(), bcachefs::DeviceError> { - let out = std::slice::from_mut(buf.as_bytes_mut()); - self.0.block_on(self.0.read(block.raw(), out)).map_err(|_| bcachefs::DeviceError::classify(&Attempted)) - } - - fn write_block(&self, block: bcachefs::BlockNum, buf: &bcachefs::BlockBuf) -> Result<(), bcachefs::DeviceError> { - let data = std::slice::from_ref(buf.as_bytes()); - self.0.block_on(self.0.write(block.raw(), data)).map_err(|_| bcachefs::DeviceError::classify(&Attempted)) - } - - fn block_count(&self) -> u64 { - self.0.blocks() - } - - fn sync(&self) -> Result<(), bcachefs::DeviceError> { - self.0.block_on(self.0.flush()).map_err(|_| bcachefs::DeviceError::classify(&Attempted)) - } -} - -/// What every device [`Error`] is to bcachefs: attempted, since nothing in -/// this server refuses on a clock. -struct Attempted; - -impl bcachefs::TransferError for Attempted { - fn refused_before_attempt(&self) -> bool { - false - } -} - #[cfg(test)] mod tests { use std::cell::Cell; diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index 75389f99024..f627372f945 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -1,1782 +1,459 @@ -//! The DATA role's volume: the bcachefs crate's format, read and written -//! through the server's cache. -//! -//! **The format's namespace is flat**: one entry per file or symlink, keyed by -//! its whole path, and a directory is only a prefix of those. So a directory -//! nothing is under yet is kept as an entry of its own, its path and a -//! trailing `/`, empty — which is what makes it outlive the server and the -//! boot — and a directory with anything under it is one whether or not that -//! entry exists, since volumes written before these entries carry none. -//! -//! **Every name is indexed in memory at mount** ([`DataVolume::names`]), with -//! what it is, kept as an open file changes, so a lookup, a stat, a listing -//! and "is this a directory" are map queries and never a search of the tree. -//! -//! **A file's data lives in the blocks its extents name**, read and written -//! past the cache ([`crate::cache`]). A reserve resolves (allocating where it -//! must) the block of each page a write will touch and moves the length; the -//! write-back puts the pages; the new length and extents reach the file's -//! entry at its close, an fsync, or the volume's sync, and a sync writes every -//! such entry before the cache is flushed. An fsync's commit writes its own -//! file's and those of the files it carries ([`Volume::carried`]): any whose -//! entry is written, or whose blocks were taken, since the last. A length that -//! shrinks is recorded before the blocks past it are freed, so a failure -//! between the two leaks blocks rather than leaving an entry naming freed ones. -//! -//! **What a kill costs.** A sync is the format's commit (`bcachefs`'s -//! `Mounted::sync`): a server killed anywhere leaves every name, and every -//! file's length and extents, as the last sync that finished or as the one it -//! was inside, whole either way; a file's bytes are not so held, since a page -//! written over is written in place. A directory's rename is one operation of -//! the format's (`Mounted::rename_all`): it moves whole, or is refused with -//! nothing moved. -//! -//! Every name a client chose is bounded by the format ([`FsError::NameTooLong`]) -//! before it reaches the tree. +//! Writable bcachefs DATA. Descriptions hold inode identities; buffered +//! sizes and capacity reservations stay here until their pages are written. use std::collections::BTreeMap; -use std::ops::Bound; use std::rc::Rc; - -use bcachefs::{Extent, Formatted, FsError, Mounted, ReadWrite}; +use bcachefs::engine::{self, DirentType, Mount, MountOptions, New, Owner, Reservation}; use toyos::fs::Error; - -use toyos_blockdev::{Block, BlockDev, BLOCK}; - -use crate::cache::{Cache, Disk, Shared}; -use crate::volume::{join, parent, Identities, Kind, Listed, Meta, Node, OpenHow, Out, Reserved, Volume}; - -/// The longest symlink target read back: the wire's path bound. -const MAX_LINK: u64 = toyos::fs::MAX_PATH as u64; - -/// The most blocks one data request carries. -const RUN: u64 = 256; - -/// One file open, while any client holds it. +use toyos_blockdev::{BlockDev, BLOCK}; +use crate::cache::{Cache, Disk}; +use crate::volume::{parent, Kind, Listed, Meta, Node, OpenHow, Out, Reserved, Volume}; + +const NODES: usize = 512; +const READ_CHUNK: usize = 256 << 10; +const OWNER: Owner = Owner { perm: 0o644, uid: 0, gid: 0 }; +struct OsRandom; +impl toyos_crypto::Random for OsRandom { + fn fill(&mut self, out: &mut [u8]) -> Result<(), toyos_crypto::Error> { + getrandom::fill(out).map_err(|_| toyos_crypto::Error::Entropy) + } +} +struct Pending { ticket: u64, reservation: Reservation } struct Open { - path: String, - extents: Vec, + holders: usize, size: u64, mtime: u64, - /// Length or extents changed since the entry was last written. - dirty: bool, - /// The entry was written, or blocks taken, since the last commit - /// ([`Volume::carried`]). - carried: bool, - /// Unlinked or renamed over: its blocks are no longer its own. - gone: bool, - holders: u32, + changed: bool, + pages: BTreeMap, } - pub struct DataVolume { - fs: Mounted, ReadWrite>, + fs: Mount>>, cache: Rc>, - /// Every name on the volume, a directory entry's without its `/`, and - /// what it is: for a file its length and time, an open node's as it - /// changes ([`noted`]), which its entry holds once written. - names: BTreeMap, - /// Directories that exist whatever the volume holds: each connection's root. roots: Vec, open: BTreeMap, - by_path: BTreeMap, - next: Node, + next_ticket: u64, clock: fn() -> u64, - /// By path: the format keeps nothing that names a file across a rename. - ids: Identities, - /// Where a data request's blocks land, kept so one allocates nothing. - span: Vec, } - -/// What a DATA partition turned out to be, as the kernel's mount decided it. -pub enum Probed { - /// A volume of ours, mounted, or a designated one freshly formatted. - Mounted(DataVolume), - /// A volume of ours that does not mount; nothing stands in and nothing is - /// written to it. - Unmountable(String), - /// No volume of ours and no designation stamp: never written. - Foreign, -} - -/// Where DATA's partition is, from the TOYOS-DATA partitions the block -/// service lists, or why it would not list them. +pub enum Probed { Mounted(DataVolume), Unmountable(String), Foreign } #[derive(Debug, PartialEq, Eq)] -pub enum Located { - /// This machine has none: memory stands in. - Nowhere, - /// The one partition the block service serves, by its unique GUID. - Served([u8; 16]), - /// Two or more, never guessed between, or none countable: DATA is absent. - Refused(String), -} - +pub enum Located { Nowhere, Served([u8; 16]), Refused(String) } pub fn find(served: Result<&[[u8; 16]], String>) -> Located { match served { Err(why) => Located::Refused(why), Ok([]) => Located::Nowhere, Ok([one]) => Located::Served(*one), - Ok(served) => Located::Refused(format!( - "this machine has {} DATA partitions the block service serves, and a volume is one", - served.len() - )), + Ok(served) => Located::Refused(format!("this machine has {} DATA partitions; none was selected", served.len())), } } - -pub(crate) fn refusal(e: &FsError) -> Error { +fn word(e: engine::Error) -> Error { match e { - FsError::NotFound => Error::NotFound, - FsError::NoSpace { .. } => Error::NoSpace, - FsError::EntryTooLarge { .. } | FsError::ListTooLong { .. } => Error::LimitExceeded, - FsError::NameTooLong { .. } => Error::NameTooLong, - FsError::DeviceRead(..) - | FsError::DeviceWrite(..) - | FsError::DeviceSync(_) - | FsError::BadMagic { .. } - | FsError::UnsupportedVersion(_) - | FsError::ChecksumMismatch { .. } - | FsError::CorruptedKey(_) - | FsError::CorruptedNode(_) - | FsError::BlockOffDevice { .. } - | FsError::NotEnoughBlocks { .. } - | FsError::TreeTooDeep(_) - | FsError::BadSuperblock { .. } - | FsError::NodeOverfull { .. } - | FsError::TargetTooLong { .. } => Error::Io, + engine::Error::NotFound => Error::NotFound, + engine::Error::NotDirectory => Error::NotADirectory, + engine::Error::Exists => Error::AlreadyExists, + engine::Error::IsDirectory => Error::IsADirectory, + engine::Error::NotEmpty => Error::NotEmpty, + engine::Error::Stale => Error::Stale, + engine::Error::NoSpace(_) => Error::NoSpace, + engine::Error::ReadOnly(_) | engine::Error::Device(toyos_blockdev::Error::ReadOnly) => Error::ReadOnly, + engine::Error::Invalid(_) => Error::InvalidName, + engine::Error::Unsupported(_) => Error::Unsupported, + _ => Error::Io, } } - -/// Logs the format's own account, answers the word a client can act on. -fn mapped(what: &str, path: &str, r: Result) -> Result { - r.map_err(|e| { - println!("fileserver: {what} of '{path}' failed: {e:?}"); - refusal(&e) - }) -} - -/// A block service that ended is a device that did not do it, as far as a -/// client of this volume can act on. -fn disk_word(e: toyos_blockdev::Error) -> Error { - match e { - toyos_blockdev::Error::Device | toyos_blockdev::Error::Range | toyos_blockdev::Error::Gone => Error::Io, - toyos_blockdev::Error::ReadOnly => Error::ReadOnly, - } +fn mapped(operation: &str, result: engine::Result) -> Result { + result.map_err(|error| { println!("fileserver: DATA {operation}: {error:?}"); word(error) }) } - -/// What a directory is: the format keeps no length or time for one. -const DIR: Meta = Meta { kind: Kind::Dir, size: 0, mtime: 0 }; - -/// An open file's length and time, now its name's too: a lookup or a listing -/// reads the index alone. -fn noted(names: &mut BTreeMap, open: &Open) { - if let Some(meta) = names.get_mut(&open.path) { - (meta.size, meta.mtime) = (open.size, open.mtime); +fn kind(inode: &engine::Inode) -> Result { + match inode.kind() { + Some(DirentType::Regular) => Ok(Kind::File), + Some(DirentType::Directory) => Ok(Kind::Dir), + Some(DirentType::Symlink) => Ok(Kind::Symlink), + _ => Err(Error::Unsupported), } } - -/// The block holding page `page`, if the extents reach it. -fn block_for(extents: &[Extent], page: u64) -> Option { - let mut at = 0u64; - for e in extents { - let end = at.checked_add(e.block_count as u64)?; - if page < end { - return e.start_block.checked_add(page - at); - } - at = end; - } - None -} - -/// Keep the first `keep` blocks of `extents`; answer the runs dropped. -fn keep_blocks(extents: &mut Vec, keep: u64) -> Vec { - let mut dropped = Vec::new(); - let mut left = keep; - let mut kept = Vec::with_capacity(extents.len()); - for e in extents.drain(..) { - let n = e.block_count as u64; - if left >= n { - left -= n; - kept.push(e); - } else { - if left > 0 { - kept.push(Extent { start_block: e.start_block, block_count: left as u32, _reserved: 0 }); - } - dropped.push(Extent { start_block: e.start_block + left, block_count: (n - left) as u32, _reserved: 0 }); - left = 0; - } - } - *extents = kept; - dropped -} - -/// Whether block 0 carries a designation stamp naming this disk's size. -fn designated(cache: &Cache) -> bool { - let mut block0 = [[0u8; BLOCK]]; - if cache.block_on(cache.read(0, &mut block0)).is_err() { - println!("fileserver: block 0 would not read; this disk is not ours to format"); - return false; - } - let [block0] = block0; - let Some(stamped) = bcachefs::designated(&block0) else { return false }; - if stamped != cache.blocks() { - println!( - "fileserver: a designation stamp at block 0 names {stamped} blocks, and this partition has {}; ignoring it", - cache.blocks() - ); - return false; - } - true -} - +fn time(value: u64) -> i64 { value.min(i64::MAX as u64) as i64 } impl DataVolume { - /// Mount the volume on `disk`, format it if it is designated, and refuse - /// it otherwise — the kernel's DATA mount's decisions, unchanged. pub fn probe(disk: D, clean_limit: usize, roots: &[&str], clock: fn() -> u64) -> Probed { let cache = Rc::new(Cache::new(disk, clean_limit)); - match Mounted::, ReadWrite>::open(Shared(Rc::clone(&cache))) { - Ok(fs) => { - println!("fileserver: mounted the DATA volume"); - return Self::over(fs, cache, roots, clock).map_or_else(Probed::Unmountable, Probed::Mounted); + let first = cache.block_on(Mount::open(Rc::clone(&cache), MountOptions { cache_nodes: NODES }, Box::new(OsRandom))); + let fs = match first { + Ok(fs) => fs, + Err(error) => { + let mut block = [[0; BLOCK]; 2]; + if let Err(read) = cache.block_on(cache.read(0, &mut block)) { + return Probed::Unmountable(format!("mount {error:?}; designation read {read:?}")); + } + if bcachefs::designated(&block[0]) != Some(cache.blocks()) { + return if bcachefs::carries_volume(block.as_flattened()) { + Probed::Unmountable(format!("{error:?}; an existing volume cannot be reformatted")) + } else { Probed::Foreign }; + } + println!("fileserver: formatting the explicitly designated DATA partition as bcachefs"); + let options = engine::Options { label: b"ToyOS DATA", time: clock(), encryption: None }; + if let Err(error) = cache.block_on(engine::mkfs(&cache, &options, &engine::Tree::new(), &mut OsRandom)) { + return Probed::Unmountable(format!("format: {error:?}")); + } + match cache.block_on(Mount::open(Rc::clone(&cache), MountOptions { cache_nodes: NODES }, Box::new(OsRandom))) { + Ok(fs) => fs, + Err(error) => return Probed::Unmountable(format!("new volume: {error:?}")), + } } - Err(e) if e.disowns_volume() => {} - Err(e) => return Probed::Unmountable(format!("{e:?}")), - } - if !designated(&cache) { - println!("fileserver: no volume of ours and no designation stamp; nothing will be written to this partition"); - return Probed::Foreign; - } - println!("fileserver: block 0 designates this partition for ToyOS; formatting it"); - match Formatted::format(Shared(Rc::clone(&cache))) { - Ok(fs) => Self::over(fs.mount(), cache, roots, clock).map_or_else(Probed::Unmountable, Probed::Mounted), - Err(e) => Probed::Unmountable(format!("the format failed: {e:?}")), - } - } - - /// A fresh volume on `disk`, which holds nothing of anyone's: memory. - pub fn format(disk: D, clean_limit: usize, roots: &[&str], clock: fn() -> u64) -> Result { - let cache = Rc::new(Cache::new(disk, clean_limit)); - let fs = Formatted::format(Shared(Rc::clone(&cache))).map_err(|e| format!("{e:?}"))?; - Self::over(fs.mount(), cache, roots, clock) - } - - fn over( - fs: Mounted, ReadWrite>, - cache: Rc>, - roots: &[&str], - clock: fn() -> u64, - ) -> Result { - let listed = fs.list(usize::MAX, &|_| true).map_err(|e| format!("the volume would not list: {e:?}"))?; - let mut names = BTreeMap::new(); - for (name, recorded) in listed { - let meta = match (name.strip_suffix('/'), recorded.symlink) { - (Some(dir), _) => { - names.insert(dir.to_string(), DIR); - continue; + }; + Self::over(fs, cache, roots, clock).map_or_else(Probed::Unmountable, Probed::Mounted) + } + fn over(fs: Mount>>, cache: Rc>, roots: &[&str], clock: fn() -> u64) -> Result { + // Mount recovery precedes every live description. + cache.block_on(fs.release_orphans()).map_err(|error| format!("orphan recovery: {error:?}"))?; + let mut volume = Self { fs, cache, roots: roots.iter().map(|root| root.to_string()).collect(), open: BTreeMap::new(), next_ticket: 1, clock }; + for root in roots { + let mut current = String::new(); + for part in root.split('/').filter(|part| !part.is_empty()) { + if !current.is_empty() { current.push('/'); } + current.push_str(part); + match volume.lstat(¤t) { + Ok(meta) if meta.kind == Kind::Dir => {} + Err(Error::NotFound) => volume.mkdir(¤t).map_err(|error| format!("root {current}: {error:?}"))?, + other => return Err(format!("root {current} is not a directory: {other:?}")), } - (None, true) => Meta { kind: Kind::Symlink, size: recorded.size, mtime: recorded.mtime }, - (None, false) => Meta { kind: Kind::File, size: recorded.size, mtime: recorded.mtime }, - }; - names.insert(name, meta); - } - Ok(Self { - fs, - cache, - names, - roots: roots.iter().map(|r| r.to_string()).collect(), - open: BTreeMap::new(), - by_path: BTreeMap::new(), - next: 1, - clock, - ids: Identities::new(), - span: Vec::new(), - }) - } - - /// `path` is gone, and with it every directory above it that existed only - /// for having something beneath. - fn forget(&mut self, path: &str) { - self.ids.forget(&path.to_string()); - self.forget_emptied(path); - } - - /// Every directory above `path` that existed only for having something - /// beneath, and has nothing now. - fn forget_emptied(&mut self, path: &str) { - let mut dir = parent(path); - while !dir.is_empty() && !self.is_dir(dir) { - self.ids.forget(&dir.to_string()); - dir = parent(dir); + } } + volume.cache.block_on(volume.fs.commit()).map_err(|error| format!("mount commit: {error:?}"))?; + Ok(volume) } - - /// Whether anything is named beneath `dir`. - fn has_children(&self, dir: &str) -> bool { - let prefix = format!("{dir}/"); - self.names.range(prefix.clone()..).next().is_some_and(|(name, _)| name.starts_with(&prefix)) - } - - fn kind(&self, path: &str) -> Option { - self.names.get(path).map(|meta| meta.kind) - } - - fn is_dir(&self, path: &str) -> bool { - path.is_empty() - || self.roots.iter().any(|r| r == path) - || self.kind(path) == Some(Kind::Dir) - || (!self.names.contains_key(path) && self.has_children(path)) - } - - fn entry(&mut self, node: Node) -> Result<&mut Open, Error> { - let open = self.open.get_mut(&node).ok_or(Error::Stale)?; - if open.gone { - return Err(Error::Stale); + fn lookup(&self, path: &str) -> Result { + self.cache.block_on(async { + let mut ino = self.fs.root(); + for name in path.split('/').filter(|name| !name.is_empty()) { + ino = self.fs.lookup(ino, name.as_bytes()).await?.ok_or(engine::Error::NotFound)?.ino; + } + self.fs.inode(ino).await + }).map_err(word) + } + fn destination<'a>(&self, path: &'a str) -> Result<(u64, &'a [u8]), Error> { + let name = path.rsplit('/').next().filter(|name| !name.is_empty()).ok_or(Error::Root)?; + let dir = self.lookup(parent(path))?; + if !dir.is_dir() { return Err(Error::NotADirectory); } + Ok((dir.ino, name.as_bytes())) + } + fn meta(&self, inode: &engine::Inode) -> Result { + let kind = kind(inode)?; + let (size, mtime) = self.open.get(&inode.ino).map(|open| (open.size, open.mtime)).unwrap_or((inode.size, inode.mtime.max(0) as u64)); + Ok(Meta { kind, size: if kind == Kind::Dir { 0 } else { size }, mtime }) + } + fn protected(&self, path: &str) -> bool { path.is_empty() || self.roots.iter().any(|root| root == path) } + fn released(&self, removed: Option) -> Result<(), Error> { + if let Some(removed) = removed.filter(|removed| removed.last_link && !self.open.contains_key(&removed.node)) { + match self.cache.block_on(self.fs.inode(removed.node)) { + Ok(inode) if !inode.is_dir() => mapped("release", self.cache.block_on(self.fs.release(removed.node)))?, + Ok(_) | Err(engine::Error::NotFound) => {} + Err(error) => return Err(word(error)), + } } - Ok(open) + Ok(()) } - - /// Write one open file's length and extents to its entry. fn persist(&mut self, node: Node) -> Result<(), Error> { - let Some(open) = self.open.get_mut(&node) else { return Ok(()) }; - if !open.dirty || open.gone { - return Ok(()); - } - mapped("the entry", &open.path, self.fs.update_metadata(&open.path, &open.extents, open.size, open.mtime))?; - open.dirty = false; - open.carried = true; - if let Some(meta) = self.names.get_mut(&open.path) { - (meta.size, meta.mtime) = (open.size, open.mtime); - } + let open = self.open.get(&node).ok_or(Error::Stale)?; + if !open.pages.is_empty() { return Err(Error::Io); } + if !open.changed { return Ok(()); } + let (size, mtime) = (open.size, open.mtime); + mapped("length", self.cache.block_on(self.fs.truncate(node, size, time(mtime))))?; + mapped("mtime", self.cache.block_on(self.fs.set_mtime(node, time(mtime))))?; + self.open.get_mut(&node).expect("open inode").changed = false; Ok(()) } - - /// The disk as a stop now leaves it: what the last flush wrote. #[cfg(test)] - pub(crate) fn stopped(self) -> D { - drop(self.fs); - Rc::try_unwrap(self.cache).ok().expect("one owner").into_disk() - } - - /// A commit landed: no entry it made durable is owed to the next. - fn committed(&mut self) { - for open in self.open.values_mut().filter(|open| !open.dirty) { - open.carried = false; - } - } - - /// Forget a node nobody holds and whose entry is written. - fn release(&mut self, node: Node) { - let open = self.open.remove(&node).expect("an open node"); - if self.by_path.get(&open.path) == Some(&node) { - self.by_path.remove(&open.path); - } - } - - /// Hand every open file at `path` its end: its blocks go with the name. - fn orphan(&mut self, path: &str) { - if let Some(node) = self.by_path.remove(path) { - if let Some(open) = self.open.get_mut(&node) { - open.gone = true; - } - } - } - - fn new_node(&mut self, path: &str, extents: Vec, size: u64, mtime: u64) -> Node { - let node = self.next; - self.next += 1; - self.open.insert(node, Open { path: path.to_string(), extents, size, mtime, dirty: false, carried: false, gone: false, holders: 0 }); - self.by_path.insert(path.to_string(), node); - node + pub(crate) fn format(disk: D, clean_limit: usize, roots: &[&str], clock: fn() -> u64) -> Result { + let cache = Rc::new(Cache::new(disk, clean_limit)); + let options = engine::Options { label: b"test DATA", time: clock(), encryption: None }; + cache.block_on(engine::mkfs(&cache, &options, &engine::Tree::new(), &mut OsRandom)).map_err(|error| format!("{error:?}"))?; + let fs = cache.block_on(Mount::open(Rc::clone(&cache), MountOptions { cache_nodes: NODES }, Box::new(OsRandom))).map_err(|error| format!("{error:?}"))?; + Self::over(fs, cache, roots, clock) } - - /// A name is made under directories that need not exist yet — the format - /// makes every prefix one, as the kernel's mount did — but never under a - /// file or a symlink, which would then be two things at once. - fn require_parent_dir(&self, path: &str) -> Result<(), Error> { - let mut dir = parent(path); - while !dir.is_empty() { - if matches!(self.kind(dir), Some(Kind::File | Kind::Symlink)) { - return Err(Error::NotADirectory); - } - dir = parent(dir); - } - Ok(()) + #[cfg(test)] + pub(crate) fn stopped(self) -> D { + let Self { fs, cache, .. } = self; + drop(fs); + Rc::try_unwrap(cache).ok().expect("mount dropped its device").into_disk() } } - impl Volume for DataVolume { - fn writable(&self) -> bool { - true - } - - fn lstat(&mut self, path: &str) -> Result { - match self.names.get(path) { - Some(&recorded) => Ok(recorded), - None if self.is_dir(path) => Ok(DIR), - None => Err(Error::NotFound), - } - } - + fn writable(&self) -> bool { true } + fn lstat(&mut self, path: &str) -> Result { self.meta(&self.lookup(path)?) } fn read_link(&mut self, path: &str) -> Result { - if self.kind(path) != Some(Kind::Symlink) { - return Err(if self.names.contains_key(path) || self.is_dir(path) { Error::NotALink } else { Error::NotFound }); - } - mapped("readlink", path, self.fs.read_link(path, MAX_LINK))?.ok_or(Error::Io) + let inode = self.lookup(path)?; + if kind(&inode)? != Kind::Symlink { return Err(Error::NotALink); } + let bytes = mapped("readlink", self.cache.block_on(self.fs.readlink(inode.ino)))?; + String::from_utf8(bytes).map_err(|_| Error::Io) } - fn list(&mut self, dir: &str) -> Result, Error> { - if !self.is_dir(dir) { - return Err(if self.names.contains_key(dir) { Error::NotADirectory } else { Error::NotFound }); - } - // The names beneath a child directory are passed over whole: a - // listing costs the directory's children, not everything below it. - let prefix = if dir.is_empty() { String::new() } else { format!("{dir}/") }; - let mut children = Vec::new(); - let mut from = prefix.clone(); + let inode = self.lookup(dir)?; + if !inode.is_dir() { return Err(Error::NotADirectory); } + let mut cursor = 0; + let mut listed = Vec::new(); loop { - let mut past = None; - for (name, &recorded) in self.names.range::((Bound::Included(from.as_str()), Bound::Unbounded)) { - let Some(rest) = name.strip_prefix(&prefix) else { break }; - let Some((child, _)) = rest.split_once('/') else { - children.push((rest.to_string(), recorded)); - continue; - }; - // A child with an entry of its own was listed at it. - if !self.names.contains_key(&format!("{prefix}{child}")) { - children.push((child.to_string(), DIR)); - } - // Every name beneath `child/`, and nothing else, sorts before - // `child0`: `0` follows `/`. - past = Some(format!("{prefix}{child}0")); - break; - } - match past { - Some(next) => from = next, - None => break, + let (entries, end) = mapped("readdir", self.cache.block_on(self.fs.readdir(inode.ino, cursor, 256)))?; + if entries.is_empty() && !end { return Err(Error::Io); } + for entry in entries { + cursor = entry.next; + let inode = mapped("directory entry", self.cache.block_on(self.fs.inode(entry.ino)))?; + listed.push(Listed { name: String::from_utf8(entry.name).map_err(|_| Error::Io)?, meta: self.meta(&inode)?, identity: entry.ino }); } + if end { break; } } - children.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); - Ok(children.into_iter().map(|(name, meta)| Listed { identity: self.ids.of(join(dir, &name)), name, meta }).collect()) + listed.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + Ok(listed) } - fn open(&mut self, path: &str, how: OpenHow) -> Result { - if self.is_dir(path) { - return Err(Error::IsADirectory); - } - let node = match self.by_path.get(path).copied() { - Some(_) if how.create_new => return Err(Error::AlreadyExists), - Some(node) => node, - None => match self.names.get(path).copied() { - Some(_) if how.create_new => return Err(Error::AlreadyExists), - Some(Meta { kind: Kind::File, mtime, .. }) => { - let (extents, size) = - mapped("open", path, self.fs.file_extents(path))?.ok_or(Error::NotFound)?; - self.new_node(path, extents, size, mtime) - } - // A link the resolver did not follow is one with nothing behind it. - Some(_) => return Err(Error::NotFound), - None if how.create || how.create_new => { - self.require_parent_dir(path)?; - let now = (self.clock)(); - mapped("create", path, self.fs.create(path, &[], now))?; - self.names.insert(path.to_string(), Meta { kind: Kind::File, size: 0, mtime: now }); - self.new_node(path, Vec::new(), 0, now) - } - None => return Err(Error::NotFound), - }, + let inode = match self.lookup(path) { + Ok(_) if how.create_new => return Err(Error::AlreadyExists), + Ok(inode) => inode, + Err(Error::NotFound) if how.create || how.create_new => { + let (dir, name) = self.destination(path)?; + mapped("create", self.cache.block_on(self.fs.create(dir, name, New::File, OWNER, time((self.clock)()))))? + } + Err(error) => return Err(error), }; - self.open.get_mut(&node).expect("just found or made").holders += 1; + match kind(&inode)? { Kind::Dir => return Err(Error::IsADirectory), Kind::Symlink => return Err(Error::NotFound), Kind::File => {} } + let node = inode.ino; + self.open.entry(node).or_insert(Open { holders: 0, size: inode.size, mtime: inode.mtime.max(0) as u64, changed: false, pages: BTreeMap::new() }).holders += 1; if how.truncate { - if let Err(e) = self.truncate(node, 0) { - // A refused close is said, and its node kept for the next sync. - let _ = self.close(node); - return Err(e); - } + if let Err(error) = self.truncate(node, 0) { let _ = self.close(node); return Err(error); } } Ok(node) } - fn close(&mut self, node: Node) -> Result<(), Error> { let Some(open) = self.open.get_mut(&node) else { return Ok(()) }; - open.holders -= 1; - if open.holders > 0 { - return Ok(()); - } - if let Err(e) = self.persist(node) { - println!("fileserver: node {node}'s entry was not written at its last close ({e:?}); the next sync writes it"); - return Err(e); - } - self.release(node); + open.holders = open.holders.checked_sub(1).expect("one close per open"); + if open.holders != 0 { return Ok(()); } + self.persist(node)?; + let inode = mapped("close", self.cache.block_on(self.fs.inode(node)))?; + if inode.nlink == 0 { mapped("orphan close", self.cache.block_on(self.fs.release(node)))?; } + self.open.remove(&node); Ok(()) } - fn node_meta(&mut self, node: Node) -> Result { - let open = self.entry(node)?; + let open = self.open.get(&node).ok_or(Error::Stale)?; Ok(Meta { kind: Kind::File, size: open.size, mtime: open.mtime }) } - - fn identity(&mut self, path: &str) -> Result { - if !self.names.contains_key(path) && !self.is_dir(path) { - return Err(Error::NotFound); - } - Ok(self.ids.of(path.to_string())) - } - + fn linked(&mut self, node: Node) -> Result { Ok(mapped("link count", self.cache.block_on(self.fs.inode(node)))?.nlink != 0) } + fn identity(&mut self, path: &str) -> Result { Ok(self.lookup(path)?.ino) } fn read(&mut self, node: Node, offset: u64, out: &mut dyn Out) -> Result { - let cache = Rc::clone(&self.cache); - let mut span = std::mem::take(&mut self.span); - let open = self.entry(node)?; - if offset >= open.size { - self.span = span; - return Ok(0); - } - let n = out.len().min((open.size - offset) as usize); + let size = self.open.get(&node).ok_or(Error::Stale)?.size; + let len = out.len().min(size.saturating_sub(offset).min(usize::MAX as u64) as usize); + let mut bytes = vec![0; len.min(READ_CHUNK)]; let mut done = 0; - let mut read = Ok(()); - while done < n && read.is_ok() { - let at = offset + done as u64; - let page = at / BLOCK as u64; - let within = (at % BLOCK as u64) as usize; - let take = (BLOCK - within).min(n - done); - let Some(first) = block_for(&open.extents, page) else { - // Past the extents: a hole, whose bytes are zeros. - out.zero(done, take); - done += take; - continue; - }; - // The pages from here whose blocks are consecutive go as one read. - let need = (within + (n - done)).div_ceil(BLOCK) as u64; - let mut run = 1u64; - while run < need.min(RUN) && block_for(&open.extents, page + run) == Some(first + run) { - run += 1; - } - span.resize(run as usize, [0; BLOCK]); - read = cache.block_on(cache.read_data(first, &mut span[..run as usize])).map_err(disk_word); - if read.is_ok() { - let bytes = &span[..run as usize].as_flattened()[within..]; - let take = bytes.len().min(n - done); - out.put(done, &bytes[..take]); - done += take; - } + while done < len { + let take = bytes.len().min(len - done); + bytes[..take].fill(0); + mapped("read", self.cache.block_on(self.fs.read(node, offset + done as u64, &mut bytes[..take])))?; + out.put(done, &bytes[..take]); + done += take; } - self.span = span; - read.map(|()| n) + Ok(len) } - fn reserve(&mut self, node: Node, offset: u64, len: u64) -> Result { let end = offset.checked_add(len).ok_or(Error::FileTooLarge)?; - let now = (self.clock)(); - let open = self.open.get_mut(&node).ok_or(Error::Stale)?; - if open.gone { - return Err(Error::Stale); - } - // Every page past the extents is allocated before the size moves, so - // a write the file's one entry could not name is refused whole. - let covered: u64 = open.extents.iter().map(|e| e.block_count as u64).sum(); - let reserved = Reserved { size: open.size, held: covered }; - let last = end.div_ceil(BLOCK as u64).saturating_sub(1); - let grown = match len != 0 && last >= covered { - false => Ok(()), - true => u32::try_from(offset / BLOCK as u64) - .and_then(|first| Ok(first..=u32::try_from(last)?)) - .map_err(|_| Error::FileTooLarge) - // Runs a block-ring request long, so each costs the flush one request. - .and_then(|pages| mapped("an allocation", &open.path, self.fs.allocate(&mut open.extents, pages, toyos_blockring::MAX_REQUEST_BLOCKS))) - .and_then(|()| match bcachefs::file_entry_fits(&open.path, &open.extents) { - true => Ok(()), - false => { - println!( - "fileserver: a write to '{}' is refused: its entry would name {} runs of blocks, more than it holds", - open.path, - open.extents.len() - ); - Err(Error::LimitExceeded) - } - }), - }; - if let Err(e) = grown { - let dropped = keep_blocks(&mut open.extents, covered); - mapped("a refused write's free", &open.path, self.fs.free_extents(&dropped))?; - return Err(e); + let open = self.open.get(&node).ok_or(Error::Stale)?; + let ticket = self.next_ticket; + self.next_ticket = ticket.checked_add(1).expect("reservation generation exhausted"); + let reserved = Reserved { size: open.size, contents: open.size, ticket }; + if len == 0 { return Ok(reserved); } + for page in offset / BLOCK as u64..end.div_ceil(BLOCK as u64) { + if self.open[&node].pages.contains_key(&page) { continue; } + match self.cache.block_on(self.fs.reserve(node, page * BLOCK as u64, BLOCK)) { + Ok(reservation) => { self.open.get_mut(&node).unwrap().pages.insert(page, Pending { ticket, reservation }); } + Err(error) => { self.unreserve(node, reserved, 0)?; return Err(word(error)); } + } } + let open = self.open.get_mut(&node).unwrap(); open.size = open.size.max(end); - open.mtime = now; - open.dirty = true; - open.carried |= open.extents.iter().map(|e| e.block_count as u64).sum::() > covered; - noted(&mut self.names, open); + open.mtime = (self.clock)(); + open.changed = true; Ok(reserved) } - fn unreserve(&mut self, node: Node, reserved: Reserved, landed: u64) -> Result<(), Error> { let open = self.open.get_mut(&node).ok_or(Error::Stale)?; - // A block a write took in a hole below the old end lies inside the - // old size, so it goes back by what the file held, not by the size. - let dropped = keep_blocks(&mut open.extents, reserved.held.max(landed.div_ceil(BLOCK as u64))); + let cancelled: Vec<_> = open.pages.iter().filter_map(|(&page, pending)| (pending.ticket == reserved.ticket && page * BLOCK as u64 >= landed).then_some(page)).collect(); + for page in cancelled { open.pages.remove(&page); } open.size = reserved.size.max(landed); - noted(&mut self.names, open); - // No entry names them: only the file's own calls write its entry. - let path = open.path.clone(); - mapped("a refused write's free", &path, self.fs.free_extents(&dropped)) + Ok(()) } - fn write_back(&mut self, node: Node, offset: u64, data: &[u8]) -> Result<(), Error> { - let cache = Rc::clone(&self.cache); - let mut span = std::mem::take(&mut self.span); - let open = self.entry(node)?; - let written = (|| { - let end = offset.checked_add(data.len() as u64).ok_or(Error::FileTooLarge)?; - if !offset.is_multiple_of(BLOCK as u64) || end > open.size { - println!("fileserver: a write-back of '{}' at {offset}+{} lies outside its {} bytes", open.path, data.len(), open.size); - return Err(Error::Io); - } - let first = offset / BLOCK as u64; - let pages = data.len().div_ceil(BLOCK); - let mut k = 0; - while k < pages { - let block = block_for(&open.extents, first + k as u64).ok_or(Error::Io)?; - let mut run = 1; - while k + run < pages && run < RUN as usize && block_for(&open.extents, first + (k + run) as u64) == Some(block + run as u64) { - run += 1; - } - span.clear(); - span.resize(run, [0; BLOCK]); - let bytes = &data[k * BLOCK..((k + run) * BLOCK).min(data.len())]; - span.as_flattened_mut()[..bytes.len()].copy_from_slice(bytes); - cache.block_on(cache.write_data(block, &span)).map_err(disk_word)?; - k += run; - } - Ok(()) - })(); - self.span = span; - written + let open = self.open.get_mut(&node).ok_or(Error::Stale)?; + if !offset.is_multiple_of(BLOCK as u64) || offset.checked_add(data.len() as u64).is_none_or(|end| end > open.size) { return Err(Error::InvalidRequest); } + for (i, bytes) in data.chunks(BLOCK).enumerate() { + let page = offset / BLOCK as u64 + i as u64; + let pending = open.pages.get_mut(&page).ok_or(Error::Io)?; + let written = mapped("writeback", self.cache.block_on(self.fs.write_reserved(&mut pending.reservation, page * BLOCK as u64, bytes, time(open.mtime))))?; + if written != bytes.len() { return Err(Error::Io); } + open.pages.remove(&page); + } + Ok(()) } - fn truncate(&mut self, node: Node, size: u64) -> Result<(), Error> { - let cache = Rc::clone(&self.cache); + let open = self.open.get(&node).ok_or(Error::Stale)?; + if open.size == size { return Ok(()); } let now = (self.clock)(); - let open = self.open.get_mut(&node).ok_or(Error::Stale)?; - if open.gone { - return Err(Error::Stale); - } - if size >= open.size { - open.size = size; - open.mtime = now; - open.dirty = true; - noted(&mut self.names, open); - return Ok(()); - } - // What the last kept page holds past the new end is zeroed, so a file - // that grows again reads zeros there and not what it held before. - let within = (size % BLOCK as u64) as usize; - if within != 0 { - if let Some(block) = block_for(&open.extents, size / BLOCK as u64) { - let mut page: [Block; 1] = [[0; BLOCK]]; - cache.block_on(cache.read_data(block, &mut page)).map_err(disk_word)?; - page[0][within..].fill(0); - cache.block_on(cache.write_data(block, &page)).map_err(disk_word)?; - } - } - let dropped = keep_blocks(&mut open.extents, size.div_ceil(BLOCK as u64)); + mapped("truncate", self.cache.block_on(self.fs.truncate(node, size, time(now))))?; + let open = self.open.get_mut(&node).unwrap(); + open.pages.split_off(&size.div_ceil(BLOCK as u64)); open.size = size; open.mtime = now; - // Recorded first, freed second. - mapped("a truncate", &open.path, self.fs.update_metadata(&open.path, &open.extents, size, now))?; - open.dirty = false; - open.carried = true; - if let Some(meta) = self.names.get_mut(&open.path) { - (meta.size, meta.mtime) = (size, now); - } - let path = open.path.clone(); - mapped("a truncate's free", &path, self.fs.free_extents(&dropped)) + open.changed = true; + Ok(()) } - fn set_mtime(&mut self, node: Node, mtime: u64) -> Result<(), Error> { - let open = self.open.get_mut(&node).filter(|open| !open.gone).ok_or(Error::Stale)?; + let open = self.open.get_mut(&node).ok_or(Error::Stale)?; open.mtime = mtime; - open.dirty = true; - noted(&mut self.names, open); + open.changed = true; Ok(()) } - fn mkdir(&mut self, path: &str) -> Result<(), Error> { - if self.names.contains_key(path) || self.is_dir(path) { - return Err(Error::AlreadyExists); - } - self.require_parent_dir(path)?; - let now = (self.clock)(); - mapped("mkdir", path, self.fs.create(&format!("{path}/"), &[], now))?; - self.names.insert(path.to_string(), DIR); - Ok(()) + let (dir, name) = self.destination(path)?; + mapped("mkdir", self.cache.block_on(self.fs.create(dir, name, New::Dir, Owner { perm: 0o755, ..OWNER }, time((self.clock)())))).map(drop) } - fn rmdir(&mut self, path: &str) -> Result<(), Error> { - if self.roots.iter().any(|r| r == path) || path.is_empty() { - return Err(Error::Root); - } - if !self.is_dir(path) { - return Err(if self.names.contains_key(path) { Error::NotADirectory } else { Error::NotFound }); - } - if self.has_children(path) { - return Err(Error::NotEmpty); - } - let marker = format!("{path}/"); - if !mapped("rmdir", path, self.fs.delete(&marker))? { - return Err(Error::NotFound); - } - self.names.remove(path); - self.forget(path); - Ok(()) + if self.protected(path) { return Err(Error::Root); } + if !self.lookup(path)?.is_dir() { return Err(Error::NotADirectory); } + let (dir, name) = self.destination(path)?; + mapped("rmdir", self.cache.block_on(self.fs.unlink(dir, name, true, time((self.clock)())))).map(drop) } - fn unlink(&mut self, path: &str) -> Result<(), Error> { - match self.kind(path) { - Some(Kind::File) | Some(Kind::Symlink) => {} - Some(Kind::Dir) => return Err(Error::IsADirectory), - None if self.is_dir(path) => return Err(Error::IsADirectory), - None => return Err(Error::NotFound), - } - // Asked of the volume first: a delete the device refused leaves every - // holder of the file its file. - if !mapped("unlink", path, self.fs.delete(path))? { - return Err(Error::NotFound); - } - self.orphan(path); - self.names.remove(path); - self.forget(path); - Ok(()) + if self.protected(path) { return Err(Error::Root); } + let (dir, name) = self.destination(path)?; + let removed = mapped("unlink", self.cache.block_on(self.fs.unlink(dir, name, false, time((self.clock)()))))?; + self.released(Some(removed)) } - fn rename(&mut self, from: &str, to: &str) -> Result<(), Error> { - if from == to { - return self.lstat(from).map(drop); - } - let kind = self.lstat(from)?.kind; - self.require_parent_dir(to)?; - if self.roots.iter().any(|r| r == from || r == to) { - return Err(Error::Root); - } - match kind { - Kind::File | Kind::Symlink => { - if self.is_dir(to) { - return Err(Error::IsADirectory); - } - // Written first, so a file renamed while open keeps its - // length under its new name. - if let Some(node) = self.by_path.get(from).copied() { - self.persist(node)?; - } - // Orphaned only once the format has taken the rename: a refused - // one leaves `to` naming its file, holders and unsynced length whole. - mapped("rename", from, self.fs.rename(from, to))?; - self.orphan(to); - let moved = self.names.remove(from).expect("a file the volume names"); - self.names.insert(to.to_string(), moved); - if let Some(node) = self.by_path.remove(from) { - self.open.get_mut(&node).expect("indexed").path = to.to_string(); - self.by_path.insert(to.to_string(), node); - } - self.ids.moved(&from.to_string(), to.to_string()); - self.forget_emptied(from); - Ok(()) - } - Kind::Dir => { - if self.names.contains_key(to) || self.is_dir(to) { - return Err(Error::AlreadyExists); - } - if to.starts_with(&format!("{from}/")) { - return Err(Error::IntoItself); - } - // Every entry beneath, and the directory's own, in one - // operation of the format's: the directory moves whole or not at all. - let prefix = format!("{from}/"); - let moving: Vec<(String, Meta)> = self - .names - .get_key_value(from) - .into_iter() - .chain(self.names.range(prefix.clone()..).take_while(|(n, _)| n.starts_with(&prefix))) - .map(|(n, k)| (n.clone(), *k)) - .collect(); - for (name, _) in &moving { - if let Some(node) = self.by_path.get(name).copied() { - self.persist(node)?; - } - } - let renames: Vec<(String, String)> = moving - .iter() - .map(|(name, meta)| { - let moved = format!("{to}{}", &name[from.len()..]); - match meta.kind { - Kind::Dir => (format!("{name}/"), format!("{moved}/")), - _ => (name.clone(), moved), - } - }) - .collect(); - let pairs: Vec<(&str, &str)> = renames.iter().map(|(a, b)| (a.as_str(), b.as_str())).collect(); - mapped("rename", from, self.fs.rename_all(&pairs))?; - for (name, _) in moving { - let moved = format!("{to}{}", &name[from.len()..]); - // The record as it is now, which the persists above updated. - let meta = self.names.remove(&name).expect("a name the volume holds"); - self.names.insert(moved.clone(), meta); - if let Some(node) = self.by_path.remove(&name) { - self.open.get_mut(&node).expect("indexed").path = moved.clone(); - self.by_path.insert(moved, node); - } - } - self.ids.moved_tree(from, to); - self.forget_emptied(from); - Ok(()) - } - } + if self.protected(from) || self.protected(to) { return Err(Error::Root); } + let source = self.destination(from)?; + let target = self.destination(to)?; + let removed = mapped("rename", self.cache.block_on(self.fs.rename(source, target, engine::Rename::Replace, time((self.clock)()))))?; + self.released(removed) } - fn symlink(&mut self, path: &str, target: &str) -> Result<(), Error> { - if self.names.contains_key(path) || self.is_dir(path) { - return Err(Error::AlreadyExists); - } - self.require_parent_dir(path)?; - mapped("symlink", path, self.fs.create_symlink(path, target))?; - // What the format's symlink entry records: its target's length, undated. - self.names.insert(path.to_string(), Meta { kind: Kind::Symlink, size: target.len() as u64, mtime: 0 }); - Ok(()) + let (dir, name) = self.destination(path)?; + mapped("symlink", self.cache.block_on(self.fs.create(dir, name, New::Symlink(target.as_bytes().to_vec()), OWNER, time((self.clock)())))).map(drop) } - fn sync(&mut self) -> Result, Error> { - let mut unwritten = Vec::new(); - let nodes: Vec = self.open.keys().copied().collect(); - for node in nodes { - match self.persist(node) { - Err(e) => unwritten.push((node, e)), - Ok(()) if self.open[&node].holders == 0 => self.release(node), - Ok(()) => {} - } - } - mapped("sync", "", self.fs.sync())?; - self.committed(); - Ok(unwritten) + let nodes: Vec<_> = self.open.keys().copied().collect(); + let mut refused = Vec::new(); + for node in nodes { if let Err(error) = self.persist(node) { refused.push((node, error)); } } + mapped("sync", self.cache.block_on(self.fs.commit()))?; + Ok(refused) } - - fn carried(&self, node: Node) -> bool { - self.open.get(&node).is_some_and(|open| open.carried && !open.gone) - } - - fn owes(&self) -> bool { - self.open.values().any(|open| open.dirty && !open.gone) - } - + fn owes(&self) -> bool { self.open.values().any(|open| open.changed || !open.pages.is_empty()) } fn commit(&mut self, node: Option) -> Result<(), Error> { - if let Some(node) = node { - self.persist(node)?; - } - let carried: Vec = self.open.keys().copied().filter(|&n| self.carried(n)).collect(); - for n in carried { - match self.persist(n) { - Err(e) => println!("fileserver: node {n}'s entry was not written at a commit ({e:?}); the next one writes it"), - Ok(()) if self.open[&n].holders == 0 => self.release(n), - Ok(()) => {} - } - } - mapped("sync", "", self.fs.sync())?; - self.committed(); - Ok(()) + if let Some(node) = node { self.persist(node)?; } + mapped("fsync", self.cache.block_on(self.fs.commit())) } - - fn room(&mut self) -> Result { - Ok(toyos::fs::Room { free: self.fs.free_bytes(), unit: BLOCK as u64 }) - } - + fn room(&mut self) -> Result { Ok(toyos::fs::Room { free: self.fs.space().1, unit: BLOCK as u64 }) } fn describe(&self) -> String { - let c = self.cache.counts(); - format!( - "bcachefs, {} names, {} files open; cache {} blocks ({} dirty), {} of {} reads hit", - self.names.len(), - self.open.len(), - c.cached, - c.dirty, - c.hits, - c.reads - ) + let (total, free) = self.fs.space(); + format!("bcachefs DATA: {free}/{total} bytes free, {} open inodes", self.open.len()) } } #[cfg(test)] mod tests { - use std::collections::HashMap; - use super::*; - use crate::volume::Writes; use crate::ram::Ram; - use crate::volume::Buf; - - fn clock() -> u64 { - 1_000_000_000 - } - - /// The clean blocks every volume's cache here keeps: 64 MiB. - const CLEAN_LIMIT: usize = 16 * 1024; - - fn vol() -> DataVolume { - DataVolume::format(Ram::new(4096), CLEAN_LIMIT, &["home", "apps"], clock).unwrap() - } - + use crate::volume::{Buf, Writes}; const CREATE: OpenHow = OpenHow { create: true, create_new: false, truncate: false }; - const PLAIN: OpenHow = OpenHow { create: false, create_new: false, truncate: false }; - - /// Two DATA partitions are refused, and a block service that would not - /// list what it has leaves DATA absent: memory stands in only where it - /// says there is none. - #[test] - fn data_is_one_partition_the_block_service_serves() { - let (a, b) = ([1; 16], [2; 16]); - assert_eq!(find(Ok(&[])), Located::Nowhere); - assert_eq!(find(Ok(&[a])), Located::Served(a)); - let Located::Refused(why) = find(Ok(&[a, b])) else { panic!("two located one") }; - assert!(why.starts_with("this machine has 2 DATA partitions"), "{why}"); - assert_eq!(find(Err("unlisted".into())), Located::Refused("unlisted".into())); - } - - /// **A symlink where a grant's root is, is no directory**, so a connection - /// granted that root reaches nothing through it: the folder it named is - /// listed by nobody and nothing is made under it, and the directory the - /// link names is untouched. The supervisor checks a folder before it - /// grants it; this is what holds once a declared row replaces it after. - #[test] - fn a_symlink_at_a_grant_s_root_is_followed_by_nothing() { - let mut v = vol(); - v.mkdir("home/toy/Apps/other").unwrap(); - let kept = v.open("home/toy/Apps/other/kept", CREATE).unwrap(); - v.close(kept).unwrap(); - v.symlink("home/toy/Games", "Apps/other").unwrap(); - let root = "home/toy/Games"; - assert_eq!(v.lstat(root).unwrap().kind, Kind::Symlink); - assert_eq!(v.list(root).map(drop), Err(Error::NotADirectory)); - assert_eq!(v.lstat("home/toy/Games/kept"), Err(Error::NotFound)); - assert_eq!(v.open("home/toy/Games/kept", PLAIN).map(drop), Err(Error::NotFound)); - assert_eq!(v.open("home/toy/Games/made", CREATE).map(drop), Err(Error::NotADirectory)); - assert_eq!(v.mkdir("home/toy/Games/sub"), Err(Error::NotADirectory)); - let listed: Vec = v.list("home/toy/Apps/other").unwrap().into_iter().map(|e| e.name).collect(); - assert_eq!(listed, ["kept"]); + const EXISTING: OpenHow = OpenHow { create: false, create_new: false, truncate: false }; + fn clock() -> u64 { 1_700_000_000_000_000_000 } + fn volume() -> DataVolume { + DataVolume::format(Ram::new(16384), 256, &["home", "apps"], clock).unwrap() } - - /// A file's pages are the paged objects' to keep: written and read, they - /// leave the block cache holding the volume's structures and no more. - #[test] - fn a_files_pages_pass_the_cache_by() { - let mut v = vol(); - let n = v.open("home/big", CREATE).unwrap(); - v.write(n, 0, &vec![7u8; 256 * BLOCK]).unwrap(); - v.sync().unwrap(); - let before = v.cache.counts().cached; - let mut out = vec![0u8; 256 * BLOCK]; - assert_eq!(v.read(n, 0, &mut Buf(&mut out)).unwrap(), out.len()); - assert!(out.iter().all(|&b| b == 7)); - assert_eq!(v.cache.counts().cached, before, "a data read keeps nothing"); - assert!(before < 64, "{before} blocks held for one file's structures"); - } - - #[test] - fn a_file_reads_back_what_was_written_across_pages_and_holes() { - let mut v = vol(); - let n = v.open("home/a", CREATE).unwrap(); - let data: Vec = (0..10_000u32).map(|i| i as u8).collect(); - v.write(n, 100, &data).unwrap(); - v.write(n, 20_000, b"tail").unwrap(); - let mut out = vec![0xFFu8; 20_004]; - assert_eq!(v.read(n, 0, &mut Buf(&mut out)).unwrap(), 20_004); - assert!(out[..100].iter().all(|&b| b == 0)); - assert_eq!(&out[100..10_100], &data[..]); - assert!(out[10_100..20_000].iter().all(|&b| b == 0), "a hole reads zeros"); - assert_eq!(&out[20_000..], b"tail"); + fn read(volume: &mut DataVolume, node: Node) -> Vec { + let mut bytes = vec![0xa5; volume.node_meta(node).unwrap().size as usize]; + assert_eq!(volume.read(node, 0, &mut Buf(&mut bytes)).unwrap(), bytes.len()); + bytes } - - #[test] - fn a_closed_file_keeps_its_length_across_a_remount() { - let mut v = vol(); - v.mkdir("home/toy").unwrap(); - let n = v.open("home/toy/x", CREATE).unwrap(); - v.write(n, 0, &[7; 5000]).unwrap(); - v.close(n).unwrap(); - v.sync().unwrap(); - let mut again = remount(v); - assert_eq!(again.lstat("home/toy/x").unwrap().size, 5000); - assert_eq!(again.lstat("home/toy").unwrap().kind, Kind::Dir, "an empty directory outlives the mount"); - let n = again.open("home/toy/x", PLAIN).unwrap(); - let mut out = vec![0u8; 5000]; - again.read(n, 0, &mut Buf(&mut out)).unwrap(); - assert_eq!(out, vec![7; 5000]); - } - - #[test] - fn a_file_unlinked_while_open_answers_gone() { - let mut v = vol(); - let n = v.open("home/a", CREATE).unwrap(); - v.write(n, 0, b"x").unwrap(); - v.unlink("home/a").unwrap(); - assert_eq!(v.read(n, 0, &mut Buf(&mut [0u8; 1])), Err(Error::Stale)); - assert_eq!(v.lstat("home/a"), Err(Error::NotFound)); - } - - #[test] - fn a_shrink_zeroes_what_it_cut_and_regrowth_reads_zeros() { - let mut v = vol(); - let n = v.open("home/a", CREATE).unwrap(); - v.write(n, 0, &[9; 8192]).unwrap(); - v.truncate(n, 10).unwrap(); - v.truncate(n, 8192).unwrap(); - let mut out = vec![0xFFu8; 8192]; - v.read(n, 0, &mut Buf(&mut out)).unwrap(); - assert_eq!(&out[..10], &[9; 10]); - assert!(out[10..].iter().all(|&b| b == 0)); - } - - #[test] - fn directories_are_listed_and_refuse_what_posix_refuses() { - let mut v = vol(); - let f = v.open("home/nodir/a", CREATE).unwrap(); - v.close(f).unwrap(); - assert_eq!(v.lstat("home/nodir").unwrap().kind, Kind::Dir, "a create makes its directories"); - assert_eq!(v.open("home/nodir/a/b", CREATE), Err(Error::NotADirectory), "never under a file"); - v.mkdir("home/d").unwrap(); - assert_eq!(v.mkdir("home/d"), Err(Error::AlreadyExists)); - let n = v.open("home/d/f", CREATE).unwrap(); - v.close(n).unwrap(); - assert_eq!(v.rmdir("home/d"), Err(Error::NotEmpty)); - let names: Vec = v.list("home").unwrap().into_iter().map(|e| e.name).collect(); - assert_eq!(names, ["d", "nodir"]); - v.unlink("home/d/f").unwrap(); - v.rmdir("home/d").unwrap(); - assert_eq!(v.lstat("home/d"), Err(Error::NotFound)); - assert_eq!(v.rmdir("home"), Err(Error::Root), "a root stays"); - } - - /// A file's number is one across renames, its own directory's and one - /// over another file; every file that exists at once has its own; and a - /// directory that held only what was removed is gone with its number. - #[test] - fn a_file_is_one_number_across_renames_and_another_files_is_another() { - let mut v = vol(); - for path in ["home/a/f", "home/a/g", "home/x/only"] { - let n = v.open(path, CREATE).unwrap(); - v.close(n).unwrap(); + fn reopen(volume: DataVolume) -> DataVolume { + match DataVolume::probe(volume.stopped(), 256, &["home", "apps"], clock) { + Probed::Mounted(volume) => volume, + _ => panic!("committed DATA must mount"), } - v.mkdir("home/empty").unwrap(); - let ids: Vec = ["home/a/f", "home/a/g", "home/a", "home/x", "home/x/only", "home/empty", "home", ""] - .iter() - .map(|p| v.identity(p).unwrap()) - .collect(); - assert!(ids.iter().all(|&n| n != 0 && ids.iter().filter(|&&m| m == n).count() == 1), "{ids:?}"); - let (f, g, a, x) = (ids[0], ids[1], ids[2], ids[3]); - assert_eq!(v.identity("home/nothing"), Err(Error::NotFound)); - - v.rename("home/a", "home/b").unwrap(); - assert_eq!(v.identity("home/b"), Ok(a), "a directory's rename keeps its number"); - assert_eq!(v.identity("home/b/f"), Ok(f), "and every file's beneath it"); - v.rename("home/b/g", "home/b/f").unwrap(); - assert_eq!(v.identity("home/b/f"), Ok(g), "a rename over a file takes its name, not its number"); - - v.unlink("home/x/only").unwrap(); - assert_eq!(v.identity("home/x"), Err(Error::NotFound), "a directory that held only the file went"); - let n = v.open("home/x/only", CREATE).unwrap(); - v.close(n).unwrap(); - let (dir, file) = (v.identity("home/x").unwrap(), v.identity("home/x/only").unwrap()); - assert!(!ids.contains(&dir) && !ids.contains(&file), "made again, each is a new file: {dir}, {file} of {ids:?}"); - assert_ne!(dir, x); } - #[test] - fn a_rename_moves_an_open_file_and_a_directory_with_its_contents() { - let mut v = vol(); - v.mkdir("home/a").unwrap(); - let n = v.open("home/a/f", CREATE).unwrap(); - v.write(n, 0, b"hello").unwrap(); - v.rename("home/a", "home/b").unwrap(); - assert_eq!(v.lstat("home/b/f").unwrap().size, 5); - let mut out = [0u8; 5]; - v.read(n, 0, &mut Buf(&mut out)).unwrap(); - assert_eq!(&out, b"hello"); - v.close(n).unwrap(); - let m = v.open("home/b/g", CREATE).unwrap(); - v.close(m).unwrap(); - v.rename("home/b/g", "home/b/f").unwrap(); - assert_eq!(v.lstat("home/b/f").unwrap().size, 0, "replaced"); - } - - /// Every name the index records is what the format's own entry holds, - /// for each closed file and symlink: the stat a client is answered from - /// memory is the one a search of the tree would answer. - fn assert_index_is_the_format_s(v: &DataVolume) { - for (name, meta) in &v.names { - if meta.kind == Kind::Dir || v.by_path.contains_key(name) { - continue; - } - let (_, size) = v.fs.file_extents(name).unwrap().unwrap_or_else(|| panic!("{name} has no entry")); - let mtime = v.fs.file_mtime(name).unwrap().unwrap(); - assert_eq!((meta.size, meta.mtime), (size, mtime), "{name}"); - assert_eq!(meta.kind == Kind::Symlink, v.fs.is_symlink(name).unwrap(), "{name}"); - } - } - - /// A clock that reads a later nanosecond at every call, so a time left - /// stale anywhere is a time that differs. - fn advancing_clock() -> u64 { - static NOW: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1_000_000_000); - NOW.fetch_add(1, std::sync::atomic::Ordering::Relaxed) - } - - #[test] - fn a_stat_from_the_index_is_what_the_entry_records() { - let mut v = DataVolume::format(Ram::new(4096), CLEAN_LIMIT, &["home"], advancing_clock).unwrap(); - let a = v.open("home/a", CREATE).unwrap(); - v.write(a, 0, &[1; 5000]).unwrap(); - assert_eq!(v.lstat("home/a").unwrap().size, 5000, "an open file's own length, before its entry has it"); - v.close(a).unwrap(); - assert_index_is_the_format_s(&v); - - let b = v.open("home/b", CREATE).unwrap(); - v.write(b, 0, &[2; 9000]).unwrap(); - v.truncate(b, 100).unwrap(); - v.close(b).unwrap(); - v.symlink("home/l", "a").unwrap(); - v.rename("home/a", "home/sub/c").unwrap(); - v.rename("home/b", "home/sub/c").unwrap(); - // Written and still open while its directory moves. - let open = v.open("home/sub/open", CREATE).unwrap(); - v.write(open, 0, &[3; 7000]).unwrap(); - let written = v.lstat("home/sub/open").unwrap(); - v.mkdir("home/d").unwrap(); - v.rename("home/sub", "home/d/moved").unwrap(); - v.close(open).unwrap(); - assert_index_is_the_format_s(&v); - assert_eq!(v.lstat("home/d/moved/open").unwrap(), written, "the open file's length and time moved with it"); - assert_eq!(v.lstat("home/d/moved/c").unwrap().size, 100); - assert_eq!(v.lstat("home/l").unwrap(), Meta { kind: Kind::Symlink, size: 1, mtime: 0 }); - v.unlink("home/l").unwrap(); - assert_eq!(v.lstat("home/l"), Err(Error::NotFound)); - - let before: Vec<(String, Meta)> = v.names.iter().map(|(n, m)| (n.clone(), *m)).collect(); + fn sparse_data_and_shrunk_tails_survive_remount() { + let mut v = volume(); + let f = v.open("home/f", CREATE).unwrap(); + v.write(f, 2 * BLOCK as u64 + 17, b"value").unwrap(); + let bytes = read(&mut v, f); + assert!(bytes[..2 * BLOCK + 17].iter().all(|&b| b == 0)); + assert_eq!(&bytes[2 * BLOCK + 17..], b"value"); + v.truncate(f, 2 * BLOCK as u64 + 19).unwrap(); + v.truncate(f, 3 * BLOCK as u64).unwrap(); v.sync().unwrap(); - let DataVolume { fs, cache, .. } = v; - drop(fs); - let disk = Rc::try_unwrap(cache).ok().expect("one owner").into_disk(); - let Probed::Mounted(again) = DataVolume::probe(disk, CLEAN_LIMIT, &["home"], advancing_clock) else { panic!("remount") }; - let after: Vec<(String, Meta)> = again.names.iter().map(|(n, m)| (n.clone(), *m)).collect(); - assert_eq!(after, before, "the index a mount reads is the one the server kept"); - assert_index_is_the_format_s(&again); + let mut v = reopen(v); + let f = v.open("home/f", EXISTING).unwrap(); + let bytes = read(&mut v, f); + assert_eq!(&bytes[2 * BLOCK + 17..2 * BLOCK + 19], b"va"); + assert!(bytes[2 * BLOCK + 19..].iter().all(|&b| b == 0)); } - - /// A listing passes over what is beneath a child directory and names - /// each child once: `x` with an entry of its own and names beneath, `y` - /// only the prefix of two, and the names beside `x` that sort among its - /// own (`-` and `.` before `/`, `0` after). #[test] - fn a_listing_names_each_child_once_and_nothing_beneath_one() { - let mut v = vol(); - v.mkdir("home/x").unwrap(); - for path in ["home/x-a", "home/x.b", "home/x/deep/er/f", "home/x/g", "home/x0", "home/y/z", "home/y/z2"] { - let n = v.open(path, CREATE).unwrap(); - v.close(n).unwrap(); - } - v.mkdir("home/x/kept").unwrap(); - v.mkdir("home/e").unwrap(); - let listed: Vec<(String, Kind)> = v.list("home").unwrap().into_iter().map(|l| (l.name, l.meta.kind)).collect(); - let want = [("e", Kind::Dir), ("x", Kind::Dir), ("x-a", Kind::File), ("x.b", Kind::File), ("x0", Kind::File), ("y", Kind::Dir)]; - assert_eq!(listed, want.map(|(n, k)| (n.to_string(), k))); - let inner: Vec = v.list("home/x").unwrap().into_iter().map(|l| l.name).collect(); - assert_eq!(inner, ["deep", "g", "kept"]); - let root: Vec = v.list("").unwrap().into_iter().map(|l| l.name).collect(); - assert_eq!(root, ["home"]); - } - - /// A clock that reads no whole second. - fn nanosecond_clock() -> u64 { - 1_717_245_296_123_456_789 - } - - /// A commit leaves owed the entry of a file it did not carry, an overwrite - /// within the file's own blocks, and nothing once that entry is committed. - #[test] - fn a_commit_owes_the_entry_of_a_file_it_did_not_carry() { - let mut v = vol(); - let a = v.open("home/a", CREATE).unwrap(); - let b = v.open("home/b", CREATE).unwrap(); - v.write(a, 0, b"a").unwrap(); - v.write(b, 0, b"b").unwrap(); - v.commit(Some(a)).unwrap(); - assert!(!v.owes(), "b took blocks, so the commit carried its entry"); - v.write(b, 0, b"B").unwrap(); - v.commit(Some(a)).unwrap(); - assert!(v.owes(), "b's overwrite took no block and was not carried"); - v.commit(Some(b)).unwrap(); - assert!(!v.owes()); - } - - /// DATA stamps a write with the nanosecond its clock read, and the entry - /// keeps it whole across a remount: a stamp of whole seconds would round - /// it away. - #[test] - fn a_write_s_stamp_keeps_its_nanosecond_across_a_remount() { - let mut v = DataVolume::format(Ram::new(4096), CLEAN_LIMIT, &["home"], nanosecond_clock).unwrap(); - let n = v.open("home/stamped", CREATE).unwrap(); - v.write(n, 0, b"stamped").unwrap(); - v.close(n).unwrap(); + fn unlink_and_replacement_keep_live_inode_identity() { + let mut v = volume(); + let old = v.open("home/name", CREATE).unwrap(); + v.write(old, 0, b"old").unwrap(); + let new = v.open("home/new", CREATE).unwrap(); + v.write(new, 0, b"new").unwrap(); + v.rename("home/new", "home/name").unwrap(); + assert_eq!(v.identity("home/name").unwrap(), new); + assert!(!v.linked(old).unwrap()); + assert_eq!(read(&mut v, old), b"old"); + v.write(old, 0, b"OLD").unwrap(); + assert_eq!(read(&mut v, new), b"new"); + v.unlink("home/name").unwrap(); + assert_eq!(v.lstat("home/name"), Err(Error::NotFound)); + assert_eq!(read(&mut v, new), b"new"); + v.close(old).unwrap(); + v.close(new).unwrap(); v.sync().unwrap(); - let DataVolume { fs, cache, .. } = v; - drop(fs); - let disk = Rc::try_unwrap(cache).ok().expect("one owner").into_disk(); - let Probed::Mounted(mut again) = DataVolume::probe(disk, CLEAN_LIMIT, &["home"], nanosecond_clock) else { panic!("remount") }; - assert_eq!(again.lstat("home/stamped").unwrap().mtime, nanosecond_clock()); - } - - fn remount(v: DataVolume) -> DataVolume { - let disk = v.stopped(); - let Probed::Mounted(again) = DataVolume::probe(disk, CLEAN_LIMIT, &["home"], clock) else { panic!("remount") }; - again - } - - /// A volume whose free blocks are all apart, so every run a file is given - /// is one block: every block a file's data may take, taken in one run, - /// every other one given back, and the run the reserve kept - /// (`bcachefs`'s `NODE_RESERVE`, 16) taken after. - fn fragmented() -> DataVolume { - let mut v = DataVolume::format(Ram::new(1024), CLEAN_LIMIT, &["home"], clock).unwrap(); - let mut run = Vec::new(); - let mut page = 0; - while v.fs.allocate(&mut run, page..=page, 1).is_ok() { - page += 1; - } - let taken: Vec = run.iter().flat_map(|e| e.start_block..e.start_block + e.block_count as u64).collect(); - let holes: Vec = - taken.iter().step_by(2).map(|&b| Extent { start_block: b, block_count: 1, _reserved: 0 }).collect(); - v.fs.free_extents(&holes).unwrap(); - let mut tail = vec![Extent { start_block: *taken.last().unwrap(), block_count: 1, _reserved: 0 }]; - v.fs.allocate(&mut tail, 16..=16, 1).unwrap(); - v + let mut v = reopen(v); + assert!(v.list("home").unwrap().is_empty()); } - - fn page_count(v: &mut DataVolume, path: &str) -> u64 { - v.lstat(path).unwrap().size / BLOCK as u64 - } - - /// The free count a sync leaves in the superblock. - fn free_blocks(v: &mut DataVolume) -> u64 { - assert_eq!(v.sync(), Ok(Vec::new())); - bcachefs::Superblock::read(v.fs.io()).unwrap().free_blocks - } - - fn write_into(file: &mut Vec, offset: u64, data: &[u8]) { - let end = offset as usize + data.len(); - if file.len() < end { - file.resize(end, 0); - } - file[offset as usize..end].copy_from_slice(data); - } - - /// Differential against a plain map of what each accepted write made - /// each file: three files grow in turn past twice the runs one entry - /// names at a block a run, through overwrites, holes, a shrink and writes - /// the volume refuses, and every byte the map holds is what the remounted - /// volume reads. - #[test] - fn every_file_reads_back_as_a_plain_map_of_its_accepted_writes() { - const PAST: usize = (2 * 250 + 20) * BLOCK; - let paths = ["home/a", "home/b", "home/c"]; - let mut v = vol(); - let nodes: Vec = paths.iter().map(|p| v.open(p, CREATE).unwrap()).collect(); - let mut model: HashMap<&str, Vec> = paths.iter().map(|p| (*p, Vec::new())).collect(); - let mut state = 0x2545_F491_4F6C_DD1Du64; - let mut next = move || { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - state - }; - let mut round = 0u64; - while paths.iter().any(|p| model[p].len() < PAST) { - for (i, (&path, &node)) in paths.iter().zip(&nodes).enumerate() { - let held = model.get_mut(path).unwrap(); - let len = held.len() as u64; - let (offset, n) = match (round + i as u64) % 11 { - 3 if len > 0 => (next() % len, (next() % (3 * BLOCK as u64)) as usize + 1), - 7 => (len + next() % (2 * BLOCK as u64), BLOCK), - _ => (len, BLOCK + (next() % 64) as usize), - }; - let data: Vec = (0..n).map(|_| next() as u8).collect(); - assert_eq!(v.write(node, offset, &data), Ok(()), "{path}: {n} bytes at {offset}, of {len} held"); - write_into(held, offset, &data); - } - if round % 97 == 50 { - let at = round as usize % paths.len(); - assert_eq!( - v.write(nodes[at], 64 << 20, b"past every block"), - Err(Error::NoSpace), - "{}: a write the volume has no blocks for", - paths[at] - ); - } - if round == 300 { - let b = model.get_mut("home/b").unwrap(); - let to = b.len() / 3 + 17; - v.truncate(nodes[1], to as u64).unwrap(); - b.truncate(to); - } - round += 1; - } - assert_eq!(v.sync(), Ok(Vec::new())); - for n in nodes { - v.close(n).unwrap(); - } - let mut v = remount(v); - let listed: Vec = v.list("home").unwrap().into_iter().map(|e| format!("home/{}", e.name)).collect(); - assert_eq!(listed, paths); - for path in paths { - let held = &model[path]; - assert_eq!(v.lstat(path).unwrap().size, held.len() as u64, "{path}'s length"); - let n = v.open(path, PLAIN).unwrap(); - let mut out = vec![0xEEu8; held.len()]; - assert_eq!(v.read(n, 0, &mut Buf(&mut out)), Ok(held.len())); - let first = out.iter().zip(held).position(|(a, b)| a != b); - assert_eq!(first, None, "{path}: the first byte the volume and the map disagree on"); - } - } - - /// On a volume whose free blocks are all apart, a file's entry fills: the - /// write that would take it past what one entry names is refused whole, - /// and every write before it reaches the volume. - #[test] - fn a_write_its_entry_could_not_name_is_refused_and_every_accepted_one_kept() { - let mut v = fragmented(); - let a = v.open("home/a", CREATE).unwrap(); - let mut pages = 0u64; - let refused = loop { - match v.write(a, pages * BLOCK as u64, &[3; BLOCK]) { - Ok(()) => pages += 1, - Err(e) => break e, - } - }; - assert_eq!(refused, Error::LimitExceeded); - // One block a run: (4064 − 24 − 19 − "home/a".len()) / 16 runs is - // what the one entry holding `home/a` names. - assert_eq!(pages, 250, "the entry filled, not the volume"); - assert_eq!(page_count(&mut v, "home/a"), pages, "the refused write changed nothing"); - let before = free_blocks(&mut v); - assert_eq!(v.write(a, pages * BLOCK as u64, &[4; 4 * BLOCK]), Err(Error::LimitExceeded)); - assert_eq!(free_blocks(&mut v), before, "a refused write's blocks went back"); - v.close(a).unwrap(); - let mut v = remount(v); - assert_eq!(page_count(&mut v, "home/a"), pages); - let a = v.open("home/a", PLAIN).unwrap(); - let mut out = vec![0u8; pages as usize * BLOCK]; - v.read(a, 0, &mut Buf(&mut out)).unwrap(); - assert!(out.iter().all(|&b| b == 3)); - } - - /// One file whose entry is refused leaves every other file's sync whole, - /// its own close refused, and its writes kept until a sync writes them. #[test] - fn an_entry_refused_costs_only_its_own_file() { - let mut v = vol(); - let a = v.open("home/a", CREATE).unwrap(); - let b = v.open("home/b", CREATE).unwrap(); - v.write(a, 0, &[1; BLOCK]).unwrap(); - v.write(b, 0, &[2; 3 * BLOCK]).unwrap(); - // More runs than the entry names, as no write can give it. - let real = v.open[&a].extents.clone(); - let too_many = vec![real[0]; 300]; - v.open.get_mut(&a).unwrap().extents = too_many.clone(); - - assert_eq!(v.sync(), Ok(vec![(a, Error::LimitExceeded)])); - v.close(b).unwrap(); - assert_eq!(v.close(a), Err(Error::LimitExceeded), "a close that lost the writes is refused"); - assert_eq!(v.sync(), Ok(vec![(a, Error::LimitExceeded)]), "unheld, it is named again"); - assert!(v.open.contains_key(&a), "and kept again"); - assert_eq!(page_count(&mut v, "home/a"), 1, "the refused file is still what its writes made it"); - - v.open.get_mut(&a).unwrap().extents = real; - assert_eq!(v.sync(), Ok(Vec::new())); - assert!(v.open.is_empty(), "written, the unheld node goes"); - let mut v = remount(v); - assert_eq!(page_count(&mut v, "home/a"), 1); - assert_eq!(page_count(&mut v, "home/b"), 3); - } - - /// A rename over an open, unsynced file that the format refuses leaves - /// that file whole: readable through its holder, and its length reaching - /// the volume at the next sync. The refusal is `EntryTooLarge`: `from`'s - /// extents fit its own short name and not `to`'s long one. - #[test] - fn a_refused_rename_over_an_open_file_keeps_its_unsynced_writes() { - let mut v = fragmented(); - let from = v.open("home/f", CREATE).unwrap(); - for page in 0..240u64 { - v.write(from, page * BLOCK as u64, &[1; BLOCK]).unwrap(); - } - let to_path = format!("home/{}", "t".repeat(300)); - let to = v.open(&to_path, CREATE).unwrap(); - v.write(to, 0, b"written and not synced").unwrap(); - - assert_eq!(v.rename("home/f", &to_path), Err(Error::LimitExceeded)); - - let mut back = [0u8; 22]; - assert_eq!(v.read(to, 0, &mut Buf(&mut back)), Ok(22), "the holder of `to` still reads it"); - assert_eq!(&back, b"written and not synced"); - assert_eq!(v.sync(), Ok(Vec::new())); - v.close(to).unwrap(); - assert_eq!(v.lstat(&to_path).unwrap().size, 22, "`to`'s length reached the volume"); - } - - /// A disk whose blocks are a map the test can copy, that keeps the first - /// `keep` block writes and loses every later one — the server killed there - /// — or refuses the one numbered `refuse` and keeps the rest. A request of - /// several blocks counts each, so a kill can land inside one. - type Image = BTreeMap>; - - struct Stops { - blocks: u64, - /// Shared, so a test reads what the disk holds while the server runs. - image: Rc>, - reads: std::cell::Cell, - writes: std::cell::Cell, - flushes: std::cell::Cell, - keep: usize, - refuse: Option, - /// A block whose read the disk refuses. - refuse_read: std::cell::Cell>, - } - - impl Stops { - fn new(blocks: u64) -> Self { - Self { blocks, image: Rc::default(), reads: Default::default(), writes: Default::default(), flushes: Default::default(), keep: usize::MAX, refuse: None, refuse_read: Default::default() } - } - - fn from(image: &Image, blocks: u64, keep: usize, refuse: Option) -> Self { - Self { blocks, image: Rc::new(image.clone().into()), reads: Default::default(), writes: Default::default(), flushes: Default::default(), keep, refuse, refuse_read: Default::default() } - } - } - - impl BlockDev for Stops { - fn blocks(&self) -> u64 { - self.blocks - } - - async fn read(&self, lba: u64, out: &mut [Block]) -> Result<(), toyos_blockdev::Error> { - self.reads.set(self.reads.get() + out.len()); - if self.refuse_read.get().is_some_and(|b| (lba..lba + out.len() as u64).contains(&b)) { - return Err(toyos_blockdev::Error::Device); - } - let image = self.image.borrow(); - for (block, out) in (lba..).zip(out) { - match image.get(&block) { - Some(data) => *out = **data, - None => out.fill(0), - } - } - Ok(()) - } - - async fn write(&self, lba: u64, data: &[Block]) -> Result<(), toyos_blockdev::Error> { - for (block, data) in (lba..).zip(data) { - let n = self.writes.get(); - self.writes.set(n + 1); - if Some(n) == self.refuse { - return Err(toyos_blockdev::Error::Device); - } - if n < self.keep { - self.image.borrow_mut().insert(block, Box::new(*data)); - } - } - Ok(()) - } - - async fn flush(&self) -> Result<(), toyos_blockdev::Error> { - self.flushes.set(self.flushes.get() + 1); - Ok(()) - } - } - - impl Disk for Stops {} - - /// A sync with nothing written since the last asks the disk for nothing, - /// and one after a write over a page the file holds writes the page and - /// makes it durable. The sync after a commit still writes the blocks that - /// commit freed, which are marked free only once it has landed. - #[test] - fn a_sync_with_nothing_new_asks_the_disk_for_nothing() { - let mut v = DataVolume::format(Stops::new(1024), CLEAN_LIMIT, &["home", "apps"], clock).unwrap(); - let n = v.open("home/f", CREATE).unwrap(); - v.write(n, 0, &[1; BLOCK]).unwrap(); - assert_eq!(v.sync(), Ok(Vec::new())); - assert_eq!(v.sync(), Ok(Vec::new())); - let asked = |v: &DataVolume| (v.cache.disk().writes.get(), v.cache.disk().flushes.get()); - let synced = asked(&v); - assert_eq!(v.sync(), Ok(Vec::new())); - assert_eq!(asked(&v), synced); - - v.write(n, 0, &[2; BLOCK]).unwrap(); - assert_eq!(v.sync(), Ok(Vec::new())); - let (writes, flushes) = asked(&v); - assert!(writes > synced.0 && flushes > synced.1, "the write over the page asked {:?} after {synced:?}", (writes, flushes)); - let block = block_for(&v.open[&n].extents, 0).unwrap(); - assert_eq!(v.cache.disk().image.borrow()[&block][..], [2; BLOCK][..]); - } - - /// A write refused after its pages were allocated gives them back: a file - /// extended over where it would have reached reads zeros there, not the - /// bytes a deleted file left in the blocks the write was given. - #[test] - fn a_write_refused_after_allocating_leaves_no_old_bytes_past_the_end() { - // Every block the volume does not write holds what an earlier tenant left. - let stale: Image = (0..1024).map(|b| (b, Box::new([0xAB; BLOCK]))).collect(); - let mut v = DataVolume::format(Stops::from(&stale, 1024, usize::MAX, None), 0, &["home", "apps"], clock).unwrap(); - - let n = v.open("home/f", CREATE).unwrap(); - v.write(n, 0, &[1; BLOCK]).unwrap(); - assert_eq!(v.sync(), Ok(Vec::new())); - let first = block_for(&v.open[&n].extents, 0).unwrap(); - v.cache.disk().refuse_read.set(Some(first)); - // Half of page 0, whose read is refused, and page 1, allocated first. - let room = v.room().unwrap().free; - assert_eq!(v.write(n, BLOCK as u64 / 2, &[2; BLOCK]), Err(Error::Io)); - assert_eq!(v.room().unwrap().free, room, "the refused write kept blocks, or gave back more"); - v.cache.disk().refuse_read.set(None); - - v.truncate(n, 2 * BLOCK as u64).unwrap(); - let mut out = vec![0u8; BLOCK]; - assert_eq!(v.read(n, BLOCK as u64, &mut Buf(&mut out)).unwrap(), BLOCK); - assert!(out.iter().all(|&b| b == 0), "page 1 reads {:#04x}, not zeros", out[out.iter().position(|&b| b != 0).unwrap()]); - } - - /// The same inside the size: a refused write into a hole the file was - /// extended over gives back the block it took there, which no cut back to - /// the old size would, and the hole still reads zeros. - #[test] - fn a_write_refused_in_a_hole_leaves_the_hole_reading_zeros() { - let stale: Image = (0..1024).map(|b| (b, Box::new([0xAB; BLOCK]))).collect(); - let mut v = DataVolume::format(Stops::from(&stale, 1024, usize::MAX, None), 0, &["home", "apps"], clock).unwrap(); - - let n = v.open("home/f", CREATE).unwrap(); - v.write(n, 0, &[1; BLOCK]).unwrap(); - v.truncate(n, 3 * BLOCK as u64).unwrap(); - assert_eq!(v.sync(), Ok(Vec::new())); - let first = block_for(&v.open[&n].extents, 0).unwrap(); - v.cache.disk().refuse_read.set(Some(first)); - // Half of page 0, whose read is refused, and page 1, in the hole. - let room = v.room().unwrap().free; - assert_eq!(v.write(n, BLOCK as u64 / 2, &[2; BLOCK]), Err(Error::Io)); - assert_eq!(v.room().unwrap().free, room, "the refused write kept the hole's block, or gave back more"); - v.cache.disk().refuse_read.set(None); - - assert_eq!(v.node_meta(n).unwrap().size, 3 * BLOCK as u64); - let mut out = vec![0u8; BLOCK]; - assert_eq!(v.read(n, BLOCK as u64, &mut Buf(&mut out)).unwrap(), BLOCK); - assert!(out.iter().all(|&b| b == 0), "page 1 reads {:#04x}, not zeros", out[out.iter().position(|&b| b != 0).unwrap()]); - } - - /// A byte written into a hole below the old end: the rest of its page - /// reads zeros, not what the block its reserve took last held. - #[test] - fn a_write_in_part_into_a_hole_reads_zeros_around_it() { - let stale: Image = (0..1024).map(|b| (b, Box::new([0xAB; BLOCK]))).collect(); - let mut v = DataVolume::format(Stops::from(&stale, 1024, usize::MAX, None), 0, &["home", "apps"], clock).unwrap(); - - let n = v.open("home/f", CREATE).unwrap(); - v.write(n, 0, &[1; BLOCK]).unwrap(); - v.truncate(n, 3 * BLOCK as u64).unwrap(); - assert_eq!(v.sync(), Ok(Vec::new())); - v.write(n, 6000, &[2]).unwrap(); - - let mut out = vec![0u8; BLOCK]; - assert_eq!(v.read(n, BLOCK as u64, &mut Buf(&mut out)).unwrap(), BLOCK); - let mut want = vec![0u8; BLOCK]; - want[6000 - BLOCK] = 2; - assert!(out == want, "page 1 reads {:#04x} where nothing was written", out[out.iter().zip(&want).position(|(a, b)| a != b).unwrap()]); - } - - fn take(v: DataVolume) -> Stops { - let DataVolume { fs, cache, .. } = v; - drop(fs); - Rc::try_unwrap(cache).ok().expect("one owner").into_disk() - } - - /// Every file under `dir` with its bytes, by its path beneath `dir`; a - /// directory with nothing under it is listed with a trailing `/`. - fn tree(v: &mut DataVolume, dir: &str) -> Option>> { - let listed = v.list(dir).ok()?; - let mut out = BTreeMap::new(); - if listed.is_empty() { - out.insert("/".to_string(), Vec::new()); - } - for Listed { name, meta, .. } in listed { - let path = join(dir, &name); - match meta.kind { - Kind::Dir => { - for (below, bytes) in tree(v, &path)? { - let below = if below == "/" { format!("{name}/") } else { format!("{name}/{below}") }; - out.insert(below, bytes); - } - } - _ => { - let n = v.open(&path, PLAIN).ok()?; - let mut bytes = vec![0u8; meta.size as usize]; - let read = v.read(n, 0, &mut Buf(&mut bytes)); - v.close(n).ok()?; - (read.ok()? == bytes.len()).then_some(())?; - out.insert(name, bytes); - } - } - } - Some(out) - } - - const STAGED: &str = "home/staged"; - const INSTALLED: &str = "apps/pkg"; - - /// Where the directory is: `Ok(true)` whole under its new name and absent - /// under its old, `Ok(false)` the reverse, and otherwise what was found. - fn whole_under_one(v: &mut DataVolume, want: &BTreeMap>) -> Result { - let (old, new) = (tree(v, STAGED), tree(v, INSTALLED)); - match (&old, &new) { - (Some(old), None) if old == want => Ok(false), - (None, Some(new)) if new == want => Ok(true), - _ => Err(format!( - "{} names under {STAGED} and {} under {INSTALLED}", - old.as_ref().map_or(0, |t| t.len()), - new.as_ref().map_or(0, |t| t.len()) - )), - } - } - - /// A staged package: files enough to fill several leaves, a subdirectory, - /// an empty one, and names beside it the rename must not move; synced. - fn staged() -> (Image, BTreeMap>) { - let mut v = DataVolume::format(Stops::new(1024), CLEAN_LIMIT, &["home", "apps"], clock).unwrap(); - for i in 0..80 { - let n = v.open(&format!("home/beside{i}"), CREATE).unwrap(); - v.write(n, 0, format!("beside {i}").as_bytes()).unwrap(); - v.close(n).unwrap(); - } - v.mkdir(STAGED).unwrap(); - v.mkdir(&format!("{STAGED}/empty")).unwrap(); - for i in 0..60 { - let path = if i % 4 == 0 { format!("{STAGED}/sub/f{i}") } else { format!("{STAGED}/f{i}") }; - let n = v.open(&path, CREATE).unwrap(); - v.write(n, 0, format!("file {i} of the package").as_bytes()).unwrap(); - v.close(n).unwrap(); - } - assert_eq!(v.sync(), Ok(Vec::new())); - let want = tree(&mut v, STAGED).unwrap(); - (take(v).image.take(), want) - } - - fn mounted(disk: Stops) -> Result, String> { - match DataVolume::probe(disk, CLEAN_LIMIT, &["home", "apps"], clock) { - Probed::Mounted(v) => Ok(v), - Probed::Unmountable(why) => Err(format!("unmountable: {why}")), - Probed::Foreign => Err("foreign".into()), - } - } - - /// The rename and the sync that makes it durable, against a disk that - /// stops at every block write the two make in turn: what the disk then - /// holds mounts, and names the directory whole under exactly one name. - #[test] - fn a_directory_rename_is_whole_under_one_name_wherever_the_server_is_killed() { - let (image, want) = staged(); - let blocks = 1024; - let mut v = mounted(Stops::from(&image, blocks, usize::MAX, None)).unwrap(); - v.rename(STAGED, INSTALLED).unwrap(); - assert_eq!(v.sync(), Ok(Vec::new())); - assert_eq!(whole_under_one(&mut v, &want), Ok(true)); - let writes = take(v).writes.get(); - assert!(writes > 4, "the rename and its sync wrote {writes} blocks"); - - let mut torn = Vec::new(); - for keep in 0..=writes { - let mut v = mounted(Stops::from(&image, blocks, keep, None)).unwrap(); - let _ = v.rename(STAGED, INSTALLED); - let _ = v.sync(); - let verdict = mounted(take(v)).and_then(|mut again| whole_under_one(&mut again, &want)); - if let Err(why) = verdict { - torn.push(format!("killed after {keep} of {writes} writes: {why}")); - } - } - assert!(torn.is_empty(), "{} of {} kill points tear the directory:\n{}", torn.len(), writes + 1, torn.join("\n")); - } - - /// The same, with the one write numbered `n` refused and the server - /// alive: what it answers and what it then serves agree, the disk holds - /// the directory whole under one name past the refused sync, and the next - /// sync makes the answer what the disk holds. - #[test] - fn a_directory_rename_is_whole_under_one_name_whichever_write_is_refused() { - let (image, want) = staged(); - let blocks = 1024; - let mut v = mounted(Stops::from(&image, blocks, usize::MAX, None)).unwrap(); - v.rename(STAGED, INSTALLED).unwrap(); - assert_eq!(v.sync(), Ok(Vec::new())); - let writes = take(v).writes.get(); - let held = |on_disk: &Image| { - mounted(Stops::from(on_disk, blocks, usize::MAX, None)).and_then(|mut v| whole_under_one(&mut v, &want)) - }; - - let mut torn = Vec::new(); - for refuse in 0..writes { - let disk = Stops::from(&image, blocks, usize::MAX, Some(refuse)); - let on_disk = Rc::clone(&disk.image); - let mut v = mounted(disk).unwrap(); - let renamed = v.rename(STAGED, INSTALLED).is_ok(); - let first = v.sync(); - let between = held(&on_disk.borrow()); - let served = whole_under_one(&mut v, &want); - let second = v.sync(); - let after = held(&on_disk.borrow()); - if served != Ok(renamed) || between.is_err() || second != Ok(Vec::new()) || after != Ok(renamed) { - torn.push(format!( - "write {refuse} of {writes} refused: answered {renamed}, served {served:?}, \ - the sync {first:?} left {between:?}, the next {second:?} left {after:?}" - )); - } - } - assert!(torn.is_empty(), "{} of {writes} refusals tear the directory:\n{}", torn.len(), torn.join("\n")); + fn directory_moves_preserve_descendants_and_symlinks() { + let mut v = volume(); + v.mkdir("home/dir").unwrap(); + let f = v.open("home/dir/file", CREATE).unwrap(); + v.write(f, 0, b"kept").unwrap(); + v.symlink("home/dir/link", "file").unwrap(); + v.rename("home/dir", "home/moved").unwrap(); + assert_eq!(v.identity("home/moved/file").unwrap(), f); + assert_eq!(v.read_link("home/moved/link").unwrap(), "file"); + assert_eq!(read(&mut v, f), b"kept"); + assert_eq!(v.rmdir("home/moved"), Err(Error::NotEmpty)); + assert_eq!(v.list("home").unwrap()[0].name, "moved"); + v.sync().unwrap(); + let mut v = reopen(v); + assert_eq!(v.read_link("home/moved/link").unwrap(), "file"); } - - /// A rename the format refuses part-way — `z`'s extents fit its own name - /// and not the long one — leaves the directory whole under its old name. #[test] - fn a_directory_rename_the_format_refuses_part_way_moves_nothing() { - let mut v = fragmented(); - let a = v.open("home/staged/a", CREATE).unwrap(); - v.write(a, 0, b"small").unwrap(); - v.close(a).unwrap(); - let z = v.open("home/staged/z", CREATE).unwrap(); - for page in 0..240u64 { - v.write(z, page * BLOCK as u64, &[1; BLOCK]).unwrap(); - } - v.close(z).unwrap(); - let to = format!("home/{}", "t".repeat(300)); - - assert_eq!(v.rename("home/staged", &to), Err(Error::LimitExceeded)); - - let names = |v: &mut DataVolume, dir: &str| -> Vec { - v.list(dir).map(|l| l.into_iter().map(|e| e.name).collect()).unwrap_or_default() - }; - assert_eq!(names(&mut v, "home/staged"), ["a", "z"]); - assert_eq!(v.lstat(&to), Err(Error::NotFound)); - assert_eq!(v.sync(), Ok(Vec::new())); - let mut v = remount(v); - assert_eq!(names(&mut v, "home/staged"), ["a", "z"]); - assert_eq!(v.lstat(&to), Err(Error::NotFound)); + fn cancelling_one_reservation_keeps_an_earlier_dirty_page() { + let mut v = volume(); + let f = v.open("home/f", CREATE).unwrap(); + v.reserve(f, 0, 4).unwrap(); + let before = v.room().unwrap().free; + let later = v.reserve(f, BLOCK as u64, 4).unwrap(); + v.unreserve(f, later, 0).unwrap(); + assert_eq!(v.room().unwrap().free, before); + assert_eq!(v.node_meta(f).unwrap().size, 4); + v.write_back(f, 0, b"kept").unwrap(); + v.commit(Some(f)).unwrap(); + let mut v = reopen(v); + let f = v.open("home/f", EXISTING).unwrap(); + assert_eq!(read(&mut v, f), b"kept"); } - - /// What a write takes is what the room answers less, block for block, - /// and the room never names the blocks the reserve keeps for a delete. #[test] - fn a_write_takes_its_blocks_out_of_the_room() { - let mut v = vol(); - let n = v.open("apps/x", CREATE).unwrap(); - let before = v.room().unwrap(); - assert_eq!(before.unit, BLOCK as u64); - v.write(n, 0, &vec![7u8; 3 * BLOCK + 1]).unwrap(); - assert_eq!(v.room().unwrap().free, before.free - 4 * BLOCK as u64); - v.close(n).unwrap(); - let free = free_blocks(&mut v); - assert!(v.room().unwrap().free < free * BLOCK as u64, "the reserve's blocks were counted as room"); + fn designation_is_required_before_any_format_write() { + let disk = Ram::new(16384); + assert!(matches!(DataVolume::probe(disk, 256, &["home"], clock), Probed::Foreign)); + let disk = Ram::new(16384); + toyos_blockdev::block_on_ready(disk.write(0, &[bcachefs::designation(16384)])).unwrap(); + assert!(matches!(DataVolume::probe(disk, 256, &["home"], clock), Probed::Mounted(_))); } } diff --git a/system/fileserver/src/fat.rs b/system/fileserver/src/fat.rs index 9bd1a71cea2..58863e9ed3b 100644 --- a/system/fileserver/src/fat.rs +++ b/system/fileserver/src/fat.rs @@ -332,7 +332,7 @@ impl Volume for FatVolume { open.dirty = true; open.written = Some(now); // A chain has no holes: it holds a block for every page of its size. - let reserved = Reserved { size: open.file.len(), held: open.file.len().div_ceil(toyos_blockdev::BLOCK as u64) }; + let reserved = Reserved { size: open.file.len(), contents: open.file.len(), ticket: 0 }; self.cache.block_on(self.fs.reserve(&mut open.file, offset, len)).map_err(|e| logged("reserve", &open.path, e))?; Ok(reserved) } diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index 216f35ec08b..a1759181b45 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -253,7 +253,7 @@ fn clock() -> u64 { } fn volume() -> DataVolume { - DataVolume::format(Ram::new(4096), 256, &["home"], clock).unwrap() + DataVolume::format(Ram::new(16384), 256, &["home"], clock).unwrap() } const CREATE: OpenHow = OpenHow { create: true, create_new: false, truncate: false }; @@ -398,8 +398,8 @@ fn a_write_refused_part_way_is_cut_back_to_what_it_put() { #[test] fn a_write_refused_in_a_hole_gives_back_the_block_it_took() { // Every block the volume does not write holds what an earlier tenant left. - let disk = Ram::new(4096); - let stale = vec![[0xAB; toyos_blockdev::BLOCK]; 4096]; + let disk = Ram::new(16384); + let stale = vec![[0xAB; toyos_blockdev::BLOCK]; 16384]; toyos_blockdev::block_on(toyos_blockdev::BlockDev::write(&disk, 0, &stale), || ()).unwrap(); let mut v = DataVolume::format(disk, 256, &["home"], clock).unwrap(); let fake = Fake::new(64, 16); @@ -424,8 +424,8 @@ fn a_write_refused_in_a_hole_gives_back_the_block_it_took() { /// the block its reserve took last held. #[test] fn a_write_in_part_into_a_hole_supplies_zeros_around_it() { - let disk = Ram::new(4096); - let stale = vec![[0xAB; toyos_blockdev::BLOCK]; 4096]; + let disk = Ram::new(16384); + let stale = vec![[0xAB; toyos_blockdev::BLOCK]; 16384]; toyos_blockdev::block_on(toyos_blockdev::BlockDev::write(&disk, 0, &stale), || ()).unwrap(); let mut v = DataVolume::format(disk, 256, &["home"], clock).unwrap(); let fake = Fake::new(64, 16); @@ -572,8 +572,8 @@ fn a_write_of_many_calls_is_read_whole_or_not_at_all() { /// A volume whose every block holds what an earlier tenant left, 0xAB. fn stale_volume() -> DataVolume { - let disk = Ram::new(4096); - let stale = vec![[0xAB; toyos_blockdev::BLOCK]; 4096]; + let disk = Ram::new(16384); + let stale = vec![[0xAB; toyos_blockdev::BLOCK]; 16384]; toyos_blockdev::block_on(toyos_blockdev::BlockDev::write(&disk, 0, &stale), || ()).unwrap(); DataVolume::format(disk, 256, &["home"], clock).unwrap() } @@ -673,7 +673,7 @@ fn ticking() -> u64 { /// open, a write, one refused, a truncate and a stamp, each is the volume's. #[test] fn an_objects_size_and_mtime_are_the_volumes_after_every_change() { - let mut v = DataVolume::format(Ram::new(4096), 256, &["home"], ticking).unwrap(); + let mut v = DataVolume::format(Ram::new(16384), 256, &["home"], ticking).unwrap(); let fake = Fake::new(64, 16); let mut files = Files::new(Some(fake.clone())); let n = v.open("home/f", CREATE).unwrap(); diff --git a/system/fileserver/src/volume.rs b/system/fileserver/src/volume.rs index f6ada02f691..c24d27a07f6 100644 --- a/system/fileserver/src/volume.rs +++ b/system/fileserver/src/volume.rs @@ -4,8 +4,7 @@ //! //! **An open file is a [`Node`]**, one per path while any client holds it //! open, so two opens of one file write one set of pages and see one length. A -//! node whose file was unlinked or renamed over answers `Stale` from then on: -//! its blocks may be another file's. +//! node is distinct from the path that was used to open it. use std::collections::BTreeMap; use std::ops::Bound; @@ -223,16 +222,17 @@ pub trait Volume { #[derive(Clone, Copy, Debug)] pub struct Reserved { pub size: u64, - /// The pages it held blocks for. Past them a page below `size` is a hole, - /// whose block, if the reserve took one, holds what another file left. - pub held: u64, + /// Old bytes that can be read safely, including holes that read as zero. + pub contents: u64, + /// Volume-owned identifier for cancelling capacity taken by this write. + pub ticket: u64, } impl Reserved { /// Where the file's old bytes on the volume end: a page in part reads /// what it held below this, and zeros from here. pub fn contents(&self) -> u64 { - self.size.min(self.held.saturating_mul(toyos_blockdev::BLOCK as u64)) + self.contents } } From add7a002e2822a0db3a9598ea55e5fcbc0887074 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 17:06:03 +0100 Subject: [PATCH 12/93] Add bcachefs copy-on-write file data and capacity reservations Write checksummed or authenticated file blocks through the existing journal and bucket allocator, retaining checksum slices of overwritten extents. Reserve concrete unpublished data slots and conservative metadata capacity before accepting dirty pages; keep pinned slots safe through bucket emptying. Resume on-disk logged truncates, clear truncated tails, and reclaim orphan extents. Keep read epochs alive through file-data I/O. Validation: eight focused host tests pass, including torn/reordered write recovery, encrypted version reservation, capacity refusal followed by reserved writes, and a pinned bucket's generation transition. no_std clippy passes with warnings denied. Metadata reservation bounds remain conservative; broader regression and pressure checks continue in the integration task. --- lib/bcachefs/src/engine/bucket.rs | 26 +- lib/bcachefs/src/engine/mod.rs | 2 +- lib/bcachefs/src/engine/mount/allocator.rs | 89 +++- lib/bcachefs/src/engine/mount/data.rs | 585 +++++++++++++++++++++ lib/bcachefs/src/engine/mount/mod.rs | 51 +- lib/bcachefs/src/engine/mount/ops.rs | 34 +- lib/bcachefs/src/engine/mount/reclaim.rs | 16 +- lib/bcachefs/src/engine/read.rs | 9 +- lib/bcachefs/src/engine/tests/data.rs | 218 ++++++++ lib/bcachefs/src/engine/tests/mod.rs | 1 + lib/bcachefs/src/engine/value.rs | 22 + 11 files changed, 999 insertions(+), 54 deletions(-) create mode 100644 lib/bcachefs/src/engine/mount/data.rs create mode 100644 lib/bcachefs/src/engine/tests/data.rs diff --git a/lib/bcachefs/src/engine/bucket.rs b/lib/bcachefs/src/engine/bucket.rs index 1430644e960..90341797312 100644 --- a/lib/bcachefs/src/engine/bucket.rs +++ b/lib/bcachefs/src/engine/bucket.rs @@ -1,9 +1,7 @@ //! A bucket's state as a transaction changes it, and the keys each change //! writes: its alloc key, its freespace, need_discard and fragmentation-LRU //! entries, its generation and the device's per-state accounting (A6, -//! A7.12). v1 changes buckets only by btree nodes: an empty bucket takes -//! nodes, and a btree bucket that loses its last node waits in need_discard -//! until its emptying is durable, when it takes nodes again (A6.19). +//! A7.12). Empty buckets wait until their emptying is durable before reuse. use super::bkey::Bpos; use super::value::{class, data_type, AllocV4}; @@ -118,19 +116,23 @@ impl Bucket { /// leave it (A6.19). `Corrupt` when the alloc key holds the bucket for /// anything else. pub(crate) fn fill(&mut self, node: u32, io_time: u64, bucket_sectors: u32) -> Result<()> { + self.fill_as(data_type::BTREE, node, io_time, bucket_sectors) + } + + pub(crate) fn fill_as(&mut self, kind: u8, sectors: u32, io_time: u64, bucket_sectors: u32) -> Result<()> { if matches!(self.data_type, data_type::FREE | data_type::NEED_DISCARD) { self.owed_raise(); - self.data_type = data_type::BTREE; + self.data_type = kind; self.needs_gen_increment = true; self.need_discard = true; self.io_time = io_time.max(1); self.journal_seq_nonempty = 0; self.journal_seq_empty = 0; } - if self.data_type != data_type::BTREE || self.dirty + node > bucket_sectors { - return Err(Error::Corrupt("a btree node placed in a bucket its alloc key holds otherwise")); + if self.data_type != kind || sectors > bucket_sectors.saturating_sub(self.dirty) { + return Err(Error::Corrupt("data placed in a bucket its alloc key holds otherwise")); } - self.dirty += node; + self.dirty += sectors; Ok(()) } @@ -139,10 +141,14 @@ impl Bucket { /// empty sequence. Whether it was emptied; `Corrupt` when the alloc key /// does not hold the node. pub(crate) fn drain(&mut self, node: u32) -> Result { - if self.data_type != data_type::BTREE || self.dirty < node { - return Err(Error::Corrupt("a btree node released from a bucket its alloc key does not hold it in")); + self.drain_as(data_type::BTREE, node) + } + + pub(crate) fn drain_as(&mut self, kind: u8, sectors: u32) -> Result { + if self.data_type != kind || self.dirty < sectors { + return Err(Error::Corrupt("data released from a bucket its alloc key does not hold it in")); } - self.dirty -= node; + self.dirty -= sectors; if self.dirty > 0 { return Ok(false); } diff --git a/lib/bcachefs/src/engine/mod.rs b/lib/bcachefs/src/engine/mod.rs index 67f79c4d33d..0c6ddaab777 100644 --- a/lib/bcachefs/src/engine/mod.rs +++ b/lib/bcachefs/src/engine/mod.rs @@ -29,7 +29,7 @@ pub use bkey::{Bpos, Key, Version}; pub use crypt::MIN_KDF; pub use fsck::{fsck, Finding}; pub use mkfs::{fit, mkfs, Encryption, Options}; -pub use mount::{Link, Mount, MountOptions, New, Owner, Removed, Rename}; +pub use mount::{Link, Mount, MountOptions, New, Owner, Removed, Rename, Reservation}; pub use read::{read_file, DirEntry, Inode, Volume}; pub use superblock::{carries_volume, Kdf, Superblock}; pub use tree::{Meta, NodeRef, Tree}; diff --git a/lib/bcachefs/src/engine/mount/allocator.rs b/lib/bcachefs/src/engine/mount/allocator.rs index 3c06ec782b2..251eeb35e17 100644 --- a/lib/bcachefs/src/engine/mount/allocator.rs +++ b/lib/bcachefs/src/engine/mount/allocator.rs @@ -28,6 +28,7 @@ pub(crate) enum Draw { /// Removes names or nodes: unlink, rmdir, an orphan's release. It takes /// the freeing reserve too, so a full volume can still be emptied (§2.3). Freeing, + Reserved(u64), } /// Node slots one freeing transaction can need: each node it dirties @@ -47,6 +48,10 @@ pub(super) struct Allocator { per_bucket: u64, /// The bucket being filled, its next slot and its generation. open: Option<(u64, u64, u8)>, + data: Option<(u64, u64, u8)>, + pins: BTreeMap)>, + holes: BTreeSet<(u64, u64)>, + durable: u64, /// Buckets taken whose filling is not yet applied, which the indexes /// still list as empty. taken: BTreeSet, @@ -65,6 +70,10 @@ impl Allocator { Allocator { per_bucket, open: None, + data: None, + pins: BTreeMap::new(), + holes: BTreeSet::new(), + durable: 0, taken: BTreeSet::new(), free, returning: BTreeMap::new(), @@ -79,13 +88,14 @@ impl Allocator { /// Node slots there are or will be once every emptying is durable. fn slots(&self) -> u64 { - self.open_slots() + self.per_bucket * (self.free + self.returning.values().sum::()) + self.open_slots() + self.per_bucket * (self.free + self.returning.values().sum::() - self.pins.values().filter(|(_, empty)| empty.is_some()).count() as u64) } /// Node slots to be had now, past `blocked` buckets that a traversal /// may still lead into. pub(super) fn ready(&self, blocked: usize) -> u64 { - self.open_slots() + self.per_bucket * self.free.saturating_sub(blocked as u64) + let pinned = self.pins.values().filter(|(_, empty)| empty.is_some_and(|seq| seq <= self.durable)).count() as u64; + self.open_slots() + self.per_bucket * self.free.saturating_sub(blocked as u64 + pinned) } /// Whether an emptying waits to be durable. @@ -101,7 +111,7 @@ impl Allocator { /// nothing here counts on it. pub(super) fn fits(&self, nodes: usize, reclaim_room: usize, draw: Draw) -> bool { let reserve = match draw { - Draw::Ordinary => FREEING_RESERVE, + Draw::Ordinary | Draw::Reserved(_) => FREEING_RESERVE, Draw::Freeing => 0, }; (COMMON_REWRITE * (nodes + reclaim_room)) as u64 + OVERSIZED + reserve <= self.slots() @@ -115,7 +125,7 @@ impl Allocator { } pub(super) fn is_taken(&self, b: u64) -> bool { - self.taken.contains(&b) + self.taken.contains(&b) || self.pins.contains_key(&b) } /// `b`, empty at generation `gen`, opened for nodes. @@ -125,9 +135,74 @@ impl Allocator { self.open = Some((b, 0, gen)); } + pub(super) fn take_data(&mut self, b: u64, gen: u8) { + self.taken.insert(b); + self.free = self.free.saturating_sub(1); + self.data = Some((b, 0, gen)); + } + + pub(super) fn next_data(&mut self, bucket_sectors: u64, sectors: u64) -> Option<(u64, u8)> { + if let Some((_, sector)) = self.holes.pop_first() { + return Some((sector, 0)); + } + let (b, next, gen) = self.data?; + if sectors > bucket_sectors - next { + self.data = None; + return None; + } + self.data = (next + sectors < bucket_sectors).then_some((b, next + sectors, gen)); + Some((b * bucket_sectors + next, gen)) + } + + pub(super) fn data_fits(&self, nodes: usize, reclaim_room: usize) -> bool { + let need = (COMMON_REWRITE * (nodes + reclaim_room)) as u64 + OVERSIZED + FREEING_RESERVE; + need + self.per_bucket <= self.slots() + } + + pub(super) fn free_bytes(&self, bucket_bytes: u64) -> u64 { + let pinned = self.pins.values().filter(|(_, empty)| empty.is_some_and(|seq| seq <= self.durable)).count() as u64; + self.free.saturating_sub(pinned) * bucket_bytes + } + + pub(super) fn pin(&mut self, b: u64) { + self.pins.entry(b).or_insert((0, None)).0 += 1; + } + + pub(super) fn unpin(&mut self, b: u64) { + let entry = self.pins.get_mut(&b).expect("a reserved bucket is pinned"); + entry.0 -= 1; + if entry.0 == 0 { self.pins.remove(&b); } + } + + pub(super) fn cancel_data(&mut self, sector: u64, bucket: u64) { + let b = sector / bucket; + let empty = self.pins[&b].1.is_some(); + self.unpin(b); + if empty { return; } + if self.taken.contains(&b) && !self.pins.contains_key(&b) { + self.taken.remove(&b); + self.free += 1; + if self.data.is_some_and(|(open, _, _)| open == b) { self.data = None; } + self.holes.retain(|(bucket, _)| *bucket != b); + } else { + self.holes.insert((b, sector)); + } + } + /// `b`'s filling applied: the indexes no longer list it. pub(super) fn filled(&mut self, b: u64) { self.taken.remove(&b); + if let Some((_, empty)) = self.pins.get_mut(&b) { + if let Some(seq) = empty.take() { + if seq <= self.durable { + self.free -= 1; + } else { + let count = self.returning.get_mut(&seq).expect("a pinned emptying is returning"); + *count -= 1; + if *count == 0 { self.returning.remove(&seq); } + } + } + } } /// `b` emptied by sequence `seq`. @@ -135,11 +210,17 @@ impl Allocator { // A round's own nodes go in the open bucket first, so the round // that releases its last node fills it again. assert!(self.open.is_none_or(|(open, _, _)| open != b), "the open bucket {b} emptied"); + if self.data.is_some_and(|(open, _, _)| open == b) { + self.data = None; + } + if let Some((_, empty)) = self.pins.get_mut(&b) { *empty = Some(seq); } + self.holes.retain(|(bucket, _)| *bucket != b); *self.returning.entry(seq).or_default() += 1; } /// The journal written through `seq`: what it emptied is free. pub(super) fn durable_through(&mut self, seq: u64) { + self.durable = seq; let later = self.returning.split_off(&(seq + 1)); self.free += self.returning.values().sum::(); self.returning = later; diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs new file mode 100644 index 00000000000..59ed13a3451 --- /dev/null +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -0,0 +1,585 @@ +//! Copy-on-write file data. A pointer is journaled only after its bytes +//! have completed; journal preflush orders those bytes before the pointer. + +use super::super::crypt::Nonce; +use super::super::value::{csum_type, Extent}; +use super::*; + +const VERSIONS: u64 = 1024; + +#[derive(Clone, Copy)] +pub(super) struct ReservedBlock { + ino: u64, + sector: u64, +} + +#[derive(Default)] +pub(super) struct Reservations { + pub(super) active: BTreeMap, + canceled: Vec, + next: u64, +} + +impl Reservations { + pub(super) fn pending(&self, draw: Draw) -> usize { + let own = match draw { + Draw::Reserved(id) => usize::from(self.active.contains_key(&id)), + _ => 0, + }; + (self.active.len() - own) * FOOTPRINT + } +} + +/// Capacity for one filesystem block. Dropping an unused reservation +/// returns its physical slot and metadata credit on the next operation. +pub struct Reservation { + owner: Rc>, + id: u64, + first: u64, + bytes: usize, +} + +impl Reservation { + pub fn range(&self) -> (u64, u64) { + (self.first, self.first + self.bytes as u64) + } +} + +impl Drop for Reservation { + fn drop(&mut self) { + let mut pool = self.owner.borrow_mut(); + if let Some(block) = pool.active.remove(&self.id) { + pool.canceled.push(block.sector); + } + } +} + +pub(super) struct DataSlot<'a, D>(&'a Mount); + +impl Drop for DataSlot<'_, D> { + fn drop(&mut self) { + self.0.data_held.set(false); + self.0.data_queue.wake(); + } +} + +fn start(k: &Key) -> Result { + k.pos.offset.checked_sub(u64::from(k.size)).and_then(|s| s.checked_mul(SECTOR as u64)).ok_or(Error::Corrupt("extent start past its end")) +} + +fn regular(inode: &InodeValue) -> Result<()> { + match inode.mode() & mode::S_IFMT { + mode::S_IFREG | mode::S_IFLNK => Ok(()), + mode::S_IFDIR => Err(Error::IsDirectory), + _ => Err(Error::Invalid("file data on a special inode")), + } +} + +impl Mount { + pub(super) fn cancel_dropped(&self) { + let canceled = core::mem::take(&mut self.reservations.borrow_mut().canceled); + let mut st = self.st.borrow_mut(); + for sector in canceled { + st.alloc.cancel_data(sector, u64::from(self.sb.member.bucket_sectors)); + } + } + + /// Reserves the complete block containing `[offset, offset + len)`. + /// A range crossing a filesystem block needs separate reservations. + pub async fn reserve(&self, ino: u64, offset: u64, len: usize) -> Result { + let bytes = self.sb.block_bytes(); + let first = offset / bytes as u64 * bytes as u64; + if len == 0 || offset.checked_add(len as u64).is_none_or(|end| end > first + bytes as u64 || end > i64::MAX as u64) { + return Err(Error::Invalid("reservation outside one filesystem block")); + } + let _data = self.data_slot().await; + let id = self + .transact(1, FOOTPRINT, Draw::Ordinary, async |txn: &mut Txn| { + regular(&self.inode_in(txn, ino).await?)?; + self.block_extents(ino, first, first + bytes as u64).await?; + let (sector, _) = self.data_sector(self.sb.block_bytes() as u64 / SECTOR as u64).await?; + self.st.borrow_mut().alloc.pin(sector / u64::from(self.sb.member.bucket_sectors)); + let mut pool = self.reservations.borrow_mut(); + let id = pool.next; + pool.next = pool.next.checked_add(1).expect("reservation identifiers exhausted"); + pool.active.insert(id, ReservedBlock { ino, sector }); + Ok(id) + }) + .await?; + Ok(Reservation { owner: Rc::clone(&self.reservations), id, first, bytes }) + } + + /// Consumes a reservation with one write inside its block. Capacity + /// cannot fail after acceptance; device failures remain errors. + pub async fn write_reserved(&self, reservation: &mut Reservation, offset: u64, data: &[u8], now: i64) -> Result { + if !Rc::ptr_eq(&reservation.owner, &self.reservations) { + return Err(Error::Invalid("reservation belongs to another mount")); + } + if offset < reservation.first || data.is_empty() || offset.checked_add(data.len() as u64).is_none_or(|end| end > reservation.range().1) { + return Err(Error::Invalid("write outside its reserved block")); + } + let _slot = self.data_slot().await; + let block = self.reservations.borrow().active.get(&reservation.id).copied().ok_or(Error::Invalid("reservation already consumed"))?; + self.observe(now); + let version = self.data_version().await?; + let out = self.write_chunk(block.ino, offset, data, now, version, Some((reservation.id, block.sector))).await; + if !self.reservations.borrow().active.contains_key(&reservation.id) { + let mut st = self.st.borrow_mut(); + if out.is_ok() { + st.alloc.unpin(block.sector / u64::from(self.sb.member.bucket_sectors)); + } else { + st.alloc.cancel_data(block.sector, u64::from(self.sb.member.bucket_sectors)); + } + } + out + } + + pub(super) async fn create_symlink(&self, dir: u64, name: &[u8], target: &[u8], owner: Owner, now: i64) -> Result { + if target.is_empty() || target.len() >= 4096 || target.contains(&0) { + return Err(Error::Invalid("symlink target empty, holding NUL or past 4095 bytes")); + } + let ino = self.create_unnamed(dir, owner, now).await?.ino; + let out = async { + self.transact(K_TX / 2, 2, Draw::Ordinary, async |txn: &mut Txn| { + let mut inode = self.inode_in(txn, ino).await?; + inode.flags = (inode.flags & !(u64::from(mode::S_IFMT) << 36)) | u64::from(mode::S_IFLNK) << 36; + self.set_inode(txn, ino, &inode).await + }) + .await?; + let mut bytes = target.to_vec(); + bytes.push(0); + if self.write(ino, 0, &bytes, now).await? != bytes.len() { + return Err(Error::NoSpace("symlink target")); + } + self.link(ino, dir, name, Link::New, now).await?; + self.inode(ino).await + } + .await; + if out.is_err() { + self.release(ino).await?; + } + out + } + + pub(super) async fn data_slot(&self) -> DataSlot<'_, D> { + self.data_queue.wait(|| (!self.data_held.get()).then(|| self.data_held.set(true))).await; + DataSlot(self) + } + + pub async fn read(&self, ino: u64, offset: u64, out: &mut [u8]) -> Result { + Source::read(self, ino, offset, out).await + } + + pub async fn readlink(&self, ino: u64) -> Result> { + Source::readlink(self, ino).await + } + + /// Writes as many bytes as fit. A later chunk's failure returns the + /// completed byte count; `commit` is the durability barrier. + pub async fn write(&self, ino: u64, offset: u64, data: &[u8], now: i64) -> Result { + offset.checked_add(data.len() as u64).filter(|n| *n <= i64::MAX as u64).ok_or(Error::Invalid("file size past 2^63-1"))?; + let _slot = self.data_slot().await; + self.observe(now); + let mut done = 0; + while done < data.len() { + let version = self.data_version().await; + let out = match version { + Ok(version) => self.write_chunk(ino, offset + done as u64, &data[done..], now, version, None).await, + Err(e) => Err(e), + }; + match out { + Ok(n) => done += n, + Err(_) if done > 0 => return Ok(done), + Err(e) => return Err(e), + } + } + Ok(done) + } + + pub async fn set_mtime(&self, ino: u64, now: i64) -> Result<()> { + self.observe(now); + self.transact(K_TX / 2, 2, Draw::Ordinary, async |txn: &mut Txn| { + let mut inode = self.inode_in(txn, ino).await?; + inode.set(inode_field::MTIME, self.stored_time(now)); + self.set_inode(txn, ino, &inode).await + }) + .await + } + + /// Total volume bytes and bytes in currently free, reusable buckets. + pub fn space(&self) -> (u64, u64) { + let bucket = u64::from(self.sb.member.bucket_sectors) * SECTOR as u64; + (self.sb.member.nbuckets * bucket, self.st.borrow().alloc.free_bytes(bucket)) + } + + /// The high-water mark is durable before any version beneath it is + /// used, including a ciphertext write whose metadata never commits. + async fn data_version(&self) -> Result { + if self.key.is_none() { + return Ok(Version::default()); + } + if self.st.borrow().next_version == self.st.borrow().key_version { + self.transact(1, 0, Draw::Ordinary, async |_txn: &mut Txn| { + let mut st = self.st.borrow_mut(); + st.key_version = st.key_version.checked_add(VERSIONS).ok_or(Error::NoSpace("encrypted extent versions"))?; + Ok(()) + }) + .await?; + self.commit().await?; + } + let mut st = self.st.borrow_mut(); + st.next_version += 1; + Ok(Version { hi: 0, lo: st.next_version }) + } + + async fn extent_after(&self, ino: u64, byte: u64) -> Result)>> { + let mut found = None; + Source::walk(self, Btree::Extents, Bpos::new(ino, byte / SECTOR as u64 + 1, 0), |k, v| { + if k.pos.inode != ino { + return Ok(false); + } + if k.pos.snapshot == ROOT_SNAPSHOT { + found = Some((*k, v.to_vec())); + } + Ok(found.is_none()) + }) + .await?; + Ok(found) + } + + async fn write_chunk(&self, ino: u64, offset: u64, input: &[u8], now: i64, version: Version, reserved: Option<(u64, u64)>) -> Result { + let draw = reserved.map_or(Draw::Ordinary, |(id, _)| Draw::Reserved(id)); + self.transact(K_TX, FOOTPRINT, draw, async |txn: &mut Txn| { + let mut inode = self.inode_in(txn, ino).await?; + regular(&inode)?; + let block = self.sb.block_bytes() as u64; + let first = offset / block * block; + let mut bytes = alloc::vec![0u8; block as usize]; + Source::read(self, ino, first, &mut bytes).await?; + let at = (offset - first) as usize; + let n = input.len().min(bytes.len() - at); + bytes[at..at + n].copy_from_slice(&input[..n]); + self.replace_block(txn, ino, &mut inode, first, &bytes, version, reserved.map(|(_, sector)| sector)).await?; + inode.size = inode.size.max(offset + n as u64); + self.touch(&mut inode, now); + self.set_inode(txn, ino, &inode).await?; + Ok(n) + }) + .await + } + + async fn data_sector(&self, sectors: u64) -> Result<(u64, u8)> { + let bucket = u64::from(self.sb.member.bucket_sectors); + if sectors > bucket { + return Err(Error::Unsupported("data extent larger than a bucket")); + } + if let Some(at) = self.st.borrow_mut().alloc.next_data(bucket, sectors) { + return Ok(at); + } + { + let st = self.st.borrow(); + let held = st.cache.dirty + st.cache.unpointed + st.granted + self.reservations.borrow().pending(Draw::Ordinary); + if !st.alloc.data_fits(held, st.reclaim_room) { + return Err(Error::NoSpace("no data bucket beyond metadata and freeing reserves")); + } + } + self.take_bucket(true).await?; + Ok(self.st.borrow_mut().alloc.next_data(bucket, sectors).expect("a fresh bucket holds the extent")) + } + + async fn block_extents(&self, ino: u64, first: u64, end: u64) -> Result)>> { + let mut old = Vec::new(); + Source::walk(self, Btree::Extents, Bpos::new(ino, first / SECTOR as u64 + 1, 0), |k, v| { + if k.pos.inode != ino || start(k)? >= end { + return Ok(false); + } + if k.pos.snapshot == ROOT_SNAPSHOT { + old.push((*k, v.to_vec())); + } + Ok(true) + }) + .await?; + if old.len() > 1 { + return Err(Error::Unsupported("multiple data extents in one filesystem block")); + } + for (k, v) in &old { + match k.kind { + kind::EXTENT => { + checked_extent(v)?; + } + kind::INLINE_DATA | kind::WHITEOUT | kind::EXTENT_WHITEOUT => {} + _ => return Err(Error::Unsupported("modifying an extent type this writer cannot account")), + } + } + Ok(old) + } + + #[allow(clippy::too_many_arguments)] + async fn replace_block(&self, txn: &mut Txn, ino: u64, inode: &mut InodeValue, first: u64, bytes: &[u8], version: Version, reserved: Option) -> Result<()> { + let end = first + bytes.len() as u64; + let old = self.block_extents(ino, first, end).await?; + let mut retained = Vec::new(); + for (k, v) in &old { + let from = start(k)?; + let to = k.pos.offset * SECTOR as u64; + if from < first { + retained.push(slice(k, v, from, first)?); + } + if to > end { + retained.push(slice(k, v, end, to)?); + } + } + let blocks = bytes.len().div_ceil(self.sb.block_bytes()) * self.sb.block_bytes() / BLOCK; + let mut data = alloc::vec![[0u8; BLOCK]; blocks]; + data.as_flattened_mut()[..bytes.len()].copy_from_slice(bytes); + let physical = (blocks * BLOCK / SECTOR) as u32; + let sector = match reserved { + Some(sector) => sector, + None => self.data_sector(u64::from(physical)).await?.0, + }; + if reserved.is_none() { + self.st.borrow_mut().alloc.pin(sector / u64::from(self.sb.member.bucket_sectors)); + txn.allocated_data.push(sector); + } + let gen = self.bucket(txn, sector / u64::from(self.sb.member.bucket_sectors)).await?.reused_gen(); + let ptr = Ptr { sector, dev: 0, gen, cached: false, unwritten: false }; + let value = if let Some(master) = &self.key { + let nonce = Nonce::extent(version.hi, version.lo, 0); + master.xor(nonce, data.as_flattened_mut())?; + let tag = master.mac(nonce, data.as_flattened())?; + Extent::authenticated(physical, tag, self.sb.flag(1, 9, 1) != 0, ptr) + } else { + if physical > 512 { + return Err(Error::Unsupported("unencrypted extent past crc64 maximum")); + } + Extent::encode(physical, csum_type::CRC32C, crate::crc32c::crc32c_zero(data.as_flattened()), ptr) + }; + let pos = Bpos::new(ino, (first + bytes.len() as u64) / SECTOR as u64, ROOT_SNAPSHOT); + let key = Key { size: (bytes.len() / SECTOR) as u32, version, ..Key::new(kind::EXTENT, pos) }; + self.reissue("file data write failed", async || { + self.dev.write_data(sector * SECTOR as u64 / BLOCK as u64, &data).await?; + Ok(()) + }) + .await?; + self.st.borrow_mut().clocks[1] += u64::from(physical); + let mut buckets = BTreeMap::new(); + self.data_counted(txn, &mut buckets, &key, &value, true).await?; + for (k, v) in old { + self.data_counted(txn, &mut buckets, &k, &v, false).await?; + self.set(txn, Btree::Extents, k.pos, None).await?; + inode.sectors = inode.sectors.checked_sub(if k.kind == kind::EXTENT { u64::from(k.size) } else { 0 }).ok_or(Error::Corrupt("inode sectors below its extents"))?; + } + for (k, v) in retained { + self.data_counted(txn, &mut buckets, &k, &v, true).await?; + if k.kind == kind::EXTENT { + inode.sectors += u64::from(k.size); + } + self.set(txn, Btree::Extents, k.pos, Some((k, v))).await?; + } + self.set(txn, Btree::Extents, pos, Some((key, value))).await?; + inode.sectors += u64::from(key.size); + self.data_buckets(txn, buckets).await?; + Ok(()) + } + + async fn data_counted(&self, txn: &mut Txn, buckets: &mut BTreeMap, k: &Key, v: &[u8], added: bool) -> Result<()> { + if k.kind != kind::EXTENT { + return Ok(()); + } + let extent = checked_extent(v)?; + let crc = extent.crc.unwrap(); + let bucket_sectors = u64::from(self.sb.member.bucket_sectors); + let b = extent.ptr.sector / bucket_sectors; + if let alloc::collections::btree_map::Entry::Vacant(slot) = buckets.entry(b) { + let before = self.bucket(txn, b).await?; + slot.insert((before, 0)); + } + let delta = &mut buckets.get_mut(&b).unwrap().1; + if added { + *delta += i64::from(k.size); + txn.filled.push(b); + } else { + *delta -= i64::from(k.size); + } + let pos = Bpos::new(0, (extent.ptr.sector << self.bp_shift()) + u64::from(crc.offset), 0); + let bp = added.then(|| (Key::new(kind::BACKPOINTER, pos), value::backpointer(Btree::Extents as u8, 0, data_type::USER, extent.ptr.gen, k.size, k.pos).to_vec())); + self.put(txn, Btree::Backpointers, 0, pos, bp).await?; + let sign = |v: u64| if added { v } else { v.wrapping_neg() }; + self.account(txn, &[class::REPLICAS, data_type::USER, 1, 1, 0], &[sign(u64::from(k.size))]).await?; + let mut record = alloc::vec![class::INUM]; + record.extend_from_slice(&k.pos.inode.to_le_bytes()); + self.account(txn, &record, &[sign(1), sign(u64::from(k.size)), sign(u64::from(k.size))]).await + } + + async fn data_buckets(&self, txn: &mut Txn, buckets: BTreeMap) -> Result<()> { + for (b, (before, delta)) in buckets { + if delta == 0 { + continue; + } + let mut after = before; + if delta > 0 { + after.fill_as(data_type::USER, delta as u32, self.st.borrow().clocks[1], self.sb.member.bucket_sectors as u32)?; + } else { + after.drain_as(data_type::USER, (-delta) as u32)?; + } + self.bucket_changed(txn, b, before, after).await?; + } + Ok(()) + } + + pub async fn truncate(&self, ino: u64, size: u64, now: i64) -> Result<()> { + if size > i64::MAX as u64 { + return Err(Error::Invalid("file size past 2^63-1")); + } + let _slot = self.data_slot().await; + self.observe(now); + let op = self + .transact(K_TX / 2, 3, Draw::Freeing, async |txn: &mut Txn| { + let mut inode = self.inode_in(txn, ino).await?; + regular(&inode)?; + self.touch(&mut inode, now); + if size >= inode.size { + inode.size = size; + self.set_inode(txn, ino, &inode).await?; + return Ok(None); + } + self.set_inode(txn, ino, &inode).await?; + let mut pos = Bpos::MIN; + while self.get_in(txn, Btree::LoggedOps, pos).await?.is_some() { + pos.offset = pos.offset.checked_add(1).ok_or(Error::NoSpace("logged operation slots"))?; + } + let mut value = alloc::vec![0u8; 24]; + value[..4].copy_from_slice(&mkfs::ROOT_SUBVOLUME.to_le_bytes()); + value[8..16].copy_from_slice(&ino.to_le_bytes()); + value[16..24].copy_from_slice(&size.to_le_bytes()); + self.put(txn, Btree::LoggedOps, 0, pos, Some((Key::new(kind::LOGGED_OP_TRUNCATE, pos), value))).await?; + Ok(Some(pos)) + }) + .await?; + if let Some(pos) = op { + let out = self.finish_truncate(pos, ino, size).await; + if out.is_err() { + self.stop("an interrupted truncate needs recovery"); + } + out?; + } + Ok(()) + } + + async fn finish_truncate(&self, pos: Bpos, ino: u64, size: u64) -> Result<()> { + self.transact(K_TX / 2, 2, Draw::Freeing, async |txn: &mut Txn| { + let mut inode = self.inode_in(txn, ino).await?; + regular(&inode)?; + inode.size = size; + self.set_inode(txn, ino, &inode).await + }) + .await?; + self.trim_extents(ino, size).await?; + self.transact(K_TX / 2, 1, Draw::Freeing, async |txn: &mut Txn| self.put(txn, Btree::LoggedOps, 0, pos, None).await).await + } + + pub(super) async fn trim_extents(&self, ino: u64, size: u64) -> Result<()> { + let block = self.sb.block_bytes() as u64; + let end = size.div_ceil(block) * block; + if !size.is_multiple_of(block) && self.extent_after(ino, size).await?.is_some_and(|(k, _)| start(&k).is_ok_and(|first| first < end)) { + let version = self.data_version().await?; + self.transact(K_TX, FOOTPRINT, Draw::Freeing, async |txn: &mut Txn| { + let mut inode = self.inode_in(txn, ino).await?; + let first = size / block * block; + let mut bytes = alloc::vec![0u8; block as usize]; + Source::read(self, ino, first, &mut bytes).await?; + bytes[(size - first) as usize..].fill(0); + self.replace_block(txn, ino, &mut inode, first, &bytes, version, None).await?; + self.set_inode(txn, ino, &inode).await + }) + .await?; + } + loop { + let Some((key, value)) = self.extent_after(ino, end).await? else { + return Ok(()); + }; + let first = start(&key)?; + self.transact(K_TX, FOOTPRINT, Draw::Freeing, async |txn: &mut Txn| { + let mut inode = self.inode_in(txn, ino).await?; + let mut buckets = BTreeMap::new(); + self.data_counted(txn, &mut buckets, &key, &value, false).await?; + self.set(txn, Btree::Extents, key.pos, None).await?; + inode.sectors = inode.sectors.checked_sub(if key.kind == kind::EXTENT { u64::from(key.size) } else { 0 }).ok_or(Error::Corrupt("inode sectors below its extents"))?; + if first < end { + let (k, v) = slice(&key, &value, first, end)?; + self.data_counted(txn, &mut buckets, &k, &v, true).await?; + if k.kind == kind::EXTENT { + inode.sectors += u64::from(k.size); + } + self.set(txn, Btree::Extents, k.pos, Some((k, v))).await?; + } + self.data_buckets(txn, buckets).await?; + self.set_inode(txn, ino, &inode).await + }) + .await?; + } + } + + pub(super) async fn resume_truncates(&self) -> Result<()> { + loop { + let mut op = None; + Source::walk(self, Btree::LoggedOps, Bpos::MIN, |k, v| { + if k.pos.inode == 0 { + op = Some((*k, v.to_vec())); + } + Ok(false) + }) + .await?; + let Some((k, v)) = op else { + return Ok(()); + }; + let (ino, size) = truncate_value(&k, &v)?; + self.finish_truncate(k.pos, ino, size).await?; + } + } +} + +fn checked_extent(v: &[u8]) -> Result { + let extent = Extent::decode(v)?; + let crc = extent.crc.ok_or(Error::Unsupported("modifying data without a checksum entry"))?; + if extent.ptr.dev != 0 || extent.ptr.cached || extent.ptr.unwritten || crc.compression != 0 || crc.compressed != crc.uncompressed { + return Err(Error::Unsupported("modifying cached, unwritten, compressed or other-device data")); + } + Ok(extent) +} + +fn slice(k: &Key, v: &[u8], first: u64, end: u64) -> Result<(Key, Vec)> { + let old_first = start(k)?; + let skip = ((first - old_first) / SECTOR as u64) as u32; + let size = ((end - first) / SECTOR as u64) as u32; + let value = match k.kind { + kind::EXTENT => { + let extent = checked_extent(v)?; + let mut crc = extent.crc.unwrap(); + crc.offset += skip; + Extent::with_crc(&crc, extent.ptr) + } + kind::INLINE_DATA => { + let at = skip as usize * SECTOR; + let n = (size as usize * SECTOR).min(v.len().saturating_sub(at)); + v.get(at..at + n).unwrap_or(&[]).to_vec() + } + kind::WHITEOUT | kind::EXTENT_WHITEOUT => Vec::new(), + _ => return Err(Error::Unsupported("slicing an extent type this writer cannot account")), + }; + Ok((Key { pos: Bpos { offset: end / SECTOR as u64, ..k.pos }, size, ..*k }, value)) +} + +pub(super) fn truncate_value(k: &Key, v: &[u8]) -> Result<(u64, u64)> { + if k.kind != kind::LOGGED_OP_TRUNCATE { + return Err(Error::Unsupported("logged operation other than truncate")); + } + if v.len() != 24 || super::super::le32(v, 0) != mkfs::ROOT_SUBVOLUME || super::super::le32(v, 4) != 0 { + return Err(Error::Corrupt("logged truncate value")); + } + let size = super::super::le64(v, 16); + if size > i64::MAX as u64 { + return Err(Error::Corrupt("logged truncate size past 2^63-1")); + } + Ok((super::super::le64(v, 8), size)) +} diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 2e8f937cf60..75517cb84b9 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -38,10 +38,12 @@ use super::value::{self, accounting_pos, class, data_type, inode_field, inode_fl use super::{kind, Btree, Error, Result, SECTOR}; mod allocator; +mod data; mod ops; mod reclaim; use allocator::{Allocator, Draw, Readers}; +pub use data::Reservation; pub use ops::{Link, Removed, Rename}; pub(crate) use reclaim::Order; #[cfg(test)] @@ -56,12 +58,10 @@ pub struct MountOptions { pub cache_nodes: usize, } -/// u64s one ordinary transaction journals at most: a namespace operation's -/// inode, dirent and accounting keys, names of 512 bytes included. +/// u64s one ordinary transaction journals at most. pub(crate) const K_TX: u64 = 512; -/// Nodes any ordinary transaction dirties at most: the largest grant of -/// the operations' (`mount/ops.rs`). -pub(crate) const FOOTPRINT: usize = 10; +/// Nodes any ordinary transaction dirties at most. +pub(crate) const FOOTPRINT: usize = 28; /// Nodes one reclaim batch writes. const BATCH: usize = 2; /// u64s a reclaim share holds for each node that can pin the journal: @@ -87,7 +87,7 @@ const BLACKLIST_FEATURE: u64 = 1 << 5; pub enum New { File, Dir, - /// Its target is file data, which B7's data path writes. + /// The target, without its on-disk NUL terminator. Symlink(Vec), } @@ -112,6 +112,7 @@ struct State { new_roots: BTreeMap, journal: Writer, key_version: u64, + next_version: u64, clocks: [u64; 2], /// Seconds since the epoch of the newest operation: each entry's /// datetime. @@ -164,6 +165,9 @@ pub struct Mount { /// waiting on a node and by reclaim waiting on a bucket. written: Broadcast, readers: RefCell, + data_held: Cell, + data_queue: Queue, + reservations: Rc>, root: u64, } @@ -201,6 +205,7 @@ struct Txn { emptied: Vec<(u64, Bucket)>, /// Buckets taken that it fills. filled: Vec, + allocated_data: Vec, /// New roots. roots: Vec, /// Written nodes made reachable, installed in the cache at apply. @@ -321,6 +326,7 @@ impl Mount { /// `empty` is the volume's count of empty buckets. async fn from(parts: Parts, empty: u64, options: MountOptions, random: Box) -> Result { let Parts { dev, mut sb, key, roots, key_version, replay, root } = parts; + let key_version = key_version.checked_add(if replay.is_some() && key.is_some() { 65_536 } else { 0 }).ok_or(Error::NoSpace("encrypted extent versions"))?; let ring: Vec = sb.journal.iter().flat_map(|&(first, count)| first..first + count).collect(); let bucket_sectors = u64::from(sb.member.bucket_sectors); let mut journal = Writer::new(ring, bucket_sectors, sb.block_bytes(), K_TX); @@ -405,6 +411,7 @@ impl Mount { new_roots: BTreeMap::new(), journal, key_version, + next_version: key_version, clocks, now: sb.header.write_time, granted: 0, @@ -435,12 +442,16 @@ impl Mount { journal_queue: Queue::default(), written: Broadcast::default(), readers: RefCell::default(), + data_held: Cell::new(false), + data_queue: Queue::default(), + reservations: Rc::default(), root, }; mount.go_read_write(restate).await?; if let Some((_, replayed)) = replay { mount.install_replay(replayed).await?; } + mount.resume_truncates().await?; Ok(mount) } @@ -547,18 +558,14 @@ impl Mount { } } -/// Refuses a volume holding a logged operation, which a mount must resume -/// before writing (A8.17, A8.18) and this one cannot yet. +/// Refuses logged operations this writer cannot resume. async fn no_logged_ops(volume: &Volume) -> Result<()> { - let mut found = false; - Source::walk(volume, Btree::LoggedOps, Bpos::MIN, |k, _| { - found = k.pos.inode == 0; - Ok(false) + Source::walk(volume, Btree::LoggedOps, Bpos::MIN, |k, v| { + if k.pos.inode != 0 { return Ok(false); } + data::truncate_value(k, v)?; + Ok(true) }) .await?; - if found { - return Err(Error::Unsupported("writing a volume with a logged operation to resume")); - } Ok(()) } @@ -941,6 +948,8 @@ impl Mount { /// room left could not hold even with nothing else dirty is refused /// `NoSpace`, once nothing written or emptied can change that. fn grant(&self, class: Class, draw: Draw, journal: u64, nodes: usize) -> Result { + self.cancel_dropped(); + let pending = self.reservations.borrow().pending(draw); let mut st = self.st.borrow_mut(); if let Some(why) = st.read_only { return Err(Error::ReadOnly(why)); @@ -948,14 +957,14 @@ impl Mount { let room = match class { Class::Ordinary => { let held = st.cache.dirty + st.cache.unpointed + st.granted; - if !st.alloc.fits(nodes, st.reclaim_room, draw) && held == 0 && st.reclaim_granted == 0 && !st.alloc.returning() { + if !st.alloc.fits(nodes + pending, st.reclaim_room, draw) && held == 0 && st.reclaim_granted == 0 && !st.alloc.returning() { return Err(Error::NoSpace(match draw { - Draw::Ordinary => "no room left for the btree nodes a change may write", + Draw::Ordinary | Draw::Reserved(_) => "no room left for the btree nodes a change may write", Draw::Freeing => "no room left for the btree nodes a removal may write", })); } // Nothing more lands in a node reclaim has yet to split. - held + nodes <= st.limit && st.cache.oversized == 0 && st.alloc.fits(held + nodes, st.reclaim_room, draw) + held + nodes <= st.limit && st.cache.oversized == 0 && st.alloc.fits(held + nodes + pending, st.reclaim_room, draw) } Class::Reclaim => st.reclaim_granted + nodes <= st.reclaim_room, }; @@ -1045,6 +1054,7 @@ impl Mount { } self.ungrant(&mut st, Class::Ordinary, bound, nodes); }; + if let Draw::Reserved(id) = draw { self.reservations.borrow_mut().active.remove(&id).expect("admission consumes a reservation"); } let mut txn = Txn::default(); let out = prepare(&mut txn).await; self.finish(txn, out.is_ok(), Class::Ordinary, bound, nodes)?; @@ -1054,7 +1064,7 @@ impl Mount { /// Applies `txn` when `apply`, else drops it; either way releases its /// holds and its grant. `ReadOnly` when the mount stopped writing while /// it was prepared. - fn finish(&self, txn: Txn, apply: bool, class: Class, bound: u64, nodes: usize) -> Result<()> { + fn finish(&self, mut txn: Txn, apply: bool, class: Class, bound: u64, nodes: usize) -> Result<()> { let mut st = self.st.borrow_mut(); let st = &mut *st; // An ordinary grant is each operation's bound; reclaim's is capped @@ -1065,10 +1075,13 @@ impl Mount { st.read_only.get_or_insert("reclaim dirtying more nodes than its cache room"); } let stopped = st.read_only; + let allocated = core::mem::take(&mut txn.allocated_data); if apply && stopped.is_none() { self.apply(st, txn, bound); + for sector in allocated { st.alloc.unpin(sector / u64::from(self.sb.member.bucket_sectors)); } } else { release(st, &txn.held); + for sector in allocated { st.alloc.cancel_data(sector, u64::from(self.sb.member.bucket_sectors)); } } #[cfg(test)] { diff --git a/lib/bcachefs/src/engine/mount/ops.rs b/lib/bcachefs/src/engine/mount/ops.rs index 870e1926b9c..343ee2880f6 100644 --- a/lib/bcachefs/src/engine/mount/ops.rs +++ b/lib/bcachefs/src/engine/mount/ops.rs @@ -34,7 +34,7 @@ const RENAME: usize = 4 + 2 + 1 + 3; const RELEASE: usize = 1 + 1 + 2; /// The whiteouts and one accounting key. const DROP_WHITEOUTS: usize = WHITEOUTS + 1; -const _: () = assert!(RENAME == FOOTPRINT && CREATE <= FOOTPRINT && LINK <= FOOTPRINT && UNLINK <= FOOTPRINT && DROP_WHITEOUTS <= FOOTPRINT); +const _: () = assert!(RENAME <= FOOTPRINT && CREATE <= FOOTPRINT && LINK <= FOOTPRINT && UNLINK <= FOOTPRINT && DROP_WHITEOUTS <= FOOTPRINT); /// What a new name links. pub enum Link { @@ -107,7 +107,7 @@ fn one_link_less(dir: &mut InodeValue) -> Result<()> { Ok(()) } -fn inode_pos(ino: u64) -> Bpos { +pub(super) fn inode_pos(ino: u64) -> Bpos { Bpos::new(0, ino, ROOT_SNAPSHOT) } @@ -140,13 +140,13 @@ impl Mount { /// A new file or directory `name` in directory `dir`, owned by `owner`, /// its times `now`, in nanoseconds since the Unix epoch. Its number is /// the inode cursor's next free one (A8.7), its name in the first free - /// slot from its hash (A5.25). A symlink waits for the data path. + /// slot from its hash (A5.25). pub async fn create(&self, dir: u64, name: &[u8], new: New, owner: Owner, now: i64) -> Result { valid_name(name)?; let (type_bits, dirent_type) = match new { New::File => (mode::S_IFREG, DirentType::Regular), New::Dir => (mode::S_IFDIR, DirentType::Directory), - New::Symlink(_) => return Err(Error::Unsupported("a symlink, whose target is file data")), + New::Symlink(target) => return self.create_symlink(dir, name, &target, owner, now).await, }; let seed = self.nonzero()?; self.observe(now); @@ -352,6 +352,9 @@ impl Mount { /// `deleted_inodes` key go (A8.13). A node with a name is left as it is; /// a key that names no orphan goes alone. pub async fn release(&self, node: u64) -> Result<()> { + let _data = self.data_slot().await; + let orphan = self.inode_opt(&Txn::default(), node).await?.is_some_and(|v| v.flags & inode_flag::UNLINKED != 0 && !is_dir(&v)); + if orphan { self.trim_extents(node, 0).await?; } self.transact(K_TX / 2, RELEASE, Draw::Freeing, async |txn: &mut Txn| { let orphan = self.inode_opt(txn, node).await?.filter(|v| v.flags & inode_flag::UNLINKED != 0 && !is_dir(v)); if orphan.is_some() { @@ -393,11 +396,11 @@ impl Mount { Ok(v) } - async fn inode_in(&self, txn: &Txn, ino: u64) -> Result { + pub(super) async fn inode_in(&self, txn: &Txn, ino: u64) -> Result { self.inode_opt(txn, ino).await?.ok_or(Error::NotFound) } - async fn inode_opt(&self, txn: &Txn, ino: u64) -> Result> { + pub(super) async fn inode_opt(&self, txn: &Txn, ino: u64) -> Result> { match self.get_in(txn, Btree::Inodes, inode_pos(ino)).await? { Some((k, v)) if k.kind == kind::INODE_V3 => Ok(Some(InodeValue::decode(&v)?)), Some((k, _)) if matches!(k.kind, kind::INODE | kind::INODE_V2) => Err(Error::Unsupported("inode in a format before inode_v3")), @@ -421,7 +424,7 @@ impl Mount { } /// A directory's change and modification times set to `now`. - fn touch(&self, dir: &mut InodeValue, now: i64) { + pub(super) fn touch(&self, dir: &mut InodeValue, now: i64) { let time = self.stored_time(now); dir.set(inode_field::MTIME, time); dir.set(inode_field::CTIME, time); @@ -616,7 +619,7 @@ impl Mount { /// Sets inode `ino` to `v`, its journal-sequence field the /// transaction's (A5.14). - async fn set_inode(&self, txn: &mut Txn, ino: u64, v: &InodeValue) -> Result<()> { + pub(super) async fn set_inode(&self, txn: &mut Txn, ino: u64, v: &InodeValue) -> Result<()> { let pos = inode_pos(ino); self.set(txn, Btree::Inodes, pos, Some((Key::new(kind::INODE_V3, pos), v.encode()))).await?; if !txn.stamped.contains(&pos) { @@ -635,7 +638,7 @@ impl Mount { /// accounting the change makes: a key of a snapshot btree counts, and /// its bytes, in its snapshot's class (A7.14); an inode counts once /// more (A7.10). - async fn set(&self, txn: &mut Txn, btree: Btree, pos: Bpos, kv: Option<(Key, Vec)>) -> Result<()> { + pub(super) async fn set(&self, txn: &mut Txn, btree: Btree, pos: Bpos, kv: Option<(Key, Vec)>) -> Result<()> { let old = self.get_in(txn, btree, pos).await?; let counted = |kv: &Option<(Key, Vec)>| match kv { Some((k, v)) if matches!(btree, Btree::Extents | Btree::Inodes | Btree::Dirents | Btree::Xattrs | Btree::Damage) && !matches!(k.kind, kind::DELETED | kind::WHITEOUT | kind::EXTENT_WHITEOUT) => { @@ -644,13 +647,18 @@ impl Mount { _ => None, }; let (before, after) = (counted(&old), counted(&kv)); - if before != after { + let external = |kv: &Option<(Key, Vec)>| match kv { + Some((k, _)) if btree == Btree::Extents && k.kind == kind::EXTENT => u64::from(k.size), + _ => 0, + }; + let sectors = external(&kv).wrapping_sub(external(&old)); + if before != after || sectors != 0 { let mut record = alloc::vec![class::SNAPSHOT]; record.extend_from_slice(&ROOT_SNAPSHOT.to_le_bytes()); record.extend_from_slice(&(btree as u32).to_le_bytes()); let keys = u64::from(after.is_some()).wrapping_sub(u64::from(before.is_some())); let bytes = after.unwrap_or(0).wrapping_sub(before.unwrap_or(0)); - self.account(txn, &record, &[keys, bytes, 0]).await?; + self.account(txn, &record, &[keys, bytes, sectors]).await?; if btree == Btree::Inodes && before.is_some() != after.is_some() { self.account(txn, &[class::NR_INODES], &[keys]).await?; } @@ -675,14 +683,14 @@ impl Mount { } /// `now` in the volume's units since its time base (A5.11). - fn stored_time(&self, now: i64) -> u64 { + pub(super) fn stored_time(&self, now: i64) -> u64 { let precision = i128::from(self.sb.header.time_precision); (i128::from(now) / precision - i128::from(self.sb.header.time_base) / precision) as i64 as u64 } /// Notes an operation at `now`, nanoseconds since the epoch: entries /// carry the newest as their datetime. - fn observe(&self, now: i64) { + pub(super) fn observe(&self, now: i64) { let mut st = self.st.borrow_mut(); st.now = st.now.max((now / 1_000_000_000).max(0) as u64); } diff --git a/lib/bcachefs/src/engine/mount/reclaim.rs b/lib/bcachefs/src/engine/mount/reclaim.rs index c56a1cb1ea7..631a80289e1 100644 --- a/lib/bcachefs/src/engine/mount/reclaim.rs +++ b/lib/bcachefs/src/engine/mount/reclaim.rs @@ -397,7 +397,7 @@ impl Mount { if let Some(slot) = self.st.borrow_mut().alloc.next_slot(bucket, node) { return Ok(slot); } - self.take_bucket().await?; + self.take_bucket(false).await?; } } @@ -405,7 +405,7 @@ impl Mount { /// and that no traversal may lead into (A6.14, A6.19), else the lowest /// free one. Its alloc key must agree it is empty, and no backpointer /// lead into it; the nodes cached there are unreachable, and go. - async fn take_bucket(&self) -> Result<()> { + pub(super) async fn take_bucket(&self, data: bool) -> Result<()> { let durable = self.st.borrow().journal.durable; let mut emptied = None; Source::walk(self, Btree::NeedDiscard, Bpos::MIN, |k, _| { @@ -469,7 +469,11 @@ impl Mount { if !st.cache.drop_unreachable(b * sectors, (b + 1) * sectors) { return Err(Error::Corrupt("a bucket an index lists as empty holding a changed btree node")); } - st.alloc.take(b, bucket.reused_gen()); + if data { + st.alloc.take_data(b, bucket.reused_gen()); + } else { + st.alloc.take(b, bucket.reused_gen()); + } #[cfg(test)] { st.alloc.reused += usize::from(emptied.is_some()); @@ -478,7 +482,7 @@ impl Mount { } /// Low bits of a backpointer's offset below its sector (A6.25). - fn bp_shift(&self) -> u32 { + pub(super) fn bp_shift(&self) -> u32 { match self.sb.flag(6, 40, 8) { 0 => 10, s => s as u32, @@ -599,7 +603,7 @@ impl Mount { } /// Bucket `b` as `txn` would leave it: its alloc key, or a hole. - async fn bucket(&self, txn: &Txn, b: u64) -> Result { + pub(super) async fn bucket(&self, txn: &Txn, b: u64) -> Result { match self.get_in(txn, Btree::Alloc, Bpos::new(0, b, 0)).await? { Some((k, v)) if k.kind == kind::ALLOC_V4 => { let a = AllocV4::decode(&v)?; @@ -614,7 +618,7 @@ impl Mount { /// key, its freespace, LRU and bucket_gens entries, and its state's /// accounting (A6.12–A6.24, A7.12). An emptied bucket's alloc key and /// need_discard entry wait for the transaction's sequence. - async fn bucket_changed(&self, txn: &mut Txn, b: u64, before: Bucket, after: Bucket) -> Result<()> { + pub(super) async fn bucket_changed(&self, txn: &mut Txn, b: u64, before: Bucket, after: Bucket) -> Result<()> { let bucket_sectors = self.sb.member.bucket_sectors as u32; let alloc_pos = Bpos::new(0, b, 0); if after.data_type == data_type::NEED_DISCARD && before.data_type != data_type::NEED_DISCARD { diff --git a/lib/bcachefs/src/engine/read.rs b/lib/bcachefs/src/engine/read.rs index 3dce4d95b55..de00382cbfb 100644 --- a/lib/bcachefs/src/engine/read.rs +++ b/lib/bcachefs/src/engine/read.rs @@ -477,6 +477,7 @@ pub(crate) trait Source { /// Bytes of file `ino` from `offset` into `out`, as many as the file /// holds there. Holes read as zeros. async fn read(&self, ino: u64, offset: u64, out: &mut [u8]) -> Result { + let _reading = Reading(self, self.begin_read()); let inode = self.inode(ino).await?; let len = inode.size.saturating_sub(offset).min(out.len() as u64) as usize; let out = &mut out[..len]; @@ -533,6 +534,7 @@ pub(crate) trait Source { /// in one run of whole blocks from the file's first byte to its last: /// inline, sparse, encrypted, unchecksummed or split apart. async fn in_place(&self, ino: u64) -> Result> { + let _reading = Reading(self, self.begin_read()); let inode = self.inode(ino).await?; if self.superblock().encrypted() || inode.kind() != Some(DirentType::Regular) || inode.size == 0 { return Ok(None); @@ -596,7 +598,12 @@ pub(crate) trait Source { if self.superblock().encrypted() && !matches!(crc.csum_type, csum::CHACHA_POLY_80 | csum::CHACHA_POLY_128) { return Err(Error::Corrupt("unauthenticated extent on an encrypted volume")); } - let mut data = read_bytes(self.dev(), extent.ptr.sector * SECTOR as u64, crc.compressed as usize * SECTOR).await?; + let at = extent.ptr.sector * SECTOR as u64; + let len = crc.compressed as usize * SECTOR; + let skip_block = (at % BLOCK as u64) as usize; + let mut blocks = alloc::vec![[0u8; BLOCK]; (skip_block + len).div_ceil(BLOCK)]; + self.dev().read_data(at / BLOCK as u64, &mut blocks).await?; + let mut data = blocks.as_flattened()[skip_block..skip_block + len].to_vec(); match crc.csum_type { csum::NONE => {} // Over the whole encoded extent, the value zero-extended to the diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs new file mode 100644 index 00000000000..4ef9cc8163f --- /dev/null +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -0,0 +1,218 @@ +use super::mount::{mounted, random, MEMORY, NOW, OWNER}; +use super::*; + +#[test] +fn file_data_overwrites_sparse_ranges_and_truncation_survive_remount() { + let dev = formatted(64 << 20, None); + let mut want = vec![0u8; 180_003]; + let ino = mounted(&dev, async |m: &Mount<&Image>| { + let ino = m.create(m.root(), b"data", New::File, OWNER, NOW).await.unwrap().ino; + let content: Vec = (0..90_005).map(|i| (i * 73) as u8).collect(); + assert_eq!(m.write(ino, 1007, &content, NOW).await.unwrap(), content.len()); + want[1007..1007 + content.len()].copy_from_slice(&content); + assert_eq!(m.write(ino, 4003, b"replacement", NOW).await.unwrap(), 11); + want[4003..4014].copy_from_slice(b"replacement"); + m.truncate(ino, want.len() as u64, NOW).await.unwrap(); + let mut got = vec![0u8; want.len()]; + assert_eq!(m.read(ino, 0, &mut got).await.unwrap(), want.len()); + assert_eq!(got, want); + m.truncate(ino, 41_003, NOW).await.unwrap(); + m.truncate(ino, want.len() as u64, NOW).await.unwrap(); + want[41_003..].fill(0); + m.commit().await.unwrap(); + ino + }); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(read_all(&v, ino).unwrap(), want); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} + +#[test] +fn orphan_release_reclaims_file_data_and_keeps_other_files() { + let dev = formatted(64 << 20, None); + mounted(&dev, async |m: &Mount<&Image>| { + let a = m.create(m.root(), b"a", New::File, OWNER, NOW).await.unwrap().ino; + let b = m.create(m.root(), b"b", New::File, OWNER, NOW).await.unwrap().ino; + m.write(a, 0, &[1; 4096], NOW).await.unwrap(); + m.write(b, 0, &[2; 8192], NOW).await.unwrap(); + m.unlink(m.root(), b"a", false, NOW).await.unwrap(); + let mut bytes = [0u8; 4096]; + assert_eq!(m.read(a, 0, &mut bytes).await.unwrap(), bytes.len()); + assert_eq!(bytes, [1; 4096]); + m.release(a).await.unwrap(); + assert_eq!(m.inode(a).await, Err(Error::NotFound)); + }); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(read_all(&v, path(&v, "b")).unwrap(), [2; 8192]); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} + +#[test] +fn encrypted_data_uses_authenticated_extents_and_durable_version_ranges() { + let dev = formatted(64 << 20, Some(Encryption { passphrase: b"test", kdf: MIN_KDF })); + let m = block_on_ready(Mount::unlock(&dev, b"test", LIMITS, MEMORY, random())).unwrap(); + let ino = block_on_ready(async { + let ino = m.create(m.root(), b"secret", New::File, OWNER, NOW).await.unwrap().ino; + m.write(ino, 0, &[0x45; 8192], NOW).await.unwrap(); + m.write(ino, 100, b"changed", NOW).await.unwrap(); + m.truncate(ino, 200, NOW).await.unwrap(); + ino + }); + block_on_ready(m.unmount()).unwrap(); + let v = block_on_ready(Volume::unlock(&dev, b"test", LIMITS)).unwrap(); + let mut want = vec![0x45; 200]; + want[100..107].copy_from_slice(b"changed"); + assert_eq!(read_all(&v, ino).unwrap(), want); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); + let first = block_on_ready(v.get(Btree::Extents, Bpos::new(ino, 8, u32::MAX))).unwrap().unwrap(); + let old_version = first.0.version.lo; + assert!(old_version > 0 && old_version <= v.key_version()); + drop(v); + let m = block_on_ready(Mount::unlock(&dev, b"test", LIMITS, MEMORY, random())).unwrap(); + block_on_ready(m.write(ino, 0, b"next", NOW)).unwrap(); + block_on_ready(m.unmount()).unwrap(); + let v = block_on_ready(Volume::unlock(&dev, b"test", LIMITS)).unwrap(); + let first = block_on_ready(v.get(Btree::Extents, Bpos::new(ino, 8, u32::MAX))).unwrap().unwrap(); + assert!(first.0.version.lo > old_version); +} + +fn interrupted_data(encrypted: bool) { + use super::crash::Watched; + use core::cell::Cell; + use toyos_blockdev::model::{Model, Overlay}; + + let dev = formatted(64 << 20, encrypted.then_some(Encryption { passphrase: b"test", kdf: MIN_KDF })); + let initial = if encrypted { block_on_ready(Mount::unlock(&dev, b"test", LIMITS, MEMORY, random())).unwrap() } else { block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap() }; + let ino = block_on_ready(initial.create(initial.root(), b"file", New::File, OWNER, NOW)).unwrap().ino; + block_on_ready(initial.write(ino, 0, &[0x33; 8192], NOW)).unwrap(); + block_on_ready(initial.unmount()).unwrap(); + let mut states = vec![vec![0x33; 8192]]; + let mut next = states[0].clone(); + next[100..4196].fill(0x75); + states.push(next.clone()); + next.truncate(1003); + states.push(next.clone()); + next.resize(8192, 0); + states.push(next); + let applied = Cell::new(0); + let committed = Cell::new(0); + let requests = Cell::new(0u64); + let model = Model::new(dev.into_inner(), 512); + let watched = Watched { + model: &model, + after: Box::new(|model| { + requests.set(requests.get() + 1); + model + .check(16, requests.get(), |bytes| { + let dev = Overlay::new(bytes); + let m = if encrypted { block_on_ready(Mount::unlock(&dev, b"test", LIMITS, MEMORY, random())) } else { block_on_ready(Mount::open(&dev, MEMORY, random())) } + .map_err(|e| format!("mount: {e:?}"))?; + let size = block_on_ready(m.inode(ino)).map_err(|e| format!("inode: {e:?}"))?.size; + let mut got = vec![0; size as usize]; + block_on_ready(m.read(ino, 0, &mut got)).map_err(|e| format!("read: {e:?}"))?; + if !(committed.get()..=applied.get()).any(|i| states[i] == got) { + return Err(format!("unexpected bytes at request {} ({} bytes), applied {}, committed {}", requests.get(), got.len(), applied.get(), committed.get())); + } + block_on_ready(m.unmount()).map_err(|e| format!("unmount: {e:?}"))?; + let v = if encrypted { block_on_ready(Volume::unlock(&dev, b"test", LIMITS)) } else { block_on_ready(Volume::open(&dev)) }.map_err(|e| format!("reopen: {e:?}"))?; + let findings = block_on_ready(fsck(&v)).map_err(|e| format!("fsck: {e:?}"))?; + if !findings.is_empty() { + return Err(format!("fsck: {findings:?}")); + } + Ok(()) + }) + .unwrap_or_else(|e| panic!("{e}")); + }), + }; + let m = if encrypted { block_on_ready(Mount::unlock(&watched, b"test", LIMITS, MEMORY, random())).unwrap() } else { block_on_ready(Mount::open(&watched, MEMORY, random())).unwrap() }; + applied.set(1); + block_on_ready(m.write(ino, 100, &[0x75; 4096], NOW)).unwrap(); + block_on_ready(m.commit()).unwrap(); + committed.set(1); + applied.set(2); + block_on_ready(m.truncate(ino, 1003, NOW)).unwrap(); + block_on_ready(m.commit()).unwrap(); + committed.set(2); + applied.set(3); + block_on_ready(m.truncate(ino, 8192, NOW)).unwrap(); + block_on_ready(m.commit()).unwrap(); + committed.set(3); + block_on_ready(m.unmount()).unwrap(); + assert!(requests.get() > 10); +} + +#[test] +fn data_and_logged_truncate_survive_torn_reordered_writes() { + interrupted_data(false); +} + +#[test] +fn encrypted_data_and_logged_truncate_survive_torn_reordered_writes() { + interrupted_data(true); +} + +#[test] +fn reserved_blocks_write_after_capacity_refusal_and_dropped_tokens_return_capacity() { + let dev = formatted(64 << 20, None); + let ino = mounted(&dev, async |m: &Mount<&Image>| { + let ino = m.create(m.root(), b"reserved", New::File, OWNER, NOW).await.unwrap().ino; + let mut tokens = Vec::new(); + loop { + match m.reserve(ino, tokens.len() as u64 * BLOCK as u64, BLOCK).await { + Ok(token) => tokens.push(token), + Err(Error::NoSpace(_)) => break, + Err(e) => panic!("reserve: {e:?}"), + } + assert!(tokens.len() < 1000); + } + assert!(tokens.len() > 1); + let count = tokens.len(); + drop(tokens); + let mut tokens = Vec::new(); + for i in 0..count { + tokens.push(m.reserve(ino, i as u64 * BLOCK as u64, BLOCK).await.unwrap()); + } + for (i, token) in tokens.iter_mut().enumerate().rev() { + assert_eq!(m.write_reserved(token, i as u64 * BLOCK as u64, &[i as u8; BLOCK], NOW).await.unwrap(), BLOCK); + } + m.commit().await.unwrap(); + ino + }); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + let bytes = read_all(&v, ino).unwrap(); + for (i, block) in bytes.chunks_exact(BLOCK).enumerate() { + assert_eq!(block, &[i as u8; BLOCK]); + } + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} + +#[test] +fn a_reserved_slot_survives_its_bucket_emptying_and_generation_change() { + let dev = formatted(64 << 20, None); + mounted(&dev, async |m: &Mount<&Image>| { + let a = m.create(m.root(), b"a", New::File, OWNER, NOW).await.unwrap().ino; + let b = m.create(m.root(), b"b", New::File, OWNER, NOW).await.unwrap().ino; + let mut ta = m.reserve(a, 0, BLOCK).await.unwrap(); + let mut tb = m.reserve(b, 0, BLOCK).await.unwrap(); + m.write_reserved(&mut ta, 0, &[1; BLOCK], NOW).await.unwrap(); + m.unlink(m.root(), b"a", false, NOW).await.unwrap(); + m.release(a).await.unwrap(); + m.commit().await.unwrap(); + m.write_reserved(&mut tb, 0, &[2; BLOCK], NOW).await.unwrap(); + }); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(read_all(&v, path(&v, "b")).unwrap(), [2; BLOCK]); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} + +#[test] +fn symlink_targets_are_file_data_and_read_after_remount() { + let dev = formatted(64 << 20, None); + mounted(&dev, async |m: &Mount<&Image>| { + let ino = m.create(m.root(), b"link", New::Symlink(b"../target".to_vec()), OWNER, NOW).await.unwrap().ino; + assert_eq!(m.readlink(ino).await.unwrap(), b"../target"); + }); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(block_on_ready(v.readlink(path(&v, "link"))).unwrap(), b"../target"); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} diff --git a/lib/bcachefs/src/engine/tests/mod.rs b/lib/bcachefs/src/engine/tests/mod.rs index 5562501ec53..08c527ffbf7 100644 --- a/lib/bcachefs/src/engine/tests/mod.rs +++ b/lib/bcachefs/src/engine/tests/mod.rs @@ -1259,6 +1259,7 @@ fn read_file_reads_a_file_whole_and_refuses_other_nodes() { mod admission; mod buckets; mod crash; +mod data; mod interleave; mod journal; mod mount; diff --git a/lib/bcachefs/src/engine/value.rs b/lib/bcachefs/src/engine/value.rs index 20e4476c951..a4dc6a178ce 100644 --- a/lib/bcachefs/src/engine/value.rs +++ b/lib/bcachefs/src/engine/value.rs @@ -308,6 +308,28 @@ const CRC32_SECTORS: u32 = 128; const CRC64_SECTORS: u32 = 512; impl Extent { + pub(crate) fn with_crc(crc: &Crc, ptr: Ptr) -> Vec { + let (compressed, uncompressed) = (u64::from(crc.compressed - 1), u64::from(crc.uncompressed - 1)); + let (offset, nonce, ty, compression) = (u64::from(crc.offset), u64::from(crc.nonce), u64::from(crc.csum_type), u64::from(crc.compression)); + let words = match Self::smallest_entry(crc) { + 4 => alloc::vec![2 | compressed << 2 | uncompressed << 9 | offset << 16 | ty << 24 | compression << 28 | u64::from(u32::from_le_bytes(crc.csum[..4].try_into().unwrap())) << 32, ptr.encode()], + 10 => alloc::vec![4 | compressed << 3 | uncompressed << 12 | offset << 21 | nonce << 30 | ty << 40 | compression << 44 | u64::from(u16::from_le_bytes(crc.csum[8..10].try_into().unwrap())) << 48, u64::from_le_bytes(crc.csum[..8].try_into().unwrap()), ptr.encode()], + _ => alloc::vec![8 | compressed << 4 | uncompressed << 17 | offset << 30 | nonce << 43 | ty << 56 | compression << 60, u64::from_le_bytes(crc.csum[..8].try_into().unwrap()), u64::from_le_bytes(crc.csum[8..].try_into().unwrap()), ptr.encode()], + }; + words.iter().flat_map(|w| w.to_le_bytes()).collect() + } + + pub(crate) fn authenticated(sectors: u32, tag: [u8; 16], wide: bool, ptr: Ptr) -> Vec { + let n = u64::from(sectors - 1); + let low = u64::from_le_bytes(tag[..8].try_into().unwrap()); + let words = if wide || sectors > CRC64_SECTORS { + alloc::vec![8 | n << 4 | n << 17 | u64::from(csum_type::CHACHA_POLY_128) << 56, low, u64::from_le_bytes(tag[8..].try_into().unwrap()), ptr.encode()] + } else { + alloc::vec![4 | n << 3 | n << 12 | u64::from(csum_type::CHACHA_POLY_80) << 40 | u64::from(u16::from_le_bytes(tag[8..10].try_into().unwrap())) << 48, low, ptr.encode()] + }; + words.iter().flat_map(|w| w.to_le_bytes()).collect() + } + /// An unencrypted, uncompressed extent of `sectors` at `ptr`, its /// checksum `csum` of `csum_type` in the smallest entry that holds it: /// a crc32 entry up to 128 sectors, else a crc64 one. From 0b81b5672c49041742641ffafcc3f258acb951b9 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 19:35:06 +0100 Subject: [PATCH 13/93] Preserve live DATA objects and defer unwritten inode sizes Correct readdir continuation, retain paged descriptions across unlink, release unheld orphans, and deduplicate sync failures. Adapt fsync tests to real bcachefs transactions: dirty lengths are not published before data. The outstanding large-write capacity regression remains under repair. --- system/fileserver/src/data.rs | 24 ++++++++++++++++++------ system/fileserver/src/pages.rs | 22 +++++++++++++--------- system/fileserver/src/pages/tests.rs | 8 ++++---- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index f627372f945..a32f3bc3efa 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -191,14 +191,14 @@ impl Volume for DataVolume { let mut cursor = 0; let mut listed = Vec::new(); loop { - let (entries, end) = mapped("readdir", self.cache.block_on(self.fs.readdir(inode.ino, cursor, 256)))?; - if entries.is_empty() && !end { return Err(Error::Io); } + let (entries, more) = mapped("readdir", self.cache.block_on(self.fs.readdir(inode.ino, cursor, 256)))?; + if entries.is_empty() && more { return Err(Error::Io); } for entry in entries { cursor = entry.next; let inode = mapped("directory entry", self.cache.block_on(self.fs.inode(entry.ino)))?; listed.push(Listed { name: String::from_utf8(entry.name).map_err(|_| Error::Io)?, meta: self.meta(&inode)?, identity: entry.ino }); } - if end { break; } + if !more { break; } } listed.sort_unstable_by(|a, b| a.name.cmp(&b.name)); Ok(listed) @@ -225,9 +225,11 @@ impl Volume for DataVolume { let Some(open) = self.open.get_mut(&node) else { return Ok(()) }; open.holders = open.holders.checked_sub(1).expect("one close per open"); if open.holders != 0 { return Ok(()); } - self.persist(node)?; let inode = mapped("close", self.cache.block_on(self.fs.inode(node)))?; - if inode.nlink == 0 { mapped("orphan close", self.cache.block_on(self.fs.release(node)))?; } + if inode.nlink == 0 { + self.open.get_mut(&node).unwrap().pages.clear(); + mapped("orphan close", self.cache.block_on(self.fs.release(node)))?; + } else { self.persist(node)?; } self.open.remove(&node); Ok(()) } @@ -338,7 +340,17 @@ impl Volume for DataVolume { fn sync(&mut self) -> Result, Error> { let nodes: Vec<_> = self.open.keys().copied().collect(); let mut refused = Vec::new(); - for node in nodes { if let Err(error) = self.persist(node) { refused.push((node, error)); } } + for node in nodes { + match self.persist(node) { + Err(error) => refused.push((node, error)), + Ok(()) if self.open[&node].holders == 0 => { + let inode = mapped("unheld inode", self.cache.block_on(self.fs.inode(node)))?; + if inode.nlink == 0 { mapped("orphan release", self.cache.block_on(self.fs.release(node)))?; } + self.open.remove(&node); + } + Ok(()) => {} + } + } mapped("sync", self.cache.block_on(self.fs.commit()))?; Ok(refused) } diff --git a/system/fileserver/src/pages.rs b/system/fileserver/src/pages.rs index 288a6307633..7c130f5e616 100644 --- a/system/fileserver/src/pages.rs +++ b/system/fileserver/src/pages.rs @@ -252,20 +252,19 @@ impl Files

{ } } - /// A name was removed or replaced. Memory inodes remain usable through - /// every open description and paged holder until the last one ends. + /// A removed name does not invalidate an inode the volume still holds. pub fn unlinked(&mut self, volume: &mut dyn Volume, identity: u64) -> Result<(), Error> { - if volume.memory() { - self.collect_unlinked(volume, identity) - } else { - self.forget(volume, identity); - Ok(()) + let Some(f) = self.files.get(&identity) else { return Ok(()) }; + match volume.node_meta(f.node) { + Ok(_) => self.collect_unlinked(volume, identity), + Err(Error::Stale) => { self.forget(volume, identity); Ok(()) } + Err(error) => Err(error), } } fn collect_unlinked(&mut self, volume: &mut dyn Volume, identity: u64) -> Result<(), Error> { let Some(f) = self.files.get(&identity) else { return Ok(()) }; - if volume.memory() && f.opens == 0 && !f.held && !volume.linked(f.node)? { + if f.opens == 0 && !f.held && !volume.linked(f.node)? { self.forget(volume, identity); } Ok(()) @@ -623,7 +622,12 @@ impl Files

{ } } let mut unwritten = volume.sync()?; - unwritten.extend(refused); + for (node, error) in refused { + match unwritten.iter_mut().find(|(held, _)| *held == node) { + Some((_, why)) => *why = error, + None => unwritten.push((node, error)), + } + } Ok(unwritten) } diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index 3b11903ef5b..75b72eb5c1f 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -686,7 +686,7 @@ fn an_fsync_makes_its_file_durable_and_no_other_files_overwrite() { /// another: its pages reach the volume first and its entry goes with them, /// so after a stop it reads its own bytes, never an earlier tenant's. #[test] -fn an_fsync_writes_first_the_pages_of_every_file_its_commit_carries() { +fn an_fsync_commits_its_file_without_publishing_another_files_dirty_size() { let page = PAGE as usize; let mut v = stale_volume(); let mut files = Files::new(Some(Fake::new(64, 16))); @@ -696,11 +696,11 @@ fn an_fsync_writes_first_the_pages_of_every_file_its_commit_carries() { files.write(&mut v, c.1, c.0, 0, &pattern(3 * page, 5)).unwrap(); files.fsync(&mut v, Some((a.1, a.0))).unwrap(); - assert!(!files.owes(&v), "the fsync left a change owed"); + assert!(files.owes(&v), "the other file still has buffered data"); let mut v = after_a_stop(v); assert!(on_volume(&mut v, "home/a") == pattern(page, 4), "the fsynced file is not as written"); let carried = on_volume(&mut v, "home/c"); - assert!(carried == pattern(3 * page, 5), "the carried file reads {} bytes, not its own", carried.len()); + assert!(carried.is_empty(), "unwritten data must not become a durable file length"); } /// A file whose pages a sync could not write is owed after an fsync of @@ -717,7 +717,7 @@ fn a_file_a_sync_could_not_write_back_stays_owed_past_another_s_fsync() { files.write(&mut v, f.1, f.0, 0, &pattern(page, 6)).unwrap(); fake.0.borrow_mut().unwritable = Some(files.files[&f.1].object); assert_eq!(files.sync(&mut v).map(|u| u.len()), Ok(1), "the sync wrote the refused file's pages"); - assert!(!v.owes(), "the sync left f's entry owed"); + assert!(v.owes(), "the failed file's buffered metadata must stay pending"); files.write(&mut v, a.1, a.0, 0, &pattern(page, 7)).unwrap(); files.fsync(&mut v, Some((a.1, a.0))).unwrap(); From 17934a82d237fdb83b4670dc385c97f56151cb9c Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 19:23:52 +0100 Subject: [PATCH 14/93] Export shared filesystem descriptions to spawned children Use the existing fileserver EXPORT protocol for inherited file descriptors, including readable stdin. Transfer each local offset to the server before export, and use that shared offset thereafter. Ordinary std Command spawns export a fresh connection each time; forwarding an already exported connection is explicitly one-shot. POSIX file-action dup2 aliases carry a process-local group in their fd label so the child adopts the protocol connection once and shares one descriptor object. Independent launch refuses ambiguous reuse of one raw exported connection instead of attaching it twice. SDK host tests (153), clippy with fd-table, and libc module tests (106) pass. Extend served_file_offset with guest checks for repeated input spawns, dup2 reads/seeks/writes, exact offset sharing and open-unlinked metadata. Guest execution follows the combined storage integration build. --- sdk/libc/src/posix_io.rs | 12 +-- sdk/libc/src/spawn.rs | 3 +- sdk/std/os/process.rs | 18 ++++- sdk/std/sys/fs.rs | 8 +- sdk/std/sys/pal/mod.rs | 1 + sdk/std/sys/process.rs | 15 +++- sdk/toyos/src/fd.rs | 49 ++++++++++-- sdk/toyos/src/fd/table.rs | 21 +++++ sdk/toyos/src/files.rs | 20 +++++ sdk/toyos/src/files/owner.rs | 45 ++++++++++- sdk/toyos/src/launch.rs | 8 +- sdk/toyos/src/spawn.rs | 19 ++++- system/supervisor/src/main.rs | 3 + .../rust-tests/src/bin/served_file_offset.rs | 78 ++++++++++++++++++- 14 files changed, 268 insertions(+), 32 deletions(-) diff --git a/sdk/libc/src/posix_io.rs b/sdk/libc/src/posix_io.rs index 261fe7e187b..ac36ef8efed 100644 --- a/sdk/libc/src/posix_io.rs +++ b/sdk/libc/src/posix_io.rs @@ -112,17 +112,17 @@ pub unsafe extern "C" fn openat(dirfd: i32, path: *const u8, flags: i32, _mode: } } -/// A spawn's `addopen` file exported as a server stream for the child. -pub(crate) fn open_handle(path: &[u8], flags: i32, _mode: u32) -> Result { +/// A spawn's `addopen` file exported as a shared description for the child. +pub(crate) fn open_handle(path: &[u8], flags: i32, _mode: u32) -> Result<(RawHandle, toyos::fd::Kind), Errno> { let how = crate::atreq::how(flags)?; if path == DEV_NULL { - return syscall::null(null_rights(how.read, how.write)).map_err(crate::errcode::of); + return syscall::null(null_rights(how.read, how.write)).map(|h| (h, toyos::fd::Kind::Handle)).map_err(crate::errcode::of); } let fd = files::open(At::Cwd, path, how).map_err(crate::errcode::of_files)?; - let stream = files::stream(fd).map(|h| h.into_raw()).map_err(crate::errcode::of_files); - // SAFETY: this function's descriptor, after the server exported the stream. + let exported = files::export(fd).map(|h| (h.into_raw(), toyos::fd::Kind::File)).map_err(crate::errcode::of_files); + // SAFETY: this function's descriptor, after the server exported its description. let _ = unsafe { toyos::fd::close(fd) }; - stream + exported } /// POSIX's null device, which libc answers with a kernel null handle: the diff --git a/sdk/libc/src/spawn.rs b/sdk/libc/src/spawn.rs index d78aa9b4197..72749fc06d5 100644 --- a/sdk/libc/src/spawn.rs +++ b/sdk/libc/src/spawn.rs @@ -298,7 +298,7 @@ fn spawn_counted(path: &str, argv: &[u8], env: &[u8], actions: &[Action i32 { // A holder past what netstack has places for is a descriptor too many. Refused::Export(toyos::net::NetError::ResourceExhausted) => crate::errcode::EMFILE, Refused::Export(_) => crate::errcode::EIO, + Refused::File(e) => crate::errcode::of_files(e), Refused::Syscall(e) => crate::errcode::of(e), } } diff --git a/sdk/std/os/process.rs b/sdk/std/os/process.rs index 5cf209dcc03..537814159e7 100644 --- a/sdk/std/os/process.rs +++ b/sdk/std/os/process.rs @@ -29,6 +29,13 @@ pub trait CommandExt { #[stable(feature = "toyos_ext", since = "1.0.0")] fn inherit_exported(&mut self, child_fd: u32, end: crate::os::toyos::io::BorrowedHandle<'_>) -> &mut Self; + /// Give the child a connection from `toyos::files::export`, naming the + /// same filesystem description and offset. The command duplicates it for + /// its next spawn only; a second spawn refuses this consumed export. + /// Ordinary `Stdio::from(File)` exports afresh on every spawn. + #[stable(feature = "toyos_ext", since = "1.0.0")] + fn inherit_file(&mut self, child_fd: u32, end: crate::os::toyos::io::BorrowedHandle<'_>) -> &mut Self; + /// Give the child a handle under a name it can look itself up by. /// /// The handle is **moved**: into the command, then by its next spawn into @@ -65,13 +72,13 @@ pub trait CommandExt { /// The bytes a prepared spawn starts, as [`prepare`](CommandExt::prepare) /// read them off a file server: what a caller holds a program to before it /// runs, with no second read between the two. `None` before a prepare and - /// for a program the kernel opens itself. + /// for a boot image or an independent launch. #[stable(feature = "toyos_ext", since = "1.0.0")] fn prepared_image(&self) -> Option<&[u8]>; /// Run the program at `path` rather than the one the command names, which - /// stays the child's `argv[0]`: the kernel opens `path`, or the spawn reads - /// it when a file server serves it, and nothing opens `argv[0]`. + /// stays the child's `argv[0]`: the spawn reads `path` through its + /// filesystem capability, and nothing opens `argv[0]`. /// /// **A helper's alone.** The launcher runs the program its manifest row /// names, so an [`independent`](CommandExt::independent) command that @@ -147,6 +154,11 @@ impl CommandExt for crate::process::Command { self } + fn inherit_file(&mut self, child_fd: u32, end: crate::os::toyos::io::BorrowedHandle<'_>) -> &mut Self { + self.as_inner_mut().inherit_file(child_fd, end); + self + } + fn endow(&mut self, label: &str, handle: crate::os::toyos::io::OwnedHandle) -> &mut Self { self.as_inner_mut().endow(label, handle); self diff --git a/sdk/std/sys/fs.rs b/sdk/std/sys/fs.rs index 589f80e1d54..adf0586eb99 100644 --- a/sdk/std/sys/fs.rs +++ b/sdk/std/sys/fs.rs @@ -327,10 +327,10 @@ impl File { crate::sys::on_fd(self.0, |h| Ok(f(h.raw()))) } - /// A child's output handle: a copied generic object or a pipe the server - /// writes at this open description's offset. - pub fn as_child_stdio(&self) -> io::Result { - files::stream(self.0).map_err(files_error) + /// A child gets a shared file description, or a copied generic object. + pub fn as_child_stdio(&self) -> io::Result<(toyos::OwnedHandle, toyos::fd::Kind)> { + toyos::fd::copy(self.0).map_err(crate::sys::refused)? + .ok_or_else(|| io::Error::from(io::ErrorKind::InvalidInput)) } pub fn open(path: &Path, opts: &OpenOptions) -> io::Result { diff --git a/sdk/std/sys/pal/mod.rs b/sdk/std/sys/pal/mod.rs index e2d59454146..c8a41ede92d 100644 --- a/sdk/std/sys/pal/mod.rs +++ b/sdk/std/sys/pal/mod.rs @@ -141,6 +141,7 @@ pub fn refused(refused: toyos::spawn::Refused) -> crate::io::Error { toyos::net::NetError::ResourceExhausted => Error::from(ErrorKind::QuotaExceeded), _ => Error::from(ErrorKind::Other), }, + Refused::File(e) => files_error(e), Refused::NotStarted => Error::from(ErrorKind::Other), Refused::Syscall(e) => to_io_error(e), } diff --git a/sdk/std/sys/process.rs b/sdk/std/sys/process.rs index 96d32767b5c..d0b8d18fc90 100644 --- a/sdk/std/sys/process.rs +++ b/sdk/std/sys/process.rs @@ -57,6 +57,8 @@ enum Given { /// The caller's stream or socket, kept, of which each spawn exports a /// holder for its child alone. Holder(toyos::net::Exporter), + /// An already-exported file connection is forwarded at most once. + File(Option>), } /// Whom a child is placed under: [`toyos::spawn::Parent`], with a place the @@ -246,6 +248,11 @@ impl Command { self.inherit(child_fd, Given::Copy(copy(end).map(|h| (h, toyos::fd::Kind::Exported)))); } + /// Forward an exported filesystem description to this command's child. + pub fn inherit_file(&mut self, child_fd: u32, end: crate::os::toyos::io::BorrowedHandle<'_>) { + self.inherit(child_fd, Given::File(Some(copy(end)))); + } + fn inherit(&mut self, child_fd: u32, given: Given) { self.extra_fds.retain(|(fd, _)| *fd != child_fd); self.extra_fds.push((child_fd, given)); @@ -381,12 +388,16 @@ impl Command { } // A stream's or a socket's holder is this spawn's alone: exported // now, and gone with the spawn. - for (fd, given) in &self.extra_fds { + for (fd, given) in &mut self.extra_fds { match given { Given::Holder(kept) => { let end = kept.export().map_err(|e| refused(toyos::spawn::Refused::Export(e)))?; held.push((*fd, end, toyos::fd::Kind::Exported)); } + Given::File(file) => { + let handle = file.take().ok_or_else(|| io::const_error!(io::ErrorKind::InvalidInput, "an exported file connection was already forwarded"))??; + held.push((*fd, handle, toyos::fd::Kind::File)); + } Given::Copy(Err(e)) => return Err(io::Error::new(e.kind(), e.to_string())), Given::Copy(Ok(_)) => {} } @@ -501,7 +512,7 @@ impl Command { *parent_pipe = Some(ours); handle(copy(&theirs)?) } - Stdio::InheritFile(file) => handle(file.as_child_stdio()?), + Stdio::InheritFile(file) => Some(file.as_child_stdio()?), Stdio::InheritPipe(pipe) => handle(copy(pipe)?), Stdio::ParentStdout => inherit(1)?, Stdio::ParentStderr => inherit(2)?, diff --git a/sdk/toyos/src/fd.rs b/sdk/toyos/src/fd.rs index abce3576bcc..3a82b0c54ab 100644 --- a/sdk/toyos/src/fd.rs +++ b/sdk/toyos/src/fd.rs @@ -303,13 +303,16 @@ pub unsafe fn replace(fd: Fd, object: Object) -> Result<(), Error> { /// What a child is given for `fd`, and its kind: a copy of the kernel handle /// it names, or for a stream, a listener or a datagram socket an end of a new -/// holder of it (`net::stream::export`, `net::socket::export`). `None` for a +/// holder of it (`net::stream::export`, `net::socket::export`), or a fresh +/// filesystem connection sharing its open description. `None` for a /// descriptor that is not open, as a child of POSIX's holds no closed one. -/// Refused for an object that is no kernel handle, for one the kernel will -/// not duplicate, and for a socket netstack makes no holder of. +/// Refused when the object cannot be exported or duplicated. pub fn copy(fd: Fd) -> Result, crate::spawn::Refused> { use crate::spawn::Refused; let copied = with(fd, |o| match (o.handle(), o.kernel_handle()) { + (None, _) if o.tag() == SERVED_FILE => { + crate::files::export_held(o).map(|end| (end, Kind::File)).map_err(Refused::File) + } (Some(bell), _) if o.tag() == SOCKET && crate::net::socket::is_socket(bell) => { crate::net::socket::export(bell).map(|end| (end, Kind::Exported)).map_err(Refused::Export) } @@ -331,6 +334,8 @@ pub enum Kind { /// A stream's, a listener's or a datagram socket's exported end, which /// the child takes up as its holder: `fd::exported`. Exported, + /// An exported filesystem connection holding description 1: `fd::file`. + File, } impl Kind { @@ -339,6 +344,7 @@ impl Kind { match self { Self::Handle => "", Self::Exported => ":exported", + Self::File => ":file", } } } @@ -361,11 +367,23 @@ pub fn endowed_fd(label: &str) -> Option<(Fd, Kind)> { let (digits, kind) = match rest.split_once(':') { None => (rest, Kind::Handle), Some((digits, "exported")) => (digits, Kind::Exported), + Some((digits, "file")) => (digits, Kind::File), + Some((digits, suffix)) if suffix.strip_prefix("file:").and_then(number).is_some() => (digits, Kind::File), Some(_) => return None, }; + Some((number(digits)?, kind)) +} + +fn number(digits: &str) -> Option { let canonical = digits.bytes().all(|b| b.is_ascii_digit()) && (digits == "0" || !digits.starts_with('0')); let n: Fd = digits.parse().ok().filter(|_| canonical)?; - (0..MAX_FDS as Fd).contains(&n).then_some((n, kind)) + (0..MAX_FDS as Fd).contains(&n).then_some(n) +} + +/// File aliases carry a process-local group selected by the spawning SDK. +#[cfg_attr(not(all(feature = "fd-table", not(test))), allow(dead_code))] +fn file_group(label: &str) -> Option { + label.split_once(":file:").and_then(|(_, group)| number(group)) } /// The table itself, in the copy that owns it. @@ -414,11 +432,26 @@ mod owner { /// closed: how many were. fn endow(table: &mut Table, endowed: &Endowments) -> Result { let mut refused = 0; + let mut files = alloc::collections::BTreeMap::new(); for label in endowed.labels() { let Some((fd, kind)) = endowed_fd(label) else { continue }; let Some(handle) = endowed.take::(label) else { continue }; + let group = (kind == Kind::File).then(|| file_group(label)).flatten(); + if let Some(source) = group.and_then(|group| files.get(&group)).copied() { + drop(handle); + match table.alias_at(source, fd) { + Ok(()) => {}, + Err(Fault::Refused(_)) => refused += 1, + Err(Fault::Broken) => return Err(Fault::Broken), + } + continue; + } let object = match kind { Kind::Handle => Object::handle(handle), + Kind::File => match crate::files::adopt_exported(handle) { + Ok(object) => object, + Err(_) => { refused += 1; continue; } + }, Kind::Exported => match crate::net::adopt_exported(handle) { Some(object) => object, None => { @@ -430,7 +463,7 @@ mod owner { let record = object.into_record(); match table.install_at(fd, record, true) { // The table owns the object from here on. - Ok(()) => {} + Ok(()) => { if let Some(group) = group { files.insert(group, fd); } } Err(Fault::Refused(_)) => { // SAFETY: refused, so still this function's alone. unsafe { record.release() }; @@ -539,12 +572,14 @@ mod tests { assert_eq!(endowed_fd("fd:0"), Some((0, Kind::Handle))); assert_eq!(endowed_fd("fd:4095"), Some((4095, Kind::Handle))); assert_eq!(endowed_fd("fd:3:exported"), Some((3, Kind::Exported))); - for kind in [Kind::Handle, Kind::Exported] { + assert_eq!(endowed_fd("fd:4:file:1"), Some((4, Kind::File))); + assert_eq!(file_group("fd:4:file:1"), Some(1)); + for kind in [Kind::Handle, Kind::Exported, Kind::File] { assert_eq!(endowed_fd(&format!("fd:7{}", kind.suffix())), Some((7, kind)), "a label of its own"); } for other in [ "fd:4096", "fd:-1", "fd:", "fd:1x", "svc", "fd1", "dev:fd:1", "fd:+3", "fd:03", "fd:00", "fd: 3", - "fd:3:", "fd:3:file", "fd:3:stream", "fd:3:exported:", "fd::exported", "fd:4096:exported", + "fd:3:", "fd:3:file:", "fd:3:file:01", "fd:3:file:4096", "fd:3:stream", "fd:3:exported:", "fd::exported", "fd:4096:exported", ] { assert_eq!(endowed_fd(other), None, "{other}"); } diff --git a/sdk/toyos/src/fd/table.rs b/sdk/toyos/src/fd/table.rs index 9c2008ad473..941278113ff 100644 --- a/sdk/toyos/src/fd/table.rs +++ b/sdk/toyos/src/fd/table.rs @@ -113,6 +113,16 @@ impl Table { self.set(i, slot, inheritable) } + /// An endowed alias at its exact number, without displacing an earlier + /// endowment. Both descriptors share one object and one final release. + pub(super) fn alias_at(&mut self, source: Fd, fd: Fd) -> Result<(), Fault> { + let i = usize::try_from(fd).map_err(|_| Error::BadFd)?; + if self.fds.get(i) != Some(&0) { return Err(Error::BadFd.into()); } + let (_, object, _) = self.entry(source)?; + self.reference(object)?; + self.set(i, object, true) + } + /// A second descriptor on `fd`'s object, the lowest free at or above /// `floor`. pub(super) fn dup(&mut self, fd: Fd, floor: Fd, inheritable: bool) -> Result { @@ -287,6 +297,17 @@ mod tests { assert_eq!(t.install(o(3), false, 0), Ok(0), "a refusal took a number or an object"); } + #[test] + fn endowed_aliases_share_one_object_and_cannot_displace_an_endowment() { + let mut t = T::new(); + t.install_at(3, o(1), true).unwrap(); + t.alias_at(3, 0).unwrap(); + assert_eq!(held(&t), 1); + assert_eq!(t.alias_at(3, 0), Err(Error::BadFd.into())); + assert_eq!(w(t.close(3)), Ok(None)); + assert_eq!(w(t.close(0)), Ok(Some(1))); + } + /// A descriptor naming no object is a broken table, which the table /// answers rather than panics on: the owner panics once its lock is let /// go. diff --git a/sdk/toyos/src/files.rs b/sdk/toyos/src/files.rs index bd43c34a005..d2d37ea8063 100644 --- a/sdk/toyos/src/files.rs +++ b/sdk/toyos/src/files.rs @@ -227,6 +227,7 @@ extern "C" { fn toyos_fs_truncate(fd: Fd, size: u64) -> i64; fn toyos_fs_fsync(fd: Fd) -> i64; fn toyos_fs_stream(fd: Fd) -> i64; + fn toyos_fs_export(word: u64) -> i64; fn toyos_fs_image(fd: Fd) -> i64; fn toyos_fs_mkdir(at: Fd, path: *const u8, len: usize) -> i64; fn toyos_fs_unlink(at: Fd, path: *const u8, len: usize, dir: bool) -> i64; @@ -311,6 +312,25 @@ pub fn stream(fd: Fd) -> Result { decode(unsafe { toyos_fs_stream(fd) }).map(|h| OwnedHandle(toyos_abi::RawHandle(h as u32))) } +/// Export a served description for another process, sharing its access mode +/// and offset. The returned connection becomes an `fd::file` endowment. +pub fn export(fd: Fd) -> Result { + fd::with(fd, export_held).map_err(Error::Fd)? +} + +/// The object remains held across the owner ABI call, including if its +/// descriptor is concurrently closed and its number reused. +pub(crate) fn export_held(object: fd::Lent<'_>) -> Result { + if object.tag() != fd::SERVED_FILE { return Err(Error::NotAFile); } + // SAFETY: the caller's Lent keeps this object alive for the call. + decode(unsafe { toyos_fs_export(object.word()) }).map(|h| OwnedHandle(toyos_abi::RawHandle(h as u32))) +} + +#[cfg(all(feature = "fd-table", not(test)))] +pub(crate) fn adopt_exported(handle: OwnedHandle) -> Result { + owner::adopt_exported(handle) +} + pub fn mkdir(at: At, path: &[u8]) -> Result<(), Error> { // SAFETY: `path` is live for the call. decode(unsafe { toyos_fs_mkdir(at.word(), path.as_ptr(), path.len()) }).map(drop) diff --git a/sdk/toyos/src/files/owner.rs b/sdk/toyos/src/files/owner.rs index 3cc04cfb94d..adb29873edc 100644 --- a/sdk/toyos/src/files/owner.rs +++ b/sdk/toyos/src/files/owner.rs @@ -75,8 +75,8 @@ fn is_directory(_: &str) -> Result { Err(Error::Served(fs::Error::I /// description is shared with another process, which is for good, and a /// directory's from the start, as the server refuses to move one. /// -/// [`Position::stream`] is the one request that shares a description, so it -/// hands the offset over itself; another that ever does belongs beside it. +/// [`Position::export`] and [`Position::stream`] share descriptions, so each +/// hands the offset over before another process can use it. /// /// Reached only under the capability's turn, so a read or write and the /// offset's move are one step against every other thread's. @@ -153,6 +153,17 @@ impl Position { Ok(landed) } + /// Share this description with another process. Once handed over, every + /// read, write and seek uses the server's one offset. + fn export(&self, dir: &mut Dir, fid: Fid) -> Result { + let mut at = self.at.lock(); + if let Offset::Here(offset) = *at { + dir.seek(fid, fs::SEEK_START, offset as i64)?; + *at = Offset::Server; + } + dir.export(&[fid]) + } + /// A stream the server writes at the description's offset for another /// process ([`Dir::stream`]): the offset is handed to the server first, /// and is its from then on. @@ -350,6 +361,10 @@ fn on_fd_held( /// A descriptor for `held`; on a full table it is let go again, which takes /// its capability's turn, so no turn may be held here. fn install(held: Held, cloexec: bool) -> Result { + fd::install(object(held), !cloexec, 0) +} + +fn object(held: Held) -> Object { let held = Arc::new(held); let word = { let mut open = OPEN.lock(); @@ -364,7 +379,31 @@ fn install(held: Held, cloexec: bool) -> Result { } } }; - fd::install(Object::tagged(SERVED_FILE, word as u64, release_held), !cloexec, 0) + Object::tagged(SERVED_FILE, word as u64, release_held) +} + +/// Take up the one description exported for an endowed file descriptor. +/// This constructs an object without entering the descriptor table again. +pub(super) fn adopt_exported(handle: crate::OwnedHandle) -> Result { + let mut dir = Dir::adopt(crate::ipc::Connection::received(handle)).map_err(Error::Served)?; + let fid = Fid { id: 1, generation: dir.generation() }; + dir.fstat(fid).map_err(Error::Served)?; + Ok(object(Held::Served(Served { + cap: Arc::new(Capability { dir: Turnstile::new(dir) }), + fid, + position: Position { append: false, at: Lock::new(Offset::Server) }, + holder: None, + }))) +} + +#[no_mangle] +extern "C" fn toyos_fs_export(word: u64) -> i64 { + let held = held(word).expect("a held descriptor names this file object"); + let exported = match &*held { + Held::Served(file) => file.position.export(&mut file.cap.dir.lock(), file.fid).map_err(Error::Served), + Held::Directory(_) => Err(Error::Served(fs::Error::Unsupported)), + }; + answer(exported.map(|conn| u64::from(conn.into_raw().0))) } /// What a server answered, as this module says it. diff --git a/sdk/toyos/src/launch.rs b/sdk/toyos/src/launch.rs index 6a742bf6def..bd0026f0e95 100644 --- a/sdk/toyos/src/launch.rs +++ b/sdk/toyos/src/launch.rs @@ -83,6 +83,7 @@ const fn kind_word(kind: Kind) -> u32 { match kind { Kind::Handle => 0, Kind::Exported => 1, + Kind::File => 2, } } @@ -91,6 +92,7 @@ const fn kind_of(word: u32) -> Option { match word { 0 => Some(Kind::Handle), 1 => Some(Kind::Exported), + 2 => Some(Kind::File), _ => None, } } @@ -467,14 +469,14 @@ mod tests { env: b"", cwd: "/", extras: Vec::new(), - fds: vec![(0, (), Kind::Handle), (1, (), Kind::Exported)], + fds: vec![(0, (), Kind::Handle), (1, (), Kind::Exported), (2, (), Kind::File)], parent: Parent::Detached, job: (), }; let len = request.encode(&mut buf).expect("encode a launch"); let decoded = Request::decode(&buf[..len]).expect("decode it"); - assert_eq!(decoded.fds().collect::>(), [(0, Some(Kind::Handle)), (1, Some(Kind::Exported))]); - assert_eq!(decoded.handle_count(), 3, "two descriptors and the job"); + assert_eq!(decoded.fds().collect::>(), [(0, Some(Kind::Handle)), (1, Some(Kind::Exported)), (2, Some(Kind::File))]); + assert_eq!(decoded.handle_count(), 4, "three descriptors and the job"); let second_kind = HEADER + FD_BYTES + 4; buf[second_kind..second_kind + 4].copy_from_slice(&7u32.to_le_bytes()); let decoded = Request::decode(&buf[..len]).expect("decode it"); diff --git a/sdk/toyos/src/spawn.rs b/sdk/toyos/src/spawn.rs index 75ab96fc79d..57c364c6d30 100644 --- a/sdk/toyos/src/spawn.rs +++ b/sdk/toyos/src/spawn.rs @@ -101,6 +101,8 @@ pub enum Refused { Stream(RawHandle), /// netstack made no holder of a stream the child was to be given. Export(crate::net::NetError), + /// The filesystem refused to export a shared description. + File(crate::files::Error), /// The launcher did not start a program it permits, or its answer was lost. NotStarted, /// The kernel refused. @@ -215,6 +217,7 @@ impl Helper<'_> { None => None, }; let mut number = [0u8; 10]; + let mut group_number = [0u8; 10]; let mut put = |label: &[&[u8]], handle: RawHandle, count: &mut usize, used: &mut usize| { let len: usize = label.iter().map(|part| part.len()).sum(); let room = labels.get_mut(*used..*used + len).ok_or(Refused::Invalid("endowment labels past what a spawn carries"))?; @@ -238,8 +241,15 @@ impl Helper<'_> { for &(fd, handle, kind) in s.fds { let copy = handle.try_clone_to_owned().map_err(Refused::Syscall)?; let digits = decimal(fd, &mut number); - let label = [FD_PREFIX.as_bytes(), digits, kind.suffix().as_bytes()]; - put(&label, copy.as_handle().raw(), &mut count, &mut used)?; + if kind == FdKind::File { + // Duplicated file actions may name one exported connection + // several times. The child adopts it once and aliases its fd. + let group = s.fds.iter().find(|(_, h, k)| *k == FdKind::File && h.raw() == handle.raw()) + .expect("this descriptor is in the spawn").0; + put(&[FD_PREFIX.as_bytes(), digits, b":file:", decimal(group, &mut group_number)], copy.as_handle().raw(), &mut count, &mut used)?; + } else { + put(&[FD_PREFIX.as_bytes(), digits, kind.suffix().as_bytes()], copy.as_handle().raw(), &mut count, &mut used)?; + } moved.push(copy); } let (image, image_len) = image.map_or((0, 0), Image::spawn_words); @@ -292,6 +302,11 @@ impl Independent<'_> { /// carries. A job the spawn endowed goes itself, moved as an endowment is. pub fn launch(self) -> Result { let mut s = self.0; + for (i, &(_, handle, kind)) in s.fds.iter().enumerate() { + if kind == FdKind::File && s.fds[..i].iter().any(|(_, prior, kind)| *kind == FdKind::File && prior.raw() == handle.raw()) { + return Err(Refused::Invalid("a launch needs a distinct exported connection for each file descriptor")); + } + } let held = match s.launcher { Some(named) => named.raw(), None => crate::endow::launcher().ok_or(Refused::Permission("this process holds no launcher"))?.as_handle().raw(), diff --git a/system/supervisor/src/main.rs b/system/supervisor/src/main.rs index 8430ac2f27e..1c3394d9cde 100644 --- a/system/supervisor/src/main.rs +++ b/system/supervisor/src/main.rs @@ -3178,6 +3178,9 @@ fn start<'a>( LaunchFd::Caller(handle, toyos::fd::Kind::Handle) => { command.inherit_handle(*fd, handle.as_handle().into()); } + LaunchFd::Caller(handle, toyos::fd::Kind::File) => { + command.inherit_file(*fd, handle.as_handle().into()); + } LaunchFd::Caller(handle, toyos::fd::Kind::Exported) => { command.inherit_exported(*fd, handle.as_handle().into()); } diff --git a/tests/guest/rust-tests/src/bin/served_file_offset.rs b/tests/guest/rust-tests/src/bin/served_file_offset.rs index c88d42bd4b8..319b8e5b64c 100644 --- a/tests/guest/rust-tests/src/bin/served_file_offset.rs +++ b/tests/guest/rust-tests/src/bin/served_file_offset.rs @@ -13,10 +13,32 @@ use std::fs::{self, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; use std::process::{Command, Stdio}; +use std::os::fd::AsRawFd; const FILE: &str = "/home/served-file-offset"; fn main() { + match std::env::args().nth(1).as_deref() { + Some("--read-three") => { + let mut bytes = [0; 3]; + assert_eq!(toyos::files::read(0, &mut bytes).unwrap(), bytes.len()); + std::io::stdout().write_all(&bytes).unwrap(); + return; + } + Some("--aliases") => { + let mut bytes = [0; 3]; + assert_eq!(toyos::files::read(0, &mut bytes).unwrap(), 3); + assert_eq!(&bytes, b"abc"); + assert_eq!(toyos::files::read(7, &mut bytes).unwrap(), 3); + assert_eq!(&bytes, b"def"); + toyos::files::seek(7, toyos::files::SeekFrom::Start(1)).unwrap(); + assert_eq!(toyos::files::read(0, &mut bytes[..1]).unwrap(), 1); + assert_eq!(bytes[0], b'b'); + assert_eq!(toyos::files::write(7, b"Z").unwrap(), 1); + return; + } + _ => {} + } let _ = fs::remove_file(FILE); let mut f = OpenOptions::new().read(true).write(true).create(true).truncate(true).open(FILE).expect("open"); f.write_all(b"0123456789").expect("write"); @@ -58,7 +80,61 @@ fn main() { assert!(status.success(), "echo: {status}"); f.write_all(b"!").unwrap(); assert_eq!(fs::read(FILE).unwrap(), b"01abCHILD\n!y", "the child's bytes at the shared offset, then the parent's"); - drop(f); + f.set_len(0).unwrap(); + f.rewind().unwrap(); + f.write_all(b"abcdefghij").unwrap(); + f.rewind().unwrap(); + let own = std::env::args().next().unwrap(); + let mut command = Command::new(&own); + command.arg("--read-three").stdin(Stdio::from(f.try_clone().unwrap())); + for expected in [b"abc", b"def"] { + let out = command.output().expect("repeat a command with the same file input"); + assert!(out.status.success()); + assert_eq!(out.stdout, expected); + } + assert_eq!(f.stream_position().unwrap(), 6, "children consume exactly what they read"); + f.read_exact(&mut buf[..2]).unwrap(); + assert_eq!(&buf[..2], b"gh", "parent resumes at the shared offset"); + f.rewind().unwrap(); + posix_aliases(&own, f.as_raw_fd()); + assert_eq!(f.stream_position().unwrap(), 3, "dup2 aliases and parent share one offset"); + assert_eq!(fs::read(FILE).unwrap(), b"abZdefghij"); fs::remove_file(FILE).unwrap(); + assert_eq!(toyos::files::fstat(f.as_raw_fd()).unwrap().nlink, 0, "an open unlinked file has no cached name count"); + f.rewind().unwrap(); + assert_eq!(f.read(&mut buf).unwrap(), 3); + assert_eq!(&buf, b"abZ", "open description survives unlink"); + drop(f); println!("served_file_offset: passed"); } + +#[repr(C)] +struct FileActions { actions: usize, reserved: [usize; 7] } + +#[link(name = "toyos_c", kind = "static")] +unsafe extern "C" { + fn posix_spawn_file_actions_init(fa: *mut FileActions) -> i32; + fn posix_spawn_file_actions_adddup2(fa: *mut FileActions, fd: i32, newfd: i32) -> i32; + fn posix_spawn_file_actions_destroy(fa: *mut FileActions) -> i32; + fn posix_spawn(pid: *mut i32, path: *const u8, fa: *const FileActions, attr: *const u8, argv: *const *const u8, envp: *const *const u8) -> i32; + fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; +} + +fn posix_aliases(path: &str, fd: i32) { + let path = std::ffi::CString::new(path).unwrap(); + let arg = c"--aliases"; + let argv = [path.as_ptr().cast(), arg.as_ptr().cast(), std::ptr::null()]; + let env = [std::ptr::null()]; + let mut actions = FileActions { actions: 0, reserved: [0; 7] }; + let (mut pid, mut status) = (0, 0); + // SAFETY: initialized file-actions storage and live, terminated C arrays. + unsafe { + assert_eq!(posix_spawn_file_actions_init(&mut actions), 0); + assert_eq!(posix_spawn_file_actions_adddup2(&mut actions, fd, 0), 0); + assert_eq!(posix_spawn_file_actions_adddup2(&mut actions, 0, 7), 0); + assert_eq!(posix_spawn(&mut pid, path.as_ptr().cast(), &actions, std::ptr::null(), argv.as_ptr(), env.as_ptr()), 0); + assert_eq!(posix_spawn_file_actions_destroy(&mut actions), 0); + assert_eq!(waitpid(pid, &mut status, 0), pid); + } + assert_eq!(status, 0, "child checks both dup2 aliases"); +} From 265de0efa45e27f0c737b2edf7565523ce3242fc Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 19:35:01 +0100 Subject: [PATCH 15/93] Finish partial truncation without allocating file data Store the final partial sector as inline data with its suffix zeroed, retaining earlier checksummed slices and removing all later extents. This keeps shrinking possible after ordinary data capacity is exhausted and makes logged-truncate recovery independent of obtaining a fresh data block. Enable the format's inline-data feature and account inline sectors in the inode. Refuse overflowing reservation offsets before adding the block size. Add coverage for full-capacity truncate, imported extent slicing, reserved I/O failure, and the maximum-offset boundary. All twelve focused data tests pass, including plaintext/encrypted torn-write recovery. The per-token metadata reservation cost remains an unresolved integration blocker. --- lib/bcachefs/src/engine/fsck.rs | 7 +- lib/bcachefs/src/engine/mount/data.rs | 54 +++++++------ lib/bcachefs/src/engine/mount/mod.rs | 2 +- lib/bcachefs/src/engine/tests/data.rs | 105 +++++++++++++++++++++++++- 4 files changed, 140 insertions(+), 28 deletions(-) diff --git a/lib/bcachefs/src/engine/fsck.rs b/lib/bcachefs/src/engine/fsck.rs index e2cfb00c847..f6478178594 100644 --- a/lib/bcachefs/src/engine/fsck.rs +++ b/lib/bcachefs/src/engine/fsck.rs @@ -86,7 +86,7 @@ fn superblock(sb: &Superblock) -> Vec { let clean = sb.clean_flag(); let ext = sb.field(field_type::EXT).unwrap_or(&[]); let flags = sb.member.flags; - let expected_features = FEATURES | if sb.blacklist.is_empty() { 0 } else { 1 << 5 }; + let expected_features = FEATURES | h.features[0] & (1 << 8) | if sb.blacklist.is_empty() { 0 } else { 1 << 5 }; // Going read-write leaves the compat bits as they are (A1.41). let expected_compat = COMPAT_INITIALIZED | COMPAT_CLEAN; let has_replicas = |t: u8| sb.replicas.iter().any(|r| r.data_type == t && r.devs == [0] && r.required == 1); @@ -570,8 +570,11 @@ impl<'a> Walk<'a> { // Each inode's extents: their sectors, the furthest end, overlaps. let mut data: BTreeMap = BTreeMap::new(); let mut before: Option<(u64, u64)> = None; - for (&(ino, end), &(_, size)) in &self.extents { + for (&(ino, end), &(kind_, size)) in &self.extents { let pos = Bpos::new(ino, end, mkfs::ROOT_SNAPSHOT); + if kind_ == kind::INLINE_DATA && sb.header.features[0] & (1 << 8) == 0 { + found.push(Finding::Key { btree: Btree::Extents, pos, rule: "inline data without its feature bit" }); + } let start = end.saturating_sub(u64::from(size)); if before.is_some_and(|(i, e)| i == ino && start < e) { found.push(Finding::Key { btree: Btree::Extents, pos, rule: "overlapping the extent before it" }); diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index 59ed13a3451..4c4843d314c 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -89,7 +89,7 @@ impl Mount { pub async fn reserve(&self, ino: u64, offset: u64, len: usize) -> Result { let bytes = self.sb.block_bytes(); let first = offset / bytes as u64 * bytes as u64; - if len == 0 || offset.checked_add(len as u64).is_none_or(|end| end > first + bytes as u64 || end > i64::MAX as u64) { + if len == 0 || offset.checked_add(len as u64).is_none_or(|end| end > i64::MAX as u64 || end > first.saturating_add(bytes as u64)) { return Err(Error::Invalid("reservation outside one filesystem block")); } let _data = self.data_slot().await; @@ -299,7 +299,7 @@ impl Mount { Ok(true) }) .await?; - if old.len() > 1 { + if old.len() > 2 || (old.len() == 2 && !old.iter().any(|(k, _)| k.kind == kind::INLINE_DATA)) { return Err(Error::Unsupported("multiple data extents in one filesystem block")); } for (k, v) in &old { @@ -362,22 +362,22 @@ impl Mount { }) .await?; self.st.borrow_mut().clocks[1] += u64::from(physical); + retained.push((key, value)); + self.replace_data_keys(txn, inode, old, retained).await + } + + async fn replace_data_keys(&self, txn: &mut Txn, inode: &mut InodeValue, old: Vec<(Key, Vec)>, new: Vec<(Key, Vec)>) -> Result<()> { let mut buckets = BTreeMap::new(); - self.data_counted(txn, &mut buckets, &key, &value, true).await?; for (k, v) in old { self.data_counted(txn, &mut buckets, &k, &v, false).await?; self.set(txn, Btree::Extents, k.pos, None).await?; - inode.sectors = inode.sectors.checked_sub(if k.kind == kind::EXTENT { u64::from(k.size) } else { 0 }).ok_or(Error::Corrupt("inode sectors below its extents"))?; + inode.sectors = inode.sectors.checked_sub(allocated_sectors(&k)).ok_or(Error::Corrupt("inode sectors below its extents"))?; } - for (k, v) in retained { + for (k, v) in new { self.data_counted(txn, &mut buckets, &k, &v, true).await?; - if k.kind == kind::EXTENT { - inode.sectors += u64::from(k.size); - } + inode.sectors += allocated_sectors(&k); self.set(txn, Btree::Extents, k.pos, Some((k, v))).await?; } - self.set(txn, Btree::Extents, pos, Some((key, value))).await?; - inode.sectors += u64::from(key.size); self.data_buckets(txn, buckets).await?; Ok(()) } @@ -479,17 +479,23 @@ impl Mount { } pub(super) async fn trim_extents(&self, ino: u64, size: u64) -> Result<()> { - let block = self.sb.block_bytes() as u64; - let end = size.div_ceil(block) * block; - if !size.is_multiple_of(block) && self.extent_after(ino, size).await?.is_some_and(|(k, _)| start(&k).is_ok_and(|first| first < end)) { - let version = self.data_version().await?; + let end = size.div_ceil(SECTOR as u64) * SECTOR as u64; + if !size.is_multiple_of(SECTOR as u64) && self.extent_after(ino, size).await?.is_some_and(|(k, _)| start(&k).is_ok_and(|first| first < end)) { self.transact(K_TX, FOOTPRINT, Draw::Freeing, async |txn: &mut Txn| { let mut inode = self.inode_in(txn, ino).await?; - let first = size / block * block; - let mut bytes = alloc::vec![0u8; block as usize]; + let first = size / SECTOR as u64 * SECTOR as u64; + let (key, value) = self.extent_after(ino, first).await?.ok_or(Error::Corrupt("truncate tail extent disappeared"))?; + let mut bytes = alloc::vec![0u8; SECTOR]; Source::read(self, ino, first, &mut bytes).await?; bytes[(size - first) as usize..].fill(0); - self.replace_block(txn, ino, &mut inode, first, &bytes, version, None).await?; + let mut retained = Vec::new(); + let from = start(&key)?; + if from < first { + retained.push(slice(&key, &value, from, first)?); + } + let pos = Bpos::new(ino, end / SECTOR as u64, ROOT_SNAPSHOT); + retained.push((Key { size: 1, ..Key::new(kind::INLINE_DATA, pos) }, bytes)); + self.replace_data_keys(txn, &mut inode, alloc::vec![(key, value)], retained).await?; self.set_inode(txn, ino, &inode).await }) .await?; @@ -504,13 +510,11 @@ impl Mount { let mut buckets = BTreeMap::new(); self.data_counted(txn, &mut buckets, &key, &value, false).await?; self.set(txn, Btree::Extents, key.pos, None).await?; - inode.sectors = inode.sectors.checked_sub(if key.kind == kind::EXTENT { u64::from(key.size) } else { 0 }).ok_or(Error::Corrupt("inode sectors below its extents"))?; + inode.sectors = inode.sectors.checked_sub(allocated_sectors(&key)).ok_or(Error::Corrupt("inode sectors below its extents"))?; if first < end { let (k, v) = slice(&key, &value, first, end)?; self.data_counted(txn, &mut buckets, &k, &v, true).await?; - if k.kind == kind::EXTENT { - inode.sectors += u64::from(k.size); - } + inode.sectors += allocated_sectors(&k); self.set(txn, Btree::Extents, k.pos, Some((k, v))).await?; } self.data_buckets(txn, buckets).await?; @@ -548,6 +552,14 @@ fn checked_extent(v: &[u8]) -> Result { Ok(extent) } +fn allocated_sectors(k: &Key) -> u64 { + if matches!(k.kind, kind::EXTENT | kind::INLINE_DATA) { + u64::from(k.size) + } else { + 0 + } +} + fn slice(k: &Key, v: &[u8], first: u64, end: u64) -> Result<(Key, Vec)> { let old_first = start(k)?; let skip = ((first - old_first) / SECTOR as u64) as u32; diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 75517cb84b9..d96a068f34a 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -79,7 +79,7 @@ const MIN_NODE: usize = 32 << 10; /// Retries of a failed node or journal write, each of the same bytes. const RETRIES: usize = 2; /// Feature bits a volume carries once written read-write (A1.41). -const RW_FEATURES: u64 = 1 << 9 | 1 << 12 | 1 << 13 | 1 << 17 | 1 << 18; +const RW_FEATURES: u64 = 1 << 8 | 1 << 9 | 1 << 12 | 1 << 13 | 1 << 17 | 1 << 18; /// Feature bit 5: the superblock holds a blacklist. const BLACKLIST_FEATURE: u64 = 1 << 5; diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index 4ef9cc8163f..1ee55fad109 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -19,6 +19,8 @@ fn file_data_overwrites_sparse_ranges_and_truncation_survive_remount() { m.truncate(ino, 41_003, NOW).await.unwrap(); m.truncate(ino, want.len() as u64, NOW).await.unwrap(); want[41_003..].fill(0); + m.write(ino, 41_000, b"new tail", NOW).await.unwrap(); + want[41_000..41_008].copy_from_slice(b"new tail"); m.commit().await.unwrap(); ino }); @@ -51,12 +53,13 @@ fn orphan_release_reclaims_file_data_and_keeps_other_files() { fn encrypted_data_uses_authenticated_extents_and_durable_version_ranges() { let dev = formatted(64 << 20, Some(Encryption { passphrase: b"test", kdf: MIN_KDF })); let m = block_on_ready(Mount::unlock(&dev, b"test", LIMITS, MEMORY, random())).unwrap(); - let ino = block_on_ready(async { + let (ino, old_version) = block_on_ready(async { let ino = m.create(m.root(), b"secret", New::File, OWNER, NOW).await.unwrap().ino; m.write(ino, 0, &[0x45; 8192], NOW).await.unwrap(); m.write(ino, 100, b"changed", NOW).await.unwrap(); + let old_version = super::super::read::Source::get(&m, Btree::Extents, Bpos::new(ino, 8, u32::MAX)).await.unwrap().unwrap().0.version.lo; m.truncate(ino, 200, NOW).await.unwrap(); - ino + (ino, old_version) }); block_on_ready(m.unmount()).unwrap(); let v = block_on_ready(Volume::unlock(&dev, b"test", LIMITS)).unwrap(); @@ -64,8 +67,6 @@ fn encrypted_data_uses_authenticated_extents_and_durable_version_ranges() { want[100..107].copy_from_slice(b"changed"); assert_eq!(read_all(&v, ino).unwrap(), want); assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); - let first = block_on_ready(v.get(Btree::Extents, Bpos::new(ino, 8, u32::MAX))).unwrap().unwrap(); - let old_version = first.0.version.lo; assert!(old_version > 0 && old_version <= v.key_version()); drop(v); let m = block_on_ready(Mount::unlock(&dev, b"test", LIMITS, MEMORY, random())).unwrap(); @@ -216,3 +217,99 @@ fn symlink_targets_are_file_data_and_read_after_remount() { assert_eq!(block_on_ready(v.readlink(path(&v, "link"))).unwrap(), b"../target"); assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } + +#[test] +fn partial_truncate_needs_no_new_data_block_at_capacity() { + let dev = formatted(64 << 20, None); + let ino = mounted(&dev, async |m: &Mount<&Image>| { + let ino = m.create(m.root(), b"f", New::File, OWNER, NOW).await.unwrap().ino; + m.write(ino, 0, &[0x77; 8192], NOW).await.unwrap(); + let mut tokens = Vec::new(); + loop { + match m.reserve(ino, 0, BLOCK).await { + Ok(token) => tokens.push(token), + Err(Error::NoSpace(_)) => break, + Err(e) => panic!("reserve: {e:?}"), + } + } + m.truncate(ino, 700, NOW).await.unwrap(); + drop(tokens); + m.truncate(ino, 8192, NOW).await.unwrap(); + ino + }); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + let mut want = vec![0; 8192]; + want[..700].fill(0x77); + assert_eq!(read_all(&v, ino).unwrap(), want); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} + +#[test] +fn a_reservation_past_the_maximum_file_size_is_refused_without_overflow() { + let dev = formatted(64 << 20, None); + mounted(&dev, async |m: &Mount<&Image>| { + assert!(matches!(m.reserve(m.root(), u64::MAX - 1, 1).await, Err(Error::Invalid(_)))); + }); +} + +#[test] +fn a_failed_reserved_data_write_keeps_the_old_contents_and_reports_failure() { + use core::cell::Cell; + use toyos_blockdev::Block; + struct Failing<'a> { + dev: &'a Image, + attempts: Cell, + } + impl BlockDev for Failing<'_> { + fn blocks(&self) -> u64 { + self.dev.blocks() + } + async fn read(&self, lba: u64, out: &mut [Block]) -> core::result::Result<(), toyos_blockdev::Error> { + self.dev.read(lba, out).await + } + async fn write(&self, lba: u64, data: &[Block]) -> core::result::Result<(), toyos_blockdev::Error> { + self.dev.write(lba, data).await + } + async fn flush(&self) -> core::result::Result<(), toyos_blockdev::Error> { + self.dev.flush().await + } + async fn write_data(&self, _: u64, _: &[Block]) -> core::result::Result<(), toyos_blockdev::Error> { + self.attempts.set(self.attempts.get() + 1); + Err(toyos_blockdev::Error::Device) + } + } + let dev = formatted(64 << 20, None); + let ino = mounted(&dev, async |m: &Mount<&Image>| { + let ino = m.create(m.root(), b"f", New::File, OWNER, NOW).await.unwrap().ino; + m.write(ino, 0, &[0x22; BLOCK], NOW).await.unwrap(); + ino + }); + let failing = Failing { dev: &dev, attempts: Cell::new(0) }; + let m = block_on_ready(Mount::open(&failing, MEMORY, random())).unwrap(); + let mut token = block_on_ready(m.reserve(ino, 0, BLOCK)).unwrap(); + assert_eq!(block_on_ready(m.write_reserved(&mut token, 0, &[0x44; BLOCK], NOW)), Err(Error::Device(toyos_blockdev::Error::Device))); + assert_eq!(failing.attempts.get(), 3); + assert!(matches!(block_on_ready(m.commit()), Err(Error::ReadOnly(_)))); + let mut got = [0; BLOCK]; + block_on_ready(m.read(ino, 0, &mut got)).unwrap(); + assert_eq!(got, [0x22; BLOCK]); +} + +#[test] +fn a_large_imported_extent_is_sliced_without_rewriting_its_neighbors() { + let original: Vec = (0..50_000).map(|i| (i * 11) as u8).collect(); + let mut tree = Tree::new(); + tree.file(Tree::ROOT, b"f", &original, Meta { perm: 0o644, uid: 0, gid: 0 }).unwrap(); + let dev = populated(64 << 20, &tree, 13); + let ino = mounted(&dev, async |m: &Mount<&Image>| { + let ino = m.lookup(m.root(), b"f").await.unwrap().unwrap().ino; + let mut token = m.reserve(ino, 4500, 100).await.unwrap(); + m.write_reserved(&mut token, 4500, &[0x66; 100], NOW).await.unwrap(); + ino + }); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + let mut want = original; + want[4500..4600].fill(0x66); + assert_eq!(read_all(&v, ino).unwrap(), want); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} From 70d982f85737cebd42cf404495e750ab993b2d31 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 19:36:36 +0100 Subject: [PATCH 16/93] std: remove obsolete ToyOS filesystem routing export All ToyOS filesystem paths now use the process namespace in userland. Remove the platform dispatch reexport of the deleted kernel-versus-served routing predicate. --- rust | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust b/rust index 8004d62da51..aa0d9be2c2b 160000 --- a/rust +++ b/rust @@ -1 +1 @@ -Subproject commit 8004d62da51296c71857a71ee956defa43fea105 +Subproject commit aa0d9be2c2b2b04fc51ad966a4a5d8ba137c589e From 4d73d0aa416f63c3f60b69e84c2b26c28e818c00 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 19:41:27 +0100 Subject: [PATCH 17/93] Simplify file-server allocation and keep lint checks clean Box the mounted probe result, use a concrete flush-future type, and clean affected file-server lint findings. Empty direct writes take no reservation, and metadata failure cancels an accepted reservation. Validation: fileserver library clippy with warnings denied passes. --- system/fileserver/src/bounds.rs | 6 +- system/fileserver/src/cache.rs | 11 +- system/fileserver/src/data.rs | 310 ++++++++++++++++++++------ system/fileserver/src/descriptions.rs | 2 +- system/fileserver/src/fat.rs | 20 +- system/fileserver/src/main.rs | 2 +- system/fileserver/src/pages/tests.rs | 2 +- system/fileserver/src/volume.rs | 10 +- 8 files changed, 265 insertions(+), 98 deletions(-) diff --git a/system/fileserver/src/bounds.rs b/system/fileserver/src/bounds.rs index 77a026c0139..e5071b901c0 100644 --- a/system/fileserver/src/bounds.rs +++ b/system/fileserver/src/bounds.rs @@ -110,10 +110,8 @@ pub fn hello( of_share: usize, window: impl FnOnce() -> Option>, ) -> Hello { - if slot == Slot::AtHello { - if let Err(why) = served_slot(served, of_share) { - return Hello::Refused(why); - } + if slot == Slot::AtHello && let Err(why) = served_slot(served, of_share) { + return Hello::Refused(why); } match window() { None => Hello::Dropped("its hello carried no window"), diff --git a/system/fileserver/src/cache.rs b/system/fileserver/src/cache.rs index 9f87465dc81..94a47a85ad2 100644 --- a/system/fileserver/src/cache.rs +++ b/system/fileserver/src/cache.rs @@ -35,7 +35,6 @@ use std::cell::RefCell; use std::collections::{BTreeMap, VecDeque}; use std::future::{poll_fn, Future}; -use std::pin::Pin; use std::rc::Rc; use std::task::{Poll, Waker}; @@ -210,11 +209,9 @@ impl Cache { match inner.slots.get(&at) { Some(slot) => *data = **slot.data, None => { - if inner.order.try_reserve(1).is_ok() { - if let Some(data) = boxed(data) { - inner.slots.insert(at, Slot { data: Rc::new(data), dirty: false, version: 0 }); - inner.order.push_back(at); - } + if inner.order.try_reserve(1).is_ok() && let Some(data) = boxed(data) { + inner.slots.insert(at, Slot { data: Rc::new(data), dirty: false, version: 0 }); + inner.order.push_back(at); } } } @@ -378,7 +375,7 @@ impl BlockDev for Cache { self.inner.borrow_mut().flushing = true; let _flushing = Flushing(self); let mut runs = self.dirty_runs().into_iter(); - let mut flight: Vec> + '_>>> = Vec::new(); + let mut flight = Vec::new(); let mut refused = Ok(()); poll_fn(|cx| loop { while refused.is_ok() && flight.len() < self.disk.depth() { diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index a32f3bc3efa..e5b497760b8 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -1,13 +1,13 @@ //! Writable bcachefs DATA. Descriptions hold inode identities; buffered //! sizes and capacity reservations stay here until their pages are written. +use crate::cache::{Cache, Disk}; +use crate::volume::{Kind, Listed, Meta, Node, OpenHow, Out, Reserved, Volume, parent}; +use bcachefs::engine::{self, DirentType, Mount, MountOptions, New, Owner, Reservation}; use std::collections::BTreeMap; use std::rc::Rc; -use bcachefs::engine::{self, DirentType, Mount, MountOptions, New, Owner, Reservation}; use toyos::fs::Error; -use toyos_blockdev::{BlockDev, BLOCK}; -use crate::cache::{Cache, Disk}; -use crate::volume::{parent, Kind, Listed, Meta, Node, OpenHow, Out, Reserved, Volume}; +use toyos_blockdev::{BLOCK, BlockDev}; const NODES: usize = 512; const READ_CHUNK: usize = 256 << 10; @@ -18,7 +18,10 @@ impl toyos_crypto::Random for OsRandom { getrandom::fill(out).map_err(|_| toyos_crypto::Error::Entropy) } } -struct Pending { ticket: u64, reservation: Reservation } +struct Pending { + ticket: u64, + reservation: Reservation, +} struct Open { holders: usize, size: u64, @@ -34,9 +37,17 @@ pub struct DataVolume { next_ticket: u64, clock: fn() -> u64, } -pub enum Probed { Mounted(DataVolume), Unmountable(String), Foreign } +pub enum Probed { + Mounted(Box>), + Unmountable(String), + Foreign, +} #[derive(Debug, PartialEq, Eq)] -pub enum Located { Nowhere, Served([u8; 16]), Refused(String) } +pub enum Located { + Nowhere, + Served([u8; 16]), + Refused(String), +} pub fn find(served: Result<&[[u8; 16]], String>) -> Located { match served { Err(why) => Located::Refused(why), @@ -61,7 +72,10 @@ fn word(e: engine::Error) -> Error { } } fn mapped(operation: &str, result: engine::Result) -> Result { - result.map_err(|error| { println!("fileserver: DATA {operation}: {error:?}"); word(error) }) + result.map_err(|error| { + println!("fileserver: DATA {operation}: {error:?}"); + word(error) + }) } fn kind(inode: &engine::Inode) -> Result { match inode.kind() { @@ -71,11 +85,14 @@ fn kind(inode: &engine::Inode) -> Result { _ => Err(Error::Unsupported), } } -fn time(value: u64) -> i64 { value.min(i64::MAX as u64) as i64 } +fn time(value: u64) -> i64 { + value.min(i64::MAX as u64) as i64 +} impl DataVolume { pub fn probe(disk: D, clean_limit: usize, roots: &[&str], clock: fn() -> u64) -> Probed { let cache = Rc::new(Cache::new(disk, clean_limit)); - let first = cache.block_on(Mount::open(Rc::clone(&cache), MountOptions { cache_nodes: NODES }, Box::new(OsRandom))); + let first = + cache.block_on(Mount::open(Rc::clone(&cache), MountOptions { cache_nodes: NODES }, Box::new(OsRandom))); let fs = match first { Ok(fs) => fs, Err(error) => { @@ -86,33 +103,51 @@ impl DataVolume { if bcachefs::designated(&block[0]) != Some(cache.blocks()) { return if bcachefs::carries_volume(block.as_flattened()) { Probed::Unmountable(format!("{error:?}; an existing volume cannot be reformatted")) - } else { Probed::Foreign }; + } else { + Probed::Foreign + }; } println!("fileserver: formatting the explicitly designated DATA partition as bcachefs"); let options = engine::Options { label: b"ToyOS DATA", time: clock(), encryption: None }; - if let Err(error) = cache.block_on(engine::mkfs(&cache, &options, &engine::Tree::new(), &mut OsRandom)) { + if let Err(error) = cache.block_on(engine::mkfs(&cache, &options, &engine::Tree::new(), &mut OsRandom)) + { return Probed::Unmountable(format!("format: {error:?}")); } - match cache.block_on(Mount::open(Rc::clone(&cache), MountOptions { cache_nodes: NODES }, Box::new(OsRandom))) { + match cache.block_on(Mount::open( + Rc::clone(&cache), + MountOptions { cache_nodes: NODES }, + Box::new(OsRandom), + )) { Ok(fs) => fs, Err(error) => return Probed::Unmountable(format!("new volume: {error:?}")), } } }; - Self::over(fs, cache, roots, clock).map_or_else(Probed::Unmountable, Probed::Mounted) + Self::over(fs, cache, roots, clock).map_or_else(Probed::Unmountable, |volume| Probed::Mounted(Box::new(volume))) } fn over(fs: Mount>>, cache: Rc>, roots: &[&str], clock: fn() -> u64) -> Result { // Mount recovery precedes every live description. cache.block_on(fs.release_orphans()).map_err(|error| format!("orphan recovery: {error:?}"))?; - let mut volume = Self { fs, cache, roots: roots.iter().map(|root| root.to_string()).collect(), open: BTreeMap::new(), next_ticket: 1, clock }; + let mut volume = Self { + fs, + cache, + roots: roots.iter().map(|root| root.to_string()).collect(), + open: BTreeMap::new(), + next_ticket: 1, + clock, + }; for root in roots { let mut current = String::new(); for part in root.split('/').filter(|part| !part.is_empty()) { - if !current.is_empty() { current.push('/'); } + if !current.is_empty() { + current.push('/'); + } current.push_str(part); match volume.lstat(¤t) { Ok(meta) if meta.kind == Kind::Dir => {} - Err(Error::NotFound) => volume.mkdir(¤t).map_err(|error| format!("root {current}: {error:?}"))?, + Err(Error::NotFound) => { + volume.mkdir(¤t).map_err(|error| format!("root {current}: {error:?}"))? + } other => return Err(format!("root {current} is not a directory: {other:?}")), } } @@ -121,26 +156,36 @@ impl DataVolume { Ok(volume) } fn lookup(&self, path: &str) -> Result { - self.cache.block_on(async { - let mut ino = self.fs.root(); - for name in path.split('/').filter(|name| !name.is_empty()) { - ino = self.fs.lookup(ino, name.as_bytes()).await?.ok_or(engine::Error::NotFound)?.ino; - } - self.fs.inode(ino).await - }).map_err(word) + self.cache + .block_on(async { + let mut ino = self.fs.root(); + for name in path.split('/').filter(|name| !name.is_empty()) { + ino = self.fs.lookup(ino, name.as_bytes()).await?.ok_or(engine::Error::NotFound)?.ino; + } + self.fs.inode(ino).await + }) + .map_err(word) } fn destination<'a>(&self, path: &'a str) -> Result<(u64, &'a [u8]), Error> { let name = path.rsplit('/').next().filter(|name| !name.is_empty()).ok_or(Error::Root)?; let dir = self.lookup(parent(path))?; - if !dir.is_dir() { return Err(Error::NotADirectory); } + if !dir.is_dir() { + return Err(Error::NotADirectory); + } Ok((dir.ino, name.as_bytes())) } fn meta(&self, inode: &engine::Inode) -> Result { let kind = kind(inode)?; - let (size, mtime) = self.open.get(&inode.ino).map(|open| (open.size, open.mtime)).unwrap_or((inode.size, inode.mtime.max(0) as u64)); + let (size, mtime) = self + .open + .get(&inode.ino) + .map(|open| (open.size, open.mtime)) + .unwrap_or((inode.size, inode.mtime.max(0) as u64)); Ok(Meta { kind, size: if kind == Kind::Dir { 0 } else { size }, mtime }) } - fn protected(&self, path: &str) -> bool { path.is_empty() || self.roots.iter().any(|root| root == path) } + fn protected(&self, path: &str) -> bool { + path.is_empty() || self.roots.iter().any(|root| root == path) + } fn released(&self, removed: Option) -> Result<(), Error> { if let Some(removed) = removed.filter(|removed| removed.last_link && !self.open.contains_key(&removed.node)) { match self.cache.block_on(self.fs.inode(removed.node)) { @@ -153,8 +198,12 @@ impl DataVolume { } fn persist(&mut self, node: Node) -> Result<(), Error> { let open = self.open.get(&node).ok_or(Error::Stale)?; - if !open.pages.is_empty() { return Err(Error::Io); } - if !open.changed { return Ok(()); } + if !open.pages.is_empty() { + return Err(Error::Io); + } + if !open.changed { + return Ok(()); + } let (size, mtime) = (open.size, open.mtime); mapped("length", self.cache.block_on(self.fs.truncate(node, size, time(mtime))))?; mapped("mtime", self.cache.block_on(self.fs.set_mtime(node, time(mtime))))?; @@ -165,8 +214,12 @@ impl DataVolume { pub(crate) fn format(disk: D, clean_limit: usize, roots: &[&str], clock: fn() -> u64) -> Result { let cache = Rc::new(Cache::new(disk, clean_limit)); let options = engine::Options { label: b"test DATA", time: clock(), encryption: None }; - cache.block_on(engine::mkfs(&cache, &options, &engine::Tree::new(), &mut OsRandom)).map_err(|error| format!("{error:?}"))?; - let fs = cache.block_on(Mount::open(Rc::clone(&cache), MountOptions { cache_nodes: NODES }, Box::new(OsRandom))).map_err(|error| format!("{error:?}"))?; + cache + .block_on(engine::mkfs(&cache, &options, &engine::Tree::new(), &mut OsRandom)) + .map_err(|error| format!("{error:?}"))?; + let fs = cache + .block_on(Mount::open(Rc::clone(&cache), MountOptions { cache_nodes: NODES }, Box::new(OsRandom))) + .map_err(|error| format!("{error:?}"))?; Self::over(fs, cache, roots, clock) } #[cfg(test)] @@ -177,28 +230,44 @@ impl DataVolume { } } impl Volume for DataVolume { - fn writable(&self) -> bool { true } - fn lstat(&mut self, path: &str) -> Result { self.meta(&self.lookup(path)?) } + fn writable(&self) -> bool { + true + } + fn lstat(&mut self, path: &str) -> Result { + self.meta(&self.lookup(path)?) + } fn read_link(&mut self, path: &str) -> Result { let inode = self.lookup(path)?; - if kind(&inode)? != Kind::Symlink { return Err(Error::NotALink); } + if kind(&inode)? != Kind::Symlink { + return Err(Error::NotALink); + } let bytes = mapped("readlink", self.cache.block_on(self.fs.readlink(inode.ino)))?; String::from_utf8(bytes).map_err(|_| Error::Io) } fn list(&mut self, dir: &str) -> Result, Error> { let inode = self.lookup(dir)?; - if !inode.is_dir() { return Err(Error::NotADirectory); } + if !inode.is_dir() { + return Err(Error::NotADirectory); + } let mut cursor = 0; let mut listed = Vec::new(); loop { let (entries, more) = mapped("readdir", self.cache.block_on(self.fs.readdir(inode.ino, cursor, 256)))?; - if entries.is_empty() && more { return Err(Error::Io); } + if entries.is_empty() && more { + return Err(Error::Io); + } for entry in entries { cursor = entry.next; let inode = mapped("directory entry", self.cache.block_on(self.fs.inode(entry.ino)))?; - listed.push(Listed { name: String::from_utf8(entry.name).map_err(|_| Error::Io)?, meta: self.meta(&inode)?, identity: entry.ino }); + listed.push(Listed { + name: String::from_utf8(entry.name).map_err(|_| Error::Io)?, + meta: self.meta(&inode)?, + identity: entry.ino, + }); + } + if !more { + break; } - if !more { break; } } listed.sort_unstable_by(|a, b| a.name.cmp(&b.name)); Ok(listed) @@ -209,27 +278,50 @@ impl Volume for DataVolume { Ok(inode) => inode, Err(Error::NotFound) if how.create || how.create_new => { let (dir, name) = self.destination(path)?; - mapped("create", self.cache.block_on(self.fs.create(dir, name, New::File, OWNER, time((self.clock)()))))? + mapped( + "create", + self.cache.block_on(self.fs.create(dir, name, New::File, OWNER, time((self.clock)()))), + )? } Err(error) => return Err(error), }; - match kind(&inode)? { Kind::Dir => return Err(Error::IsADirectory), Kind::Symlink => return Err(Error::NotFound), Kind::File => {} } + match kind(&inode)? { + Kind::Dir => return Err(Error::IsADirectory), + Kind::Symlink => return Err(Error::NotFound), + Kind::File => {} + } let node = inode.ino; - self.open.entry(node).or_insert(Open { holders: 0, size: inode.size, mtime: inode.mtime.max(0) as u64, changed: false, pages: BTreeMap::new() }).holders += 1; - if how.truncate { - if let Err(error) = self.truncate(node, 0) { let _ = self.close(node); return Err(error); } + self.open + .entry(node) + .or_insert(Open { + holders: 0, + size: inode.size, + mtime: inode.mtime.max(0) as u64, + changed: false, + pages: BTreeMap::new(), + }) + .holders += 1; + if how.truncate + && let Err(error) = self.truncate(node, 0) + { + let _ = self.close(node); + return Err(error); } Ok(node) } fn close(&mut self, node: Node) -> Result<(), Error> { let Some(open) = self.open.get_mut(&node) else { return Ok(()) }; open.holders = open.holders.checked_sub(1).expect("one close per open"); - if open.holders != 0 { return Ok(()); } + if open.holders != 0 { + return Ok(()); + } let inode = mapped("close", self.cache.block_on(self.fs.inode(node)))?; if inode.nlink == 0 { self.open.get_mut(&node).unwrap().pages.clear(); mapped("orphan close", self.cache.block_on(self.fs.release(node)))?; - } else { self.persist(node)?; } + } else { + self.persist(node)?; + } self.open.remove(&node); Ok(()) } @@ -237,8 +329,12 @@ impl Volume for DataVolume { let open = self.open.get(&node).ok_or(Error::Stale)?; Ok(Meta { kind: Kind::File, size: open.size, mtime: open.mtime }) } - fn linked(&mut self, node: Node) -> Result { Ok(mapped("link count", self.cache.block_on(self.fs.inode(node)))?.nlink != 0) } - fn identity(&mut self, path: &str) -> Result { Ok(self.lookup(path)?.ino) } + fn linked(&mut self, node: Node) -> Result { + Ok(mapped("link count", self.cache.block_on(self.fs.inode(node)))?.nlink != 0) + } + fn identity(&mut self, path: &str) -> Result { + Ok(self.lookup(path)?.ino) + } fn read(&mut self, node: Node, offset: u64, out: &mut dyn Out) -> Result { let size = self.open.get(&node).ok_or(Error::Stale)?.size; let len = out.len().min(size.saturating_sub(offset).min(usize::MAX as u64) as usize); @@ -259,12 +355,21 @@ impl Volume for DataVolume { let ticket = self.next_ticket; self.next_ticket = ticket.checked_add(1).expect("reservation generation exhausted"); let reserved = Reserved { size: open.size, contents: open.size, ticket }; - if len == 0 { return Ok(reserved); } + if len == 0 { + return Ok(reserved); + } for page in offset / BLOCK as u64..end.div_ceil(BLOCK as u64) { - if self.open[&node].pages.contains_key(&page) { continue; } + if self.open[&node].pages.contains_key(&page) { + continue; + } match self.cache.block_on(self.fs.reserve(node, page * BLOCK as u64, BLOCK)) { - Ok(reservation) => { self.open.get_mut(&node).unwrap().pages.insert(page, Pending { ticket, reservation }); } - Err(error) => { self.unreserve(node, reserved, 0)?; return Err(word(error)); } + Ok(reservation) => { + self.open.get_mut(&node).unwrap().pages.insert(page, Pending { ticket, reservation }); + } + Err(error) => { + self.unreserve(node, reserved, 0)?; + return Err(word(error)); + } } } let open = self.open.get_mut(&node).unwrap(); @@ -275,26 +380,50 @@ impl Volume for DataVolume { } fn unreserve(&mut self, node: Node, reserved: Reserved, landed: u64) -> Result<(), Error> { let open = self.open.get_mut(&node).ok_or(Error::Stale)?; - let cancelled: Vec<_> = open.pages.iter().filter_map(|(&page, pending)| (pending.ticket == reserved.ticket && page * BLOCK as u64 >= landed).then_some(page)).collect(); - for page in cancelled { open.pages.remove(&page); } + let cancelled: Vec<_> = open + .pages + .iter() + .filter_map(|(&page, pending)| { + (pending.ticket == reserved.ticket && page * BLOCK as u64 >= landed).then_some(page) + }) + .collect(); + for page in cancelled { + open.pages.remove(&page); + } open.size = reserved.size.max(landed); Ok(()) } fn write_back(&mut self, node: Node, offset: u64, data: &[u8]) -> Result<(), Error> { let open = self.open.get_mut(&node).ok_or(Error::Stale)?; - if !offset.is_multiple_of(BLOCK as u64) || offset.checked_add(data.len() as u64).is_none_or(|end| end > open.size) { return Err(Error::InvalidRequest); } + if !offset.is_multiple_of(BLOCK as u64) + || offset.checked_add(data.len() as u64).is_none_or(|end| end > open.size) + { + return Err(Error::InvalidRequest); + } for (i, bytes) in data.chunks(BLOCK).enumerate() { let page = offset / BLOCK as u64 + i as u64; let pending = open.pages.get_mut(&page).ok_or(Error::Io)?; - let written = mapped("writeback", self.cache.block_on(self.fs.write_reserved(&mut pending.reservation, page * BLOCK as u64, bytes, time(open.mtime))))?; - if written != bytes.len() { return Err(Error::Io); } + let written = mapped( + "writeback", + self.cache.block_on(self.fs.write_reserved( + &mut pending.reservation, + page * BLOCK as u64, + bytes, + time(open.mtime), + )), + )?; + if written != bytes.len() { + return Err(Error::Io); + } open.pages.remove(&page); } Ok(()) } fn truncate(&mut self, node: Node, size: u64) -> Result<(), Error> { let open = self.open.get(&node).ok_or(Error::Stale)?; - if open.size == size { return Ok(()); } + if open.size == size { + return Ok(()); + } let now = (self.clock)(); mapped("truncate", self.cache.block_on(self.fs.truncate(node, size, time(now))))?; let open = self.open.get_mut(&node).unwrap(); @@ -312,30 +441,61 @@ impl Volume for DataVolume { } fn mkdir(&mut self, path: &str) -> Result<(), Error> { let (dir, name) = self.destination(path)?; - mapped("mkdir", self.cache.block_on(self.fs.create(dir, name, New::Dir, Owner { perm: 0o755, ..OWNER }, time((self.clock)())))).map(drop) + mapped( + "mkdir", + self.cache.block_on(self.fs.create( + dir, + name, + New::Dir, + Owner { perm: 0o755, ..OWNER }, + time((self.clock)()), + )), + ) + .map(drop) } fn rmdir(&mut self, path: &str) -> Result<(), Error> { - if self.protected(path) { return Err(Error::Root); } - if !self.lookup(path)?.is_dir() { return Err(Error::NotADirectory); } + if self.protected(path) { + return Err(Error::Root); + } + if !self.lookup(path)?.is_dir() { + return Err(Error::NotADirectory); + } let (dir, name) = self.destination(path)?; mapped("rmdir", self.cache.block_on(self.fs.unlink(dir, name, true, time((self.clock)())))).map(drop) } fn unlink(&mut self, path: &str) -> Result<(), Error> { - if self.protected(path) { return Err(Error::Root); } + if self.protected(path) { + return Err(Error::Root); + } let (dir, name) = self.destination(path)?; let removed = mapped("unlink", self.cache.block_on(self.fs.unlink(dir, name, false, time((self.clock)()))))?; self.released(Some(removed)) } fn rename(&mut self, from: &str, to: &str) -> Result<(), Error> { - if self.protected(from) || self.protected(to) { return Err(Error::Root); } + if self.protected(from) || self.protected(to) { + return Err(Error::Root); + } let source = self.destination(from)?; let target = self.destination(to)?; - let removed = mapped("rename", self.cache.block_on(self.fs.rename(source, target, engine::Rename::Replace, time((self.clock)()))))?; + let removed = mapped( + "rename", + self.cache.block_on(self.fs.rename(source, target, engine::Rename::Replace, time((self.clock)()))), + )?; self.released(removed) } fn symlink(&mut self, path: &str, target: &str) -> Result<(), Error> { let (dir, name) = self.destination(path)?; - mapped("symlink", self.cache.block_on(self.fs.create(dir, name, New::Symlink(target.as_bytes().to_vec()), OWNER, time((self.clock)())))).map(drop) + mapped( + "symlink", + self.cache.block_on(self.fs.create( + dir, + name, + New::Symlink(target.as_bytes().to_vec()), + OWNER, + time((self.clock)()), + )), + ) + .map(drop) } fn sync(&mut self) -> Result, Error> { let nodes: Vec<_> = self.open.keys().copied().collect(); @@ -345,7 +505,9 @@ impl Volume for DataVolume { Err(error) => refused.push((node, error)), Ok(()) if self.open[&node].holders == 0 => { let inode = mapped("unheld inode", self.cache.block_on(self.fs.inode(node)))?; - if inode.nlink == 0 { mapped("orphan release", self.cache.block_on(self.fs.release(node)))?; } + if inode.nlink == 0 { + mapped("orphan release", self.cache.block_on(self.fs.release(node)))?; + } self.open.remove(&node); } Ok(()) => {} @@ -354,12 +516,18 @@ impl Volume for DataVolume { mapped("sync", self.cache.block_on(self.fs.commit()))?; Ok(refused) } - fn owes(&self) -> bool { self.open.values().any(|open| open.changed || !open.pages.is_empty()) } + fn owes(&self) -> bool { + self.open.values().any(|open| open.changed || !open.pages.is_empty()) + } fn commit(&mut self, node: Option) -> Result<(), Error> { - if let Some(node) = node { self.persist(node)?; } + if let Some(node) = node { + self.persist(node)?; + } mapped("fsync", self.cache.block_on(self.fs.commit())) } - fn room(&mut self) -> Result { Ok(toyos::fs::Room { free: self.fs.space().1, unit: BLOCK as u64 }) } + fn room(&mut self) -> Result { + Ok(toyos::fs::Room { free: self.fs.space().1, unit: BLOCK as u64 }) + } fn describe(&self) -> String { let (total, free) = self.fs.space(); format!("bcachefs DATA: {free}/{total} bytes free, {} open inodes", self.open.len()) @@ -373,7 +541,9 @@ mod tests { use crate::volume::{Buf, Writes}; const CREATE: OpenHow = OpenHow { create: true, create_new: false, truncate: false }; const EXISTING: OpenHow = OpenHow { create: false, create_new: false, truncate: false }; - fn clock() -> u64 { 1_700_000_000_000_000_000 } + fn clock() -> u64 { + 1_700_000_000_000_000_000 + } fn volume() -> DataVolume { DataVolume::format(Ram::new(16384), 256, &["home", "apps"], clock).unwrap() } @@ -384,7 +554,7 @@ mod tests { } fn reopen(volume: DataVolume) -> DataVolume { match DataVolume::probe(volume.stopped(), 256, &["home", "apps"], clock) { - Probed::Mounted(volume) => volume, + Probed::Mounted(volume) => *volume, _ => panic!("committed DATA must mount"), } } diff --git a/system/fileserver/src/descriptions.rs b/system/fileserver/src/descriptions.rs index 1d9bd8ea140..ab556c0f157 100644 --- a/system/fileserver/src/descriptions.rs +++ b/system/fileserver/src/descriptions.rs @@ -143,7 +143,7 @@ impl Ids { pub fn moved(path: &mut String, from: &str, to: &str) { let beneath = path.len() > from.len() && path.starts_with(from) && path.as_bytes()[from.len()] == b'/'; if path == from || beneath { - *path = crate::volume::join(to, &path[from.len()..].trim_start_matches('/').to_string()); + *path = crate::volume::join(to, path[from.len()..].trim_start_matches('/')); } } diff --git a/system/fileserver/src/fat.rs b/system/fileserver/src/fat.rs index 58863e9ed3b..576a87fc592 100644 --- a/system/fileserver/src/fat.rs +++ b/system/fileserver/src/fat.rs @@ -173,10 +173,8 @@ impl FatVolume { } fn orphan(&mut self, path: &str) { - if let Some(node) = self.by_path.remove(path) { - if let Some(open) = self.open.get_mut(&node) { - open.gone = true; - } + if let Some(node) = self.by_path.remove(path) && let Some(open) = self.open.get_mut(&node) { + open.gone = true; } } @@ -197,10 +195,8 @@ impl Volume for FatVolume { if path.is_empty() { return Ok(Meta { kind: Kind::Dir, size: 0, mtime: 0 }); } - if let Some(&node) = self.by_path.get(path) { - if self.open.contains_key(&node) { - return self.node_meta(node); - } + if let Some(&node) = self.by_path.get(path) && self.open.contains_key(&node) { + return self.node_meta(node); } let meta = self.cache.block_on(self.fs.metadata(path)).map_err(|e| match e { Error::NotADirectory => fs::Error::NotFound, @@ -284,11 +280,9 @@ impl Volume for FatVolume { if open.holders > 0 { return Ok(()); } - if self.writable { - if let Err(e) = self.level(node) { - println!("fileserver: node {node} was not brought level at its last close ({e:?}); the next sync does it"); - return Err(e); - } + if self.writable && let Err(e) = self.level(node) { + println!("fileserver: node {node} was not brought level at its last close ({e:?}); the next sync does it"); + return Err(e); } self.release(node); Ok(()) diff --git a/system/fileserver/src/main.rs b/system/fileserver/src/main.rs index aad95ffc2ac..3f92ed07ec4 100644 --- a/system/fileserver/src/main.rs +++ b/system/fileserver/src/main.rs @@ -391,7 +391,7 @@ fn absent(roots: &[&str], why: String) -> Box { fn data_on(disk: D, roots: &[&str]) -> Box { match DataVolume::probe(disk, CLEAN_BLOCKS, roots, fileserver::volume::now_nanos) { - Probed::Mounted(volume) => Box::new(volume), + Probed::Mounted(volume) => volume, Probed::Unmountable(why) => { println!("fileserver: the DATA volume is ours and does not mount ({why}); it is absent this boot"); Box::new(Absent::new(roots, why)) diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index 75b72eb5c1f..fc5a65ac0bb 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -654,7 +654,7 @@ fn stale_volume() -> DataVolume { /// `v` stopped now and mounted again: what its last commit left. fn after_a_stop(v: DataVolume) -> DataVolume { match DataVolume::probe(v.stopped(), 256, &["home"], clock) { - crate::data::Probed::Mounted(v) => v, + crate::data::Probed::Mounted(v) => *v, _ => panic!("the volume does not mount after a stop"), } } diff --git a/system/fileserver/src/volume.rs b/system/fileserver/src/volume.rs index 3b9a4df8de7..e17e3105a8f 100644 --- a/system/fileserver/src/volume.rs +++ b/system/fileserver/src/volume.rs @@ -63,6 +63,7 @@ pub type Node = u64; /// through a reference, or a buffer of this process's own. pub trait Out { fn len(&self) -> usize; + fn is_empty(&self) -> bool { self.len() == 0 } /// `bytes` at `at`, inside `len()`. fn put(&mut self, at: usize, bytes: &[u8]); /// `len` zeros at `at`, inside `len()`: a hole's bytes. @@ -250,8 +251,15 @@ impl Reserved { pub trait Writes: Volume { fn write(&mut self, node: Node, offset: u64, data: &[u8]) -> Result<(), Error> { const PAGE: u64 = toyos_blockdev::BLOCK as u64; + if data.is_empty() { return Ok(()); } let reserved = self.reserve(node, offset, data.len() as u64)?; - let size = self.node_meta(node)?.size; + let size = match self.node_meta(node) { + Ok(meta) => meta.size, + Err(error) => { + self.unreserve(node, reserved, 0)?; + return Err(error); + } + }; let end = offset + data.len() as u64; let mut landed = 0; for start in (offset / PAGE * PAGE..end).step_by(PAGE as usize) { From 538da31bc609fb5fa90bd46fc3bd0db0d127fcc8 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 19:44:20 +0100 Subject: [PATCH 18/93] Guard persistent open-unlinked file lifetime through paged holders The file stays readable after unlink and description close while a holder remains, and its pages and inode are released after the final holder notice. --- system/fileserver/src/pages/tests.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index fc5a65ac0bb..ab222c2b967 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -336,6 +336,26 @@ fn memory_bytes_survive_close_sync_and_namespace_replacement_until_the_last_hold assert_eq!(fake.0.borrow().held(), 0, "anonymous bytes need no writeback slots"); } +#[test] +fn persistent_unlink_preserves_dirty_bytes_until_the_last_holder() { + let mut v = volume(); + let fake = Fake::new(64, 16); + let mut files = Files::new(Some(fake)); + let (node, id) = open(&mut files, &mut v, "home/held"); + files.write(&mut v, id, node, 0, b"still held").unwrap(); + files.held(id); + let object = files.object(id).unwrap(); + v.unlink("home/held").unwrap(); + files.unlinked(&mut v, id).unwrap(); + assert_eq!(read(&mut files, &mut v, (node, id), 0, 10), b"still held"); + files.closed(&mut v, id, node).unwrap(); + assert_eq!(files.object(id), Some(object)); + files.notice(&mut v, Notice::NoHolders { object }); + assert_eq!(files.object(id), None); + assert_eq!(v.node_meta(node), Err(Error::Stale)); + assert!(v.sync().unwrap().is_empty()); +} + #[test] fn memory_sparse_files_do_not_resurrect_truncated_bytes_or_evict_existing_data() { let mut volume = crate::memory::MemoryVolume::new(&[], clock); From f916f41edba44b0ae953afde232752d960e19156 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 19:44:02 +0100 Subject: [PATCH 19/93] Move filesystem consumers to served descriptors Replace kernel pathname probes with ordinary SDK filesystem operations. Root hierarchy checks use controlled namespaces, copyout probes use generic objects, and directory pagination checks cover complete served listings. Remove the retired backing-revocation actuator registration and its oracle. Keep process cwd within the bootstrap path bound and refuse oversized input before normalization. Include a complete-listing probe in the shared boot now that it no longer exhausts a kernel filesystem inventory. Validation: SDK 153 host tests, metal 81 unit tests, SDK fd-table clippy, focused path boundary tests, and host guest-harness compilation passed. Guest compilation and execution require the consolidated sysroot/image. --- sdk/abi/src/syscall.rs | 14 +- sdk/toyos/src/files.rs | 10 +- sdk/toyos/src/files/owner.rs | 3 + system/logkeeper/src/store.rs | 6 +- system/toybox/src/mv.rs | 19 +- tests/guest/rust-tests/src/arch/vector.rs | 47 +-- .../rust-tests/src/bin/abuse_cwd_growth.rs | 287 +++--------------- .../rust-tests/src/bin/abuse_dlopen_ledger.rs | 2 +- .../rust-tests/src/bin/abuse_page_straddle.rs | 5 +- .../src/bin/abuse_readonly_copyout.rs | 42 ++- .../rust-tests/src/bin/empty_dir_stat.rs | 138 ++------- .../rust-tests/src/bin/handle_lifetime.rs | 113 ++----- .../rust-tests/src/bin/hierarchy_paths.rs | 124 ++++---- .../src/bin/kernel_objects_refused.rs | 165 +++------- .../rust-tests/src/bin/pipe_other_end_gone.rs | 18 +- .../guest/rust-tests/src/bin/readdir_bound.rs | 142 ++------- .../src/bin/spawn_returns_the_census.rs | 12 +- .../guest/rust-tests/src/bin/vector_state.rs | 13 +- tests/guest/rust-tests/src/image.rs | 8 +- tests/metal/src/lib.rs | 1 - tests/toyos.rs | 29 +- 21 files changed, 281 insertions(+), 917 deletions(-) diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 04f709da147..90902c7bf44 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -565,19 +565,7 @@ pub enum SyscallError { WouldBlock = 6, ResourceExhausted = 7, NotSupported = 8, - /// The device did not do it. - /// - /// One word for a transfer that was issued and not completed, and for a - /// volume whose own structures do not decode: there is nothing a caller can - /// do differently about the two, and both are the opposite of `NotFound`. - /// The channel below it — `block::BlockDevice`, `FileBacking::read_page`, - /// `bcachefs::BlockIO`, `vfs::FileSystem` — is fallible the whole way so - /// that this is what arrives, rather than "no such file". - /// - /// It carries nothing. Which endpoint stalled, what the sense key was and - /// which block was asked for are in the kernel's own log line, where a - /// triage reads them; an enum here would be a vocabulary userland has no - /// use for and every new driver would have to guess an arm from. + /// An object operation failed while performing I/O. Io = 9, /// The object was there and its other end is not. /// diff --git a/sdk/toyos/src/files.rs b/sdk/toyos/src/files.rs index d2d37ea8063..f39a96c7920 100644 --- a/sdk/toyos/src/files.rs +++ b/sdk/toyos/src/files.rs @@ -104,7 +104,7 @@ pub struct Stat { } impl Stat { - /// What the kernel answers about a file or a handle of its own: one name, + /// What the kernel answers about an object of its own: one name, /// and of the times only when it was last written. pub fn of_kernel(stat: toyos_abi::syscall::Stat) -> Self { Self { @@ -469,6 +469,9 @@ pub fn dl_open(path: &[u8]) -> Result { /// from one) says so before it asks for a path. #[cfg_attr(not(all(feature = "fd-table", not(test))), allow(dead_code))] fn named(path: &[u8]) -> Result<&str, Error> { + if path.len() > fs::MAX_PATH { + return Err(Error::Served(fs::Error::NameTooLong)); + } match core::str::from_utf8(path) { Ok("") => Err(Error::Served(fs::Error::NotFound)), Ok(path) => Ok(path), @@ -519,14 +522,15 @@ mod tests { assert_eq!(absolute("/", "../../.."), "/"); } - /// The empty path names nothing, whoever would resolve it; any other - /// UTF-8 path is itself. + /// The path boundary refuses empty, non-UTF-8 and oversized names. #[test] fn the_empty_path_names_nothing() { assert_eq!(named(b""), Err(Error::Served(fs::Error::NotFound))); assert_eq!(named(b"."), Ok(".")); assert_eq!(named(b"/"), Ok("/")); assert_eq!(named(b"\xff"), Err(Error::NotUtf8)); + assert!(named(&alloc::vec![b'a'; fs::MAX_PATH]).is_ok()); + assert_eq!(named(&alloc::vec![b'a'; fs::MAX_PATH + 1]), Err(Error::Served(fs::Error::NameTooLong))); } #[test] diff --git a/sdk/toyos/src/files/owner.rs b/sdk/toyos/src/files/owner.rs index adb29873edc..cfb79860e8f 100644 --- a/sdk/toyos/src/files/owner.rs +++ b/sdk/toyos/src/files/owner.rs @@ -641,6 +641,9 @@ unsafe extern "C" fn toyos_fs_chdir(path: *const u8, len: usize) -> i64 { let path = unsafe { bytes(path, len) }; let changed = super::named(path).and_then(|path| { let abs = super::absolute(&cwd(), path); + if abs.len() > fs::MAX_PATH { + return Err(Error::Served(fs::Error::NameTooLong)); + } let stat = on_path(At::Cwd, abs.as_bytes(), directory_stat, |_, dir, base, rel| dir.stat(base, rel, true).map(stat_of))?; if stat.file_type != FileType::Directory { return Err(Error::NotADirectory); diff --git a/system/logkeeper/src/store.rs b/system/logkeeper/src/store.rs index 835b36642e0..6570a12dce3 100644 --- a/system/logkeeper/src/store.rs +++ b/system/logkeeper/src/store.rs @@ -1,8 +1,7 @@ //! `/log` as this program's policy: one file per boot, continuations, and //! retention. //! -//! Every byte goes through `SYS_WRITE` and `SYS_FSYNC` exactly as any other -//! program's would. +//! Writes and syncs use the same served file API as other programs. //! //! **A part is not preallocated.** Its whole length in zeros is a burst of //! writes to the stick at every part's start, and a burst that size starves a @@ -222,8 +221,7 @@ impl Volume { Ok(()) } - /// Get everything written so far onto the device, cache flush included — - /// which is what `SYS_FSYNC` means on this tree. + /// Get everything written so far onto the device, cache flush included. pub fn sync(&mut self) -> std::io::Result<()> { self.file.sync_all() } diff --git a/system/toybox/src/mv.rs b/system/toybox/src/mv.rs index 8a8868dd51e..67aec7d135c 100644 --- a/system/toybox/src/mv.rs +++ b/system/toybox/src/mv.rs @@ -2,17 +2,8 @@ use std::fs; use std::path::Path; use std::process; -/// Move a file, which here is exactly one `SYS_RENAME` and nothing else. -/// -/// There is no copy-and-delete fallback for a move between mounts, and the -/// reason is not that the pieces are missing — `cp` is right there. It is that -/// `sys_rename` in `kernel/src/syscall/fs.rs` collapses all five of -/// `Vfs::rename`'s errors into `NotFound`, and `Stat` carries no mount -/// identity, so this process cannot tell "different mounts" from "the rename -/// is broken". A fallback keyed on *any* rename failure would quietly copy its -/// way past a rename defect and pass every test that a working rename passes. -/// So the refusal names both paths, says what it could not distinguish, and -/// stops. +/// Rename a file within one served filesystem. Cross-volume moves require +/// an explicit copy and delete. pub fn main(args: Vec) { let [source, dest] = args.as_slice() else { eprintln!("Usage: mv "); @@ -29,8 +20,7 @@ pub fn main(args: Vec) { pub fn move_file(source: &Path, dest: &Path) -> Result<(), String> { let dest = crate::cp::destination(source, dest); - // Asked before the rename so the commonest cause gets its own answer - // rather than the one the kernel returns for everything. + // Report a missing source directly before attempting the rename. if let Err(e) = fs::metadata(source) { return Err(format!("mv: {}: {e}", source.display())); } @@ -39,8 +29,7 @@ pub fn move_file(source: &Path, dest: &Path) -> Result<(), String> { format!( "mv: {} -> {}: {e} — but the source is there, so it is the rename being refused\n\ mv: most likely the two paths are on different mounts, which is not one rename. \ - SYS_RENAME reports one error for every cause, so mv cannot tell that from a rename \ - defect and will not copy behind your back — use cp then rm.", + Use cp then rm for a move between filesystems.", source.display(), dest.display() ) diff --git a/tests/guest/rust-tests/src/arch/vector.rs b/tests/guest/rust-tests/src/arch/vector.rs index e0589adf010..fc95728ccf6 100644 --- a/tests/guest/rust-tests/src/arch/vector.rs +++ b/tests/guest/rust-tests/src/arch/vector.rs @@ -8,7 +8,8 @@ mod x86_64 { use core::arch::asm; use core::arch::x86_64::__cpuid_count; - use toyos_abi::syscall::{Stat, SYS_STAT}; + use toyos_abi::syscall::{Stat, SYS_FSTAT}; + use toyos::AsHandle; /// Which vector registers this machine gives a thread. Only [`width`] /// makes one, so holding one is having checked what std reports. @@ -109,12 +110,12 @@ mod x86_64 { ); } - /// Turns of the hold loop between two of its `stat`s. + /// Turns of the hold loop between two of its `fstat`s. const BURST: u64 = 100_000; - /// The hold loop: `r12` runs of [`BURST`] turns, each ending in a `stat` + /// The hold loop: `r12` runs of [`BURST`] turns, each ending in a `fstat` /// whose arguments `rdi`, `rsi`, `rdx`, `r8` and `r9` keep across every - /// `syscall`, and the `stat`s' results ORed into `r13`. + /// `syscall`, and the `fstat`s' results ORed into `r13`. macro_rules! bursts { () => { concat!( @@ -132,7 +133,7 @@ mod x86_64 { } #[target_feature(enable = "avx")] - unsafe fn hold_avx(p: *mut u8, bursts: u64, path: &[u8], stat: *mut Stat) -> u64 { + unsafe fn hold_avx(p: *mut u8, bursts: u64, handle: u32, stat: *mut Stat) -> u64 { let refused; asm!( ymm!(load 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15), @@ -142,11 +143,11 @@ mod x86_64 { inout("r12") bursts => _, inout("r13") 0u64 => refused, burst = const BURST, - in("rdi") SYS_STAT, - in("rsi") path.as_ptr(), - in("rdx") path.len(), - in("r8") stat, - in("r9") 1u64, + in("rdi") SYS_FSTAT, + in("rsi") u64::from(handle), + in("rdx") stat, + in("r8") 0u64, + in("r9") 0u64, out("rax") _, out("rcx") _, out("r11") _, @@ -157,7 +158,7 @@ mod x86_64 { } #[target_feature(enable = "avx512f")] - unsafe fn hold_avx512(p: *mut u8, bursts: u64, path: &[u8], stat: *mut Stat) -> u64 { + unsafe fn hold_avx512(p: *mut u8, bursts: u64, handle: u32, stat: *mut Stat) -> u64 { let refused; asm!( zmm!(load 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31), @@ -169,11 +170,11 @@ mod x86_64 { inout("r12") bursts => _, inout("r13") 0u64 => refused, burst = const BURST, - in("rdi") SYS_STAT, - in("rsi") path.as_ptr(), - in("rdx") path.len(), - in("r8") stat, - in("r9") 1u64, + in("rdi") SYS_FSTAT, + in("rsi") u64::from(handle), + in("rdx") stat, + in("r8") 0u64, + in("r9") 0u64, out("rax") _, out("rcx") _, out("r11") _, @@ -199,23 +200,25 @@ mod x86_64 { /// `file` loaded into every vector register, held there through `bursts` /// runs of [`BURST`] turns of a loop that touches no vector register, each - /// ending in a `stat` of `path` that the kernel runs on this thread's own + /// ending in an object `fstat` that the kernel runs on this thread's own /// kernel stack, and read back. - pub fn hold(width: Width, file: &[u8], bursts: u64, path: &[u8]) -> Vec { + pub fn hold(width: Width, file: &[u8], bursts: u64) -> Vec { assert_eq!(file.len(), width.bytes()); assert!(bursts > 0, "`dec` from zero would loop 2^64 times"); let mut held = file.to_vec(); + let object = toyos::null(toyos_abi::Rights::READ).expect("an object for fstat"); + let handle = object.as_handle().raw().0; let mut stat = Stat::default(); // SAFETY: as `read`'s; the loads and stores are `width.bytes()` long, // every vector register the block writes is declared clobbered, and - // `stat` reads `path` and writes `stat`, both live across the block. + // `fstat` uses the held object and writes `stat`, both live across the block. let refused = unsafe { match width.0 { - Kind::Avx => hold_avx(held.as_mut_ptr(), bursts, path, &mut stat), - Kind::Avx512 => hold_avx512(held.as_mut_ptr(), bursts, path, &mut stat), + Kind::Avx => hold_avx(held.as_mut_ptr(), bursts, handle, &mut stat), + Kind::Avx512 => hold_avx512(held.as_mut_ptr(), bursts, handle, &mut stat), } }; - assert_eq!(refused, 0, "a `stat` of {} was refused", String::from_utf8_lossy(path)); + assert_eq!(refused, 0, "fstat of a held object was refused"); held } } diff --git a/tests/guest/rust-tests/src/bin/abuse_cwd_growth.rs b/tests/guest/rust-tests/src/bin/abuse_cwd_growth.rs index 5d25ad13776..7e8c3444323 100644 --- a/tests/guest/rust-tests/src/bin/abuse_cwd_growth.rs +++ b/tests/guest/rust-tests/src/bin/abuse_cwd_growth.rs @@ -1,269 +1,58 @@ -//! A process's `cwd` must not be able to grow without bound. -//! -//! `MAX_USER_STR` (64 KiB) bounds every path *argument*, and its derivation -//! says the number is set by the largest allocation derived from it. But -//! `Vfs::resolve_absolute` prepends `cwd` before handing the result to -//! `normalize`, and `cwd` was bounded by nothing — so the input the constant -//! was sized against stopped being the input `normalize` saw. A bound defeated -//! by composition: the check was real, the assumption behind it had quietly -//! stopped holding. -//! -//! The amplifier is that `mkdir` does no existence check — `create_dir` just -//! inserts the resolved string into a set, and `cd` accepts anything in that -//! set. So `mkdir(rel); chdir(rel)` appends ~64 KiB per round with no -//! filesystem involvement at all, and about thirty rounds put a `format!` -//! past the kernel heap's 2 MiB single-allocation ceiling — an assert taken -//! *inside the allocator lock*, which wedges the recovered CPU's next alloc. -//! -//! Roughly sixty syscalls from plain std, no crafted ELF, no large-RAM guest. -//! Before the fix this panics the kernel; after it, every over-long `chdir` is -//! a clean error and `cwd` stays bounded. +//! The shared SDK cwd is bounded, and short getcwd buffers are never truncated. use std::fs; - -/// `kernel/src/vfs.rs`'s `MAX_PATH`, by value. The test asserts against the -/// bound's *consequences*, not this number, so a change to it there does not -/// silently make this test vacuous — but the reachability checks below need to -/// know roughly where the wall is. -const MAX_PATH: usize = 4096; +use toyos::files; +use toyos_abi::syscall::MAX_PATH; const DIR: &str = "/tmp/abuse_cwd"; -/// One component long enough that a single successful round would blow most of -/// the budget, and far longer than `MAX_PATH` on its own. -const HUGE_COMPONENT: usize = 60_000; - -/// Enough rounds that, unfixed, `cwd` passes 2 MiB well before the loop ends. -/// At ~64 KiB a round the panic lands around round 32. -const ROUNDS: usize = 64; - -/// The kernel's `cwd` length, read through the syscall rather than through -/// `std::env::current_dir`. -/// -/// `current_dir` cannot be used to measure this: std's `getcwd` passes a fixed -/// `[u8; 256]` buffer and `sys_getcwd` copies `min(cwd.len(), buf.len())` with -/// no error, so any cwd past 256 bytes comes back *silently truncated to a -/// different path*. Measuring with it made this test report 256 bytes for a -/// 2 KiB cwd — a broken instrument, and a defect of its own worth fixing where -/// the ABI contract lives. -/// -/// The buffer here is deliberately larger than `MAX_PATH`, so a cwd that -/// somehow exceeded the bound would be observed at its true length rather than -/// clamped to one that satisfies the assertions below. -fn cwd_len() -> usize { - let mut buf = [0u8; 4 * MAX_PATH]; - toyos_abi::syscall::getcwd(&mut buf) +fn cwd() -> Vec { + let mut bytes = vec![0; files::getcwd(&mut [])]; + assert_eq!(files::getcwd(&mut bytes), bytes.len()); + bytes } -/// The Route B exploit: append a huge component per round and never stop. -/// -/// Unfixed, `cwd` grows ~64 KiB a round until a `format!` in `resolve_absolute` -/// crosses the heap ceiling and the kernel dies. Fixed, every round is refused -/// and `cwd` never moves. -fn drive_cwd_growth() { - let huge = "a".repeat(HUGE_COMPONENT); - let start = cwd_len(); - let mut accepted = 0usize; - - for round in 0..ROUNDS { - // `mkdir` may legitimately succeed — it is not the operation under - // test, and it does no existence check. The bound belongs on `chdir`, - // which is what stores a path in the kernel. - let _ = fs::create_dir(&huge); - - if std::env::set_current_dir(&huge).is_err() { - break; - } - accepted += 1; - - let len = cwd_len(); - assert!( - len <= MAX_PATH, - "round {round}: chdir accepted a path of {len} bytes, past MAX_PATH {MAX_PATH}", - ); +fn refused_growth() { + let before = cwd(); + for huge in ["a".repeat(60_000), "b/".repeat(30_000)] { + assert!(fs::create_dir(&huge).is_err(), "mkdir accepted an overlong path"); + assert!(std::env::set_current_dir(&huge).is_err(), "chdir accepted an overlong path"); + assert_eq!(cwd(), before, "refused chdir moved cwd"); } - - // A single 60 KiB component cannot fit in 4096 bytes, so not one round may - // be accepted. If this ever passes, the bound is not on the path length. - assert_eq!( - accepted, 0, - "a {HUGE_COMPONENT}-byte component was accepted {accepted} times; MAX_PATH is {MAX_PATH}", - ); - assert_eq!(cwd_len(), start, "cwd moved despite every chdir being refused"); -} - -/// The same growth driven by *many small* components rather than one huge one. -/// -/// This is the shape that drives `normalize`'s `Vec<&str>` (16 bytes per -/// component) rather than the joined `String`, and it is the one a path bound -/// must also cover: without it, `"a/a/a/…"` reaches the same ceiling through a -/// different allocation. -fn drive_component_growth() { - let many = "b/".repeat(HUGE_COMPONENT / 2); - - let before = cwd_len(); - let _ = fs::create_dir_all(&many); - let refused = std::env::set_current_dir(&many).is_err(); - - assert!(refused, "chdir accepted {} bytes of components", many.len()); - assert_eq!(cwd_len(), before, "cwd moved despite the chdir being refused"); } -/// The bound must refuse over-long paths without refusing legal ones. -/// -/// Without this the test would pass just as well against a `cd` that always -/// returned an error, which would be a broken kernel rather than a fixed one. -fn legal_nesting_still_works() { +fn legal_nesting() { let root = format!("{DIR}/deep"); - fs::create_dir_all(&root).expect("create_dir_all deep"); - std::env::set_current_dir(&root).expect("chdir into a legal directory"); - - let mut depth = 0usize; - // Components small enough that several hundred still fit inside MAX_PATH. + fs::create_dir(&root).expect("create deep root"); + std::env::set_current_dir(&root).expect("chdir deep root"); for _ in 0..32 { let name = "n".repeat(64); - let _ = fs::create_dir(&name); - if std::env::set_current_dir(&name).is_err() { - break; - } - depth += 1; - } - - assert!( - depth >= 16, - "only {depth} legal nested chdirs succeeded; the bound is refusing paths it should accept", - ); - let len = cwd_len(); - assert!(len <= MAX_PATH, "legal nesting produced a {len}-byte cwd"); - assert!(len > 1024, "legal nesting only reached {len} bytes — not exercising the bound"); -} - -/// The kernel heap is intact and the VFS still resolves paths correctly. -/// -/// The defect's signature is a panic taken while holding the allocator lock, so -/// "the kernel is still allocating" is the assertion that matters — and a -/// resolved path that still finds its file proves the bound did not truncate -/// anything into naming a different one. -fn kernel_still_healthy() { - std::env::set_current_dir("/").expect("chdir back to /"); - - let mut blocks: Vec> = Vec::new(); - for i in 0..256 { - blocks.push(vec![(i % 251) as u8; 4096]); - } - for (i, b) in blocks.iter().enumerate() { - assert!(b.iter().all(|&x| x == (i % 251) as u8), "kernel heap corrupted block {i}"); - } - - let probe = format!("{DIR}/probe.txt"); - fs::write(&probe, b"resolved").expect("write probe"); - std::env::set_current_dir(DIR).expect("chdir into DIR"); - let via_relative = fs::read("probe.txt").expect("read probe by relative path"); - assert_eq!(via_relative, b"resolved", "relative resolution found the wrong file"); - std::env::set_current_dir("/").expect("chdir back to /"); -} - -/// `getcwd` reports the length the path needs, and writes nothing when it does -/// not fit. -/// -/// The old contract returned `min(cwd.len(), buf.len())` after writing a -/// prefix, so "fit exactly" and "silently truncated" were the same answer. -/// That is how `std::env::current_dir` — which passes a fixed 256-byte buffer — -/// came to hand back a shorter, valid-looking path to a *different* directory. -fn getcwd_contract() { - let root = format!("{DIR}/getcwd"); - fs::create_dir_all(&root).expect("create getcwd dir"); - std::env::set_current_dir(&root).expect("chdir into getcwd dir"); - - let truth = cwd_len(); - assert!(truth > 8, "cwd is implausibly short: {truth}"); - - // An empty buffer is a pure size query: same answer, nothing touched. - let mut empty: [u8; 0] = []; - assert_eq!( - toyos_abi::syscall::getcwd(&mut empty), - truth, - "an empty buffer must still report the required length", - ); - - // A buffer one byte short must report the requirement and write *nothing*: - // a partial path names the wrong directory. - let mut small = vec![0xAAu8; truth - 1]; - let n = toyos_abi::syscall::getcwd(&mut small); - assert_eq!(n, truth, "a short buffer must report the required length, got {n}"); - assert!( - small.iter().all(|&b| b == 0xAA), - "a short buffer was written to; a truncated path is a path to the wrong directory", - ); - - // Exactly enough succeeds, and the answer round-trips. - let mut exact = vec![0u8; truth]; - assert_eq!(toyos_abi::syscall::getcwd(&mut exact), truth); - assert_eq!( - core::str::from_utf8(&exact).expect("cwd is utf-8"), - root, - "getcwd did not return the directory we chdir'd into", - ); -} - -/// The bug the contract change exists to fix, end to end. -/// -/// A cwd past std's fixed 256-byte buffer used to come back truncated, so -/// `current_dir()` named a directory the process was not in and every path -/// built from it pointed somewhere wrong. Now std allocates and retries. -fn current_dir_survives_a_long_cwd() { - std::env::set_current_dir(DIR).expect("chdir into DIR"); - let mut depth = 0; - while cwd_len() <= 300 { - let name = "d".repeat(64); - let _ = fs::create_dir(&name); - if std::env::set_current_dir(&name).is_err() { - break; - } - depth += 1; + fs::create_dir(&name).expect("create legal nested directory"); + std::env::set_current_dir(&name).expect("chdir legal nested directory"); } - let truth_len = cwd_len(); - assert!( - truth_len > 256, - "could not build a cwd past std's 256-byte buffer (got {truth_len} at depth {depth})", - ); + let truth = cwd(); + assert!(truth.len() > 1024 && truth.len() <= MAX_PATH); + assert_eq!(std::env::current_dir().expect("long current_dir").to_str().expect("UTF-8 cwd").as_bytes(), truth); - let via_std = std::env::current_dir().expect("current_dir with a long cwd"); - assert_eq!( - via_std.to_string_lossy().len(), - truth_len, - "current_dir returned a {}-byte path for a {truth_len}-byte cwd — still truncating", - via_std.to_string_lossy().len(), - ); - std::env::set_current_dir("/").expect("chdir back to /"); -} - -/// `mkdir` reports a path it refuses instead of returning success. -/// -/// `sys_mkdir` discarded `create_dir`'s outcome and always returned 0, so a -/// bound added without changing that return would have been a silent failure: -/// the caller told nothing, the directory simply absent. -fn mkdir_refuses_overlong() { - std::env::set_current_dir("/").expect("chdir to /"); - let huge = format!("{DIR}/{}", "m".repeat(HUGE_COMPONENT)); - let err = fs::create_dir(&huge) - .expect_err("mkdir accepted a path longer than MAX_PATH"); - // The point is that it is reported at all, rather than reported as success. - let _ = err; + let mut small = vec![0xaa; truth.len() - 1]; + assert_eq!(files::getcwd(&mut small), truth.len()); + assert!(small.iter().all(|&b| b == 0xaa), "getcwd wrote a partial path"); + let mut exact = vec![0; truth.len()]; + assert_eq!(files::getcwd(&mut exact), truth.len()); + assert_eq!(exact, truth); + refused_growth(); } fn main() { - fs::create_dir_all(DIR).expect("create /home/abuse_cwd"); - std::env::set_current_dir(DIR).expect("chdir into DIR"); - - drive_cwd_growth(); - drive_component_growth(); - legal_nesting_still_works(); - getcwd_contract(); - current_dir_survives_a_long_cwd(); - mkdir_refuses_overlong(); - kernel_still_healthy(); - - let _ = fs::remove_dir_all(DIR); + fs::create_dir(DIR).expect("create cwd test root"); + std::env::set_current_dir(DIR).expect("enter cwd test root"); + refused_growth(); + legal_nesting(); + + std::env::set_current_dir(DIR).expect("return to test root"); + fs::write("probe.txt", b"resolved").expect("relative write"); + assert_eq!(fs::read(format!("{DIR}/probe.txt")).expect("absolute read"), b"resolved"); + std::env::set_current_dir("/").expect("return to root"); + fs::remove_dir_all(DIR).expect("clean cwd test root"); println!("cwd growth bounded, kernel intact"); } diff --git a/tests/guest/rust-tests/src/bin/abuse_dlopen_ledger.rs b/tests/guest/rust-tests/src/bin/abuse_dlopen_ledger.rs index ad82ade770e..766f3cd3ee4 100644 --- a/tests/guest/rust-tests/src/bin/abuse_dlopen_ledger.rs +++ b/tests/guest/rust-tests/src/bin/abuse_dlopen_ledger.rs @@ -54,7 +54,7 @@ fn main() { for i in 0..ATTEMPTS { // A name of its own for one file: the ledger is keyed by name. let name = format!("{DIR}/{i}.so"); - syscall::symlink(lib.as_bytes(), name.as_bytes()).unwrap_or_else(|e| panic!("link {name}: {e:?}")); + files::symlink(lib.as_bytes(), (files::At::Cwd, name.as_bytes())).unwrap_or_else(|e| panic!("link {name}: {e:?}")); match files::dl_open(name.as_bytes()) { Ok(handle) => { first.get_or_insert((name, handle)); diff --git a/tests/guest/rust-tests/src/bin/abuse_page_straddle.rs b/tests/guest/rust-tests/src/bin/abuse_page_straddle.rs index 47bf8228a7a..70d81156fd6 100644 --- a/tests/guest/rust-tests/src/bin/abuse_page_straddle.rs +++ b/tests/guest/rust-tests/src/bin/abuse_page_straddle.rs @@ -17,7 +17,7 @@ mod image; use toyos_abi::RawHandle; use toyos_abi::syscall::{ - self, VmAt, VmProt, OpenFlags, ProcessStats, SpawnArgs, Stat, SyscallError, SYS_FSTAT, + self, VmAt, VmProt, ProcessStats, SpawnArgs, Stat, SyscallError, SYS_FSTAT, SYS_PROCESS_STATS, }; @@ -64,8 +64,7 @@ fn main() { let base = region as u64; let boundary = base + PAGE; - let own = image::in_tmp("/system/bin/test_rs_abuse_page_straddle"); - let handle = syscall::open(own.as_bytes(), OpenFlags::READ).expect("open self"); + let handle = syscall::null(toyos_abi::Rights::READ).expect("a generic object for fstat"); let poison = |addr: u64, len: usize| { for i in 0..len { unsafe { (addr as *mut u8).add(i).write_volatile(CANARY) }; diff --git a/tests/guest/rust-tests/src/bin/abuse_readonly_copyout.rs b/tests/guest/rust-tests/src/bin/abuse_readonly_copyout.rs index 2d1b291393f..17a37eeaff3 100644 --- a/tests/guest/rust-tests/src/bin/abuse_readonly_copyout.rs +++ b/tests/guest/rust-tests/src/bin/abuse_readonly_copyout.rs @@ -15,24 +15,18 @@ //! clock page asserts in every stamp any process takes, `logkeeper`'s among them, so //! the clock arm's verdict may reach no other line. -#[path = "../image.rs"] -mod image; - use std::os::toyos::io::AsRawHandle; use std::process::Command; use toyos_abi::clock::{ClockPage, CLOCK_PAGE}; -use toyos_abi::syscall::{self, OpenFlags, VmAt, VmProt, ProcessStats, SeekFrom, SyscallError}; +use toyos_abi::syscall::{self, VmAt, VmProt, ProcessStats, SyscallError}; use toyos_abi::RawHandle; const SELF_PATH: &str = "/system/bin/test_rs_abuse_readonly_copyout"; const CHILD_ARG: &str = "reads-the-clock"; const PAGE_4K: u64 = 4096; -/// Covers a `ProcessStats`, and ends inside the file this reads. +/// Covers a complete `ProcessStats` in both copyout probes. const LEN: usize = core::mem::size_of::(); -/// The bytes the straddling `read` offers, at [`STRADDLE_AT`]. -const PROBE_PATH: &[u8] = b"/tmp/abuse_readonly_copyout.bin"; -const STRADDLE_AT: u64 = 256; /// The straddling `read`: half on the last writable page, half on the next. const STRADDLE: usize = 16; @@ -50,12 +44,14 @@ fn snapshot(addr: u64) -> [u8; LEN] { out } -/// A fresh handle on this binary at offset 0, so every `read` has bytes to give: -/// its copy in the kernel's `/tmp`, made once. -fn open_self() -> RawHandle { - static OWN: std::sync::OnceLock = std::sync::OnceLock::new(); - let own = OWN.get_or_init(|| image::in_tmp(SELF_PATH)); - syscall::open(own.as_bytes(), OpenFlags::READ).expect("open self") +/// A pipe containing a known payload, with its writer closed. The read +/// exercises the kernel's generic copyout without filesystem policy. +fn source() -> RawHandle { + let pipe = syscall::pipe().expect("a source pipe"); + syscall::write(pipe.write, &[0x5a; LEN]).expect("fill the source pipe"); + // SAFETY: this function owns the raw writer. + unsafe { syscall::close(pipe.write) }; + pipe.read } /// `len` bytes at `addr`, as the buffer a `read` is asked to fill. @@ -71,7 +67,7 @@ unsafe fn target(addr: u64, len: usize) -> &'static mut [u8] { /// both refused, and not one byte moved. fn refused(what: &str, addr: u64, stats: RawHandle) -> Result<(), String> { let before = snapshot(addr); - let fd = open_self(); + let fd = source(); let ret = syscall::read(fd, unsafe { target(addr, LEN) }); // SAFETY: no type holds this number. unsafe { syscall::close(fd) }; @@ -103,8 +99,7 @@ fn refused_mmap(stats: RawHandle) -> Result<(), String> { /// A `read` that starts on a writable page and runs into the read-only page /// after it is refused whole. fn refused_across_pages() -> Result<(), String> { - let flags = OpenFlags::READ | OpenFlags::WRITE | OpenFlags::CREATE | OpenFlags::TRUNCATE; - let probe = syscall::open(PROBE_PATH, flags).expect("create the probe file"); + let probe = syscall::pipe().expect("a source pipe"); // SAFETY: placed anywhere, so it replaces nothing. let pages = unsafe { syscall::vm_reserve(2 * PAGE_4K as usize, VmProt::ReadWrite, VmAt::ANYWHERE) }.expect("reserve two pages"); let page = pages as u64 + PAGE_4K; @@ -115,10 +110,8 @@ fn refused_across_pages() -> Result<(), String> { // The writable half is offered its own bytes and the read-only half their // complement, so a write to the read-only page shows and one below harms nothing. let offered: [u8; STRADDLE] = core::array::from_fn(|i| if i < STRADDLE / 2 { before[i] } else { !before[i] }); - syscall::seek(probe, SeekFrom::Start(STRADDLE_AT)).expect("seek the probe"); - syscall::write(probe, &offered).expect("write the straddle's bytes"); - syscall::seek(probe, SeekFrom::Start(STRADDLE_AT)).expect("seek the probe"); - let ret = syscall::read(probe, unsafe { target(at, STRADDLE) }); + syscall::write(probe.write, &offered).expect("write the straddle's bytes"); + let ret = syscall::read(probe.read, unsafe { target(at, STRADDLE) }); let verdict = if before != snapshot(at) { Err(format!("read wrote across a writable page into the read-only one at {page:#x}")) } else if ret != Err(SyscallError::BadAddress) { @@ -128,8 +121,7 @@ fn refused_across_pages() -> Result<(), String> { }; unsafe { syscall::vm_release(pages, 2 * PAGE_4K as usize) }.expect("release"); // SAFETY: no type holds this number. - unsafe { syscall::close(probe) }; - syscall::delete(PROBE_PATH).expect("delete the probe file"); + unsafe { syscall::close(probe.read); syscall::close(probe.write); }; verdict } @@ -154,9 +146,9 @@ fn main() { assert!(first.wait().expect("wait for the first reader").success(), "the first reader failed"); let stats = RawHandle(first.as_raw_handle()); let mut ok = [0u8; LEN]; - let fd = open_self(); + let fd = source(); assert_eq!(syscall::read(fd, &mut ok), Ok(LEN), "read into a writable buffer"); - assert_eq!(&ok[..4], b"\x7fELF", "read put something other than this file in the buffer"); + assert_eq!(ok, [0x5a; LEN], "read returned the offered pipe bytes"); // SAFETY: no type holds this number. unsafe { syscall::close(fd) }; syscall::process_stats(stats, &mut ProcessStats::default()).expect("process_stats into a writable value"); diff --git a/tests/guest/rust-tests/src/bin/empty_dir_stat.rs b/tests/guest/rust-tests/src/bin/empty_dir_stat.rs index bc29f1c6cae..5f025f45e9f 100644 --- a/tests/guest/rust-tests/src/bin/empty_dir_stat.rs +++ b/tests/guest/rust-tests/src/bin/empty_dir_stat.rs @@ -1,126 +1,38 @@ -//! An empty directory and a path that names nothing are different answers. -//! -//! `Vfs::list` used to give both of them `NotFound`: a directory was visible -//! only through the files under it, so `mkdir("/tmp/d")` made something no -//! later call could see until something was written into it. What that cost is -//! not `readdir` — it is every tool whose two-argument form means "into this -//! directory", because `cp x d/` on a `d` that does not stat as a directory -//! silently writes a *file* named `d`. -//! -//! Asked at the syscall boundary and again through `std`, because it took a fix -//! at each: the kernel tells the two cases apart and `std`'s `is_dir` reads -//! anything the kernel accepted as a directory. A machine carrying one half and -//! not the other passes the first block below and reds in the second, which is -//! how they are kept from drifting apart again. +//! Empty, missing and occupied directories retain distinct served answers. use std::fs; +use std::io::ErrorKind; -use toyos_abi::syscall::{self, SyscallError}; - -/// `/tmp`, because it is a tmpfs whose directories exist only in the VFS's own -/// `created_dirs` — nothing on a disk can make one of these look like a -/// directory by accident, so a pass here is the VFS answering rather than a -/// filesystem remembering. const EMPTY: &str = "/tmp/empty_dir_stat_empty"; const MISSING: &str = "/tmp/empty_dir_stat_missing"; const WITH_FILE: &str = "/tmp/empty_dir_stat_full"; -fn readdir(path: &str) -> Result { - // One byte on purpose: the kernel reports the size the listing *needs* - // whether or not it fits, so the answer is the return value and the bytes - // are not wanted. - let mut buf = [0u8; 1]; - syscall::readdir(path.as_bytes(), &mut buf) +fn listing(path: &str) -> std::io::Result { + fs::read_dir(path)?.collect::>>().map(|entries| entries.len()) } fn main() { - fs::create_dir(EMPTY).expect("mkdir the empty directory"); - fs::create_dir(WITH_FILE).expect("mkdir the directory with a file in it"); - fs::write(format!("{WITH_FILE}/f"), b"x").expect("write the file"); - - // The distinction itself, at the one layer that can make it. - assert_eq!( - readdir(EMPTY), - Ok(0), - "an empty directory must list as empty, not refuse" - ); - assert_eq!( - readdir(MISSING), - Err(SyscallError::NotFound), - "a path that names nothing must refuse, not list as empty" - ); - // Non-vacuity: a kernel that answered `Ok` for everything would satisfy the - // first assertion and be no distinction at all. - let full = readdir(WITH_FILE).expect("a directory with a file in it must list"); - assert!(full > 0, "a directory holding a file listed {full} bytes"); - - // The same question through `std`, which is how a program asks it. - let listed = fs::read_dir(EMPTY) - .expect("read_dir on an empty directory must succeed") - .count(); - assert_eq!(listed, 0, "the empty directory yielded {listed} entries"); - - let err = fs::read_dir(MISSING).expect_err("read_dir on a missing path must fail"); - assert_eq!( - err.kind(), - std::io::ErrorKind::NotFound, - "read_dir on a missing path reported {err:?}" - ); - - // `stat`, which is the question `cp x d/` actually asks and the one - // `read_dir` above does not answer: `is_dir` decided it from the *length* - // of the listing, so an empty directory came back `NotFound`. - let empty = fs::metadata(EMPTY).expect("metadata on an empty directory must succeed"); - assert!(empty.is_dir(), "an empty directory answered is_dir() == false"); - assert!(!empty.is_file(), "an empty directory also answered is_file()"); - assert!(fs::metadata(WITH_FILE).expect("metadata on an occupied directory").is_dir()); - let err = fs::metadata(MISSING).expect_err("metadata on a missing path must fail"); - assert_eq!( - err.kind(), - std::io::ErrorKind::NotFound, - "metadata on a missing path reported {err:?}" - ); - - // A directory that has had a file removed from it is empty, not gone — the - // state the `created_dirs` lookup exists for, reached by ordinary use - // rather than by never writing anything. - fs::remove_file(format!("{WITH_FILE}/f")).expect("remove the file"); - assert_eq!( - readdir(WITH_FILE), - Ok(0), - "a directory emptied by a delete must still list as a directory" - ); - assert!( - fs::metadata(WITH_FILE).expect("metadata on an emptied directory").is_dir(), - "a directory emptied by a delete stopped stat-ing as one" - ); - - // `rmdir` reports the real outcome, not a blanket success (F20): a missing - // name, a non-empty directory, and a mount point are each refused. - assert_eq!( - syscall::rmdir(MISSING.as_bytes()), - Err(SyscallError::NotFound), - "rmdir of a name that never existed reported success", - ); - fs::write(format!("{WITH_FILE}/g"), b"y").expect("refill the directory"); - assert_eq!( - syscall::rmdir(WITH_FILE.as_bytes()), - Err(SyscallError::InvalidArgument), - "rmdir of a non-empty directory reported success", - ); - fs::remove_file(format!("{WITH_FILE}/g")).expect("empty it again"); - syscall::rmdir(WITH_FILE.as_bytes()).expect("rmdir of an empty directory must succeed"); - assert_eq!( - readdir(WITH_FILE), - Err(SyscallError::NotFound), - "a directory reported as removed still listed", - ); - syscall::rmdir(EMPTY.as_bytes()).expect("rmdir of the empty directory must succeed"); - assert_eq!( - syscall::rmdir("/tmp".as_bytes()), - Err(SyscallError::InvalidArgument), - "rmdir of a mount point reported success, which erases the mount's directories", - ); + fs::create_dir(EMPTY).expect("mkdir empty"); + fs::create_dir(WITH_FILE).expect("mkdir occupied"); + fs::write(format!("{WITH_FILE}/f"), b"x").expect("write file"); + assert_eq!(listing(EMPTY).expect("empty listing"), 0); + assert_eq!(listing(MISSING).expect_err("missing listing").kind(), ErrorKind::NotFound); + assert_eq!(listing(WITH_FILE).expect("occupied listing"), 1); + + let empty = fs::metadata(EMPTY).expect("empty metadata"); + assert!(empty.is_dir() && !empty.is_file()); + assert!(fs::metadata(WITH_FILE).expect("occupied metadata").is_dir()); + assert_eq!(fs::metadata(MISSING).expect_err("missing metadata").kind(), ErrorKind::NotFound); + assert_eq!(fs::remove_dir(MISSING).expect_err("missing rmdir").kind(), ErrorKind::NotFound); + assert!(fs::remove_dir(WITH_FILE).is_err(), "removed an occupied directory"); + + fs::remove_file(format!("{WITH_FILE}/f")).expect("remove file"); + assert_eq!(listing(WITH_FILE).expect("emptied listing"), 0); + assert!(fs::metadata(WITH_FILE).expect("emptied metadata").is_dir()); + fs::remove_dir(WITH_FILE).expect("remove emptied directory"); + assert_eq!(listing(WITH_FILE).expect_err("removed listing").kind(), ErrorKind::NotFound); + fs::remove_dir(EMPTY).expect("remove empty directory"); + assert!(fs::remove_dir("/tmp").is_err(), "removed a capability root"); println!("empty dir stat: empty stats as a directory, missing refuses, emptied stays one"); println!("rmdir outcome: missing refuses, non-empty refuses, mount refuses, empty removed"); diff --git a/tests/guest/rust-tests/src/bin/handle_lifetime.rs b/tests/guest/rust-tests/src/bin/handle_lifetime.rs index 01c6f88843c..bb533f600d6 100644 --- a/tests/guest/rust-tests/src/bin/handle_lifetime.rs +++ b/tests/guest/rust-tests/src/bin/handle_lifetime.rs @@ -1,28 +1,9 @@ -//! What a handle holds is released when the *last* handle goes, and no sooner — -//! on `close` and on being killed alike. +//! Duplicated descriptors survive the first close. Killing a process releases +//! its acceptor and ring, while accepted file writes remain readable. //! -//! Duplicating a handle used to copy a `ListenerId` and a `RingId` as bare -//! numbers while `close` unregistered the service and destroyed the ring -//! unconditionally, so `dup` and then closing either handle took the object out -//! from under the survivor. A file was already refcounted; it is here because -//! the kinds are one property, and a test covering three of them says nothing -//! about the fourth. -//! -//! **The service half is now a port**, and its witness is better for it: there -//! is no name to ask about, so what says the acceptor is alive is that a client -//! connecting through the connector is accepted, and what says it is gone is -//! that the next open answers [`SyscallError::Gone`] — the kernel's own record -//! of a server that has left, rather than a name nobody re-took. -//! -//! The kill half is why this is a guest test and not a host one. This kernel -//! does not unwind, so a `Drop` reached only by an orderly `close` would be -//! decoration: `kill` runs on another CPU and drains the victim's handle -//! table itself, and that is the path each case below re-checks. -//! -//! Roles: no argument is the test; `holder ` takes one object, reports -//! what it can about it, and waits to be killed. +//! `holder ` acquires the resource, reports readiness and waits to be killed. -use std::io::{BufRead, BufReader, Write}; +use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; use std::os::toyos::process::{CommandExt, ExitStatusExt}; use std::process::{Child, ChildStdout, Command, Stdio}; @@ -31,7 +12,7 @@ use toyos::{namespace, port, AsHandle}; use toyos_abi::end::{End, Fault}; use toyos_abi::handle::ObjectKind; use toyos_abi::inbox::RingLayout; -use toyos_abi::syscall::{self, OpenFlags, SeekFrom, SyscallError, SERVE_PREFIX}; +use toyos_abi::syscall::{self, SyscallError, SERVE_PREFIX}; #[path = "../census_wait.rs"] mod census_wait; @@ -40,8 +21,8 @@ const SELF_PATH: &str = "/system/bin/test_rs_handle_lifetime"; /// The name this test's own namespaces map to the port under test. Private to /// this process and its children, which is the whole of what a namespace is. const SERVICE: &str = "handle-lifetime-service"; -const PATH: &[u8] = b"/tmp/handle-lifetime.txt"; -const KILLED_PATH: &[u8] = b"/tmp/handle-lifetime-killed.txt"; +const PATH: &str = "/tmp/handle-lifetime.txt"; +const KILLED_PATH: &str = "/tmp/handle-lifetime-killed.txt"; const PAYLOAD: &[u8] = b"a file outlives the handle that was closed first"; const KILLED_PAYLOAD: &[u8] = b"written by a process that was killed before it could close"; /// How the kernel ends a process that names a handle it does not hold. @@ -62,54 +43,28 @@ fn main() { } fn test() { - an_undefined_open_flag_bit_is_refused(); file_survives_one_close(); acceptor_survives_one_close(); ring_survives_one_close(); kill_releases_acceptor(); kill_releases_ring(); - kill_flushes_file(); + kill_preserves_file_writes(); println!("file, acceptor and ring each outlive the first close and are released by kill"); } -/// A bit `OpenFlags` does not define is a caller asking for an openness this -/// kernel has no name for, so the open is refused before the path is resolved. -/// The differential is the bit: the same open without it is the handle below. -fn an_undefined_open_flag_bit_is_refused() { - // The lowest bit the ABI does not define. - const UNDEFINED: u64 = 1 << OpenFlags::KNOWN.0.trailing_ones(); - - let asked = OpenFlags(OpenFlags::WRITE.0 | OpenFlags::CREATE.0 | OpenFlags::TRUNCATE.0); - assert_eq!( - syscall::open(PATH, OpenFlags(asked.0 | UNDEFINED)).err(), - Some(SyscallError::InvalidArgument), - "an open carrying a flag bit this ABI does not define was served", - ); - let handle = syscall::open(PATH, asked).expect("the same open without the bit"); - // SAFETY: no type holds this number. - unsafe { syscall::close(handle) }; - println!(" an undefined OpenFlags bit is InvalidArgument, and without it the same open works"); -} - fn file_survives_one_close() { - let a = syscall::open(PATH, OpenFlags::WRITE | OpenFlags::CREATE | OpenFlags::TRUNCATE) - .expect("create the file"); - // SAFETY: nothing relies on being the source's only holder. - let b = unsafe { syscall::dup(a) }.expect("dup a file handle"); - syscall::write(b, PAYLOAD).expect("write through the dup"); - // SAFETY: no type holds this number. - unsafe { syscall::close(a) }; - - // Reading through the survivor is what says the first close did not take - // the file's cache entry with it. - syscall::seek(b, SeekFrom::Start(0)).expect("seek on the surviving handle"); - let mut buf = [0u8; 128]; - let n = syscall::read(b, &mut buf).expect("read through the surviving handle"); - assert_eq!(&buf[..n], PAYLOAD, "the surviving handle no longer names the file"); - // SAFETY: no type holds this number. - unsafe { syscall::close(b) }; + let a = std::fs::OpenOptions::new().read(true).write(true).create(true).truncate(true).open(PATH).expect("create the file"); + let mut b = a.try_clone().expect("duplicate the open description"); + b.write_all(PAYLOAD).expect("write through the duplicate"); + drop(a); + b.seek(SeekFrom::Start(0)).expect("seek the surviving description"); + let mut buf = Vec::new(); + b.read_to_end(&mut buf).expect("read through the surviving description"); + assert_eq!(buf, PAYLOAD); + drop(b); + std::fs::remove_file(PATH).unwrap(); } fn acceptor_survives_one_close() { @@ -223,7 +178,7 @@ fn kill_releases_acceptor() { /// /// **Every kind and not the one this arm is about**, because the arrival check /// is what says the rings were seen and the reclaim check is about everything -/// the dead process held: a `SharedMem` or a `File` it kept would otherwise be +/// the dead process held: a `SharedMem` or a connection it kept would otherwise be /// a green run. `Census::grown_since` is the comparison the census header asks /// for. /// @@ -254,24 +209,13 @@ fn kill_releases_ring() { census_wait::released_to(&before); } -/// A killed process's dirty file must still reach the filesystem: that flush -/// used to be a hand-written arm of `close_all`, and is now the handle's -/// own drop on the same teardown path. -fn kill_flushes_file() { +/// Accepted filesystem writes survive the writer's abrupt exit, whose +/// connection teardown releases its open descriptions. +fn kill_preserves_file_writes() { let (mut child, _) = spawn_holder("file"); kill_and_reap(&mut child); - - let handle = syscall::open(KILLED_PATH, OpenFlags::READ) - .expect("the killed holder's file does not exist"); - let mut buf = [0u8; 128]; - let n = syscall::read(handle, &mut buf).expect("read the killed holder's file"); - // SAFETY: no type holds this number. - unsafe { syscall::close(handle) }; - assert_eq!( - &buf[..n], - KILLED_PAYLOAD, - "a killed process's unclosed file was not written back" - ); + assert_eq!(std::fs::read(KILLED_PATH).unwrap(), KILLED_PAYLOAD); + std::fs::remove_file(KILLED_PATH).unwrap(); } fn spawn_holder(kind: &str) -> (Child, BufReader) { @@ -339,12 +283,9 @@ fn holder(kind: &str) { println!("held"); } "file" => { - let handle = syscall::open( - KILLED_PATH, - OpenFlags::WRITE | OpenFlags::CREATE | OpenFlags::TRUNCATE, - ) - .expect("holder: open"); - syscall::write(handle, KILLED_PAYLOAD).expect("holder: write"); + let mut file = std::fs::File::create(KILLED_PATH).expect("holder: open"); + file.write_all(KILLED_PAYLOAD).expect("holder: write"); + core::mem::forget(file); println!("held"); } other => panic!("holder: unknown kind {other:?}"), diff --git a/tests/guest/rust-tests/src/bin/hierarchy_paths.rs b/tests/guest/rust-tests/src/bin/hierarchy_paths.rs index 715e305420c..9efffdf4caa 100644 --- a/tests/guest/rust-tests/src/bin/hierarchy_paths.rs +++ b/tests/guest/rust-tests/src/bin/hierarchy_paths.rs @@ -1,18 +1,15 @@ -//! `/` is a synthesized read-only directory. -//! -//! The listing of `/` is a closed set, so a stray top-level name cannot appear -//! because some volume happens to carry a directory by that name; `/` is no -//! filesystem either, so every syscall that would change what is at it is -//! refused the way a read-only mount refuses one, and the machine survives it. +//! Synthetic directories expose only ancestors of endowed filesystem capabilities. use std::fs; +use std::os::toyos::process::CommandExt; +use std::process::Command; -use toyos_abi::syscall::{self, OpenFlags, SyscallError}; +use toyos::files::{self, At, Open}; +use toyos::{endow, namespace, port}; +use toyos_abi::syscall::{FileType, SVC_LABEL}; -/// `vfs::ROOT_ENTRIES`: the kernel's set and this one are two spellings of the -/// hierarchy, so a boot that grew another entry reds here. -const ROOT_ENTRIES: [&str; 9] = - ["apps", "boot", "config", "home", "log", "media", "state", "system", "tmp"]; +const SELF: &str = "/system/bin/test_rs_hierarchy_paths"; +const PROBE: &str = "/tmp/hierarchy_paths_probe"; fn names(dir: &str) -> Vec { let mut out: Vec = fs::read_dir(dir) @@ -24,65 +21,56 @@ fn names(dir: &str) -> Vec { } fn main() { - let listed = names("/"); - assert_eq!(listed, ROOT_ENTRIES, "/ lists {listed:?}"); - println!(" PASS / lists exactly {:?}", ROOT_ENTRIES); - - root_refuses_every_way_of_changing_it(); - media_is_an_empty_directory(); - - println!("all hierarchy tests passed"); + match std::env::args().nth(1).as_deref() { + Some("empty") => { + assert!(endow::namespace().expect("namespace").names().unwrap().is_empty()); + assert!(fs::metadata("/").expect("empty root").is_dir()); + assert!(names("/").is_empty()); + std::env::set_current_dir("/").expect("cwd in empty root"); + assert!(fs::read(PROBE).is_err(), "an empty namespace reached /tmp"); + } + Some("narrow") => narrowed(), + Some(other) => panic!("unknown role {other}"), + None => { + fs::write(PROBE, b"endowed").expect("create probe"); + // The unserved connector supplies an ancestor, without granting any + // global filesystem access or requiring a filesystem implementation. + let (_acceptor, connector) = port::create().expect("port"); + let narrow = namespace::build() + .keep(endow::namespace().expect("namespace"), &["fs:/tmp"]) + .add("fs:/nested/leaf", &connector) + .finish().expect("narrow namespace"); + for (role, names) in [("narrow", narrow), ("empty", namespace::build().finish().unwrap())] { + assert!(Command::new(SELF).arg(role).endow(SVC_LABEL, names.into()) + .status().expect("spawn namespace probe").success(), "{role} child failed"); + } + fs::remove_file(PROBE).expect("remove probe"); + println!("all hierarchy tests passed"); + } + } } -/// Every syscall that can change what is at a path, aimed at `/`. One each -/// rather than one representative: they are separate entry points, so a gate on -/// `open` alone says nothing about `unlink`. None may end the machine. -fn root_refuses_every_way_of_changing_it() { - assert_eq!( - syscall::open(b"/intruder", OpenFlags::CREATE | OpenFlags::WRITE), - Err(SyscallError::PermissionDenied), - "creating a file at / was permitted" - ); - assert_eq!( - syscall::delete(b"/system"), - Err(SyscallError::PermissionDenied), - "unlinking a mount point was permitted" - ); - assert_eq!( - syscall::rename(b"/system", b"/elsewhere"), - Err(SyscallError::PermissionDenied), - "renaming a mount point was permitted" - ); - assert_eq!( - syscall::mkdir(b"/intruder"), - Err(SyscallError::PermissionDenied), - "mkdir at / was permitted" - ); - // `InvalidArgument` and not `PermissionDenied`: a mount point is not a - // directory a caller may remove even where the mount itself is writable. - assert_eq!( - syscall::rmdir(b"/tmp"), - Err(SyscallError::InvalidArgument), - "rmdir of a mount point was permitted" - ); - assert_eq!( - syscall::symlink(b"/system/etc/os-release", b"/intruder"), - Err(SyscallError::PermissionDenied), - "a symlink at / was permitted" - ); +fn narrowed() { + assert_eq!(endow::namespace().expect("namespace").names().unwrap(), ["fs:/nested/leaf", "fs:/tmp"]); + assert_eq!(names("/"), ["nested", "tmp"]); + assert_eq!(names("/nested"), ["leaf"]); + assert!(fs::metadata("/nested").expect("synthetic ancestor").is_dir()); + assert!(fs::metadata("/system").is_err(), "an unendowed root name appeared"); - // The machine is still here, and / still holds what it held. - let after = names("/"); - assert_eq!(after, ROOT_ENTRIES, "a refused operation changed / to {after:?}"); - assert!(fs::metadata("/system/etc/os-release").is_ok(), "/system is unreadable after the refusals"); - println!(" PASS create, unlink, rename, mkdir, rmdir and symlink are all refused at /"); -} + let root = files::open(At::Cwd, b"/", Open { read: true, directory: true, ..Open::default() }).expect("open root"); + assert_eq!(files::fstat(root).expect("root fstat").file_type, FileType::Directory); + let file = files::open(At::Fd(root), b"tmp/hierarchy_paths_probe", Open { read: true, ..Open::default() }).expect("open from synthetic root"); + let mut bytes = [0; 7]; + assert_eq!(files::read(file, &mut bytes).expect("read relative file"), bytes.len()); + assert_eq!(&bytes, b"endowed"); + // SAFETY: these descriptors were opened here and are held nowhere else. + unsafe { toyos::fd::close(file).unwrap(); toyos::fd::close(root).unwrap(); } -/// A bare mount point is an empty directory, not an error: foreign volumes land -/// under it at the track's stage 5. -fn media_is_an_empty_directory() { - let listed = names("/media"); - assert!(listed.is_empty(), "/media holds {listed:?}"); - assert!(fs::read("/media/anything").is_err(), "/media served a file"); - println!(" PASS /media is an empty directory"); + assert!(fs::write("/intruder", b"x").is_err()); + assert!(fs::remove_file("/nested").is_err()); + assert!(fs::rename("/nested", "/elsewhere").is_err()); + assert!(fs::create_dir("/intruder").is_err()); + assert!(fs::remove_dir("/").is_err()); + assert!(std::os::toyos::fs::symlink("/tmp", "/intruder").is_err()); + assert_eq!(names("/"), ["nested", "tmp"], "a refused mutation changed the root"); } diff --git a/tests/guest/rust-tests/src/bin/kernel_objects_refused.rs b/tests/guest/rust-tests/src/bin/kernel_objects_refused.rs index 671e56dbb37..a979b78c2c3 100644 --- a/tests/guest/rust-tests/src/bin/kernel_objects_refused.rs +++ b/tests/guest/rust-tests/src/bin/kernel_objects_refused.rs @@ -3,93 +3,52 @@ //! machine goes on: the kernel never hands a process what it holds back for //! itself, so it never panics for want of memory a process took. //! -//! The process commits every frame it may, which costs nothing while no page -//! is touched; makes ranges, then directories, until each is refused, which -//! leaves less of what charges cover than either takes; is refused one of each -//! other kind, a listing of a directory of long names it made before among -//! them, larger than all the kernel keeps for itself; and is served again once -//! it gives everything back. Nothing it does in between allocates in its own -//! heap, which could only abort it. +//! The process commits every frame it may, then fills ranges and smaller +//! null objects until each is refused. It checks the remaining generic +//! allocations and gives everything back before checking recovery. No +//! filesystem service is asked to allocate while the machine has no room. //! //! While it holds every frame, every other process's allocation is refused //! too, so it has a boot of its own: `kernel_objects_refused_alone`. use std::process::Command; -use toyos_abi::syscall::{self, OpenFlags, SyscallError, VmAt, VmProt, MAX_RANGES}; +use toyos_abi::syscall::{self, SyscallError, VmAt, VmProt, MAX_RANGES}; const PAGE: usize = 4096; -/// An offset of the /tmp file no write reached. -const HOLE: u64 = 16 * PAGE as u64; - -/// The directories made under, and the most the fill makes before it is -/// owed a refusal: what a few superframes of charges hold, many times over. -const DIRS: &[u8] = b"/tmp/kernel_objects_refused"; -const MOST_DIRS: usize = 200_000; - -/// A directory of long names whose listing, the names held twice while it is -/// built, is past the 16 MiB the kernel keeps on the guest's 4 GiB: 8,192 -/// names of 3,900 bytes, 64 MiB listed. -const LONG: &[u8] = b"/tmp/kernel_objects_refused_long"; -const LONG_NAMES: usize = 8192; -const LONG_NAME: usize = 3900; - fn main() { - // Made before anything is taken, so what follows needs no heap of its - // own: the pipe that holds no frame yet, the file not yet - // written, every list at its bound, and stdout's buffer. + // All bookkeeping storage is allocated before exhausting memory. let unwritten = syscall::pipe().expect("a pipe before anything is taken"); - let file = syscall::open(b"/tmp/kernel_objects_refused.bin", OpenFlags::READ | OpenFlags::WRITE | OpenFlags::CREATE | OpenFlags::TRUNCATE) - .expect("a /tmp file before anything is taken"); - // Its last byte written, so every page below is a hole. - syscall::pwrite(file, b"x", 64 * PAGE as u64).expect("a /tmp pwrite before anything is taken"); - syscall::mkdir(DIRS).expect("the directory the fill makes its own in"); - syscall::mkdir(LONG).expect("the directory of long names"); - for n in 0..LONG_NAMES { - let file = syscall::open(long_path(&mut [0; 4096], n), OpenFlags::WRITE | OpenFlags::CREATE).expect("a long name before anything is taken"); - // SAFETY: no type holds this number. - unsafe { syscall::close(file) }; - } - let mut listing = [0u8; 64]; + let mut small = Vec::with_capacity(toyos_abi::RawHandle::MAX_SLOTS); let mut committed: Vec<(*mut u8, usize)> = Vec::with_capacity(128); let mut ranges: Vec<*mut u8> = Vec::with_capacity(MAX_RANGES); - let mut dirs = 0usize; println!(" kernel_objects_refused: taking every frame a process may"); commit_every_frame(&mut committed); let range_refused = fill_ranges(&mut ranges); - let dir_refused = fill_dirs(&mut dirs); + let objects_refused = fill_small(&mut small); let refused = [ ("a pipe", syscall::pipe().map(|_| ())), ("a port", syscall::port_create().map(|_| ())), ("an event", syscall::event_create().map(|_| ())), - ("a directory", syscall::mkdir(b"/tmp/kernel_objects_refused_more")), + ("a null object", syscall::null(toyos_abi::Rights::READ).map(|_| ())), ("a pipe's first write, its frame", syscall::write(unwritten.write, b"x").map(|_| ())), - ("a /tmp file's page", syscall::write(file, &[0x5a; PAGE]).map(|_| ())), - ("a /tmp file's page at an offset", syscall::pwrite(file, &[0x5a; PAGE], HOLE).map(|_| ())), ("a memory object", syscall::shm_create(PAGE).map(|_| ())), // SAFETY: placed anywhere, so it replaces nothing. ("a commitment", unsafe { syscall::vm_reserve(PAGE, VmProt::ReadWrite, VmAt::ANYWHERE) }.map(|_| ())), - ("a listing of long names", syscall::readdir(LONG, &mut listing).map(|_| ())), ]; - // A hole needs no page kept for it: a pread of one answers its zeros. - let mut hole = [0xffu8; 64]; - let read_hole = syscall::pread(file, &mut hole, HOLE); - - give_back(&committed, &ranges, dirs); - assert_eq!(read_hole, Ok(64), "a pread of a /tmp file's hole with every frame taken"); - assert!(hole.iter().all(|&b| b == 0), "a pread of a hole with every frame taken read other than zeros"); + give_back(&committed, &ranges, &small); assert!(range_refused < MAX_RANGES - 64, "ranges were refused at {range_refused}, the per-space bound, not for memory"); - assert!(dir_refused < MOST_DIRS, "{MOST_DIRS} directories were made and none refused: directories are not charged"); - println!(" PASS: {range_refused} ranges, then {dir_refused} directories, and the next of each was refused"); + assert!(objects_refused < toyos_abi::RawHandle::MAX_SLOTS - 64, "the handle limit was reached before memory ran out"); + println!(" PASS: {range_refused} ranges, then {objects_refused} null objects, and the next of each was refused"); for (what, answer) in refused { assert_eq!(answer, Err(SyscallError::ResourceExhausted), "{what} with every frame taken"); println!(" PASS: {what} is refused, ResourceExhausted"); } - served_again(unwritten.write, file); + served_again(unwritten.write); println!("kernel_objects_refused: every kind is refused at its call while nothing is left, and served once it is given back"); } @@ -126,76 +85,22 @@ fn fill_ranges(ranges: &mut Vec<*mut u8>) -> usize { } } -/// Directories `DIRS/` until one is refused: how many were made. -fn fill_dirs(dirs: &mut usize) -> usize { - while *dirs < MOST_DIRS { - match syscall::mkdir(dir_path(&mut [0; 64], *dirs)) { - Ok(()) => { - *dirs += 1; - // Never silent for long: a fill that is still working says so. - if dirs.is_multiple_of(20_000) { - println!(" kernel_objects_refused: {} directories made", *dirs); - } - } - Err(SyscallError::ResourceExhausted) => return *dirs, - Err(e) => panic!("directory {} answered {e:?}", *dirs), - } - } - *dirs -} - -/// `LONG/` padded with `a` to [`LONG_NAME`] bytes, written into `buf`. -fn long_path(buf: &mut [u8; 4096], n: usize) -> &[u8] { - let at = LONG.len() + 1; - buf[..LONG.len()].copy_from_slice(LONG); - buf[LONG.len()] = b'/'; - let digits = number(n); - let len = digits.iter().position(|&d| d == 0).unwrap_or(digits.len()); - buf[at..at + len].copy_from_slice(&digits[..len]); - buf[at + len..at + LONG_NAME].fill(b'a'); - &buf[..at + LONG_NAME] -} - -/// `n` in decimal, ended by a zero byte. -fn number(n: usize) -> [u8; 21] { - let mut digits = [0u8; 21]; - let (mut n, mut len) = (n, 0); +/// Consume the smaller object charges left after a range no longer fits. +fn fill_small(objects: &mut Vec) -> usize { loop { - digits[len] = b'0' + (n % 10) as u8; - len += 1; - n /= 10; - if n == 0 { - break; + match syscall::null(toyos_abi::Rights::READ) { + Ok(object) => objects.push(object), + Err(SyscallError::ResourceExhausted) => return objects.len(), + Err(e) => panic!("a null object answered {e:?}"), } + assert!(objects.len() < objects.capacity(), "every handle filled before memory ran out"); } - digits[..len].reverse(); - digits } -/// `DIRS/`, written into `buf`. -fn dir_path(buf: &mut [u8; 64], n: usize) -> &[u8] { - buf[..DIRS.len()].copy_from_slice(DIRS); - buf[DIRS.len()] = b'/'; - let mut digits = [0u8; 20]; - let (mut n, mut len) = (n, 0); - loop { - digits[len] = b'0' + (n % 10) as u8; - len += 1; - n /= 10; - if n == 0 { - break; - } - } - let at = DIRS.len() + 1; - for (i, digit) in digits[..len].iter().rev().enumerate() { - buf[at + i] = *digit; - } - &buf[..at + len] -} - -fn give_back(committed: &[(*mut u8, usize)], ranges: &[*mut u8], dirs: usize) { - for n in 0..dirs { - syscall::rmdir(dir_path(&mut [0; 64], n)).expect("remove a directory the fill made"); +fn give_back(committed: &[(*mut u8, usize)], ranges: &[*mut u8], small: &[toyos_abi::RawHandle]) { + for &object in small { + // SAFETY: raw handles this function owns, each closed once. + unsafe { syscall::close(object) }; } for &at in ranges { // SAFETY: a range this test made, which nothing reaches. @@ -208,25 +113,23 @@ fn give_back(committed: &[(*mut u8, usize)], ranges: &[*mut u8], dirs: usize) { } /// Everything refused above is served, and a spawn runs: the machine went on. -fn served_again(unwritten: toyos_abi::handle::RawHandle, file: toyos_abi::handle::RawHandle) { +fn served_again(unwritten: toyos_abi::handle::RawHandle) { let pipe = syscall::pipe().expect("a pipe once memory is given back"); // SAFETY: no type holds this number. unsafe { syscall::close(pipe.read) }; // SAFETY: no type holds this number. unsafe { syscall::close(pipe.write) }; assert_eq!(syscall::write(unwritten, b"x"), Ok(1), "a pipe's first write once memory is given back"); - assert_eq!(syscall::write(file, &[0x5a; PAGE]), Ok(PAGE), "a /tmp write once memory is given back"); - assert_eq!(syscall::pwrite(file, &[0x5a; PAGE], HOLE), Ok(PAGE), "a /tmp pwrite once memory is given back"); - // SAFETY: no type holds this number. - unsafe { syscall::close(file) }; - syscall::delete(b"/tmp/kernel_objects_refused.bin").expect("delete the /tmp file"); - syscall::rmdir(DIRS).expect("remove the fill's directory"); - let mut listing = [0u8; 64]; - assert!(syscall::readdir(LONG, &mut listing).is_ok_and(|needed| needed > listing.len()), "a listing of long names once memory is given back"); - for n in 0..LONG_NAMES { - syscall::delete(long_path(&mut [0; 4096], n)).expect("delete a long name"); + let null = syscall::null(toyos_abi::Rights::READ).expect("a null object after release"); + let memory = syscall::shm_create(PAGE).expect("memory object after release"); + let port = syscall::port_create().expect("port after release"); + let event = syscall::event_create().expect("event after release"); + // SAFETY: each raw handle was created here and closed once. + unsafe { + syscall::close(null); syscall::close(memory); + syscall::close(port.acceptor); syscall::close(port.connector); + syscall::close(event.signal); syscall::close(event.wait); } - syscall::rmdir(LONG).expect("remove the directory of long names"); let echo = Command::new("/system/bin/echo").arg("still alive").output().expect("spawn echo once memory is given back"); assert!(echo.status.success() && String::from_utf8_lossy(&echo.stdout).trim() == "still alive", "echo answered {echo:?}"); println!(" PASS: every kind is served again once the memory is given back, and a spawn runs"); diff --git a/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs b/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs index f4e3b14b762..f391cf87719 100644 --- a/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs +++ b/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs @@ -9,7 +9,7 @@ //! waiter left parked is a hang the runner's ceiling reds. A watch made after //! the leaving is answered by the submit that makes it. //! -//! **The refusal.** A connection and a file have no other end, and a watch that +//! **The refusal.** A connection and a memory object have no other end, and a watch that //! asks for one is refused whatever else it asks. //! //! **Its own close.** An end whose holder closes it while the other end lives @@ -21,9 +21,6 @@ //! it moved to a child that still lives: the watch stays armed through each, //! and is answered once that holder goes. -#[path = "../image.rs"] -mod image; - use std::io::Read; use std::os::toyos::process::CommandExt; use std::process::{Command, Stdio}; @@ -35,7 +32,7 @@ use toyos::ipc::Connection; use toyos::poller::{Poller, OTHER_END_GONE, READABLE}; use toyos::syscap::SysCap; use toyos::{namespace, pipe_pair, port, AsHandle, BorrowedHandle, OwnedHandle, Pipe}; -use toyos_abi::syscall::{self, OpenFlags, SyscallError}; +use toyos_abi::syscall::{self, SyscallError}; #[path = "../roster.rs"] mod roster; @@ -76,13 +73,10 @@ fn test() { let (client, server) = connection(); refused(client.as_handle(), "a connection"); - let own = image::in_tmp(SELF_PATH); - let file = syscall::open(own.as_bytes(), OpenFlags::READ).expect("this binary's own file"); - // SAFETY: the kernel installed the handle for this call and no other. - let file = unsafe { OwnedHandle::from_raw(file) }; - refused(file.as_handle(), "a file"); - drop(file); - println!(" a connection and a file were refused the question"); + let memory = toyos::shm::SharedMemory::create(4096).expect("a memory object"); + refused(memory.as_handle(), "a memory object"); + drop(memory); + println!(" a connection and a memory object were refused the question"); for own in [End::Read, End::Write] { let (reader, writer) = pipe(""); diff --git a/tests/guest/rust-tests/src/bin/readdir_bound.rs b/tests/guest/rust-tests/src/bin/readdir_bound.rs index 3ce0c8d2522..e55353f4316 100644 --- a/tests/guest/rust-tests/src/bin/readdir_bound.rs +++ b/tests/guest/rust-tests/src/bin/readdir_bound.rs @@ -1,129 +1,31 @@ -//! `read_dir` must return every entry or an error, never a short listing. -//! -//! Two defects, and the second is what makes the first's bound honest. The -//! kernel built a directory listing with one entry per file and no cap, so a -//! `read_dir` over 32,769 files in one tmpfs directory panicked it — measured, -//! 1.8 s, from `fs::write` in a loop. And `SYS_READDIR` filled the caller's -//! buffer, stopped, and reported the bytes it had written, which is -//! indistinguishable from a complete listing: 4125 entries of 34,816, as -//! success. A bound plus a silent truncation is a quieter version of the same -//! defect, so both are asserted here. All below is an ordinary workload. +//! A served directory listing larger than one reply returns every entry once. +use std::collections::BTreeSet; use std::fs; -use std::process::Command; -/// `vfs::MAX_LIST_ENTRIES`. The kernel refuses a listing above it; this test -/// exists partly to prove the number is not above the real ceiling, so it must -/// track the kernel's. -const MAX_LIST_ENTRIES: usize = 16_384; - -/// Enough plain entries that the encoded listing cannot fit std's first -/// buffer (65,536 bytes). At ~15 bytes an entry this is ~90 KB, so a -/// truncating kernel returns about 4,000 of them. -const PLAIN_ENTRIES: usize = 6_000; +const DIR: &str = "/tmp/readdir_bound"; +const ENTRIES: usize = 6_000; fn main() { - plain_entries_are_all_returned(); - subdirectories_at_the_limit(); - one_past_the_limit_is_refused(); - a_directory_still_lists_in_a_mount_past_the_bound(); - system_alive(); - println!("all readdir bound tests passed"); -} - -/// A listing larger than the caller's buffer comes back whole. -/// -/// Plain file entries, in their own subdirectory, so this exercises the -/// `result` path rather than the dedup one. The count is the assertion: a -/// truncating kernel returns a valid-looking prefix of it. -fn plain_entries_are_all_returned() { - for i in 0..PLAIN_ENTRIES { - fs::write(format!("/tmp/a/f{i}"), b"").expect("create failed"); - } - let n = fs::read_dir("/tmp/a").expect("read_dir failed").count(); - assert_eq!(n, PLAIN_ENTRIES, "read_dir truncated: {n} entries of {PLAIN_ENTRIES}"); - println!(" PASS: {n} entries returned whole, past a 65,536-byte buffer"); - - for i in 0..PLAIN_ENTRIES { - fs::remove_file(format!("/tmp/a/f{i}")).expect("remove failed"); + fs::create_dir(DIR).expect("create listing directory"); + // The names alone exceed a 64 KiB reply, independently of wire overhead. + for i in 0..ENTRIES { + fs::write(format!("{DIR}/entry-{i:06}"), b"").expect("create file"); } -} - -/// Exactly `MAX_LIST_ENTRIES` entries, nearly all in the shape that costs the -/// most. -/// -/// Every name but [`PLAIN_AT_LIMIT`] is `d/f`, so listing `/tmp` puts one -/// entry per distinct subdirectory into the kernel's dedup set — the -/// allocation the bound is derived against. A directory opened is numbered -/// (`kernel/pure/dirnodes.rs`, `MAX_DIRECTORIES`, the same count), and this -/// test opens two more than it fills, `/tmp` here and `/tmp/empty` below, so -/// those two entries are plain files. It succeeding is what says the bound is -/// not above the real ceiling; deriving it on paper does not. -fn subdirectories_at_the_limit() { - for i in 0..MAX_LIST_ENTRIES - PLAIN_AT_LIMIT { - fs::write(format!("/tmp/d{i}/f"), b"").expect("create failed"); - } - for i in 0..PLAIN_AT_LIMIT { - fs::write(format!("/tmp/plain{i}"), b"").expect("create failed"); - } - let n = fs::read_dir("/tmp").expect("read_dir at the limit failed").count(); - assert_eq!(n, MAX_LIST_ENTRIES, "listing at the limit returned {n}"); - println!(" PASS: {n} entries listed at the limit, {} of them distinct subdirectories", n - PLAIN_AT_LIMIT); -} - -/// The entries at the limit that are files, leaving a directory number each -/// for the two directories the test opens. -const PLAIN_AT_LIMIT: usize = 2; - -/// The whole listing of `path`, or the error that ended it: opening the -/// directory or any read of its listing. -fn listing(path: &str) -> std::io::Result { - fs::read_dir(path)?.collect::>>().map(|entries| entries.len()) -} - -/// One more, and the answer is an error rather than a panic or a short list. -fn one_past_the_limit_is_refused() { - fs::write(format!("/tmp/d{MAX_LIST_ENTRIES}/f"), b"").expect("create failed"); - match listing("/tmp") { - Ok(n) => panic!("listing past the limit returned {n} entries"), - Err(e) => println!(" PASS: one past the limit refused ({e})"), + let mut names = BTreeSet::new(); + for entry in fs::read_dir(DIR).expect("open listing") { + let entry = entry.expect("read listing batch"); + assert!(entry.file_type().expect("entry type").is_file()); + let name = entry.file_name().into_string().expect("UTF-8 name"); + assert!(names.insert(name), "listing repeated an entry"); } -} - -/// The bound belongs to the directory, not to the mount holding it. -/// -/// `/tmp` is over it by now and cannot list its own root, which is the shape -/// asserted at the end: every directory *in* it still lists, including one -/// holding nothing — the answer a mount-wide bound cannot give, an empty -/// listing and a refusal being the same byte to a caller checking existence. -fn a_directory_still_lists_in_a_mount_past_the_bound() { - let full = fs::read_dir("/tmp/d0").expect("read_dir of a directory in a full tmpfs"); - let names: Vec = - full.map(|e| e.expect("entry").file_name().to_string_lossy().into_owned()).collect(); - assert_eq!(names, vec![String::from("f")], "/tmp/d0 listed {names:?}"); - - fs::create_dir("/tmp/empty").expect("mkdir failed"); - let n = fs::read_dir("/tmp/empty").expect("read_dir of an empty directory in a full tmpfs").count(); - assert_eq!(n, 0, "an empty directory listed {n} entries"); - println!(" PASS: /tmp serves a directory and an empty one while its own root is refused"); - - // The mount root is a directory too, and its bound is still the same one. - match listing("/tmp") { - Ok(n) => panic!("/tmp listed {n} entries after the per-directory bound"), - Err(e) => println!(" PASS: /tmp's own root is still refused ({e})"), + assert_eq!(names.len(), ENTRIES, "listing omitted entries"); + for i in 0..ENTRIES { + let name = format!("entry-{i:06}"); + assert!(names.contains(&name), "listing omitted {name}"); + fs::remove_file(format!("{DIR}/{name}")).expect("remove file"); } -} - -/// The refusal cost the caller an error and nothing else. A panic inside -/// `Vfs::list` would have stranded the VFS lock, so a spawn — which reads the -/// binary through the same lock — is the check that the filesystem still -/// works, not just that this process is still running. -fn system_alive() { - let output = Command::new("/system/bin/echo") - .arg("still alive") - .output() - .expect("failed to run echo after the refusal"); - assert!(output.status.success()); - assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "still alive"); - println!(" PASS: the VFS still serves after the refusal"); + assert_eq!(fs::read_dir(DIR).expect("emptied listing").collect::>>().expect("empty batch").len(), 0); + fs::remove_dir(DIR).expect("remove listing directory"); + println!("all readdir bound tests passed"); } diff --git a/tests/guest/rust-tests/src/bin/spawn_returns_the_census.rs b/tests/guest/rust-tests/src/bin/spawn_returns_the_census.rs index e271aca2bec..07729752ae5 100644 --- a/tests/guest/rust-tests/src/bin/spawn_returns_the_census.rs +++ b/tests/guest/rust-tests/src/bin/spawn_returns_the_census.rs @@ -1,9 +1,5 @@ -//! A spawn and the end of its child leave every kind of kernel object as it -//! was, for a program from each of three places: a file its server holds in -//! memory, which it answers as a region over those pages; a file a server -//! does not, which the spawn reads whole into a region of the spawner's own; -//! and a file of the kernel's `/tmp`, which the kernel opens itself. Whatever -//! the child is paged from, the spawner and the child must both let it go. +//! Spawns from served system, persistent and memory volumes release their +//! kernel objects when the child and its process handle are gone. //! //! Roles: no argument is the test; `held` writes a line and waits for its //! stdin to end. @@ -34,7 +30,9 @@ fn main() { let copy = in_state(SELF_PATH); spawns_return_the_census("a copy in /state", ©); std::fs::remove_file(©).unwrap_or_else(|e| panic!("remove {copy}: {e}")); - spawns_return_the_census("a copy in /tmp", &image::in_tmp(SELF_PATH)); + let copy = image::in_tmp(SELF_PATH); + spawns_return_the_census("a copy in /tmp", ©); + std::fs::remove_file(©).expect("remove /tmp executable"); } } } diff --git a/tests/guest/rust-tests/src/bin/vector_state.rs b/tests/guest/rust-tests/src/bin/vector_state.rs index 540f3063c98..672262ad874 100644 --- a/tests/guest/rust-tests/src/bin/vector_state.rs +++ b/tests/guest/rust-tests/src/bin/vector_state.rs @@ -5,7 +5,7 @@ //! //! More threads than CPUs, each spinning with its registers loaded, so every //! CPU is time-shared and each switch puts a thread with other values on it. -//! The spin stops every so often for a `stat`, whose path walk the kernel runs +//! The spin stops every so often for an object `fstat`, which the kernel runs //! on the thread's own kernel stack, below the frame of the switch that //! preempts the spin. A CPU whose save skips a component unmodified since its //! restore from the same address (SDM Vol. 1 §13.6) keeps it only where @@ -14,15 +14,11 @@ #[path = "../arch/vector.rs"] mod vector; -/// What each burst `stat`s: a file of the kernel's `/tmp`, so the kernel walks -/// its path, which it does for no path a file server holds. -const STAT_PATH: &str = "/tmp/vector_state"; - /// Rounds per thread, each with a pattern of its own, so a value left from /// the thread's own previous round is caught as surely as another thread's. const ROUNDS: u64 = 3; -/// Bursts of the hold loop, each ending in a `stat`: past a quantum on the +/// Bursts of the hold loop, each ending in an object `fstat`: past a quantum on the /// T14 and under TCG alike. const BURSTS: u64 = 1_000; @@ -42,8 +38,6 @@ fn pattern(thread: u64, round: u64, len: usize) -> Vec { fn main() { let width = vector::width(); - std::fs::write(STAT_PATH, b"x").expect("a file for the kernel to walk to"); - let path = STAT_PATH.as_bytes(); let threads = 2 * toyos_abi::syscall::cpu_count() as u64 + 1; let workers: Vec<_> = (0..threads) .map(|thread| { @@ -54,7 +48,7 @@ fn main() { } for round in 0..ROUNDS { let want = pattern(thread, round, width.bytes()); - let got = vector::hold(width, &want, BURSTS, path); + let got = vector::hold(width, &want, BURSTS); if let Some(register) = width.first_difference(&got, &want) { panic!("thread {thread} round {round}: {register} changed while it was switched out"); } @@ -65,6 +59,5 @@ fn main() { for (thread, worker) in workers.into_iter().enumerate() { assert!(worker.join().is_ok(), "thread {thread} found a vector register not its own"); } - std::fs::remove_file(STAT_PATH).expect("the walked file removed"); println!("vector_state: {threads} threads each held {width:?} through {ROUNDS} rounds"); } diff --git a/tests/guest/rust-tests/src/image.rs b/tests/guest/rust-tests/src/image.rs index 7a6d9a368eb..1d9de3c22cc 100644 --- a/tests/guest/rust-tests/src/image.rs +++ b/tests/guest/rust-tests/src/image.rs @@ -1,7 +1,4 @@ -//! What a raw `SYS_SPAWN` names as its program and what a raw `SYS_OPEN` -//! opens, now that the kernel opens no file of `/system`: a program read whole -//! into a memory object of this process's own, and a file copied to the -//! kernel's `/tmp`. +//! Read a served executable into the memory object a raw spawn accepts. // Each binary takes the half it needs. #![allow(dead_code)] @@ -18,8 +15,7 @@ pub fn image_of(path: &str) -> Image { Image::read(len, |buf| file.read(buf)).unwrap_or_else(|why| panic!("read {path} into an image: {why:?}")) } -/// `path` copied to `/tmp` under its own name, which the kernel opens: the -/// path to open. +/// `path` copied to the memory filesystem under its own name. pub fn in_tmp(path: &str) -> String { let name = path.rsplit('/').next().expect("a path names a file"); let copy = format!("/tmp/{name}"); diff --git a/tests/metal/src/lib.rs b/tests/metal/src/lib.rs index 8ef0e48db0d..772cf9e4f09 100644 --- a/tests/metal/src/lib.rs +++ b/tests/metal/src/lib.rs @@ -732,7 +732,6 @@ pub const FLASHABLE: &[&str] = &[ // the boot goes on to userland and ends the way an unarmed one does; what // an armed image leaves behind is a longer log. "pci-cap-selftest", - "revoked-backing-selftest", "lapic-spurious-selftest", "unclaimed-vector-selftest", // A probe rather than staged inputs, and a read: it runs inside the first diff --git a/tests/toyos.rs b/tests/toyos.rs index 4f33f1c52b2..4308b7aeaaf 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -183,10 +183,6 @@ const RUST_SKIP: &[&str] = &[ // `tests/images/floorcase.toml`: `memory_floor_holds_the_controls` runs it // on QEMU and the `memory_floor` metal row on the T14. "memory_floor", - // Fills /tmp to the VFS listing limit, so it needs a boot nothing else - // shares — every later `read_dir("/tmp")` in it would be refused. - // `readdir_bound` gives it one. - "readdir_bound", // They need an HDA controller, which the shared boot's machine has not: // the `hda_tone`, `hda_tone_resampled` and `hda_volume_step` metal rows // run them on the T14 and the AMD bench, and @@ -862,7 +858,7 @@ const METAL: &[(&str, metal::Metal)] = &[ ( "readdir_bound", metal::Metal { - arms: TESTCASES_READDIR, + arms: TESTCASES, judge: |b| b[0].job_passed("test_rs_readdir_bound"), }, ), @@ -1458,10 +1454,6 @@ const METAL: &[(&str, metal::Metal)] = &[ "pci_capability_walk", metal::Metal { arms: SELFTESTS, judge: |b| pci_cap_selftest(b[0].kernel().text()) }, ), - ( - "read_fault_selftests", - metal::Metal { arms: SELFTESTS, judge: |b| read_fault_probes(b[0].kernel().text()) }, - ), ( "lapic_spurious_vector", metal::Metal { arms: SELFTESTS, judge: |b| lapic_vectors(b[0].kernel().text()) }, @@ -2012,11 +2004,6 @@ const TESTCASES_REFUSED: &[metal::Arm] = const FLOORCASE: &[metal::Arm] = &[metal::once("floorcase", "tests/images/floorcase.toml", &[], &["test_rs_memory_floor"])]; -/// Its own boot: `readdir_bound` fills `/tmp` to the VFS listing limit, and -/// every later `read_dir("/tmp")` in a boot it shared would be refused. -const TESTCASES_READDIR: &[metal::Arm] = - &[metal::once("testcases-readdir", "tests/images/testcases.toml", &[], &["test_rs_readdir_bound"])]; - /// The shipping kernel with the windows' instrument and nothing else, so what /// it reads is that kernel under the herd. Three exits, a report each: /// `idle_span`'s is the boot's first, where the kernel holds; `pwd`'s reads the @@ -2137,7 +2124,6 @@ const PROCTREECASE: &[metal::Arm] = &[metal::once( /// [`ACTUATOR_TESTS`] ride, which is the same kernel. const SELFTEST_PARAMS: &[&str] = &[ "pci-cap-selftest", - "revoked-backing-selftest", "lapic-spurious-selftest", "unclaimed-vector-selftest", "sysret-ss-probe", @@ -6476,19 +6462,6 @@ fn app_view(back: &metal::Readback) -> Result<(), String> { Ok(()) } -/// A backing read after deletion is refused on both writable mounts, and a page-cache slot whose fill the device refused is unbound. -fn read_fault_probes(log: &str) -> Result<(), String> { - let probe = "revoke-selftest: /tmp/revoke_probe"; - let Some(verdict) = log.lines().find(|l| l.contains(probe)) else { - return Err(format!("{probe} never ran:\n{log}")); - }; - if !verdict.contains("PASS") { - return Err(format!("{}\n{log}", verdict.trim())); - } - eprintln!(" [read-fault] {}", verdict.trim()); - Ok(()) -} - /// The spurious vector and an unclaimed one are both gated rather than escalated to #DF. fn lapic_vectors(log: &str) -> Result<(), String> { for kind in ["spurious", "unclaimed"] { From 483562ea365a7f5cbb389390adcccc993687e770 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 19:50:30 +0100 Subject: [PATCH 20/93] Remove unused paged filesystem metadata Served fstat now reads authoritative filesystem metadata, so paged objects no longer need a timestamp, Stamp operation, or holder metadata syscall. Remove that path from the kernel, ABI, SDK and filesystem mirror while preserving size, atomic read versions, write spans and authoritative Volume timestamps. Pager create now takes backing kind after size. Namespace syscall 166 and the remaining paged operation numbers stay stable. Keep size-coherence tests and remove only assertions about the obsolete timestamp mirror. Include three mechanical file-server binary lint fixes uncovered by all-target clippy. Validation: SDK 153 and ABI 49 tests, paged 18 tests, both bare-target kernel clippy checks, SDK fd-table clippy, file-server binary check and all-target clippy passed. File-server tests 99 passed; its existing 64-page capacity test still fails with NoSpace before touching the pager. No guest image built here. --- kernel/pure/paged.rs | 62 ++++++++-------------- kernel/src/object/paged.rs | 56 +++++--------------- kernel/src/syscall/dispatch.rs | 7 ++- kernel/src/syscall/paged.rs | 24 ++------- sdk/abi/src/syscall.rs | 43 +++------------ sdk/toyos/src/paged.rs | 42 ++++++--------- system/fileserver/src/main.rs | 31 +++++------ system/fileserver/src/pages.rs | 78 +++++++++------------------- system/fileserver/src/pages/tests.rs | 41 ++++----------- tests/guest/pagerprobe/src/main.rs | 8 +-- 10 files changed, 115 insertions(+), 277 deletions(-) diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index 3b56f1c6901..210649580ca 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -227,8 +227,6 @@ pub struct Object { writing: u32, /// A call that did not take all it was given left its write under way. held: bool, - /// What its pager last set: nanoseconds, as the file's volume keeps it. - mtime: u64, } /// What a holder's read began at ([`Object::read_begin`]). @@ -253,20 +251,6 @@ impl Object { self.size.div_ceil(PAGE) } - pub fn mtime(&self) -> u64 { - self.mtime - } - - pub fn set_mtime(&mut self, mtime: u64) { - self.mtime = mtime; - } - - /// The version a holder's stat publishes: `None` while a write is under - /// way, as for a read. - pub fn published(&self) -> Option { - (self.writing == 0).then_some(self.version) - } - /// Where a holder's read begins: `None` while a write is under way. pub fn read_begin(&self) -> Option { (self.writing == 0).then_some(Version(self.version)) @@ -378,15 +362,15 @@ impl Pages { /// An object of `size` bytes, at most [`LAST_END`], named `id`: an id no /// live object of this table has, which a released one's may be, since /// [`release`](Self::release) takes every page it held. - pub fn object(&mut self, id: u32, size: u64, mtime: u64) -> Object { + pub fn object(&mut self, id: u32, size: u64) -> Object { assert!(size <= LAST_END, "an object of {size} bytes past the last page"); - Object { id, anonymous: false, size, gone: false, head: NONE, pages: 0, version: 0, writing: 0, held: false, mtime } + Object { id, anonymous: false, size, gone: false, head: NONE, pages: 0, version: 0, writing: 0, held: false } } /// An anonymous object owns its pages: no backing can supply them again, /// so writeback, failure and withdrawal cannot discard present bytes. - pub fn anonymous(&mut self, id: u32, size: u64, mtime: u64) -> Object { - Object { anonymous: true, ..self.object(id, size, mtime) } + pub fn anonymous(&mut self, id: u32, size: u64) -> Object { + Object { anonymous: true, ..self.object(id, size) } } fn hash(&self, object: u32, page: u64) -> usize { @@ -766,7 +750,7 @@ mod tests { #[test] fn a_read_counts_only_where_no_write_began_or_was_under_way() { let mut p = Pages::::new(8, 4).unwrap(); - let mut o = p.object(1, PAGE, 0); + let mut o = p.object(1, PAGE); let before = o.read_begin().expect("nothing writes yet"); o.write_begin(); assert_eq!(o.read_begin(), None, "a read meets the write in flight"); @@ -785,7 +769,7 @@ mod tests { #[test] fn a_short_call_holds_the_write_until_its_end() { let mut p = Pages::::new(8, 4).unwrap(); - let mut o = p.object(2, PAGE, 0); + let mut o = p.object(2, PAGE); assert_eq!(o.end_write(), Err(Refused::InvalidArgument), "nothing is held"); o.write_begin(); o.write_end(false); @@ -795,19 +779,17 @@ mod tests { o.write_begin(); o.write_end(true); assert_eq!(o.read_begin(), None, "a later call, short or whole, ends no held write"); - assert_eq!(o.published(), None, "nor does a stat publish one half written"); let held = o.version; assert_eq!(o.end_write(), Ok(())); let v = o.read_begin().expect("the end lets a read begin"); assert!(o.unchanged(v) && v != Version(held), "the end moved the version"); - assert_eq!(o.published(), Some(v.0), "a stat publishes the version a read begins at"); assert_eq!(o.end_write(), Err(Refused::InvalidArgument), "a write ends once"); } #[test] fn anonymous_bytes_survive_cache_operations_and_release_with_the_object() { let mut pages = Pages::new(2, 0).unwrap(); - let mut object = pages.anonymous(1, 0, 0); + let mut object = pages.anonymous(1, 0); assert_eq!(pages.supply(&mut object, 0, 42, true, PAGE), Ok(Supplied::Installed)); assert_eq!(pages.dirty_held(), 0); assert_eq!(pages.write_pin(&mut object, 0), Ok(42)); @@ -831,7 +813,7 @@ mod tests { #[test] fn a_supplied_page_is_found_and_a_second_supply_changes_nothing() { let mut p = Pages::new(8, 4).unwrap(); - let mut o = p.object(1, 3 * PAGE, 0); + let mut o = p.object(1, 3 * PAGE); assert_eq!(p.lookup(&o, 1), Lookup::Absent); assert_eq!(p.supply(&mut o, 1, 100, false, 2 * PAGE), Ok(Supplied::Installed)); assert_eq!(p.supply(&mut o, 1, 200, false, 2 * PAGE), Ok(Supplied::Present)); @@ -842,10 +824,10 @@ mod tests { #[test] fn a_clean_supply_past_the_size_is_refused_and_a_dirty_one_extends_it() { let mut p = Pages::new(8, 4).unwrap(); - let mut o = p.object(2, PAGE, 0); + let mut o = p.object(2, PAGE); assert_eq!(p.supply(&mut o, 1, 1, false, 2 * PAGE), Err(Refused::PastEnd)); assert_eq!(o.size(), PAGE); - let mut short = p.object(3, 100, 0); + let mut short = p.object(3, 100); assert_eq!(p.supply(&mut short, 0, 2, false, PAGE), Ok(Supplied::Installed)); assert_eq!(short.size(), 100, "a clean page ending past the size leaves it"); assert_eq!(p.supply(&mut o, 1, 1, true, PAGE + 10), Ok(Supplied::Installed)); @@ -858,7 +840,7 @@ mod tests { #[test] fn a_reader_racing_an_extending_write_never_sees_its_unwritten_bytes() { let mut p = Pages::new(8, 4).unwrap(); - let mut o = p.object(4, 100, 0); + let mut o = p.object(4, 100); p.supply(&mut o, 0, 7, false, 100).unwrap(); assert_eq!(p.write_pin(&mut o, 0), Ok(7)); assert_eq!(o.size(), 100, "the copy is in flight: a reader still stops at the old end"); @@ -870,7 +852,7 @@ mod tests { #[test] fn an_unpin_after_a_shrink_removed_the_page_grows_nothing() { let mut p = Pages::new(8, 4).unwrap(); - let mut o = p.object(5, PAGE, 0); + let mut o = p.object(5, PAGE); p.supply(&mut o, 0, 7, false, PAGE).unwrap(); p.write_pin(&mut o, 0).unwrap(); p.resize(&mut o, 0, &mut none); @@ -885,7 +867,7 @@ mod tests { #[test] fn the_dirty_ceiling_refuses_and_a_write_back_frees_it() { let mut p = Pages::new(8, 2).unwrap(); - let mut o = p.object(6, 4 * PAGE, 0); + let mut o = p.object(6, 4 * PAGE); for page in 0..2 { p.supply(&mut o, page, page, true, (page + 1) * PAGE).unwrap(); } @@ -900,7 +882,7 @@ mod tests { #[test] fn kept_is_exactly_the_pages_dirtied_or_pinned_since_the_begin_or_unwritten() { let mut p = Pages::new(16, 16).unwrap(); - let mut o = p.object(7, 8 * PAGE, 0); + let mut o = p.object(7, 8 * PAGE); for page in 0..6 { p.supply(&mut o, page, page, true, (page + 1) * PAGE).unwrap(); } @@ -922,7 +904,7 @@ mod tests { #[test] fn a_range_must_be_ended_exactly_as_begun_and_once() { let mut p: Pages = Pages::new(8, 8).unwrap(); - let mut o = p.object(8, PAGE * 1024, 0); + let mut o = p.object(8, PAGE * 1024); p.writeback_begin(&mut o, 0, 4).unwrap(); assert_eq!(p.writeback_begin(&mut o, 3, 2), Err(Refused::InvalidArgument), "overlapping"); assert_eq!(p.writeback_end(&mut o, 0, 3, true), Err(Refused::InvalidArgument), "not the range begun"); @@ -935,7 +917,7 @@ mod tests { #[test] fn only_clean_unheld_pages_are_withdrawn() { let mut p = Pages::new(8, 8).unwrap(); - let mut o = p.object(9, 4 * PAGE, 0); + let mut o = p.object(9, 4 * PAGE); p.supply(&mut o, 0, 10, false, PAGE).unwrap(); p.supply(&mut o, 1, 11, true, 2 * PAGE).unwrap(); p.supply(&mut o, 2, 12, true, 3 * PAGE).unwrap(); @@ -993,7 +975,7 @@ mod tests { #[test] fn a_withdrawal_of_a_long_range_walks_the_objects_pages() { let mut p = Pages::new(8, 8).unwrap(); - let mut o = p.object(1, LAST_END, 0); + let mut o = p.object(1, LAST_END); p.supply(&mut o, 7, 1, false, 8 * PAGE).unwrap(); p.supply(&mut o, 1 << 40, 2, false, ((1 << 40) + 1) * PAGE).unwrap(); let mut freed = Vec::new(); @@ -1015,7 +997,7 @@ mod tests { assert!(!within_reach(u64::MAX - PAGE + 1, PAGE), "the supply that wrapped `ends`"); assert!(!within_reach(1, u64::MAX)); let mut p = Pages::new(8, 8).unwrap(); - let mut o = p.object(1, LAST_END, 0); + let mut o = p.object(1, LAST_END); let last = LAST_END / PAGE - 1; assert_eq!(p.supply(&mut o, last, 1, true, LAST_END), Ok(Supplied::Installed), "the last page"); assert!(Pages::::new(Pages::::MAX_CAPACITY + 1, 0).is_none()); @@ -1024,7 +1006,7 @@ mod tests { #[test] fn a_shrink_removes_dirty_pages_releases_their_slots_and_names_the_tail() { let mut p = Pages::new(8, 8).unwrap(); - let mut o = p.object(11, 3 * PAGE, 0); + let mut o = p.object(11, 3 * PAGE); for page in 0..3 { p.supply(&mut o, page, page + 100, true, (page + 1) * PAGE).unwrap(); } @@ -1040,7 +1022,7 @@ mod tests { #[test] fn a_failed_page_answers_failed_until_supplied_again() { let mut p = Pages::new(8, 8).unwrap(); - let mut o = p.object(12, 2 * PAGE, 0); + let mut o = p.object(12, 2 * PAGE); p.supply(&mut o, 0, 5, false, PAGE).unwrap(); let mut freed = Vec::new(); p.fail(&mut o, 0, 2, &mut sink(&mut freed)).unwrap(); @@ -1053,7 +1035,7 @@ mod tests { #[test] fn an_invalidated_object_answers_gone_and_gives_every_frame_back() { let mut p = Pages::new(8, 8).unwrap(); - let mut o = p.object(13, 4 * PAGE, 0); + let mut o = p.object(13, 4 * PAGE); p.supply(&mut o, 0, 1, true, PAGE).unwrap(); p.supply(&mut o, 2, 3, false, 3 * PAGE).unwrap(); let mut freed = Vec::new(); @@ -1068,7 +1050,7 @@ mod tests { #[test] fn a_full_table_refuses_and_every_slot_is_reached_after_churn() { let mut p = Pages::new(64, 64).unwrap(); - let mut objects: Vec = (0..4).map(|i| p.object(i, 1 << 30, 0)).collect(); + let mut objects: Vec = (0..4).map(|i| p.object(i, 1 << 30)).collect(); let mut model: BTreeSet<(usize, u64)> = BTreeSet::new(); let mut x = 1u64; for round in 0..20_000u64 { diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index eba640bad6f..833046731a5 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -48,7 +48,7 @@ use core::sync::atomic::{AtomicU32, Ordering}; use kernel::chunked::{Chunked, Stack}; use kernel::paged::{Core, Draw, Leave, Lookup, Object, Pages, Refused, Supplied, Version, PAGE}; use kernel::sched::task::WaitClass; -use toyos_abi::syscall::{PagedNotice, PagedStat, SyscallError, NOTICE_NO_HOLDERS, NOTICE_REQUESTED}; +use toyos_abi::syscall::{PagedNotice, SyscallError, NOTICE_NO_HOLDERS, NOTICE_REQUESTED}; use crate::mm::pmm; use crate::process::Pid; @@ -276,9 +276,9 @@ impl PagerObject { unreported || state.no_holders > 0 } - /// A new object of `size` bytes last written at `mtime`: its id. + /// A new object of `size` bytes: its id. /// `ResourceExhausted` where every slot holds one. - pub fn add(&self, size: u64, mtime: u64, anonymous: bool) -> u64 { + pub fn add(&self, size: u64, anonymous: bool) -> u64 { if size > kernel::paged::LAST_END { return SyscallError::InvalidArgument.to_u64(); } @@ -287,7 +287,7 @@ impl PagerObject { return SyscallError::Gone.to_u64(); } let Some(slot) = state.free.pop() else { return SyscallError::ResourceExhausted.to_u64() }; - let object = if anonymous { state.pages.anonymous(slot, size, mtime) } else { state.pages.object(slot, size, mtime) }; + let object = if anonymous { state.pages.anonymous(slot, size) } else { state.pages.object(slot, size) }; state.entries[slot as usize] = Some(Entry { object, refs: 1, holders: 0, owned: true, no_holders: false }); u64::from(slot) } @@ -604,45 +604,18 @@ impl PagerObject { Step::Copy(count) } - /// A write call on object `id` begins, stamping it `mtime` where given: - /// its slot, held by a reference until [`Self::write_end`], so no release + /// A write call on object `id` begins: its slot is held by a reference + /// until [`Self::write_end`], so no release /// can hand it to another object. - fn write_begin(&self, id: u64, mtime: Option) -> Result { + fn write_begin(&self, id: u64) -> Result { let mut state = self.state.lock(); let slot = state.live(id)?; let entry = state.entry_mut(slot); entry.refs += 1; entry.object.write_begin(); - if let Some(mtime) = mtime { - entry.object.set_mtime(mtime); - } Ok(slot) } - /// Stamp object `id` last written at `mtime`. - pub fn stamp(&self, id: u64, mtime: u64) -> u64 { - let mut state = self.state.lock(); - match state.live(id) { - Ok(slot) => { - state.entry_mut(slot).object.set_mtime(mtime); - 0 - } - Err(e) => e.to_u64(), - } - } - - /// What the object at `slot` publishes to a holder: `WouldBlock` while a - /// write is under way. - fn stat(&self, slot: u32) -> Result { - let state = self.state.lock(); - let object = &state.entry(slot).object; - if state.gone || object.gone() { - return Err(SyscallError::Gone); - } - let version = object.published().ok_or(SyscallError::WouldBlock)?; - Ok(PagedStat { size: object.size(), mtime: object.mtime(), version }) - } - /// The write call [`Self::write_begin`] began ends, `whole` where it took /// everything it was given: its reference goes. fn write_end(&self, slot: u32, whole: bool) { @@ -671,7 +644,7 @@ impl PagerObject { if !dirty { return self.supply_pages(id, offset, src, false); } - let slot = match self.write_begin(id, None) { + let slot = match self.write_begin(id) { Ok(slot) => slot, Err(e) => return e.to_u64(), }; @@ -773,10 +746,10 @@ impl PagerObject { } /// Copy `src` into the present pages of object `id` from `offset`, dirty, - /// stamping it `mtime`: the bytes written, up to the first absent page. + /// answering the bytes written, up to the first absent page. /// The caller checked the span is in reach. - pub fn write(&self, id: u64, offset: u64, src: &UserBytes, mtime: u64) -> u64 { - let slot = match self.write_begin(id, Some(mtime)) { + pub fn write(&self, id: u64, offset: u64, src: &UserBytes) -> u64 { + let slot = match self.write_begin(id) { Ok(slot) => slot, Err(e) => return e.to_u64(), }; @@ -863,12 +836,11 @@ impl PagerObject { } } - pub fn resize(&self, id: u64, size: u64, mtime: u64) -> u64 { + pub fn resize(&self, id: u64, size: u64) -> u64 { if size > kernel::paged::LAST_END { return SyscallError::InvalidArgument.to_u64(); } let resized = self.removing(id, |pages, object, out| { - object.set_mtime(mtime); if let Some(phys) = pages.resize(object, size, &mut |f| out(f)) { let within = (size % PAGE) as usize; // SAFETY: the last page's frame, the object's while the lock is @@ -1133,10 +1105,6 @@ impl PagedHolderObject { pub fn read(&self, out: &mut UserBytesMut, offset: u64) -> u64 { self.named.pager.read(self.named.slot, out, offset, true) } - - pub fn stat(&self) -> Result { - self.named.pager.stat(self.named.slot) - } } impl ZeroHandles for PagedHolderObject { diff --git a/kernel/src/syscall/dispatch.rs b/kernel/src/syscall/dispatch.rs index c5980b840ab..a125c0b18a4 100644 --- a/kernel/src/syscall/dispatch.rs +++ b/kernel/src/syscall/dispatch.rs @@ -45,7 +45,7 @@ use super::machine::{ }; #[cfg(feature = "test-actuators")] use super::machine::lower_sysinfo_bound; -use super::paged::{sys_paged_create, sys_paged_holder, sys_paged_notices, sys_paged_op, sys_paged_read, sys_paged_stat, sys_pager_create}; +use super::paged::{sys_paged_create, sys_paged_holder, sys_paged_notices, sys_paged_op, sys_paged_read, sys_pager_create}; use super::proc::{ spawn_pager, spawn_place, sys_endowments, sys_exit, sys_nanosleep, sys_process_stats, sys_account_info, sys_job_create, sys_job_floor, sys_job_own, sys_process_quit, sys_process_wait, sys_rt_enter, sys_spawn, sys_thread_exit, sys_thread_get_affinity, sys_thread_join, sys_thread_set_affinity, sys_thread_spawn, sys_wake_latency_hold, @@ -604,8 +604,8 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> SYS_ACPI => sys_acpi(&ctx, RawHandle(a1 as u32), a2, a3), SYS_DEVICE_REG_READ => sys_device_reg_read(RawHandle(a1 as u32), a2, a3), SYS_PAGER_CREATE => sys_pager_create(a1, a2, a3, a4), - SYS_PAGED_CREATE => match a4 { - 0 | 1 => sys_paged_create(RawHandle(a1 as u32), a2, a3, a4 != 0), + SYS_PAGED_CREATE => match a3 { + 0 | 1 => sys_paged_create(RawHandle(a1 as u32), a2, a3 != 0), _ => SyscallError::InvalidArgument.to_u64(), }, SYS_PAGED_HOLDER => sys_paged_holder(RawHandle(a1 as u32), a2), @@ -614,7 +614,6 @@ pub(crate) fn syscall_dispatch(num: u64, a1: u64, a2: u64, a3: u64, a4: u64) -> sys_paged_read(RawHandle(a1 as u32), &mut buf, a4) } SYS_PAGED_OP => sys_paged_op(&ctx, RawHandle(a1 as u32), a2, UserAddr::new(a3)), - SYS_PAGED_STAT => sys_paged_stat(&ctx, RawHandle(a1 as u32), UserAddr::new(a2)), SYS_PAGED_NOTICES => { let size = core::mem::size_of::(); let Some(len) = (a3 as usize).checked_mul(size) else { return SyscallError::InvalidArgument.to_u64() }; diff --git a/kernel/src/syscall/paged.rs b/kernel/src/syscall/paged.rs index f7b49046038..bf05a078fe9 100644 --- a/kernel/src/syscall/paged.rs +++ b/kernel/src/syscall/paged.rs @@ -48,9 +48,9 @@ fn pager(h: RawHandle, rights: Rights) -> Result, u64> { process::with_process_data(|data| data.handles.get::(h, rights)).map_err(|e| e.refuse()) } -pub(super) fn sys_paged_create(h: RawHandle, size: u64, mtime: u64, anonymous: bool) -> u64 { +pub(super) fn sys_paged_create(h: RawHandle, size: u64, anonymous: bool) -> u64 { match pager(h, Rights::WRITE) { - Ok(pager) => pager.add(size, mtime, anonymous), + Ok(pager) => pager.add(size, anonymous), Err(refused) => refused, } } @@ -86,21 +86,6 @@ pub(super) fn sys_paged_read(h: RawHandle, out: &mut UserBytesMut, offset: u64) } } -/// A holder's stat, refused as its read is. -pub(super) fn sys_paged_stat(ctx: &SyscallContext<'_>, h: RawHandle, out: UserAddr) -> u64 { - let stat = match holder(h) { - Ok(holder) => holder.stat(), - Err(refused) => return refused, - }; - match stat { - Ok(stat) => match ctx.copy_out(out, &stat) { - Ok(()) => 0, - Err(_) => SyscallError::BadAddress.to_u64(), - }, - Err(e) => e.to_u64(), - } -} - fn holder(h: RawHandle) -> Result, u64> { let resolved = process::with_process_data(|data| { let KObjectRef::Paged(holder) = data.handles.get_ref(h, Rights::NONE)? else { @@ -148,13 +133,12 @@ pub(super) fn sys_paged_op(ctx: &SyscallContext<'_>, h: RawHandle, op: u64, args let Some(src) = ctx.user_bytes(UserAddr::new(args.buf), args.len) else { return SyscallError::BadAddress.to_u64() }; match x == PagedOp::Supply as u64 { true => pager.supply(id, args.offset, &src, args.flags & PAGED_DIRTY != 0), - false => pager.write(id, args.offset, &src, args.time), + false => pager.write(id, args.offset, &src), } } x if x == PagedOp::Withdraw as u64 => pager.withdraw(id, args.offset, args.len), x if x == PagedOp::Fail as u64 => pager.fail(id, args.offset, args.len), - x if x == PagedOp::Resize as u64 => pager.resize(id, args.offset, args.time), - x if x == PagedOp::Stamp as u64 => pager.stamp(id, args.time), + x if x == PagedOp::Resize as u64 => pager.resize(id, args.offset), x if x == PagedOp::Release as u64 => pager.release(id), x if x == PagedOp::EndWrite as u64 => pager.end_write(id), x if x == PagedOp::WritebackBegin as u64 || x == PagedOp::WritebackEnd as u64 => { diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 90902c7bf44..650e96d34d0 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -354,8 +354,6 @@ pub const SYS_CURRENT_CPU: u64 = 163; /// Map a firmware window a claim was granted at its mint. See /// [`device_firmware_window`]. pub const SYS_DEVICE_FIRMWARE_WINDOW: u64 = 164; -/// What a holder's paged object publishes. See [`paged_stat`]. -pub const SYS_PAGED_STAT: u64 = 165; /// List the names held by a namespace, without opening any connector. pub const SYS_NAMESPACE_NAMES: u64 = 166; @@ -3111,8 +3109,7 @@ pub enum PagedOp { /// Copy `len` bytes from `buf` into the present pages at `offset`, dirty, /// growing the size to their end as each page's bytes land. Answers the /// bytes written, stopping at the first absent page; `NotFound` when the - /// first is absent, `WouldBlock` at the dirty ceiling. `time` is the - /// file's mtime from this write, as its volume keeps it. A write that + /// first is absent, `WouldBlock` at the dirty ceiling. A write that /// does not take all it was given stays under way until /// [`PagedOp::EndWrite`]. Write = 1, @@ -3122,7 +3119,7 @@ pub enum PagedOp { /// Mark `len` pages from page `offset` failed: each read of one answers /// `Io` until it is supplied again. Fail = 3, - /// Set the size to `offset`, and the mtime to `time`. A shrink drops + /// Set the size to `offset`. A shrink drops /// every page past it, dirty or not, and zeroes the last page's tail; the /// new range of a growth reads as the pager supplies it. Resize = 4, @@ -3143,9 +3140,6 @@ pub enum PagedOp { /// End the write a short [`PagedOp::Write`] or dirty supply left under /// way; `InvalidArgument` where none is. EndWrite = 9, - /// Set the mtime to `time`, as a file's `utimens` does; the bytes and - /// the version stay. - Stamp = 10, /// Unused slots in the pager's resident-page quota. This advisory /// snapshot reserves nothing and promises no physical memory. Room = 11, @@ -3169,21 +3163,6 @@ crate::user_safe! { pub len: u64, pub buf: u64, pub flags: u64, - /// [`PagedOp::Write`]'s, [`PagedOp::Resize`]'s and [`PagedOp::Stamp`]'s - /// mtime, nanoseconds since the Unix epoch. - pub time: u64, - } -} - -crate::user_safe! { - /// What a paged object publishes to its holders ([`paged_stat`]): the - /// size and version the kernel keeps, and the mtime its pager last set. - #[derive(Clone, Copy, Default, Debug, PartialEq, Eq)] - pub struct PagedStat { - pub size: u64, - pub mtime: u64, - /// Moved by every write and resize: the object's data generation. - pub version: u64, } } @@ -3218,18 +3197,18 @@ pub fn pager_create(capacity: u64, dirty: u64, objects: u64, report_ns: u64) -> check(syscall(SYS_PAGER_CREATE, capacity, dirty, objects, report_ns)).map(|raw| RawHandle(raw as u32)) } -/// A paged object of `size` bytes last written at `mtime`, its pages absent: +/// A paged object of `size` bytes, its pages absent: /// its id, which `pager`'s calls and notices name it by. /// `ResourceExhausted` where the pager serves as many objects as it was made /// for. -pub fn paged_create(pager: RawHandle, size: u64, mtime: u64) -> Result { - check(syscall(SYS_PAGED_CREATE, pager.0 as u64, size, mtime, 0)) +pub fn paged_create(pager: RawHandle, size: u64) -> Result { + check(syscall(SYS_PAGED_CREATE, pager.0 as u64, size, 0, 0)) } /// A resizable anonymous object: its present pages are authoritative, never /// withdrawn or discarded by a failure, and writes need no writeback slot. -pub fn paged_anonymous(pager: RawHandle, size: u64, mtime: u64) -> Result { - check(syscall(SYS_PAGED_CREATE, pager.0 as u64, size, mtime, 1)) +pub fn paged_anonymous(pager: RawHandle, size: u64) -> Result { + check(syscall(SYS_PAGED_CREATE, pager.0 as u64, size, 1, 0)) } /// A holder handle on `pager`'s object `object`, carrying `READ`, which may @@ -3252,14 +3231,6 @@ pub fn paged_read(holder: RawHandle, out: &mut [u8], offset: u64) -> Result Result { - let mut out = PagedStat::default(); - check(syscall(SYS_PAGED_STAT, holder.0 as u64, &mut out as *mut PagedStat as u64, 0, 0)).map(|_| out) -} - /// One [`PagedOp`] on one of `pager`'s objects. pub fn paged_op(pager: RawHandle, op: PagedOp, args: &PagedArgs) -> Result { check(syscall(SYS_PAGED_OP, pager.0 as u64, op as u64, args as *const PagedArgs as u64, 0)) diff --git a/sdk/toyos/src/paged.rs b/sdk/toyos/src/paged.rs index 93834fa5a06..2c0a7c05546 100644 --- a/sdk/toyos/src/paged.rs +++ b/sdk/toyos/src/paged.rs @@ -12,7 +12,7 @@ use core::time::Duration; use toyos_abi::syscall::{self, PagedArgs, PagedOp, SyscallError, NOTICE_NO_HOLDERS, NOTICE_REQUESTED, PAGED_DIRTY, PAGED_WRITTEN}; -pub use toyos_abi::syscall::{PagedNotice, PagedStat, PAGED_RANGE_PAGES}; +pub use toyos_abi::syscall::{PagedNotice, PAGED_RANGE_PAGES}; use crate::shm::Bytes; use crate::{AsHandle, BorrowedHandle, OwnedHandle}; @@ -81,16 +81,15 @@ impl Pager { })) } - /// An object of `size` bytes last written at `mtime`, every page absent: - /// its id. - pub fn object(&self, size: u64, mtime: u64) -> Result { - syscall::paged_create(self.0.raw(), size, mtime).map_err(|e| Refused::of(e, None)) + /// An object of `size` bytes, every page absent: its id. + pub fn object(&self, size: u64) -> Result { + syscall::paged_create(self.0.raw(), size).map_err(|e| Refused::of(e, None)) } /// Anonymous bytes, retained until shrunk or released. No backing can /// refill a present page, so withdrawal and writeback cannot discard it. - pub fn anonymous(&self, size: u64, mtime: u64) -> Result { - syscall::paged_anonymous(self.0.raw(), size, mtime).map_err(|e| Refused::of(e, None)) + pub fn anonymous(&self, size: u64) -> Result { + syscall::paged_anonymous(self.0.raw(), size).map_err(|e| Refused::of(e, None)) } /// Unused slots in this pager's page quota, without reserving them or @@ -124,15 +123,15 @@ impl Pager { /// kept. The pages installed. pub fn supply(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result { let flags = if dirty { PAGED_DIRTY } else { 0 }; - let args = PagedArgs { object: id, offset, len: bytes.len() as u64, buf: bytes.as_ptr() as u64, flags, time: 0 }; + let args = PagedArgs { object: id, offset, len: bytes.len() as u64, buf: bytes.as_ptr() as u64, flags }; self.op(PagedOp::Supply, args, None) } /// `bytes` into the present pages of object `id` from `offset`, dirty, - /// the size growing as they land and the object stamped `mtime`: how - /// many were written before the first absent page. - pub fn write(&self, id: u64, offset: u64, bytes: &[u8], mtime: u64) -> Result { - let args = PagedArgs { object: id, offset, len: bytes.len() as u64, buf: bytes.as_ptr() as u64, flags: 0, time: mtime }; + /// the size growing as they land: how many were written before the + /// first absent page. + pub fn write(&self, id: u64, offset: u64, bytes: &[u8]) -> Result { + let args = PagedArgs { object: id, offset, len: bytes.len() as u64, buf: bytes.as_ptr() as u64, flags: 0 }; self.op(PagedOp::Write, args, Some(SyscallError::NotFound)).map(|n| n as usize) } @@ -154,15 +153,10 @@ impl Pager { self.op(PagedOp::Fail, PagedArgs { object: id, offset: first, len: count, ..PagedArgs::default() }, None).map(drop) } - /// The size and mtime, as `ftruncate` sets them: a shrink drops every + /// Set the size: a shrink drops every /// page past it, dirty or not. - pub fn resize(&self, id: u64, size: u64, mtime: u64) -> Result<(), Refused> { - self.op(PagedOp::Resize, PagedArgs { object: id, offset: size, time: mtime, ..PagedArgs::default() }, None).map(drop) - } - - /// The mtime, as `utimens` sets it. - pub fn stamp(&self, id: u64, mtime: u64) -> Result<(), Refused> { - self.op(PagedOp::Stamp, PagedArgs { object: id, time: mtime, ..PagedArgs::default() }, None).map(drop) + pub fn resize(&self, id: u64, size: u64) -> Result<(), Refused> { + self.op(PagedOp::Resize, PagedArgs { object: id, offset: size, ..PagedArgs::default() }, None).map(drop) } /// Begin writing back `count` pages from `first`: a bit per page begun. @@ -178,7 +172,7 @@ impl Pager { pub fn writeback_end(&self, id: u64, first: u64, count: u64, written: bool) -> Result<[u64; 8], Refused> { let mut bits = [0u64; 8]; let flags = if written { PAGED_WRITTEN } else { 0 }; - let args = PagedArgs { object: id, offset: first, len: count, buf: bits.as_mut_ptr() as u64, flags, time: 0 }; + let args = PagedArgs { object: id, offset: first, len: count, buf: bits.as_mut_ptr() as u64, flags }; self.op(PagedOp::WritebackEnd, args, None)?; Ok(bits) } @@ -230,10 +224,4 @@ impl Paged { pub fn read(&self, out: &mut [u8], offset: u64) -> Result { syscall::paged_read(self.0.raw(), out, offset) } - - /// The size, mtime and version its object publishes; `WouldBlock` while - /// a write of the pager's is under way, and the pager is asked. - pub fn stat(&self) -> Result { - syscall::paged_stat(self.0.raw()) - } } diff --git a/system/fileserver/src/main.rs b/system/fileserver/src/main.rs index 3f92ed07ec4..2de41a08edc 100644 --- a/system/fileserver/src/main.rs +++ b/system/fileserver/src/main.rs @@ -542,10 +542,8 @@ impl Server { for id in late { self.drop_client(id, "it never lent its window"); } - if self.writeback.take_due(now) { - if let Err(e) = self.sync() { - println!("fileserver: the write-back sync failed: {e:?}"); - } + if self.writeback.take_due(now) && let Err(e) = self.sync() { + println!("fileserver: the write-back sync failed: {e:?}"); } } } @@ -1131,7 +1129,7 @@ impl Server { // The volume truncated first, so a refused open changed // nothing; the object follows before anything writes back. if how.truncate && found.is_some() { - let truncated = self.volume.identity(&path).and_then(|identity| self.files.truncated(&mut *self.volume, identity, node)); + let truncated = self.volume.identity(&path).and_then(|identity| self.files.truncated(identity)); if let Err(e) = truncated { let _ = self.volume.close(node); return Err(e); @@ -1241,12 +1239,8 @@ impl Server { }; if held { let file = self.fid(id, r.fid)?; - let (node, identity) = { - let f = file.borrow(); - (f.node().map_err(|_| Error::Unsupported)?, f.identity) - }; + let node = file.borrow().node().map_err(|_| Error::Unsupported)?; self.volume.set_mtime(node, mtime)?; - self.files.stamped(&mut *self.volume, identity, node)?; } else { let path = match self.resolve(id, r.fid, &rel, r.flags & FOLLOW_LAST != 0)? { Resolved::Absolute(p) => return self.follow(id, r, &p), @@ -1255,9 +1249,8 @@ impl Server { if self.volume.lstat(&path)?.kind != Kind::File { return Err(Error::Unsupported); } - let identity = self.volume.identity(&path)?; let node = self.volume.open(&path, OpenHow { create: false, create_new: false, truncate: false })?; - let set = self.volume.set_mtime(node, mtime).and_then(|()| self.files.stamped(&mut *self.volume, identity, node)); + let set = self.volume.set_mtime(node, mtime); let closed = self.volume.close(node); set.and(closed)?; } @@ -1324,7 +1317,7 @@ impl Server { let mut words = vec![0u8; count * 8]; self.window(id).read(at, &mut words); let fids: Vec = - words.chunks_exact(8).map(|w| u64::from_le_bytes(w.try_into().expect("eight bytes"))).collect(); + words.as_chunks::<8>().0.iter().map(|w| u64::from_le_bytes(*w)).collect(); let parent = &self.clients[&id]; let (share, writes) = (parent.share, parent.writes); let fids = parent.fids.export(&fids)?; @@ -1373,12 +1366,12 @@ impl Server { if let Some(why) = ended { println!("fileserver: a stream is ended: {why}"); } - if let Some(stream) = self.streams.remove(&sid) { - if let Some((node, identity)) = descriptions::release(stream.file) { - // Nobody is left to answer: a refused close is said, and its - // node kept for the next sync. - let _ = self.files.closed(&mut *self.volume, identity, node); - } + if let Some(stream) = self.streams.remove(&sid) + && let Some((node, identity)) = descriptions::release(stream.file) + { + // Nobody is left to answer: a refused close is said, and its + // node kept for the next sync. + let _ = self.files.closed(&mut *self.volume, identity, node); } false } diff --git a/system/fileserver/src/pages.rs b/system/fileserver/src/pages.rs index 7c130f5e616..9151e318186 100644 --- a/system/fileserver/src/pages.rs +++ b/system/fileserver/src/pages.rs @@ -16,10 +16,9 @@ //! a call that stops short leaves it under way until its end, so no holder //! reads the file half written (`toyos::paged::Pager::end_write`). //! -//! **The volume's size and mtime are the object's**: each call that moves the volume's mtime hands -//! the object the volume's own value, at its resolution. A write reserves on the volume -//! first, so the volume's size runs past the object's only for the bytes the -//! write then puts in before it returns; nothing reads the volume for them +//! A write reserves on the volume first, so the volume's size runs past the +//! object's only for the bytes the write then puts in before it returns; +//! nothing reads the volume for them //! meanwhile, and a write refused part way is cut back to what it put. //! //! **A server with no pager** (ROOT's, whose files are immutable) reads and @@ -38,28 +37,27 @@ const RANGE: u64 = PAGED_RANGE_PAGES; /// A pager's calls on its objects, by id, as `toyos::paged::Pager` makes them. pub trait Pager { - fn object(&self, size: u64, mtime: u64) -> Result; - fn anonymous(&self, size: u64, mtime: u64) -> Result; + fn object(&self, size: u64) -> Result; + fn anonymous(&self, size: u64) -> Result; fn room(&self) -> Result; fn read(&self, id: u64, out: Bytes<'_>, offset: u64) -> Result; fn supply(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result; - fn write(&self, id: u64, offset: u64, bytes: &[u8], mtime: u64) -> Result; + fn write(&self, id: u64, offset: u64, bytes: &[u8]) -> Result; fn end_write(&self, id: u64) -> Result<(), Refused>; fn withdraw(&self, id: u64, first: u64, count: u64) -> Result; fn fail(&self, id: u64, first: u64, count: u64) -> Result<(), Refused>; - fn resize(&self, id: u64, size: u64, mtime: u64) -> Result<(), Refused>; - fn stamp(&self, id: u64, mtime: u64) -> Result<(), Refused>; + fn resize(&self, id: u64, size: u64) -> Result<(), Refused>; fn writeback_begin(&self, id: u64, first: u64, count: u64) -> Result<[u64; 8], Refused>; fn writeback_end(&self, id: u64, first: u64, count: u64, written: bool) -> Result<[u64; 8], Refused>; fn release(&self, id: u64) -> Result<(), Refused>; } impl Pager for toyos::paged::Pager { - fn object(&self, size: u64, mtime: u64) -> Result { - toyos::paged::Pager::object(self, size, mtime) + fn object(&self, size: u64) -> Result { + toyos::paged::Pager::object(self, size) } - fn anonymous(&self, size: u64, mtime: u64) -> Result { - toyos::paged::Pager::anonymous(self, size, mtime) + fn anonymous(&self, size: u64) -> Result { + toyos::paged::Pager::anonymous(self, size) } fn room(&self) -> Result { toyos::paged::Pager::room(self) @@ -70,8 +68,8 @@ impl Pager for toyos::paged::Pager { fn supply(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result { toyos::paged::Pager::supply(self, id, offset, bytes, dirty) } - fn write(&self, id: u64, offset: u64, bytes: &[u8], mtime: u64) -> Result { - toyos::paged::Pager::write(self, id, offset, bytes, mtime) + fn write(&self, id: u64, offset: u64, bytes: &[u8]) -> Result { + toyos::paged::Pager::write(self, id, offset, bytes) } fn end_write(&self, id: u64) -> Result<(), Refused> { toyos::paged::Pager::end_write(self, id) @@ -82,11 +80,8 @@ impl Pager for toyos::paged::Pager { fn fail(&self, id: u64, first: u64, count: u64) -> Result<(), Refused> { toyos::paged::Pager::fail(self, id, first, count) } - fn resize(&self, id: u64, size: u64, mtime: u64) -> Result<(), Refused> { - toyos::paged::Pager::resize(self, id, size, mtime) - } - fn stamp(&self, id: u64, mtime: u64) -> Result<(), Refused> { - toyos::paged::Pager::stamp(self, id, mtime) + fn resize(&self, id: u64, size: u64) -> Result<(), Refused> { + toyos::paged::Pager::resize(self, id, size) } fn writeback_begin(&self, id: u64, first: u64, count: u64) -> Result<[u64; 8], Refused> { toyos::paged::Pager::writeback_begin(self, id, first, count) @@ -193,14 +188,13 @@ impl Files

{ Some(f) } - /// After an open of `identity` at `node` that truncated: its object holds + /// After an open of `identity` that truncated: its object holds /// no page of before. The volume went first, so a refused open changed /// nothing, and nothing writes back between the two. - pub fn truncated(&mut self, volume: &mut dyn Volume, identity: u64, node: Node) -> Result<(), Error> { + pub fn truncated(&mut self, identity: u64) -> Result<(), Error> { let Some(f) = self.files.get(&identity) else { return Ok(()) }; let id = f.object; - let mtime = volume.node_meta(node)?.mtime; - self.pg().resize(id, 0, mtime).map_err(word)?; + self.pg().resize(id, 0).map_err(word)?; self.files.get_mut(&identity).expect("found").dirty = None; Ok(()) } @@ -226,7 +220,7 @@ impl Files

{ // Objects are bounded by count as pages are: a file nobody holds goes // to make room for another's. let object = loop { - let created = if volume.memory() { self.pg().anonymous(meta.size, meta.mtime) } else { self.pg().object(meta.size, meta.mtime) }; + let created = if volume.memory() { self.pg().anonymous(meta.size) } else { self.pg().object(meta.size) }; match created { Ok(object) => break object, Err(Refused::NoRoom) if self.drop_oldest(volume, identity) => {} @@ -425,19 +419,10 @@ impl Files

{ } let reserved = volume.reserve(node, offset, data.len() as u64)?; let old = reserved.size; - let mtime = match volume.node_meta(node) { - Ok(meta) => meta.mtime, - Err(e) => { - if let Err(again) = volume.unreserve(node, reserved, 0) { - println!("fileserver: a write of file {identity} was not cut back to {old} ({again:?}), after {e:?}"); - } - return Err(e); - } - }; self.touch(identity); let id = self.files[&identity].object; let mut put = Put { done: 0, held: false }; - let writing = Writing { identity, node, old, contents: reserved.contents(), offset, data, mtime }; + let writing = Writing { identity, node, old, contents: reserved.contents(), offset, data }; let written = self.put(volume, &writing, &mut put); if let Err(e) = &written { // What was reserved and not put would read as the blocks' old bytes. @@ -445,10 +430,9 @@ impl Files

{ let cut = old.max(landed); let resized = self .pg() - .resize(id, cut, mtime) + .resize(id, cut) .map_err(word) - .and(volume.unreserve(node, reserved, landed)) - .and_then(|()| self.stamped(volume, identity, node)); + .and(volume.unreserve(node, reserved, landed)); if let Err(again) = resized { println!("fileserver: a refused write of file {identity} was not cut back to {cut} ({again:?}), after {e:?}"); } @@ -466,13 +450,13 @@ impl Files

{ /// counting into `put` what landed and whether a call left the write /// under way. fn put(&mut self, volume: &mut dyn Volume, w: &Writing<'_>, put: &mut Put) -> Result<(), Error> { - let Writing { identity, node, old, contents, offset, data, mtime } = *w; + let Writing { identity, node, old, contents, offset, data } = *w; let end = offset + data.len() as u64; let done = &mut put.done; while *done < data.len() { let at = offset + *done as u64; let id = self.files.get(&identity).ok_or(Error::Stale)?.object; - let wrote = self.pg().write(id, at, &data[*done..], mtime); + let wrote = self.pg().write(id, at, &data[*done..]); // A first call that takes everything is the whole write; any later // call is under the write a short first one left under way. put.held |= wrote != Ok(data.len() - *done); @@ -532,15 +516,6 @@ impl Files

{ } } - /// `identity`'s volume moved its mtime, which `node` reads: its object - /// follows. - pub fn stamped(&mut self, volume: &mut dyn Volume, identity: u64, node: Node) -> Result<(), Error> { - let Some(f) = self.files.get(&identity) else { return Ok(()) }; - let id = f.object; - let mtime = volume.node_meta(node)?.mtime; - self.pg().stamp(id, mtime).map_err(word) - } - /// `node`'s file cut or grown to `size`: the volume first, so a refusal /// changes nothing, then the object, before anything can write a page /// past the size back. @@ -549,14 +524,13 @@ impl Files

{ if !self.files.contains_key(&identity) { return Ok(()); } - let mtime = volume.node_meta(node)?.mtime; let f = self.files.get_mut(&identity).expect("found"); if let Some((lo, hi)) = f.dirty { let hi = hi.min(size.div_ceil(PAGE)); f.dirty = (lo < hi).then_some((lo, hi)); } let id = f.object; - self.pg().resize(id, size, mtime).map_err(word) + self.pg().resize(id, size).map_err(word) } /// `identity`'s dirty pages written to the volume. A page dirtied again @@ -707,8 +681,6 @@ struct Writing<'a> { contents: u64, offset: u64, data: &'a [u8], - /// The file's mtime from this write, as the volume keeps it. - mtime: u64, } /// The runs of set bits among the first `count` of `bits`: where each starts diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index ab222c2b967..b4ccc410dc6 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -20,7 +20,6 @@ struct Model { next: u64, pages: BTreeMap<(u64, u64), Page>, sizes: BTreeMap, - mtimes: BTreeMap, anonymous: BTreeSet, supplies: u32, /// Every supply from here is refused. @@ -138,7 +137,7 @@ impl Pager for Fake { Ok((model.capacity - model.pages.len()) as u64) } - fn object(&self, size: u64, mtime: u64) -> Result { + fn object(&self, size: u64) -> Result { let mut m = self.0.borrow_mut(); if m.sizes.len() >= m.objects { return Err(Refused::NoRoom); @@ -146,12 +145,11 @@ impl Pager for Fake { m.next += 1; let id = m.next; m.sizes.insert(id, size); - m.mtimes.insert(id, mtime); Ok(id) } - fn anonymous(&self, size: u64, mtime: u64) -> Result { - let id = self.object(size, mtime)?; + fn anonymous(&self, size: u64) -> Result { + let id = self.object(size)?; self.0.borrow_mut().anonymous.insert(id); Ok(id) } @@ -191,8 +189,7 @@ impl Pager for Fake { answer } - fn write(&self, id: u64, offset: u64, bytes: &[u8], mtime: u64) -> Result { - self.0.borrow_mut().mtimes.insert(id, mtime); + fn write(&self, id: u64, offset: u64, bytes: &[u8]) -> Result { let answer = self.write_pages(id, offset, bytes); self.0.borrow_mut().changed(id, answer == Ok(bytes.len())); answer @@ -219,14 +216,8 @@ impl Pager for Fake { Ok(()) } - fn stamp(&self, id: u64, mtime: u64) -> Result<(), Refused> { - self.0.borrow_mut().mtimes.insert(id, mtime); - Ok(()) - } - - fn resize(&self, id: u64, size: u64, mtime: u64) -> Result<(), Refused> { + fn resize(&self, id: u64, size: u64) -> Result<(), Refused> { let mut m = self.0.borrow_mut(); - m.mtimes.insert(id, mtime); m.pages.retain(|(c, p), _| *c != id || *p < size.div_ceil(PAGE)); if let Some(page) = m.pages.get_mut(&(id, size / PAGE)) { page.bytes[(size % PAGE) as usize..].fill(0); @@ -566,7 +557,7 @@ fn a_truncating_open_leaves_the_object_no_page_of_before() { let truncate = OpenHow { create: false, create_new: false, truncate: true }; let again = v.open("home/f", truncate).unwrap(); - files.truncated(&mut v, id, again).unwrap(); + files.truncated(id).unwrap(); files.opened(&mut v, id, again).unwrap(); let object = files.object(id).unwrap(); assert_eq!((fake.0.borrow().sizes[&object], fake.pages_of(object)), (0, 0), "a holder reads the object, and finds nothing of before"); @@ -755,18 +746,11 @@ fn a_directorys_fsync_makes_a_new_name_durable() { assert_eq!(v.lstat("home/d").map(|m| m.kind), Ok(crate::volume::Kind::Dir)); } -static NOW: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1_000_000_000); - -/// A clock a step ahead at each reading, so each change stamps a new time. -fn ticking() -> u64 { - NOW.fetch_add(1_000_000_007, std::sync::atomic::Ordering::Relaxed) -} - -/// What a holder's `fstat` reads is the object's size and mtime: after an -/// open, a write, one refused, a truncate and a stamp, each is the volume's. +/// A holder stops at the volume's size after an open, write, refused write +/// or truncate. #[test] -fn an_objects_size_and_mtime_are_the_volumes_after_every_change() { - let mut v = DataVolume::format(Ram::new(16384), 256, &["home"], ticking).unwrap(); +fn an_objects_size_is_the_volumes_after_every_change() { + let mut v = volume(); let fake = Fake::new(64, 16); let mut files = Files::new(Some(fake.clone())); let n = v.open("home/f", CREATE).unwrap(); @@ -777,7 +761,7 @@ fn an_objects_size_and_mtime_are_the_volumes_after_every_change() { let published = |v: &mut DataVolume, what: &str| { let m = fake.0.borrow(); let meta = v.node_meta(node).unwrap(); - assert_eq!((m.sizes[&object], m.mtimes[&object]), (meta.size, meta.mtime), "after {what}"); + assert_eq!(m.sizes[&object], meta.size, "after {what}"); }; published(&mut v, "the open"); files.write(&mut v, id, node, 3 * PAGE, b"after").unwrap(); @@ -788,7 +772,4 @@ fn an_objects_size_and_mtime_are_the_volumes_after_every_change() { published(&mut v, "a refused write"); files.truncate(&mut v, id, node, 100).unwrap(); published(&mut v, "a truncate"); - v.set_mtime(node, 42_000_000_000).unwrap(); - files.stamped(&mut v, id, node).unwrap(); - published(&mut v, "a stamp"); } diff --git a/tests/guest/pagerprobe/src/main.rs b/tests/guest/pagerprobe/src/main.rs index 7e17537ee46..058e3b87ca7 100644 --- a/tests/guest/pagerprobe/src/main.rs +++ b/tests/guest/pagerprobe/src/main.rs @@ -40,7 +40,7 @@ fn main() { return hold(); } let pager = Pager::create(LARGE + 16, 0, 4, Duration::from_secs(1)).expect("pagerprobe: its row starts it as a pager"); - let large = pager.object(LARGE * PAGE, 0).expect("pagerprobe: the large object"); + let large = pager.object(LARGE * PAGE).expect("pagerprobe: the large object"); fill(&pager, large); assert_eq!(pager.withdraw(large, 0, u64::MAX), Ok(LARGE), "pagerprobe: the whole object withdrawn at once"); fill(&pager, large); @@ -50,10 +50,10 @@ fn main() { Err(Refused::Kernel(SyscallError::InvalidArgument)), "pagerprobe: a second release of one object was taken" ); - let left = pager.object(LARGE * PAGE, 0).expect("pagerprobe: the object the pager's end takes"); + let left = pager.object(LARGE * PAGE).expect("pagerprobe: the object the pager's end takes"); fill(&pager, left); println!("pagerprobe: more than 512 MiB in one object withdrawn, released and left for the end"); - let anonymous = pager.anonymous(32 * PAGE, 0).expect("pagerprobe: anonymous memory"); + let anonymous = pager.anonymous(32 * PAGE).expect("pagerprobe: anonymous memory"); assert_eq!(pager.supply(anonymous, 0, &vec![0x37; 16 * PAGE as usize], true), Ok(16)); assert_eq!(pager.room(), Ok(0), "the anonymous pages consumed the remaining quota"); assert_eq!(pager.withdraw(anonymous, 0, 32), Ok(0), "anonymous bytes cannot be withdrawn"); @@ -64,7 +64,7 @@ fn main() { assert_eq!(pager.read(anonymous, out, 0), Ok(bytes.len())); anonymous_contents(&bytes); assert_eq!(pager.room(), Ok(0), "a zero-hole read needs no quota"); - let object = pager.object(4096, 0).expect("pagerprobe: an object"); + let object = pager.object(4096).expect("pagerprobe: an object"); let own = pager.holder(object).expect("pagerprobe: a holder for itself"); // The handle is a holder, and this process a pager: refused by the kernel. assert_eq!( From 09ede5236a8f6e4bf812a9d2cb98726f02b6381b Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 19:54:51 +0100 Subject: [PATCH 21/93] Keep directory descriptions correct across replacement Allow an in-memory directory to replace an empty directory. Update each shared directory description once during rename, retaining removed destination descriptions and moving source aliases together. Migrate the identity guest test from implicit kernel directories to served directory lifetimes. Validation: file-server memory and description host tests pass. Guest execution follows the integrated image build. --- system/fileserver/src/descriptions.rs | 28 ++++- system/fileserver/src/main.rs | 8 +- system/fileserver/src/memory.rs | 26 +++- .../guest/rust-tests/src/bin/stat_identity.rs | 114 ++++++++---------- 4 files changed, 104 insertions(+), 72 deletions(-) diff --git a/system/fileserver/src/descriptions.rs b/system/fileserver/src/descriptions.rs index ab556c0f157..d31c6d7f146 100644 --- a/system/fileserver/src/descriptions.rs +++ b/system/fileserver/src/descriptions.rs @@ -138,9 +138,14 @@ impl Ids { } } -/// The directory at `from`, never the volume's root, is at `to` now with -/// everything beneath it: move `path` with it. -pub fn moved(path: &mut String, from: &str, to: &str) { +/// Follow a successful rename once per description. The replaced directory's +/// old description stays removed instead of naming the source now at its path. +pub fn renamed(path: &mut String, removed: &mut bool, from: &str, to: &str) { + if *removed || from == to { return; } + if path == to { + *removed = true; + return; + } let beneath = path.len() > from.len() && path.starts_with(from) && path.as_bytes()[from.len()] == b'/'; if path == from || beneath { *path = crate::volume::join(to, path[from.len()..].trim_start_matches('/')); @@ -207,11 +212,26 @@ mod tests { ]; for (held, from, to, now) in cases { let mut path = held.to_string(); - moved(&mut path, from, to); + let mut removed = false; + renamed(&mut path, &mut removed, from, to); assert_eq!(path, now, "{held} after {from} -> {to}"); + assert!(!removed); } } + #[test] + fn replacement_never_retargets_an_old_directory_description() { + let mut path = "target".to_string(); + let mut removed = false; + renamed(&mut path, &mut removed, "source", "target"); + assert!(removed); + renamed(&mut path, &mut removed, "target", "another"); + assert_eq!(path, "target"); + let mut removed = false; + renamed(&mut path, &mut removed, "target", "target"); + assert!(!removed); + } + #[test] fn a_connection_holds_at_most_max_ids() { let mut ids = Ids::default(); diff --git a/system/fileserver/src/main.rs b/system/fileserver/src/main.rs index 2de41a08edc..8037035b471 100644 --- a/system/fileserver/src/main.rs +++ b/system/fileserver/src/main.rs @@ -1055,7 +1055,7 @@ impl Server { if let Some(identity) = over { self.files.unlinked(&mut *self.volume, identity)?; } - self.each_dir(|held, _| descriptions::moved(held, &from, &to)); + self.each_dir(|held, removed| descriptions::renamed(held, removed, &from, &to)); self.dirtied(); Ok(Answer::Reply(Reply::ok())) } @@ -1085,10 +1085,14 @@ impl Server { } } - /// `f` on every held directory's path and removal, on every connection. + /// `f` once per held directory description, including exported aliases. fn each_dir(&mut self, mut f: impl FnMut(&mut String, &mut bool)) { + let mut seen = std::collections::BTreeSet::new(); for client in self.clients.values() { for shared in client.fids.dirs() { + if !seen.insert(Rc::as_ptr(shared)) { + continue; + } if let Held::Dir { path, removed } = &mut shared.borrow_mut().held { f(path, removed); } diff --git a/system/fileserver/src/memory.rs b/system/fileserver/src/memory.rs index b30529e3907..cb2b0b03976 100644 --- a/system/fileserver/src/memory.rs +++ b/system/fileserver/src/memory.rs @@ -201,8 +201,12 @@ impl Volume for MemoryVolume { let kind = self.nodes[&node].meta.kind; if to.strip_prefix(from).is_some_and(|rest| rest.starts_with('/')) { return Err(Error::IntoItself); } if let Ok(over) = self.node(to) { - if self.nodes[&over].meta.kind == Kind::Dir { return Err(Error::IsADirectory); } - if kind == Kind::Dir { return Err(Error::NotADirectory); } + match (kind == Kind::Dir, self.nodes[&over].meta.kind == Kind::Dir) { + (false, true) => return Err(Error::IsADirectory), + (true, false) => return Err(Error::NotADirectory), + (true, true) if !self.nodes[&over].children.is_empty() => return Err(Error::NotEmpty), + _ => {} + } self.detach(to); } let old_parent = self.node(parent(from))?; @@ -280,4 +284,22 @@ mod tests { assert_eq!(listed[1].meta.kind, Kind::Symlink); assert_eq!(volume.read_link("link"), Ok("/home/toy/target".into())); } + + #[test] + fn a_directory_can_replace_only_an_empty_directory() { + let mut volume = MemoryVolume::new(&[], clock); + volume.mkdir("source").unwrap(); + volume.mkdir("target").unwrap(); + let file = volume.open("source/file", CREATE).unwrap(); + let directory = volume.identity("source").unwrap(); + volume.rename("source", "target").unwrap(); + assert_eq!(volume.identity("target"), Ok(directory)); + assert_eq!(volume.identity("target/file"), Ok(file)); + assert_eq!(volume.lstat("source"), Err(Error::NotFound)); + volume.mkdir("empty").unwrap(); + assert_eq!(volume.rename("empty", "target"), Err(Error::NotEmpty)); + assert_eq!(volume.rename("target/file", "empty"), Err(Error::IsADirectory)); + assert_eq!(volume.rename("empty", "target/file"), Err(Error::NotADirectory)); + assert_eq!(volume.identity("target/file"), Ok(file)); + } } diff --git a/tests/guest/rust-tests/src/bin/stat_identity.rs b/tests/guest/rust-tests/src/bin/stat_identity.rs index a7e1ef03e01..6c21be0339c 100644 --- a/tests/guest/rust-tests/src/bin/stat_identity.rs +++ b/tests/guest/rust-tests/src/bin/stat_identity.rs @@ -1,10 +1,5 @@ -//! A file's identity through std (`std::os::toyos::fs::MetadataExt`) on the -//! kernel's own mount (`/tmp`): one file is one `(dev, ino)` through every -//! path, a symlink, its open handle, its directory's listing and a rename; -//! two files are two, as are two directories; a symlink is one of its own, -//! across a rename; a directory that ends gives its number up, and one that -//! stands keeps it; and the volume is no served one, `toyos::fs::SERVED_VOLUME` -//! clear. +//! File and directory identities survive renames and held descriptions on +//! both the userland memory volume and writable DATA. use std::fs::{self, File, Metadata}; use std::os::toyos::fs::{symlink, MetadataExt}; @@ -71,67 +66,58 @@ fn judge(dir: &str, spelling: &str) -> u64 { ida.0 } -/// On a kernel mount a directory that ends gives its number up, so the one -/// made there next is another: by `rmdir`, and by the unlink or the rename of -/// the file that alone made it one, but not by that file's truncating open. -fn an_ended_directory_is_forgotten() { - const DIR: &str = "/tmp/stat-identity-ended"; - const FILE: &str = "/tmp/stat-identity-ended/f"; - const AWAY: &str = "/tmp/stat-identity-away"; - let again = |end: &str, make: &dyn Fn(), ends: &dyn Fn()| { - make(); - let before = of(DIR); - ends(); - make(); - assert_ne!(of(DIR), before, "{end} left {DIR}'s number to the directory made there next"); - ends(); - }; - let mkdir = || fs::create_dir(DIR).unwrap(); - let file = || write(FILE); - again("rmdir", &mkdir, &|| fs::remove_dir(DIR).unwrap()); - again("an unlink", &file, &|| fs::remove_file(FILE).unwrap()); - again("a rename", &file, &|| { - fs::rename(FILE, AWAY).unwrap(); - fs::remove_file(AWAY).unwrap(); - }); - - // A truncating open puts its file back at once: the directory never ended. - file(); - let kept = of(DIR); - file(); - assert_eq!(of(DIR), kept, "a truncating open of its one file renumbered {DIR}"); - fs::remove_file(FILE).unwrap(); +/// Removing a child never removes or renumbers its directory. Removing and +/// recreating that directory must assign a new identity. +fn directory_lifetimes(root: &str) { + let dir = format!("{root}/stat-identity-lifetime"); + let file = format!("{dir}/file"); + let away = format!("{root}/stat-identity-away"); + fs::create_dir(&dir).unwrap(); + let before = of(&dir); + write(&file); + fs::remove_file(&file).unwrap(); + assert_eq!(of(&dir), before, "unlink removed or renumbered its parent"); + write(&file); + fs::rename(&file, &away).unwrap(); + assert_eq!(of(&dir), before, "rename removed or renumbered its parent"); + fs::remove_file(&away).unwrap(); + fs::remove_dir(&dir).unwrap(); + fs::create_dir(&dir).unwrap(); + assert_ne!(of(&dir), before, "a recreated directory reused a live identity"); + fs::remove_dir(&dir).unwrap(); } -/// On a kernel mount a directory that still stands after a removal beneath it -/// keeps its number, as `/tmp` above it does: one a second file still makes a -/// directory, and one `mkdir` made whose only file goes. -fn a_standing_directory_keeps_its_number() { - const DIR: &str = "/tmp/stat-identity-stands"; - const ONE: &str = "/tmp/stat-identity-stands/one"; - const TWO: &str = "/tmp/stat-identity-stands/two"; - let tmp = of("/tmp"); - - write(ONE); - write(TWO); - let before = of(DIR); - fs::remove_file(ONE).unwrap(); - assert_eq!(of(DIR), before, "an unlink beside a second file renumbered {DIR}"); - assert_eq!(of("/tmp"), tmp, "an unlink beneath it renumbered /tmp"); - fs::remove_file(TWO).unwrap(); - - fs::create_dir(DIR).unwrap(); - write(ONE); - let before = of(DIR); - fs::remove_file(ONE).unwrap(); - assert_eq!(of(DIR), before, "an unlink of its only file renumbered the made {DIR}"); - fs::remove_dir(DIR).unwrap(); +fn directory_replacement(root: &str) { + use toyos::fs::{Base, Dir, O_DIRECTORY, O_READ}; + let source = format!("{root}/stat-identity-source"); + let target = format!("{root}/stat-identity-target"); + fs::create_dir(&source).unwrap(); + fs::create_dir(&target).unwrap(); + write(&format!("{source}/file")); + let mut directory = Dir::connect(toyos::endow::namespace().unwrap(), &format!("fs:{root}")).unwrap(); + let original = directory.open(Base::Root, "stat-identity-source", O_DIRECTORY | O_READ).unwrap().fid; + let alias = directory.dup(original).unwrap(); + let replaced = directory.open(Base::Root, "stat-identity-target", O_DIRECTORY | O_READ).unwrap().fid; + fs::rename(&source, &target).unwrap(); + let now = of(&format!("{target}/file")); + for fid in [original, alias] { + assert_eq!(directory.stat(Base::Fid(fid), "file", true).unwrap().node, now.1, + "a directory alias lost its identity while its name moved"); + } + assert!(directory.stat(Base::Fid(replaced), "file", true).is_err(), + "a replaced directory description acquired the replacing directory's contents"); + for fid in [original, alias, replaced] { directory.close(fid); } + fs::remove_file(format!("{target}/file")).unwrap(); + fs::remove_dir(&target).unwrap(); } fn main() { - an_ended_directory_is_forgotten(); - a_standing_directory_keeps_its_number(); - let tmp = judge("/tmp", "/tmp/../tmp/./stat-identity-a"); - assert_eq!(tmp & toyos::fs::SERVED_VOLUME, 0, "a kernel mount answered a served volume"); + for root in ["/tmp", "/home"] { + directory_lifetimes(root); + directory_replacement(root); + let spelling = format!("{root}/../{}/./stat-identity-a", root.trim_start_matches('/')); + let volume = judge(root, &spelling); + assert_ne!(volume & toyos::fs::SERVED_VOLUME, 0, "a userland volume answered an unserved identity"); + } println!("stat_identity: ok"); } From 08ac5ed16b80214bed58220dae7da5e4dee0f316 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 19:56:56 +0100 Subject: [PATCH 22/93] Remove the copy command workaround for kernel file-cache pressure Let the file server enforce dirty-page limits; cp no longer forces fsync every megabyte. Keep the final sync before destination replacement to report delayed write failures. Update the memory benchmark label and sparse test-device description for userland storage. Validation: toybox copy host tests pass. --- system/fileserver/src/ram.rs | 3 +-- system/toybox/copies/src/copying.rs | 9 ++++----- system/toybox/src/cp.rs | 21 +-------------------- tests/guest/osbench/src/files.rs | 2 +- 4 files changed, 7 insertions(+), 28 deletions(-) diff --git a/system/fileserver/src/ram.rs b/system/fileserver/src/ram.rs index 19d73b5d6d1..2c8de36b79f 100644 --- a/system/fileserver/src/ram.rs +++ b/system/fileserver/src/ram.rs @@ -7,8 +7,7 @@ use toyos_blockdev::{within, Block, BlockDev, Error}; use crate::cache::Disk; -/// A volume in memory: the DATA role on a machine with no DATA partition, as -/// the kernel's tmpfs was. Sparse, so what nothing wrote costs nothing. +/// A sparse test block device. Unwritten blocks cost no storage. pub struct Ram { blocks: u64, written: RefCell>>, diff --git a/system/toybox/copies/src/copying.rs b/system/toybox/copies/src/copying.rs index 380cb6b8017..7d3c5b4a297 100644 --- a/system/toybox/copies/src/copying.rs +++ b/system/toybox/copies/src/copying.rs @@ -16,13 +16,12 @@ fn partials(dir: &Path) -> Vec { .collect() } -/// Larger than the flush interval twice over and no page multiple, so the -/// periodic flush and the partial tail both run; it lands whole under the -/// destination's name, and no partial is left beside it. +/// A copy spanning many buffers and a partial tail lands whole under the +/// destination's name, with no partial left beside it. #[test] -fn a_copy_past_the_flush_interval_lands_whole_and_leaves_no_partial() { +fn a_copy_spanning_many_buffers_lands_whole_and_leaves_no_partial() { let dir = TempDir::new("cp-whole"); - let data: Vec = (0..2 * FLUSH_BYTES as usize + 137).map(|i| (i.wrapping_mul(31) ^ (i >> 9)) as u8).collect(); + let data: Vec = (0..(2 << 20) + 137usize).map(|i| (i.wrapping_mul(31) ^ (i >> 9)) as u8).collect(); let (source, dest) = (dir.join("source"), dir.join("dest")); fs::write(&source, &data).unwrap(); copy(&source, &dest).unwrap(); diff --git a/system/toybox/src/cp.rs b/system/toybox/src/cp.rs index ee3d1b2f1aa..4f0df6962e0 100644 --- a/system/toybox/src/cp.rs +++ b/system/toybox/src/cp.rs @@ -7,17 +7,6 @@ use std::process; /// allocates the same 64 KiB whatever the source's length. const BUF_BYTES: usize = 64 * 1024; -/// Bytes written between fsyncs. -/// -/// Not a durability knob. `evict_if_needed` in `kernel/src/file_cache.rs` -/// declines to take a dirty page, so the kernel's dirty set is bounded only by -/// what the writer leaves un-flushed — without this the kernel holds the whole -/// copy however small the buffer above is, and a bounded buffer in front of an -/// unbounded one is not a bound. 256 pages, against a file cache whose smallest -/// possible budget is the 2048-page floor in `block::file_cache_pages`. Policy, -/// not physics; reaching it costs a flush, never a refusal. -pub const FLUSH_BYTES: u64 = 1024 * 1024; - pub fn main(args: Vec) { let [source, dest] = args.as_slice() else { eprintln!("Usage: cp "); @@ -85,7 +74,6 @@ pub fn copy(source: &Path, dest: &Path) -> Result<(), String> { fn stream(reader: &mut File, source: &Path, partial: &Path) -> Result<(), String> { let mut writer = File::create(partial).map_err(|e| format!("{}: {e}", partial.display()))?; let mut buf = vec![0u8; BUF_BYTES]; - let mut unflushed = 0u64; loop { let n = match reader.read(&mut buf) { @@ -96,19 +84,12 @@ fn stream(reader: &mut File, source: &Path, partial: &Path) -> Result<(), String writer .write_all(&buf[..n]) .map_err(|e| format!("writing {}: {e}", partial.display()))?; - unflushed += n as u64; - if unflushed >= FLUSH_BYTES { - sync(&writer, partial)?; - unflushed = 0; - } } sync(&writer, partial) } -/// Closing a file reports nothing, so a copy that never asks is a copy that -/// cannot be told the volume was full. `fsync` is the channel that -/// answers, which is why every write here goes through this function. +/// Report delayed write errors before replacing the destination's name. fn sync(writer: &File, partial: &Path) -> Result<(), String> { writer.sync_all().map_err(|e| format!("flushing {}: {e}", partial.display())) } diff --git a/tests/guest/osbench/src/files.rs b/tests/guest/osbench/src/files.rs index 2f182dafb33..edffab3f7c1 100644 --- a/tests/guest/osbench/src/files.rs +++ b/tests/guest/osbench/src/files.rs @@ -40,7 +40,7 @@ impl Target { pub fn memory() -> Target { let (dir, served_by) = if cfg!(target_os = "toyos") { - ("/tmp", "the kernel's tmpfs, through syscalls") + ("/tmp", "the userland memory filesystem") } else { ("/dev/shm", "tmpfs, through the VFS") }; From 24560b39e2c8bfd828ca20be71e6556a0bd21b2a Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:00:36 +0100 Subject: [PATCH 23/93] Remove tests of the retired private disk format The removed suites use only the deleted BCFS/BTND API. Real bcachefs operations, I/O failures, reservation pressure, and crash recovery are tested in engine/tests and the served DATA tests. Keep all upstream-generated format fixtures. --- lib/bcachefs/tests/crash.rs | 269 ------ lib/bcachefs/tests/integration.rs | 1388 ----------------------------- 2 files changed, 1657 deletions(-) delete mode 100644 lib/bcachefs/tests/crash.rs delete mode 100644 lib/bcachefs/tests/integration.rs diff --git a/lib/bcachefs/tests/crash.rs b/lib/bcachefs/tests/crash.rs deleted file mode 100644 index 357794bc8f1..00000000000 --- a/lib/bcachefs/tests/crash.rs +++ /dev/null @@ -1,269 +0,0 @@ -//! A directory rename, changes after it and the sync that commits them, -//! stopped at every block write: the crash-point model of the commit. -//! -//! **A device keeps what a flush covered and any part of what it did not, a -//! 512-byte sector at a time.** The writes since the last flush may land in -//! any order, so each stop is tried with every write of the open epoch before -//! it landed and with none of them; and the stop's write itself whole, or -//! torn at a sector boundary: its first sectors new and the rest old, or its -//! last. What the device then holds must mount, keep every name beside the -//! directory, and name the directory whole under exactly one of its two -//! names; and so must it with block 0 lost, from the backup superblock. - -use std::cell::RefCell; -use std::collections::BTreeMap; - -use bcachefs::{BlockBuf, BlockIO, BlockNum, DeviceError, Formatted, FsError, Mounted, ReadOnly, ReadWrite, Superblock, TransferError, VecBlockIO}; - -const BLOCKS: u64 = 512; -const BLOCK: usize = 4096; -const SECTOR: usize = 512; -const STAGED: &str = "staged"; -const INSTALLED: &str = "apps/pkg"; - -/// What the device was asked, in order: a block written, or a flush. -enum Event { - Write(u64, Box<[u8; BLOCK]>), - Flush, -} - -/// The device as the filesystem sees it, every write in place at once, with -/// the order it was asked in kept beside it; and the write numbered `refuse`, -/// refused. -struct Logged { - image: RefCell>, - log: RefCell>, - writes: RefCell, - refuse: Option, -} - -impl Logged { - fn new(image: &[u8], refuse: Option) -> Self { - Self { image: RefCell::new(image.to_vec()), log: RefCell::default(), writes: RefCell::default(), refuse } - } -} - -struct Refused; -impl TransferError for Refused { - fn refused_before_attempt(&self) -> bool { - false - } -} - -impl BlockIO for Logged { - fn read_block(&self, block: BlockNum, buf: &mut BlockBuf) -> Result<(), DeviceError> { - let at = block.raw() as usize * BLOCK; - buf.0.copy_from_slice(&self.image.borrow()[at..at + BLOCK]); - Ok(()) - } - - fn write_block(&self, block: BlockNum, buf: &BlockBuf) -> Result<(), DeviceError> { - let n = self.writes.replace_with(|n| *n + 1); - if Some(n) == self.refuse { - return Err(DeviceError::classify(&Refused)); - } - self.log.borrow_mut().push(Event::Write(block.raw(), Box::new(buf.0))); - let at = block.raw() as usize * BLOCK; - self.image.borrow_mut()[at..at + BLOCK].copy_from_slice(&buf.0); - Ok(()) - } - - fn block_count(&self) -> u64 { - BLOCKS - } - - fn sync(&self) -> Result<(), DeviceError> { - self.log.borrow_mut().push(Event::Flush); - Ok(()) - } -} - -fn file(i: usize) -> String { - if i.is_multiple_of(4) { format!("{STAGED}/sub/f{i}") } else { format!("{STAGED}/f{i}") } -} - -/// A committed volume: names beside the directory, and the directory with -/// files enough to span leaves, a subdirectory and an empty one. -fn staged() -> Vec { - let mut fs = Formatted::format(VecBlockIO::new(BLOCKS)).expect("format"); - for i in 0..80 { - fs.create(&format!("beside{i}"), format!("beside {i}").as_bytes(), 1).expect("beside"); - } - fs.create(&format!("{STAGED}/"), b"", 1).expect("the directory's own entry"); - fs.create(&format!("{STAGED}/empty/"), b"", 1).expect("an empty directory"); - for i in 0..60 { - fs.create(&file(i), format!("file {i} of the package").as_bytes(), 1).expect("staged"); - } - fs.into_io().expect("commit").into_vec() -} - -/// Every name `keep` accepts, with its bytes, or what would not read. -fn names(fs: &Mounted, keep: &dyn Fn(&str) -> bool) -> Result>, String> { - let listed = fs.list(usize::MAX, keep).map_err(|e| format!("the list: {e:?}"))?; - listed - .into_iter() - .map(|(name, _)| fs.read_file(&name).map(|bytes| (name.clone(), bytes)).map_err(|e| format!("{name}: {e:?}"))) - .collect() -} - -/// The directory's own entry and every name beneath it, by what follows `dir`. -fn tree(fs: &Mounted, dir: &str) -> Result>, String> { - let beneath = format!("{dir}/"); - let all = names(fs, &|n| n.starts_with(&beneath))?; - Ok(all.into_iter().map(|(name, bytes)| (name[dir.len()..].to_string(), bytes)).collect()) -} - -fn beside(fs: &Mounted) -> Result>, String> { - names(fs, &|n| n.starts_with("beside")) -} - -/// The rename: every name beneath the directory, and its own. -fn rename(fs: &mut Mounted) -> Result<(), FsError> { - let moves: Vec<(String, String)> = tree(fs, STAGED) - .expect("the directory reads") - .into_keys() - .map(|rest| (format!("{STAGED}{rest}"), format!("{INSTALLED}{rest}"))) - .collect(); - let pairs: Vec<(&str, &str)> = moves.iter().map(|(a, b)| (a.as_str(), b.as_str())).collect(); - fs.rename_all(&pairs) -} - -/// What a volume must hold: the directory's names and those beside it. -struct Want { - tree: BTreeMap>, - beside: BTreeMap>, -} - -impl Want { - fn of(image: &[u8]) -> Self { - let fs = reopened(image.to_vec()).expect("the staged volume mounts"); - Self { tree: tree(&fs, STAGED).unwrap(), beside: beside(&fs).unwrap() } - } - - /// Where the directory is: `Ok(true)` whole under its new name, - /// `Ok(false)` whole under its old, and otherwise what was found. - fn whole_under_one(&self, fs: &Mounted) -> Result { - let kept = beside(fs)?; - if kept != self.beside { - return Err(format!("{} of {} names beside the directory", kept.len(), self.beside.len())); - } - let (old, new) = (tree(fs, STAGED)?, tree(fs, INSTALLED)?); - match (old.is_empty(), new.is_empty()) { - (false, true) if old == self.tree => Ok(false), - (true, false) if new == self.tree => Ok(true), - _ => Err(format!("{} names under {STAGED} and {} under {INSTALLED}", old.len(), new.len())), - } - } - - fn held(&self, image: Vec) -> Result { - reopened(image).and_then(|fs| self.whole_under_one(&fs)) - } -} - -fn reopened(image: Vec) -> Result, String> { - Mounted::<_, ReadOnly>::open(VecBlockIO::from_vec(image)).map_err(|e| format!("unmountable: {e:?}")) -} - -#[test] -fn a_directory_rename_is_whole_under_one_name_wherever_the_device_stops() { - let image = staged(); - let want = Want::of(&image); - assert!(want.tree.len() > 60, "the directory holds {} names", want.tree.len()); - - let mut fs = Mounted::<_, ReadWrite>::open(Logged::new(&image, None)).expect("mount"); - rename(&mut fs).expect("the rename"); - // Changes after it, which take the blocks the rename gave up if they - // are handed out before the commit lands. - for i in 0..40 { - fs.create(&format!("after{i}"), &[7; BLOCK], 2).expect("a change after it"); - } - fs.sync().expect("the commit"); - assert_eq!(want.held(fs.io().image.borrow().clone()), Ok(true)); - let log = fs.io().log.replace(Vec::new()); - - let writes: Vec = (0..log.len()).filter(|&i| matches!(log[i], Event::Write(..))).collect(); - assert!(writes.len() > 10, "the run wrote {} blocks", writes.len()); - let sb = Superblock::read(&VecBlockIO::from_vec(image.clone())).expect("the staged superblock"); - let in_place: Vec = [0, BLOCKS - 1] - .into_iter() - .chain(sb.bitmap_start.raw()..sb.bitmap_start.raw() + sb.bitmap_blocks) - .collect(); - let sectors = BLOCK / SECTOR; - let parts: Vec<(String, std::ops::Range)> = std::iter::once(("whole".to_string(), 0..sectors)) - .chain((1..sectors).flat_map(|k| [(format!("its first {k} sectors"), 0..k), (format!("its last {k} sectors"), sectors - k..sectors)])) - .collect(); - let (mut torn, mut tried) = (Vec::new(), 0); - for (k, &stop) in writes.iter().enumerate() { - let epoch = log[..stop].iter().rposition(|e| matches!(e, Event::Flush)).map_or(0, |f| f + 1); - let Event::Write(block, data) = &log[stop] else { unreachable!("a stop is a write") }; - for (shape, before) in [("the epoch's writes before it landed", epoch..stop), ("alone", stop..stop)] { - let mut landed = image.clone(); - for event in log[..epoch].iter().chain(&log[before]) { - if let Event::Write(block, data) = event { - let at = *block as usize * BLOCK; - landed[at..at + BLOCK].copy_from_slice(&data[..]); - } - } - let at = *block as usize * BLOCK; - let old = landed[at..at + BLOCK].to_vec(); - for (part, range) in &parts { - let mut held = landed.clone(); - held[at + range.start * SECTOR..at + range.end * SECTOR] - .copy_from_slice(&data[range.start * SECTOR..range.end * SECTOR]); - // A tear only of a block the layout writes in place, and only - // where it leaves bytes neither the old nor the new did. - let new = &held[at..at + BLOCK]; - if range.len() < sectors && (!in_place.contains(block) || new == &old[..] || new == &data[..]) { - continue; - } - let mut lost = held.clone(); - lost[..BLOCK].fill(0); - for (copy, image) in [("", held), (", block 0 lost", lost)] { - tried += 1; - if let Err(why) = want.held(image) { - torn.push(format!("stopped at write {k} of {} (block {block}), {shape}, {part}{copy}: {why}", writes.len())); - } - } - } - } - } - assert!(torn.is_empty(), "{} of {tried} stops tear the volume:\n{}", torn.len(), torn.join("\n")); -} - -/// The same with the one write numbered `n` refused, the filesystem alive -/// after it: the rename's answer is what it serves, the device holds the -/// directory whole under one name past the refused sync, and the next sync -/// makes the answer what the device holds. -#[test] -fn a_directory_rename_is_whole_under_one_name_whichever_write_is_refused() { - let image = staged(); - let want = Want::of(&image); - let writes = { - let mut fs = Mounted::<_, ReadWrite>::open(Logged::new(&image, None)).expect("mount"); - rename(&mut fs).expect("the rename"); - fs.sync().expect("the commit"); - let n = *fs.io().writes.borrow(); - n - }; - - let mut torn = Vec::new(); - for refuse in 0..writes { - let mut fs = Mounted::<_, ReadWrite>::open(Logged::new(&image, Some(refuse))).expect("mount"); - let renamed = rename(&mut fs).is_ok(); - let served = want.whole_under_one(&fs); - let first = fs.sync(); - // Changes past a refused sync, which take the blocks of the tree it - // may have committed if they are handed out before the next lands. - let changed = (0..40).try_for_each(|i| fs.create(&format!("between{i}"), &[7; 9000], 2)); - let between = want.held(fs.io().image.borrow().clone()); - let second = fs.sync(); - let held = want.held(fs.io().image.borrow().clone()); - if served != Ok(renamed) || changed.is_err() || between.is_err() || second.is_err() || held != Ok(renamed) { - torn.push(format!( - "write {refuse} of {writes} refused: renamed {renamed}, served {served:?}, \ - the sync {first:?}, the changes past it {changed:?}, left {between:?}, the next {second:?} left {held:?}" - )); - } - } - assert!(torn.is_empty(), "{} of {writes} refusals tear the directory:\n{}", torn.len(), torn.join("\n")); -} diff --git a/lib/bcachefs/tests/integration.rs b/lib/bcachefs/tests/integration.rs deleted file mode 100644 index c1b470c08ff..00000000000 --- a/lib/bcachefs/tests/integration.rs +++ /dev/null @@ -1,1388 +0,0 @@ -use bcachefs::{BlockNum, Extent, Formatted, FsError, FsUuid, Mounted, ReadOnly, ReadWrite, VecBlockIO}; - -// --- Basic read-only tests --- - -#[test] -fn format_and_mount_empty() { - let io = VecBlockIO::new(128); - let fs = Formatted::format(io).expect("format"); - let mounted = fs.mount_readonly(); - let files = mounted.list(usize::MAX, &|_| true).expect("list failed"); - assert!(files.is_empty(), "expected empty filesystem, got {:?}", files); -} - -#[test] -fn create_single_small_file() { - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - fs.create("hello.txt", b"Hello, world!", 42).expect("create failed"); - let mounted = fs.mount_readonly(); - - let files = mounted.list(usize::MAX, &|_| true).expect("list failed"); - assert_eq!(files.len(), 1); - assert_eq!(files[0].0, "hello.txt"); - assert_eq!(files[0].1, bcachefs::Recorded { symlink: false, size: 13, mtime: 42 }); - - let data = mounted.read_file("hello.txt").expect("read failed"); - assert_eq!(data, b"Hello, world!"); - - assert_eq!(mounted.file_mtime("hello.txt").expect("mtime").unwrap_or(0), 42); -} - -#[test] -fn create_multiple_files() { - let io = VecBlockIO::new(256); - let mut fs = Formatted::format(io).expect("format"); - - fs.create("bin/shell", b"shell-binary-data", 100).expect("create shell"); - fs.create("bin/compositor", b"compositor-binary-data-longer", 200).expect("create compositor"); - fs.create("share/font.ttf", b"font-data", 300).expect("create font"); - - let mounted = fs.mount_readonly(); - - let files = mounted.list(usize::MAX, &|_| true).expect("list failed"); - assert_eq!(files.len(), 3, "expected 3 files, got: {:?}", files); - - assert_eq!(mounted.read_file("bin/shell").unwrap(), b"shell-binary-data"); - assert_eq!(mounted.read_file("bin/compositor").unwrap(), b"compositor-binary-data-longer"); - assert_eq!(mounted.read_file("share/font.ttf").unwrap(), b"font-data"); - - assert_eq!(mounted.file_mtime("bin/shell").expect("mtime").unwrap_or(0), 100); - assert_eq!(mounted.file_mtime("bin/compositor").expect("mtime").unwrap_or(0), 200); - assert_eq!(mounted.file_mtime("share/font.ttf").expect("mtime").unwrap_or(0), 300); -} - -#[test] -fn file_not_found() { - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - fs.create("exists.txt", b"data", 0).expect("create"); - let mounted = fs.mount_readonly(); - - let result = mounted.read_file("nonexistent.txt"); - assert!(result.is_err(), "expected NotFound error"); -} - -#[test] -fn file_mtime_nonexistent() { - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - fs.create("exists.txt", b"data", 999).expect("create"); - let mounted = fs.mount_readonly(); - - // file_mtime returns 0 for nonexistent files, not panic - assert_eq!(mounted.file_mtime("exists.txt").expect("mtime").unwrap_or(0), 999); - assert_eq!(mounted.file_mtime("nope.txt").expect("mtime").unwrap_or(0), 0); -} - -#[test] -fn read_link() { - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - fs.create("real.txt", b"real data", 0).expect("create file"); - fs.create_symlink("link.txt", "real.txt", 0).expect("create symlink"); - - let mounted = fs.mount_readonly(); - - let target = mounted.read_link("link.txt", u64::MAX).expect("read_link"); - assert_eq!(target.as_deref(), Some("real.txt")); - - assert_eq!(mounted.read_link("real.txt", u64::MAX).expect("read_link"), None); - assert_eq!(mounted.read_link("nope", u64::MAX).expect("read_link"), None); -} - -#[test] -fn list_includes_symlinks() { - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - fs.create("file.txt", b"data", 0).expect("create file"); - fs.create_symlink("link.txt", "file.txt", 0).expect("create symlink"); - - let mounted = fs.mount_readonly(); - let files = mounted.list(usize::MAX, &|_| true).expect("list"); - assert_eq!(files.len(), 2, "expected 2 entries (file + symlink), got: {:?}", files); - - let names: Vec<&str> = files.iter().map(|(n, _)| n.as_str()).collect(); - assert!(names.contains(&"file.txt"), "missing file.txt in {:?}", names); - assert!(names.contains(&"link.txt"), "missing link.txt in {:?}", names); - - assert!(mounted.is_symlink("link.txt").expect("is_symlink")); - assert!(!mounted.is_symlink("file.txt").expect("is_symlink")); -} - -#[test] -fn dangling_symlink_allowed() { - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - // Symlink to nonexistent target should succeed — symlinks are just strings - fs.create_symlink("dangling", "/nonexistent/path", 0).expect("create dangling symlink"); - - let mounted = fs.mount_readonly(); - assert_eq!(mounted.read_link("dangling", u64::MAX).expect("read_link").as_deref(), Some("/nonexistent/path")); - assert!(mounted.is_symlink("dangling").expect("is_symlink")); -} - -// --- File size edge cases --- - -#[test] -fn empty_file() { - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - fs.create("empty", b"", 0).expect("create empty file"); - - let mounted = fs.mount_readonly(); - let files = mounted.list(usize::MAX, &|_| true).expect("list"); - assert_eq!(files.len(), 1); - assert_eq!(files[0].0, "empty"); - assert_eq!(files[0].1.size, 0); - - let data = mounted.read_file("empty").expect("read empty"); - assert!(data.is_empty()); -} - -#[test] -fn large_file_single_extent() { - let data = vec![0xABu8; 100 * 1024]; - let io = VecBlockIO::new(512); - let mut fs = Formatted::format(io).expect("format"); - fs.create("big.bin", &data, 0).expect("create large file"); - - let mounted = fs.mount_readonly(); - let read_data = mounted.read_file("big.bin").expect("read large file"); - assert_eq!(read_data.len(), data.len()); - assert_eq!(read_data, data); -} - -#[test] -fn large_file_exact_block_boundary() { - let data = vec![0x42u8; 4096]; - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - fs.create("block.bin", &data, 0).expect("create"); - - let mounted = fs.mount_readonly(); - let read_data = mounted.read_file("block.bin").expect("read"); - assert_eq!(read_data, data); -} - -#[test] -fn large_file_crosses_block_boundary() { - let data: Vec = (0..4097).map(|i| (i % 256) as u8).collect(); - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - fs.create("cross.bin", &data, 0).expect("create"); - - let mounted = fs.mount_readonly(); - let read_data = mounted.read_file("cross.bin").expect("read"); - assert_eq!(read_data.len(), 4097); - assert_eq!(read_data, data); -} - -// --- Filename edge cases --- - -#[test] -fn long_filename_near_entry_limit() { - // 200-byte filename should work fine (fits in a leaf entry) - let name: String = (0..200).map(|i| (b'a' + (i % 26) as u8) as char).collect(); - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - fs.create(&name, b"data", 0).expect("create with 200-byte name"); - - let mounted = fs.mount_readonly(); - assert_eq!(mounted.read_file(&name).unwrap(), b"data"); - - // 513-byte filename should be rejected (MAX_NAME_LEN = 512) - let too_long: String = (0..513).map(|i| (b'a' + (i % 26) as u8) as char).collect(); - let io2 = VecBlockIO::new(128); - let mut fs2 = Formatted::format(io2).expect("format"); - let result = fs2.create(&too_long, b"data", 0); - assert!(result.is_err(), "expected NameTooLong for 513-byte filename"); -} - -#[test] -fn zero_length_filename_rejected() { - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - let result = fs.create("", b"data", 0); - assert!(result.is_err(), "empty filename should be rejected"); -} - -#[test] -fn duplicate_filename_overwrites() { - // Creating a file with the same name should overwrite, not duplicate - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - fs.create("test.txt", b"version 1", 10).expect("create v1"); - fs.create("test.txt", b"version 2", 20).expect("create v2"); - - let mounted = fs.mount_readonly(); - let files = mounted.list(usize::MAX, &|_| true).expect("list"); - assert_eq!(files.len(), 1, "duplicate filename should overwrite, not create second entry"); - assert_eq!(mounted.read_file("test.txt").unwrap(), b"version 2"); - assert_eq!(mounted.file_mtime("test.txt").expect("mtime").unwrap_or(0), 20); -} - -// --- B+ tree split correctness --- - -#[test] -fn incremental_insert_and_read() { - // Insert files one at a time, checking readability after each insert. - // This exercises node splits — hashed keys land in unpredictable order, - // and we verify no entries are lost across splits. - let io = VecBlockIO::new(2048); - let mut fs = Formatted::format(io).expect("format"); - - for i in 0..100 { - let name = format!("file_{:04}.txt", i); - let data = format!("content of file {}", i); - fs.create(&name, data.as_bytes(), i as u64) - .unwrap_or_else(|e| panic!("create {} failed: {:?}", name, e)); - - // Verify ALL previously inserted files are still readable - let mounted = fs.mount_readonly(); - for j in 0..=i { - let check_name = format!("file_{:04}.txt", j); - let expected = format!("content of file {}", j); - let read_data = mounted.read_file(&check_name).unwrap_or_else(|e| { - panic!( - "after inserting file_{:04}, cannot read {}: {:?}", - i, check_name, e - ); - }); - assert_eq!( - String::from_utf8(read_data).unwrap(), - expected, - "content mismatch for {} after inserting file_{:04}", - check_name, - i - ); - } - fs = mounted.into_formatted(); - } -} - -#[test] -fn many_files_with_large_data() { - // Simulate a realistic ROOT image: 50 files of varying sizes - let io = VecBlockIO::new(4096); - let mut fs = Formatted::format(io).expect("format"); - - let mut expected: Vec<(String, Vec)> = Vec::new(); - - for i in 0..50 { - let name = format!("bin/program_{}", i); - let size = (i + 1) * 1024; // 1KB to 50KB - let data: Vec = (0..size).map(|j| ((i + j) % 256) as u8).collect(); - fs.create(&name, &data, i as u64 * 1000).unwrap_or_else(|_| panic!("create {name}")); - expected.push((name, data)); - } - - let mounted = fs.mount_readonly(); - let files = mounted.list(usize::MAX, &|_| true).expect("list"); - assert_eq!(files.len(), 50); - - for (name, data) in &expected { - let read_data = mounted.read_file(name).unwrap_or_else(|e| { - panic!("failed to read {}: {:?}", name, e); - }); - assert_eq!(read_data.len(), data.len(), "size mismatch for {}", name); - assert_eq!(&read_data, data, "data mismatch for {}", name); - } -} - -// --- Read-write tests --- - -#[test] -fn mounted_readwrite_create_and_read() { - let io = VecBlockIO::new(256); - let fs = Formatted::format(io).expect("format"); - let mut mounted = Mounted::<_, ReadWrite>::open(fs.into_io().expect("sync")).expect("open"); - - mounted.create("test.txt", b"hello world", 100).expect("create"); - let data = mounted.read_file("test.txt").expect("read"); - assert_eq!(data, b"hello world"); - assert_eq!(mounted.file_mtime("test.txt").expect("mtime").unwrap_or(0), 100); -} - -#[test] -fn mounted_readwrite_delete() { - let io = VecBlockIO::new(256); - let fs = Formatted::format(io).expect("format"); - let mut mounted = Mounted::<_, ReadWrite>::open(fs.into_io().expect("sync")).expect("open"); - - mounted.create("a.txt", b"aaa", 0).expect("create a"); - mounted.create("b.txt", b"bbb", 0).expect("create b"); - assert_eq!(mounted.list(usize::MAX, &|_| true).unwrap().len(), 2); - - assert!(mounted.delete("a.txt").expect("delete")); - assert_eq!(mounted.list(usize::MAX, &|_| true).unwrap().len(), 1); - assert!(mounted.read_file("a.txt").is_err()); - assert_eq!(mounted.read_file("b.txt").unwrap(), b"bbb"); - - assert!(!mounted.delete("nonexistent").expect("delete")); -} - -#[test] -fn mounted_readwrite_overwrite_file() { - let io = VecBlockIO::new(256); - let fs = Formatted::format(io).expect("format"); - let mut mounted = Mounted::<_, ReadWrite>::open(fs.into_io().expect("sync")).expect("open"); - - mounted.create("test.txt", b"version 1", 10).expect("create v1"); - assert_eq!(mounted.read_file("test.txt").unwrap(), b"version 1"); - - mounted.create("test.txt", b"version 2 is longer", 20).expect("create v2"); - assert_eq!(mounted.read_file("test.txt").unwrap(), b"version 2 is longer"); - assert_eq!(mounted.file_mtime("test.txt").expect("mtime").unwrap_or(0), 20); - assert_eq!(mounted.list(usize::MAX, &|_| true).unwrap().len(), 1); -} - -#[test] -fn mounted_readwrite_symlink() { - let io = VecBlockIO::new(256); - let fs = Formatted::format(io).expect("format"); - let mut mounted = Mounted::<_, ReadWrite>::open(fs.into_io().expect("sync")).expect("open"); - - mounted.create("real.txt", b"real data", 0).expect("create"); - mounted.create_symlink("link.txt", "real.txt").expect("symlink"); - - assert_eq!(mounted.read_link("link.txt", u64::MAX).expect("read_link").as_deref(), Some("real.txt")); - assert_eq!(mounted.read_link("real.txt", u64::MAX).expect("read_link"), None); - assert!(mounted.is_symlink("link.txt").expect("is_symlink")); - assert!(!mounted.is_symlink("real.txt").expect("is_symlink")); -} - -#[test] -fn mounted_readwrite_sync_and_reopen() { - let io = VecBlockIO::new(512); - let fs = Formatted::format(io).expect("format"); - let mut mounted = Mounted::<_, ReadWrite>::open(fs.into_io().expect("sync")).expect("open"); - - mounted.create("persistent.txt", b"I survive reboots", 42).expect("create"); - mounted.sync().expect("sync"); - - let raw = mounted.into_formatted().into_io().expect("sync").into_vec(); - let io2 = VecBlockIO::from_vec(raw); - let mounted2 = Mounted::<_, ReadOnly>::open(io2).expect("reopen"); - - assert_eq!(mounted2.read_file("persistent.txt").unwrap(), b"I survive reboots"); - assert_eq!(mounted2.file_mtime("persistent.txt").expect("mtime").unwrap_or(0), 42); -} - -#[test] -fn mounted_readwrite_double_roundtrip() { - // Create → sync → reopen rw → create more → sync → reopen ro → verify all. - // Catches bitmap free count drift or state corruption across reopens. - let io = VecBlockIO::new(512); - let fs = Formatted::format(io).expect("format"); - let mut m = Mounted::<_, ReadWrite>::open(fs.into_io().expect("sync")).expect("open"); - - m.create("round1.txt", b"first round", 10).expect("create round1"); - m.sync().expect("sync"); - let raw = m.into_formatted().into_io().expect("sync").into_vec(); - - // Reopen read-write, add more - let mut m = Mounted::<_, ReadWrite>::open(VecBlockIO::from_vec(raw)).expect("reopen rw"); - assert_eq!(m.read_file("round1.txt").unwrap(), b"first round"); - m.create("round2.txt", b"second round", 20).expect("create round2"); - m.sync().expect("sync"); - let raw = m.into_formatted().into_io().expect("sync").into_vec(); - - // Final read-only verification - let m = Mounted::<_, ReadOnly>::open(VecBlockIO::from_vec(raw)).expect("reopen ro"); - assert_eq!(m.list(usize::MAX, &|_| true).unwrap().len(), 2); - assert_eq!(m.read_file("round1.txt").unwrap(), b"first round"); - assert_eq!(m.read_file("round2.txt").unwrap(), b"second round"); - assert_eq!(m.file_mtime("round1.txt").expect("mtime").unwrap_or(0), 10); - assert_eq!(m.file_mtime("round2.txt").expect("mtime").unwrap_or(0), 20); -} - -#[test] -fn mounted_readwrite_overwrite_with_smaller_data() { - // Overwrite a 4KB file with 10 bytes. Verifies old extents are freed - // and the reclaimed blocks are reusable. - let io = VecBlockIO::new(64); // tight on space - let fs = Formatted::format(io).expect("format"); - let mut m = Mounted::<_, ReadWrite>::open(fs.into_io().expect("sync")).expect("open"); - - // Fill with a large file (uses most free blocks) - let big = vec![0xBBu8; 40 * 1024]; // 10 blocks - m.create("big.bin", &big, 0).expect("create big"); - - // Overwrite with tiny data — should free the 10 blocks - m.create("big.bin", b"tiny", 0).expect("overwrite with smaller"); - assert_eq!(m.read_file("big.bin").unwrap(), b"tiny"); - assert_eq!(m.list(usize::MAX, &|_| true).unwrap().len(), 1); - - // The freed blocks should be reusable — create another large file - let big2 = vec![0xCCu8; 40 * 1024]; - m.create("big2.bin", &big2, 0).expect("create big2 with reclaimed space"); - assert_eq!(m.read_file("big2.bin").unwrap(), big2); -} - -// --- Filesystem capacity --- - -#[test] -fn filesystem_full_returns_no_space() { - // Tiny filesystem: 32 blocks total. Fill until alloc fails. - let io = VecBlockIO::new(32); - let fs = Formatted::format(io).expect("format"); - let mut mounted = Mounted::<_, ReadWrite>::open(fs.into_io().expect("sync")).expect("open"); - - let mut created = 0; - for i in 0..100 { - let name = format!("f{}", i); - let data = vec![0xFFu8; 4096]; // 1 block per file - match mounted.create(&name, &data, 0) { - Ok(()) => created += 1, - Err(_) => break, // NoSpace expected - } - } - assert!(created > 0, "should have created at least one file"); - assert!(created < 32, "should have hit NoSpace before 32 files"); - - // All previously created files should still be readable - for i in 0..created { - let name = format!("f{}", i); - let data = mounted.read_file(&name).unwrap_or_else(|e| { - panic!("file {} unreadable after NoSpace: {:?}", name, e); - }); - assert_eq!(data.len(), 4096, "data corruption in {} after NoSpace", name); - } -} - -// --- Integrity --- - -#[test] -fn superblock_backup_recovery() { - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - fs.create("test.txt", b"test data", 0).expect("create"); - let mut raw = fs.into_io().expect("sync").into_vec(); - - // Corrupt block 0 (superblock) - raw[0..4].copy_from_slice(b"JUNK"); - - let io = VecBlockIO::from_vec(raw); - let mounted = Mounted::<_, ReadOnly>::open(io).expect("mount from backup"); - let data = mounted.read_file("test.txt").expect("read after recovery"); - assert_eq!(data, b"test data"); -} - -/// `BadMagic` means no copy of the superblock claims the volume, so it is the -/// one refusal a caller may read as "not ours": a broken backup behind a lost -/// block 0 answers with its own reason, and so does a broken block 0. -#[test] -fn bad_magic_means_no_copy_claims_the_volume() { - let fs = Formatted::format(VecBlockIO::new(128)).expect("format"); - let raw = fs.into_io().expect("sync").into_vec(); - let backup = 127 * 4096; - let open = |raw: Vec| Mounted::<_, ReadOnly>::open(VecBlockIO::from_vec(raw)).err(); - - let mut both = raw.clone(); - both[0..4].copy_from_slice(b"JUNK"); - both[backup..backup + 4].copy_from_slice(b"JUNK"); - assert!(matches!(open(both), Some(FsError::BadMagic { .. })), "neither copy is a superblock"); - - let mut lost_front = raw.clone(); - lost_front[0..4].copy_from_slice(b"JUNK"); - lost_front[backup + 200] ^= 0xFF; - match open(lost_front) { - Some(FsError::ChecksumMismatch { .. }) => {} - other => panic!("the backup claims the volume and is broken: {other:?}"), - } - - let mut broken = raw; - broken[200] ^= 0xFF; - broken[backup + 200] ^= 0xFF; - match open(broken) { - Some(FsError::ChecksumMismatch { .. }) => {} - other => panic!("block 0 claims the volume and is broken: {other:?}"), - } -} - -#[test] -fn crc_verification_on_nodes() { - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - fs.create("test.txt", b"hello", 0).expect("create"); - let mut raw = fs.into_io().expect("sync").into_vec(); - - // Corrupt a byte in the root node (block 2 for small fs) - let root_offset = 2 * 4096 + 100; - raw[root_offset] ^= 0xFF; - - let io = VecBlockIO::from_vec(raw); - let mounted = Mounted::<_, ReadOnly>::open(io).expect("mount"); - let result = mounted.read_file("test.txt"); - assert!(result.is_err(), "expected checksum error, got: {:?}", result.ok().map(|d| d.len())); -} - -#[test] -fn corrupt_data_block_returns_raw_bytes() { - // Data blocks have no CRC, so corruption reads back silently. - // - // Layout for a 128-block filesystem: - // block 0: superblock - // block 1: bitmap (128 blocks / 32768 bits_per_block = 1 block) - // block 2: root btree node - // block 3+: data blocks (first file's data starts here) - // block 127: superblock backup - let io = VecBlockIO::new(128); - let mut fs = Formatted::format(io).expect("format"); - let original = vec![0xAAu8; 4096]; - fs.create("data.bin", &original, 0).expect("create"); - let mut raw = fs.into_io().expect("sync").into_vec(); - - // Corrupt byte 50 of the first data block (block 3) - let data_offset = 3 * 4096 + 50; - raw[data_offset] ^= 0xFF; - - let io = VecBlockIO::from_vec(raw); - let mounted = Mounted::<_, ReadOnly>::open(io).expect("mount"); - let data = mounted.read_file("data.bin").expect("read should succeed — no data CRC"); - assert_ne!(data, original, "corruption should be visible in read data"); - assert_eq!(data[50], 0xAA ^ 0xFF, "byte 50 should be flipped"); -} - -// --- State transitions --- - -#[test] -fn format_mount_unmount_create_mount_roundtrip() { - // Verify into_formatted preserves all state: superblock, bitmap, free count - let io = VecBlockIO::new(512); - let mut fs = Formatted::format(io).expect("format"); - - // Create files in Formatted state - fs.create("phase1.txt", b"created during format", 10).expect("create phase1"); - - // Mount readonly, verify, unmount back to Formatted - let mounted = fs.mount_readonly(); - assert_eq!(mounted.list(usize::MAX, &|_| true).unwrap().len(), 1); - assert_eq!(mounted.read_file("phase1.txt").unwrap(), b"created during format"); - fs = mounted.into_formatted(); - - // Create more files in Formatted state after round-trip - fs.create("phase2.txt", b"created after round-trip", 20).expect("create phase2"); - - // Mount readonly again, verify both files exist - let mounted = fs.mount_readonly(); - let files = mounted.list(usize::MAX, &|_| true).unwrap(); - assert_eq!(files.len(), 2, "expected 2 files after round-trip, got: {:?}", files); - assert_eq!(mounted.read_file("phase1.txt").unwrap(), b"created during format"); - assert_eq!(mounted.read_file("phase2.txt").unwrap(), b"created after round-trip"); - assert_eq!(mounted.file_mtime("phase1.txt").expect("mtime").unwrap_or(0), 10); - assert_eq!(mounted.file_mtime("phase2.txt").expect("mtime").unwrap_or(0), 20); -} - -// --- Values sized by userland: the extent list lives inline in a btree node --- - -/// A file grown a page at a time, in ascending order. -fn write_pages(fs: &mut Mounted, pages: u32) -> Vec { - let mut extents = Vec::new(); - for page in 0..pages { - fs.allocate(&mut extents, page..=page, 1).expect("allocate a block for the page"); - } - extents -} - -/// The block holding `page`, which `extents` reach. -fn block_of(extents: &[Extent], page: u32) -> u64 { - let mut cursor = 0; - for e in extents { - if page < cursor + e.block_count { - return e.start_block + u64::from(page - cursor); - } - cursor += e.block_count; - } - panic!("page {page} is past {extents:?}") -} - -#[test] -fn a_sequential_file_needs_one_extent() { - let mut fs = Formatted::format(VecBlockIO::new(4096)).expect("format").mount(); - fs.create("seq.bin", b"", 1).expect("create"); - - let extents = write_pages(&mut fs, 600); - - // 600 pages used to be 600 extents of 16 bytes — 9600 bytes into a value - // that has to fit a 4040-byte node payload. - assert_eq!( - extents.len(), 1, - "a run of consecutive blocks became {} extents: {:?}", extents.len(), extents, - ); - assert_eq!(extents[0].block_count, 600); -} - -#[test] -fn a_file_past_the_old_extent_cap_round_trips() { - // ~250 pages was where `19 + name + 16*pages` crossed the node payload and - // `write_to` underflowed `MAX_PAYLOAD - used`. This is four times that. - let mut fs = Formatted::format(VecBlockIO::new(4096)).expect("format").mount(); - fs.create("big.bin", b"", 1).expect("create"); - - let extents = write_pages(&mut fs, 1000); - let size = 1000 * 4096; - fs.update_metadata("big.bin", &extents, size, 7).expect("metadata for a 1000-page file"); - - let (back, got_size) = fs.file_extents("big.bin").expect("file_extents").expect("reopen"); - assert_eq!(got_size, size); - assert_eq!(back.len(), 1, "extents did not survive the round trip: {back:?}"); - assert_eq!(back[0].block_count, 1000); -} - -#[test] -fn a_value_too_large_for_any_node_is_an_error_not_a_panic() { - let mut fs = Formatted::format(VecBlockIO::new(8192)).expect("format").mount(); - fs.create("frag.bin", b"", 1).expect("create"); - - // Discontiguous by construction: merging cannot help, so this is the case - // that has to be *refused*. Each extent is a separate 16-byte run. - let extents: Vec = (0..400) - .map(|i| bcachefs::Extent { start_block: 3000 + i * 2, block_count: 1, _reserved: 0 }) - .collect(); - - match fs.update_metadata("frag.bin", &extents, 400 * 4096, 7) { - Err(bcachefs::FsError::EntryTooLarge { size, max }) => { - assert!(size > max, "rejected an entry that fits: {size} <= {max}"); - } - other => panic!("expected EntryTooLarge, got {other:?}"), - } - - // And the rejection did not take the file with it. update_metadata used to - // delete the old entry before attempting the insert. - assert!( - fs.file_extents("frag.bin").expect("file_extents").is_some(), - "the file was deleted by a metadata update that failed", - ); -} - -#[test] -fn rename_bounds_the_new_name() { - let mut fs = Formatted::format(VecBlockIO::new(256)).expect("format").mount(); - fs.create("short.txt", b"data", 1).expect("create"); - - // Every other name-taking entry point checked this one and rename did not. - let huge = "n".repeat(64 * 1024); - match fs.rename("short.txt", &huge) { - Err(bcachefs::FsError::NameTooLong { .. }) => {} - other => panic!("expected NameTooLong, got {other:?}"), - } - assert!(fs.file_extents("short.txt").expect("file_extents").is_some(), "rename lost the source file"); -} - -// --- Rename: the destination's blocks are freed, the source's are carried --- - -/// The blocks an extent list names, as the pairs a comparison can read. -fn runs(extents: &[bcachefs::Extent]) -> Vec<(u64, u32)> { - extents.iter().map(|e| (e.start_block, e.block_count)).collect() -} - -/// Every name on the volume, sorted. -fn names(fs: &Mounted) -> Vec { - let mut names: Vec = fs.list(usize::MAX, &|_| true).expect("list").into_iter().map(|(n, _)| n).collect(); - names.sort(); - names -} - -#[test] -fn a_rename_carries_the_file_to_the_new_name() { - // Nothing covered a *successful* rename: the only test that called it - // asserted the NameTooLong direction, so a rename that reported success - // and freed the file's blocks was green everywhere. - let mut fs = Formatted::format(VecBlockIO::new(256)).expect("format").mount(); - let data: Vec = (0..3 * 4096 + 17).map(|i: usize| (i.wrapping_mul(31) ^ 0x5A) as u8).collect(); - fs.create("a.bin", &data, 77).expect("create"); - let (before, size) = fs.file_extents("a.bin").expect("file_extents").expect("the source's extents"); - - fs.rename("a.bin", "b.bin").expect("rename"); - - assert_eq!(names(&fs), ["b.bin"], "the old name outlived the rename"); - assert!(fs.read_file("a.bin").is_err(), "the old name still resolves"); - assert_eq!(fs.read_file("b.bin").expect("read the renamed file"), data); - assert_eq!(fs.file_mtime("b.bin").expect("mtime").unwrap_or(0), 77, "the rename invented an mtime"); - - let (after, size_after) = fs.file_extents("b.bin").expect("file_extents").expect("the renamed file's extents"); - assert_eq!(runs(&after), runs(&before), "the rename moved the file's blocks"); - assert_eq!(size_after, size); - - // And on disk, not just in the tree this process is holding. - fs.sync().expect("sync"); - let raw = fs.into_formatted().into_io().expect("sync").into_vec(); - let reopened = Mounted::<_, ReadOnly>::open(VecBlockIO::from_vec(raw)).expect("reopen"); - assert_eq!(reopened.read_file("b.bin").expect("read after reopen"), data); - assert!(reopened.read_file("a.bin").is_err(), "the old name came back from disk"); -} - -#[test] -fn a_renamed_file_keeps_every_extent_it_had() { - // A file in one run survives a rename that reallocates as readily as one - // that carries the extents. Discontiguous runs do not. - let (mut fs, _) = one_block_holes(128); - let data: Vec = (0..4 * 4096 + 11).map(|i| (i % 251) as u8).collect(); - fs.create("frag.bin", &data, 3).expect("create a fragmented file"); - let (before, _) = fs.file_extents("frag.bin").expect("file_extents").expect("extents"); - assert!(before.len() > 1, "the volume is not fragmented, this proves nothing: {before:?}"); - - fs.rename("frag.bin", "moved.bin").expect("rename"); - - let (after, _) = fs.file_extents("moved.bin").expect("file_extents").expect("the renamed file's extents"); - assert_eq!(runs(&after), runs(&before), "the extent list did not survive the re-encode"); - assert_eq!(fs.read_file("moved.bin").expect("read back"), data); -} - -#[test] -fn a_rename_onto_an_existing_name_frees_that_file_and_only_it() { - let mut fs = Formatted::format(VecBlockIO::new(64)).expect("format").mount(); - let source = vec![0xA5u8; 10 * 4096]; - let doomed = vec![0x5Au8; 10 * 4096]; - fs.create("keep.bin", &source, 1).expect("create the source"); - fs.create("doomed.bin", &doomed, 2).expect("create the destination"); - let (source_blocks, _) = fs.file_extents("keep.bin").expect("file_extents").expect("the source's extents"); - - // Spend the rest, so the only free blocks after the rename are the ones - // the displaced file gave back. - let mut filler = 0; - while fs.create(&format!("filler{filler}"), &vec![0xFF; 4096], 0).is_ok() { - filler += 1; - } - - fs.rename("keep.bin", "doomed.bin").expect("rename onto an existing name"); - - assert_eq!(fs.read_file("doomed.bin").expect("read the destination"), source); - assert!(fs.read_file("keep.bin").is_err(), "the source name survived"); - assert_eq!( - names(&fs).iter().filter(|n| n.ends_with(".bin")).count(), - 1, - "the volume holds more than one entry under the two names: {:?}", - names(&fs), - ); - let (after, _) = fs.file_extents("doomed.bin").expect("file_extents").expect("extents"); - assert_eq!(runs(&after), runs(&source_blocks), "the source's blocks were not carried"); - - // Ten blocks were freed by the displaced file and nothing else was, so a - // ten-block file fits exactly once. A rename that freed the *source's* - // blocks would leave room for two. - fs.create("reclaim.bin", &vec![0xCCu8; 10 * 4096], 0).expect("the displaced blocks are free"); - assert_eq!(fs.read_file("doomed.bin").expect("read after reclaim"), source); - assert!( - fs.create("again.bin", &vec![0xDDu8; 10 * 4096], 0).is_err(), - "twenty blocks came free where one ten-block file was displaced", - ); -} - -#[test] -fn a_rename_onto_itself_keeps_the_file() { - let mut fs = Formatted::format(VecBlockIO::new(128)).expect("format").mount(); - fs.create("same.bin", b"the file that is its own destination", 9).expect("create"); - let (before, _) = fs.file_extents("same.bin").expect("file_extents").expect("extents"); - - fs.rename("same.bin", "same.bin").expect("rename onto itself"); - - assert_eq!(names(&fs), ["same.bin"]); - assert_eq!(fs.read_file("same.bin").expect("read"), b"the file that is its own destination"); - let (after, _) = fs.file_extents("same.bin").expect("file_extents").expect("extents"); - assert_eq!(runs(&after), runs(&before)); -} - -#[test] -fn a_rename_of_a_symlink_stays_a_symlink() { - let mut fs = Formatted::format(VecBlockIO::new(128)).expect("format").mount(); - fs.create_symlink("link", "/home/target").expect("create a symlink"); - - fs.rename("link", "moved-link").expect("rename"); - - assert!(fs.is_symlink("moved-link").expect("is_symlink"), "the rename turned a symlink into a file"); - assert_eq!(fs.read_link("moved-link", u64::MAX).expect("read_link").as_deref(), Some("/home/target")); - assert_eq!(fs.read_link("link", u64::MAX).expect("read_link"), None, "the old name still reads as a symlink"); - assert_eq!(names(&fs), ["moved-link"]); -} - -#[test] -fn a_file_renamed_onto_a_symlink_leaves_one_entry() { - // The two differ in key type, so the insert does not replace the - // destination — nothing but an explicit delete removes it, and what it - // leaves behind answers to the same name with blocks no name can reach. - let mut fs = Formatted::format(VecBlockIO::new(128)).expect("format").mount(); - fs.create_symlink("shadow", "/somewhere").expect("create a symlink"); - fs.create("file.bin", b"a file, not a symlink", 4).expect("create a file"); - - fs.rename("file.bin", "shadow").expect("rename a file onto a symlink"); - - assert_eq!(names(&fs), ["shadow"], "the displaced symlink is still on the volume"); - assert!(!fs.is_symlink("shadow").expect("is_symlink"), "the symlink still shadows the file that replaced it"); - assert_eq!(fs.read_file("shadow").expect("read"), b"a file, not a symlink"); -} - -#[test] -fn a_rename_with_no_source_touches_nothing() { - let mut empty = Formatted::format(VecBlockIO::new(128)).expect("format").mount(); - assert!(matches!(empty.rename("a", "b"), Err(bcachefs::FsError::NotFound))); - assert!(empty.list(usize::MAX, &|_| true).expect("list").is_empty(), "a rename created an entry from nothing"); - - let mut fs = Formatted::format(VecBlockIO::new(128)).expect("format").mount(); - fs.create("bystander.bin", b"not part of this", 6).expect("create"); - assert!(matches!(fs.rename("absent", "bystander.bin"), Err(bcachefs::FsError::NotFound))); - assert_eq!(fs.read_file("bystander.bin").expect("read"), b"not part of this"); - assert_eq!(names(&fs), ["bystander.bin"]); -} - -#[test] -fn entries_of_mixed_size_survive_node_splits() { - // Keys are hashed, so the order entries land in a leaf is not the order - // they are created in. Varying the name length by two orders of magnitude - // is the cheapest way to put non-uniform entries in front of the split - // logic, which is where halving by *count* stops being the right rule. - let mut fs = Formatted::format(VecBlockIO::new(8192)).expect("format").mount(); - let names: Vec = (0..60) - .map(|i| format!("{}{i:03}", "e".repeat(if i % 3 == 0 { 500 } else { 4 }))) - .collect(); - for (i, n) in names.iter().enumerate() { - fs.create(n, b"x", i as u64).expect("create"); - } - for n in &names { - assert_eq!(fs.read_file(n).expect("read back"), b"x", "{n} did not survive splitting"); - } -} - - -// --- A short allocation is not the allocation that was asked for --- - -/// The blocks every change but a shrink leaves free: the crate's `NODE_RESERVE`. -const NODE_RESERVE: u32 = 16; - -/// A volume whose free space is nothing but one-block holes, so the allocator -/// can only ever report a run shorter than a multi-block request: every block -/// a file's data may take, taken in one run, every other one given back, and -/// the run the reserve kept taken after. -/// -/// Returns the blocks still taken alongside it: a block the allocator did -/// *not* hand out is the ground truth for "this write landed on somebody -/// else's file". -fn one_block_holes(blocks: u64) -> (Mounted, Vec) { - let mut fs = Formatted::format(VecBlockIO::new(blocks)).expect("format").mount(); - let mut run = Vec::new(); - let mut page = 0; - while fs.allocate(&mut run, page..=page, 1).is_ok() { - page += 1; - } - let taken: Vec = run.iter().flat_map(|e| e.start_block..e.start_block + e.block_count as u64).collect(); - assert!(taken.len() > 8, "volume too small to fragment: {} blocks", taken.len()); - let holes: Vec = - taken.iter().step_by(2).map(|&b| Extent { start_block: b, block_count: 1, _reserved: 0 }).collect(); - fs.free_extents(&holes).expect("give every other block back"); - let mut tail = vec![Extent { start_block: *taken.last().unwrap(), block_count: 1, _reserved: 0 }]; - fs.allocate(&mut tail, NODE_RESERVE..=NODE_RESERVE, 1).expect("the run left for nodes"); - - let sb = bcachefs::Superblock::read(fs.io()).expect("the superblock"); - let mut bitmap = bcachefs::BlockBuf::zeroed(); - bcachefs::BlockIO::read_block(fs.io(), sb.bitmap_start, &mut bitmap).expect("the bitmap"); - let free = |b: u64| bitmap.0[b as usize / 8] & (1 << (b % 8)) == 0; - let run = (0..blocks - 1).find(|&b| free(b) && free(b + 1)); - assert_eq!(run, None, "a free run longer than one block: this proves nothing"); - (fs, taken.into_iter().skip(1).step_by(2).collect()) -} - -#[test] -fn a_sparse_write_resolves_inside_the_blocks_it_reserved() { - let (mut fs, survivors) = one_block_holes(64); - - // Page 3 of an empty file needs four blocks, and no free run here is - // longer than one. `alloc_contiguous` says so in its second return value; - // this caller used to read it as "all four, starting here". - let mut extents = Vec::new(); - fs.allocate(&mut extents, 3..=3, 1).expect("allocate page 3"); - let block = block_of(&extents, 3); - - let reserved: Vec = extents - .iter() - .flat_map(|e| (0..e.block_count as u64).map(move |i| e.start_block + i)) - .collect(); - assert!( - reserved.contains(&block), - "page 3 resolved to block {block}, outside the extents it recorded: {extents:?}", - ); - assert!(!survivors.contains(&block), "page 3 resolved to block {block}, which was already taken"); -} - -#[test] -fn every_page_of_a_fragmented_file_owns_a_distinct_block() { - let (mut fs, _) = one_block_holes(64); - - let mut extents = Vec::new(); - fs.allocate(&mut extents, 5..=5, 1).expect("allocate through page 5"); - let covered: u32 = extents.iter().map(|e| e.block_count).sum(); - assert_eq!(covered, 6, "six pages need six blocks, got {extents:?}"); - - let mut seen: Vec = Vec::new(); - for page in 0..=5 { - let block = block_of(&extents, page); - assert!(!seen.contains(&block), "page {page} shares block {block} with an earlier page"); - seen.push(block); - } - assert_eq!( - extents.iter().map(|e| e.block_count).sum::(), - 6, - "resolving a page that already has a block allocated another: {extents:?}", - ); -} - -// --- Write-path ordering: an operation that fails must not have destroyed -// what it was asked to replace, nor kept what it took. --- - -/// One-block files a fresh 64-block volume takes: its 60 free blocks, less -/// the 16 a create leaves free, less the formatted root, which the first -/// change copies and frees only at a commit, less the leaf's copy the last -/// create holds beside the one it replaces. -const ONE_BLOCK_FILES_64: usize = 60 - NODE_RESERVE as usize - 2; - -fn small_volume() -> Mounted { - Formatted::format(VecBlockIO::new(64)).expect("format").mount() -} - -/// How many one-block files this volume still has room for. -fn one_block_files_that_fit(fs: &mut Mounted) -> usize { - let mut fitted = 0; - for i in 0..ONE_BLOCK_FILES_64 * 2 { - let name = format!("p{:03}", i); - if fs.create(&name, b"x", 0).is_err() { - break; - } - fitted += 1; - } - fitted -} - -#[test] -fn a_create_that_runs_out_of_space_leaves_the_old_file_where_it_was() { - let mut fs = small_volume(); - let original = vec![0x5Au8; 5 * 4096]; - fs.create("keep.bin", &original, 7).expect("the first create fits"); - - let err = fs - .create("keep.bin", &vec![0xA5u8; 400 * 4096], 8) - .expect_err("400 blocks do not fit in a 64-block volume"); - - assert!( - matches!(err, FsError::NoSpace { .. }), - "expected NoSpace, got {err:?}", - ); - assert_eq!( - fs.read_file("keep.bin").expect("the file that was already here"), - original, - "the replacement failed and took the original with it", - ); - assert_eq!(fs.file_mtime("keep.bin").expect("mtime").unwrap_or(0), 7, "the old entry's mtime was rewritten"); -} - -#[test] -fn a_write_that_runs_out_of_space_gives_back_what_it_took() { - let mut fresh = small_volume(); - let untouched = one_block_files_that_fit(&mut fresh); - assert_eq!( - untouched, ONE_BLOCK_FILES_64, - "the baseline is wrong, so the comparison below proves nothing", - ); - - let mut fs = small_volume(); - fs.create("big.bin", &vec![0u8; 400 * 4096], 0) - .expect_err("400 blocks do not fit in a 64-block volume"); - - assert_eq!( - one_block_files_that_fit(&mut fs), - untouched, - "a create that failed kept the blocks it had already reserved", - ); -} - -#[test] -fn a_metadata_update_that_cannot_be_reinserted_leaves_the_entry_alone() { - let mut fs = small_volume(); - // Every block a file's data may take spent, then the rest on the nodes - // of names with no data, until one more needs a block the volume does - // not have — so the reinsert has no block to write to. - for i in 0..ONE_BLOCK_FILES_64 { - fs.create(&format!("f{:02}", i), b"x", 100 + i as u64).expect("fill"); - } - let mut names = 0; - while fs.create(&format!("empty{names}"), b"", 0).is_ok() { - names += 1; - } - - let (extents, _) = fs.file_extents("f00").expect("file_extents").expect("f00 is on the volume"); - let grown: Vec = (0..16).map(|_| extents[0]).collect(); - - let err = fs - .update_metadata("f00", &grown, 1, 999) - .expect_err("a 16-extent value needs a split this volume cannot pay for"); - assert!( - matches!(err, FsError::NoSpace { .. }), - "expected NoSpace, got {err:?}", - ); - - assert_eq!( - fs.read_file("f00").expect("f00 after a metadata update that failed"), - b"x", - ); - assert_eq!(fs.file_mtime("f00").expect("mtime").unwrap_or(0), 100, "the failed update left its mtime behind"); -} - -/// Fill `fs` with names enough to span leaves, then with one-block files, -/// then with names with none until a create right after a commit is refused: -/// how many files it took. -fn filled_to_the_last_name(fs: &mut Mounted) -> usize { - for i in 0..150 { - fs.create(&format!("spread{i:03}"), b"", 0).expect("a name with no data"); - if i % 50 == 49 { - fs.sync().expect("a commit"); - } - } - let mut files = 0; - while fs.create(&format!("f{files:02}"), b"x", 0).is_ok() { - files += 1; - } - let mut names = 0; - loop { - fs.sync().expect("a commit"); - let before = names; - while fs.create(&format!("empty{names}"), b"", 0).is_ok() { - names += 1; - } - if names == before { - break; - } - } - assert!(files > 4 && names > 0, "{files} files and {names} empty names filled the volume"); - assert!(fs.create("after", b"x", 0).is_err(), "the volume is full"); - files -} - -/// Shrink every one of `files` to nothing, then delete each, on a volume -/// filled to its last name; then, after a commit, take a file again. -fn shrinks_and_deletes_and_takes_again(fs: &mut Mounted, files: usize) { - for i in 0..files { - let name = format!("f{i:02}"); - let (extents, _) = fs.file_extents(&name).expect("file_extents").expect("the file is on the volume"); - fs.update_metadata(&name, &[], 0, 1).expect("a shrink of a full volume's file"); - fs.free_extents(&extents).expect("the shrink's free"); - } - for i in 0..files { - assert!(matches!(fs.delete(&format!("f{i:02}")), Ok(true)), "the delete of f{i:02}"); - } - fs.sync().expect("a commit"); - fs.create("after", b"x", 0).expect("a file in what the deletes gave back"); -} - -/// A volume filled to its last name still shrinks and deletes every file, and -/// after the next commit takes a file again. -#[test] -fn a_volume_full_of_empty_names_still_deletes_and_frees() { - let mut fs = Formatted::format(VecBlockIO::new(128)).expect("format").mount(); - let files = filled_to_the_last_name(&mut fs); - shrinks_and_deletes_and_takes_again(&mut fs, files); -} - -/// Data grown outside any operation takes no block of the reserve, even -/// where every free block lies in one run after the file's end, so the entry -/// that records the run does not grow and draws on the reserve itself. -#[test] -fn a_file_grown_past_what_the_volume_spares_leaves_the_reserve() { - let mut mkfs = Formatted::format(VecBlockIO::new(128)).expect("format"); - mkfs.create("f00", b"x", 0).expect("a one-block file"); - let mut fs = mkfs.mount(); - let (mut extents, _) = fs.file_extents("f00").expect("file_extents").expect("f00 is on the volume"); - - let grown = fs.allocate(&mut extents, 128..=128, 1); - assert!(matches!(grown, Err(FsError::NoSpace { .. })), "a page past every block of the volume: {grown:?}"); - assert_eq!(extents.len(), 1, "the run is not contiguous with the file, so this proves nothing: {extents:?}"); - let blocks = extents[0].block_count as u64; - fs.update_metadata("f00", &extents, blocks * 4096, 1).expect("the run recorded"); - fs.sync().expect("a commit"); - - let free = bcachefs::Superblock::read(fs.io()).expect("the superblock").free_blocks; - assert!(free >= NODE_RESERVE as u64, "the commit holds {free} blocks free, fewer than the reserve"); - shrinks_and_deletes_and_takes_again(&mut fs, 1); -} - -/// A volume stopped between two commits holds more blocks used than its -/// superblock counts free; a read-write mount counts its bitmap, so what the -/// stop leaked is no block a shrink or a delete is promised. -#[test] -fn a_mount_after_a_stop_counts_the_blocks_its_bitmap_holds() { - let mut fs = Formatted::format(VecBlockIO::new(128)).expect("format").mount(); - fs.sync().expect("a commit"); - fs.create("lost", &[1; 20 * 4096], 0).expect("taken and never committed"); - let image = (0..128).flat_map(|b| { - let mut buf = bcachefs::BlockBuf::zeroed(); - bcachefs::BlockIO::read_block(fs.io(), BlockNum::new(b), &mut buf).expect("read"); - buf.as_bytes().to_vec() - }); - let mut fs = Mounted::<_, ReadWrite>::open(VecBlockIO::from_vec(image.collect())).expect("the stopped volume mounts"); - assert!(matches!(fs.read_file("lost"), Err(FsError::NotFound)), "the stop came before the commit"); - - let files = filled_to_the_last_name(&mut fs); - shrinks_and_deletes_and_takes_again(&mut fs, files); -} - -// --- The device error channel: a block the device would not give back is not -// a block of zeros. --- - -/// A volume whose device refuses one chosen block. -/// -/// The only way to stage a device failure here: `VecBlockIO` cannot fail and -/// neither can QEMU's NVMe, so nothing else in the tree drives a refused -/// transfer through the filesystem at all. -struct Refuses { - inner: VecBlockIO, - read: Option, - write: Option, -} - -/// The transfer was attempted and failed — the device's own word. -struct Attempted; -impl bcachefs::TransferError for Attempted { - fn refused_before_attempt(&self) -> bool { - false - } -} - -/// Refused on the caller's budget before the attempt; still durable. -struct OnBudget; -impl bcachefs::TransferError for OnBudget { - fn refused_before_attempt(&self) -> bool { - true - } -} - -impl Refuses { - fn read(raw: Vec, block: u64) -> Self { - Self { inner: VecBlockIO::from_vec(raw), read: Some(block), write: None } - } - - fn write(raw: Vec, block: u64) -> Self { - Self { inner: VecBlockIO::from_vec(raw), read: None, write: Some(block) } - } -} - -impl bcachefs::BlockIO for Refuses { - fn read_block( - &self, - block: bcachefs::BlockNum, - buf: &mut bcachefs::BlockBuf, - ) -> Result<(), bcachefs::DeviceError> { - if self.read == Some(block.raw()) { - return Err(bcachefs::DeviceError::classify(&Attempted)); - } - self.inner.read_block(block, buf) - } - - fn write_block( - &self, - block: bcachefs::BlockNum, - buf: &bcachefs::BlockBuf, - ) -> Result<(), bcachefs::DeviceError> { - if self.write == Some(block.raw()) { - return Err(bcachefs::DeviceError::classify(&Attempted)); - } - self.inner.write_block(block, buf) - } - - fn block_count(&self) -> u64 { - self.inner.block_count() - } -} - -/// A 128-block volume holding one file of `pattern` bytes, as raw bytes. -fn volume_with(name: &str, pattern: &[u8]) -> Vec { - let mut fs = Formatted::format(VecBlockIO::new(128)).expect("format"); - fs.create(name, pattern, 5).expect("create"); - fs.into_io().expect("sync").into_vec() -} - -#[test] -fn a_data_block_the_device_refuses_is_not_a_page_of_zeros() { - let pattern = vec![0xC3u8; 3 * 4096]; - let raw = volume_with("doc.bin", &pattern); - let data_block = { - let fs = Mounted::<_, ReadOnly>::open(VecBlockIO::from_vec(raw.clone())).expect("open"); - fs.file_extents("doc.bin").expect("file_extents").expect("doc.bin").0[0].start_block - }; - - let fs = Mounted::<_, ReadOnly>::open(Refuses::read(raw, data_block)).expect("open"); - match fs.read_file("doc.bin") { - Err(FsError::DeviceRead(block, e)) => { assert_eq!(block.raw(), data_block); assert!(matches!(e, bcachefs::DeviceError::Failed(_))); } - Ok(data) => panic!( - "read_file returned {} bytes for a block the device refused; first is {:#x}", - data.len(), - data.first().copied().unwrap_or(0), - ), - Err(other) => panic!("expected DeviceRead, got {other:?}"), - } -} - -#[test] -fn a_btree_node_the_device_refuses_is_not_a_node_of_zeros() { - let raw = volume_with("doc.bin", b"small"); - let root = u64::from_le_bytes(raw[24..32].try_into().unwrap()); - - let fs = Mounted::<_, ReadOnly>::open(Refuses::read(raw, root)).expect("open"); - match fs.list(usize::MAX, &|_| true) { - Err(FsError::DeviceRead(block, _)) => assert_eq!(block.raw(), root), - other => panic!("expected DeviceRead, got {other:?}"), - } -} - -#[test] -fn a_block_zero_the_device_refuses_does_not_fall_through_to_the_backup() { - // The backup superblock is the answer to a bad superblock, not to a bad - // device. Reaching for it after a refused read mounts a volume from a - // device that is not answering. - let raw = volume_with("doc.bin", b"small"); - match Mounted::<_, ReadOnly>::open(Refuses::read(raw, 0)) { - Err(FsError::DeviceRead(block, _)) => assert_eq!(block.raw(), 0), - other => panic!("expected DeviceRead, got {:?}", other.map(|_| "a mount")), - } -} - -/// `disowns_volume` is what a caller outside this crate uses to tell "not -/// ours" from "ours, but did not mount" — every variant, so a new one left -/// out of this list is a case this test does not cover rather than one that -/// silently defaults to either answer. -#[test] -fn bad_magic_alone_disowns_the_volume() { - let block = BlockNum::new(0); - let device = bcachefs::DeviceError::classify(&Attempted); - let cases: &[(FsError, bool)] = &[ - (FsError::BadMagic { expected: *b"TOYS", got: *b"NOPE" }, true), - (FsError::UnsupportedVersion(1), false), - (FsError::ChecksumMismatch { block, stored: 0, computed: 1 }, false), - (FsError::CorruptedKey(0), false), - (FsError::CorruptedNode(block), false), - (FsError::BlockOffDevice { block: 0, device_blocks: 0 }, false), - (FsError::NotEnoughBlocks { needed: 0, available: 0 }, false), - (FsError::TreeTooDeep(block), false), - (FsError::BadSuperblock { field: "x" }, false), - (FsError::DeviceRead(block, device), false), - (FsError::DeviceWrite(block, device), false), - (FsError::DeviceSync(device), false), - (FsError::NotFound, false), - (FsError::NoSpace { requested: 0, available: 0 }, false), - (FsError::NameTooLong { len: 0, max: 0 }, false), - (FsError::EntryTooLarge { size: 0, max: 0 }, false), - (FsError::NodeOverfull { used: 0, max: 0 }, false), - (FsError::TargetTooLong { size: 0, max: 0 }, false), - (FsError::ListTooLong { limit: 0 }, false), - ]; - for (err, want) in cases { - assert_eq!(err.disowns_volume(), *want, "{err:?}"); - } - - // A device that will not answer at block 0 is not a device that says - // "not ours" — the volume stays `Unmountable`, never a candidate for the - // designation stamp. - let raw = volume_with("doc.bin", b"small"); - match Mounted::<_, ReadOnly>::open(Refuses::read(raw, 0)) { - Err(err) => assert!(!err.disowns_volume(), "{err:?}"), - Ok(_) => panic!("a device that refuses block 0 must not mount"), - } -} - -#[test] -fn a_write_the_device_refuses_is_reported_and_gives_its_blocks_back() { - let raw = volume_with("keep.bin", b"the file that was already here"); - let next_free = { - let fs = Mounted::<_, ReadOnly>::open(VecBlockIO::from_vec(raw.clone())).expect("open"); - fs.file_extents("keep.bin").expect("file_extents").expect("keep.bin").0[0].start_block + 1 - }; - - let mut fs = Mounted::<_, ReadWrite>::open(Refuses::write(raw, next_free)).expect("open"); - match fs.create("new.bin", &vec![0x11u8; 4096], 0) { - Err(FsError::DeviceWrite(block, e)) => { assert_eq!(block.raw(), next_free); assert!(matches!(e, bcachefs::DeviceError::Failed(_))); } - other => panic!("expected DeviceWrite, got {other:?}"), - } - - assert_eq!( - fs.read_file("keep.bin").expect("the file that was already here"), - b"the file that was already here", - ); - // The refused write's block came back, so the retry that lands on the next - // free block is the same one. - let (extents, _) = fs - .file_extents("keep.bin") - .expect("file_extents") - .expect("keep.bin"); - assert_eq!(extents[0].start_block + 1, next_free); -} - -/// A device that answers every transfer but postpones its cache flush on the -/// caller's budget — the NVMe reset-reclaimed silence, at this boundary. -struct Postpones { - inner: VecBlockIO, -} - -impl bcachefs::BlockIO for Postpones { - fn read_block( - &self, - block: bcachefs::BlockNum, - buf: &mut bcachefs::BlockBuf, - ) -> Result<(), bcachefs::DeviceError> { - self.inner.read_block(block, buf) - } - - fn write_block( - &self, - block: bcachefs::BlockNum, - buf: &bcachefs::BlockBuf, - ) -> Result<(), bcachefs::DeviceError> { - self.inner.write_block(block, buf) - } - - fn block_count(&self) -> u64 { - self.inner.block_count() - } - - fn sync(&self) -> Result<(), bcachefs::DeviceError> { - Err(bcachefs::DeviceError::classify(&OnBudget)) - } -} - -/// The retry discriminant crosses this crate unchanged: a `Refused` sync comes -/// out of `Mounted::sync` still `Refused`, never widened into the device's own -/// word. -#[test] -fn a_refused_sync_stays_refused_through_the_filesystem() { - let raw = volume_with("doc.bin", b"small"); - let mut fs = - Mounted::<_, ReadWrite>::open(Postpones { inner: VecBlockIO::from_vec(raw) }) - .expect("open"); - fs.create("new.bin", b"new bytes", 0).expect("create"); - match fs.sync() { - Err(FsError::DeviceSync(bcachefs::DeviceError::Refused(_))) => {} - other => panic!("expected DeviceSync(Refused), got {other:?}"), - } -} - -/// A name survives the trip to disk and back, and a volume nobody named comes -/// back unnamed: the kernel selects ROOT by comparing these sixteen bytes to -/// the one its boot parameter carries, so a name that did not persist would -/// make every candidate look like every other. -#[test] -fn a_filesystems_name_round_trips_through_its_superblock() { - let named = FsUuid::parse("0123456789abcdef0123456789abcdef").expect("32 hex digits"); - assert_eq!(named.to_string(), "0123456789abcdef0123456789abcdef"); - assert_eq!(FsUuid::parse("0123456789ABCDEF0123456789ABCDEF"), Some(named)); - for wrong in ["", "0123456789abcdef0123456789abcde", "0123456789abcdef0123456789abcdeg"] { - assert_eq!(FsUuid::parse(wrong), None, "{wrong:?} is not a filesystem name"); - } - - let mut fs = Formatted::format(VecBlockIO::new(128)).expect("format"); - fs.create("bin/supervisor", b"supervisor-binary", 0).expect("create"); - fs.set_uuid(named); - let raw = fs.into_io().expect("finish").into_vec(); - let mounted = - Mounted::<_, ReadOnly>::open(VecBlockIO::from_vec(raw)).expect("open what was written"); - assert_eq!(mounted.uuid(), named); - - let unnamed = Formatted::format(VecBlockIO::new(128)).expect("format"); - assert_eq!(unnamed.mount_readonly().uuid(), FsUuid::UNNAMED); -} From d679ad360597c1769a14fdc1863f718c74f6e750 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 19:48:41 +0100 Subject: [PATCH 24/93] Coalesce underfull btree siblings and collapse redundant roots Capture adjacent siblings as one bounded rewrite, publish their replacement pointers after all bytes complete, and transfer later changes by key range. Keep still-underfull replacements queued until their siblings can compact. Collapse a one-child root through the journal and ignore obsolete higher levels during replay after collapse. Expose the same relocation path for metadata bucket evacuation and share the minimum fill calculation with capacity admission. Existing conservative reclaim reserve assumptions remain separate integration work. Validation: four host tests cover repeated create/delete churn, clean-node relocation, changes between merged writes and pointer publication, and torn/reordered crash recovery through merges and root collapse. --- lib/bcachefs/src/engine/cache.rs | 7 +- lib/bcachefs/src/engine/mount/mod.rs | 5 +- lib/bcachefs/src/engine/mount/reclaim.rs | 266 ++++++++++++++++++----- lib/bcachefs/src/engine/tests/compact.rs | 119 ++++++++++ lib/bcachefs/src/engine/tests/mod.rs | 1 + 5 files changed, 338 insertions(+), 60 deletions(-) create mode 100644 lib/bcachefs/src/engine/tests/compact.rs diff --git a/lib/bcachefs/src/engine/cache.rs b/lib/bcachefs/src/engine/cache.rs index d517947e91e..69545bb93dc 100644 --- a/lib/bcachefs/src/engine/cache.rs +++ b/lib/bcachefs/src/engine/cache.rs @@ -49,6 +49,8 @@ pub(crate) struct Cached { /// The oldest sequence a written bset holds whose pointer update is not /// applied. pub captured: Option, + /// A published underfull node still needing a sibling rewrite. + pub compact: Option, /// A write of it is in flight: it takes no change until it completes. pub writing: bool, /// Its next write goes to a fresh node, never appended: it was read @@ -62,7 +64,7 @@ pub(crate) struct Cached { impl Cached { pub(crate) fn pin(&self) -> Option { - let oldest = self.unwritten.values().min().copied(); + let oldest = self.unwritten.values().copied().chain(self.compact).min(); match (oldest, self.captured) { (Some(a), Some(b)) => Some(a.min(b)), (a, b) => a.or(b), @@ -70,7 +72,7 @@ impl Cached { } pub(crate) fn dirty(&self) -> bool { - !self.unwritten.is_empty() + !self.unwritten.is_empty() || self.compact.is_some() } /// Whether dropping it loses nothing: a reread from the medium gives @@ -288,6 +290,7 @@ mod tests { bytes: 0, unwritten: BTreeMap::new(), captured: None, + compact: None, writing: false, fresh: false, held: 0, diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index d96a068f34a..0892a79c2c3 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -222,7 +222,8 @@ enum Install { Append { id: NodeId, sectors: u16 }, /// `id` replaced by the nodes written for it, and a root above them when /// a root split. - Replace { id: NodeId, by: Vec, root: Option }, + Replace { ids: Vec, by: Vec, root: Option }, + Collapse { id: NodeId }, } /// A new node as planned and written. @@ -648,6 +649,7 @@ impl Mount { keys: Rc::new(node.keys), unwritten: BTreeMap::new(), captured: None, + compact: None, writing: false, fresh, held: 0, @@ -716,6 +718,7 @@ impl Mount { keys: Rc::new(Vec::new()), unwritten: BTreeMap::new(), captured: None, + compact: None, writing: false, fresh: true, held: 0, diff --git a/lib/bcachefs/src/engine/mount/reclaim.rs b/lib/bcachefs/src/engine/mount/reclaim.rs index 631a80289e1..96506a344bf 100644 --- a/lib/bcachefs/src/engine/mount/reclaim.rs +++ b/lib/bcachefs/src/engine/mount/reclaim.rs @@ -23,7 +23,7 @@ pub(crate) enum Order { /// What a reclaim batch writes for one node. pub(crate) struct Plan { - id: NodeId, + ids: Vec, btree: Btree, level: u8, /// It is its btree's root. @@ -52,7 +52,7 @@ impl Plan { fn cost(&self) -> u64 { match self.written() { 0 => APPEND_COST, - k => fresh_cost(k), + k => fresh_cost(k) + (self.ids.len() - 1) as u64 * (ALLOC + 4 * SET + BUCKET_GENS), } } @@ -60,7 +60,7 @@ impl Plan { fn nodes(&self) -> usize { match self.written() { 0 => 1, - k => fresh_nodes(k), + k => fresh_nodes(k) + 7 * (self.ids.len() - 1), } } } @@ -184,25 +184,94 @@ impl Mount { } async fn write_batch_once(&self, order: Order) -> Result> { + while self.collapse_root().await? {} self.make_room().await?; let plans = { let _slot = self.slot(Class::Reclaim).await; - self.plan(order).await? + self.plan(order, None).await? }; if plans.is_empty() { return Ok(plans); } self.write_plans(&plans).await?; - { - let mut st = self.st.borrow_mut(); - for p in &plans { - st.cache.change(p.id, |n| n.writing = false); - } - } - self.written.wake(); Ok(plans) } + /// Moves the reachable node still at `expected_sector`. Bucket + /// evacuation uses the same publication and reader barriers as reclaim. + pub(crate) async fn relocate_node(&self, btree: Btree, level: u8, pos: Bpos, expected_sector: u64) -> Result { + let out = async { + while self.collapse_root().await? {} + self.make_room().await?; + let plans = { + let _slot = self.slot(Class::Reclaim).await; + if self.st.borrow().roots.get(&btree).is_none_or(|r| r.level < level) { return Ok(false); } + let id = self.find(btree, level, pos).await?; + { + let mut st = self.st.borrow_mut(); + let seq = st.journal.open.as_ref().map_or(st.journal.next_seq, |o| o.seq); + let n = st.cache.node(id); + if n.sector != expected_sector || n.writing || n.captured.is_some() || n.held != 0 { return Ok(false); } + st.cache.change(id, |n| { n.fresh = true; n.compact = Some(seq); }); + } + self.plan(Order::OldestPin, Some(id)).await? + }; + if plans.is_empty() { return Ok(false); } + self.write_plans(&plans).await?; + self.point_batch(plans).await?; + Ok(true) + }.await; + self.stopped_by(out) + } + + async fn collapse_root(&self) -> Result { + let candidate = { + let st = self.st.borrow(); + st.roots.values().find_map(|r| { + let id = st.cache.id_at(r.ptr.ptr.sector)?; + let n = st.cache.node(id); + (n.level > 0 && n.keys.len() == 1 && n.dirty() && !n.writing && n.captured.is_none() && n.held == 0).then_some((r.btree, n.sector)) + }) + }; + let Some((btree, expected)) = candidate else { return Ok(false) }; + let bound = fresh_cost(1); + let room = RECLAIM_ROOM; + self.flush_closed().await?; + self.admit(Class::Reclaim, Draw::Freeing, bound, room).await?; + let _slot = self.slot(Class::Reclaim).await; + let mut txn = Txn::default(); + let out = async { + let id = { + let st = self.st.borrow(); + if st.roots.get(&btree).is_none_or(|r| r.ptr.ptr.sector != expected) { return Ok(false); } + let Some(id) = st.cache.id_at(expected) else { return Ok(false) }; + id + }; + let (level, sector, gen, key, ptr) = { + let st = self.st.borrow(); + let n = st.cache.node(id); + if n.keys.len() != 1 || n.writing || n.captured.is_some() || n.held != 0 { return Ok(false); } + let (key, value) = &n.keys[0]; + (n.level, n.sector, n.gen, *key, NodePtr::decode(value)?) + }; + let child = self.load(btree, level - 1, &ptr, key.pos).await?; + { + let st = self.st.borrow(); + let n = st.cache.node(child); + if n.writing || n.captured.is_some() { return Ok(false); } + } + if ptr.min != Bpos::MIN || key.pos != Bpos::MAX { return Err(Error::Corrupt("only root child does not span the btree")); } + txn.roots.push(Root { btree, level: level - 1, key, ptr }); + let mut buckets = BTreeMap::new(); + self.node_counted(&mut txn, &mut buckets, btree, level, sector, gen, Bpos::MAX, false).await?; + for (b, (before, after)) in buckets { self.bucket_changed(&mut txn, b, before, after).await?; } + txn.installs.push(Install::Collapse { id }); + Ok(true) + }.await; + self.finish(txn, out.is_ok(), Class::Reclaim, bound, room)?; + out + } + /// Reclaim's second half: the written batch's pointer updates /// journaled, on reclaim's reserves and ahead in the slot. pub(crate) async fn point_batch(&self, plans: Vec) -> Result<()> { @@ -242,8 +311,8 @@ impl Mount { /// written-but-unpointed, and no change taken until its write completes. /// A node is appended to when it may be and has room; otherwise it is /// rewritten to fresh nodes, split past two thirds full. - async fn plan(&self, order: Order) -> Result> { - let chosen: Vec = { + async fn plan(&self, order: Order, only: Option) -> Result> { + let chosen: Vec = if let Some(id) = only { alloc::vec![id] } else { let st = self.st.borrow(); // An oversized node first; then by `order`. let mut dirty: Vec<(bool, u8, u64, NodeId)> = st @@ -261,8 +330,7 @@ impl Mount { }) .collect(); dirty.sort_unstable(); - let first = dirty.first().map(|d| (d.0, d.1)); - dirty.into_iter().filter(|d| Some((d.0, d.1)) == first).take(BATCH).map(|d| d.3).collect() + dirty.into_iter().map(|d| d.3).collect() }; let node_bytes = self.sb.node_sectors() as usize * SECTOR; let block = self.sb.block_bytes(); @@ -270,21 +338,27 @@ impl Mount { let mut plans = Vec::new(); let mut spent = 0; for id in chosen { - let (btree, level, root, append, groups) = { + if plans.len() == BATCH { break; } + if self.st.borrow().cache.node(id).captured.is_some() { continue; } + let ids = self.rebalance_pair(id).await?; + if ids.is_empty() { continue; } + let (btree, level, root, min, max, keys, changes, append, groups) = { let st = self.st.borrow(); let n = st.cache.node(id); let root = st.new_roots.get(&n.btree) == Some(&id) || st.roots.get(&n.btree).is_some_and(|r| r.ptr.ptr.sector == n.sector); let later = node::later_record_bytes(n.unwritten.keys().map(|pos| changed(n, *pos).len())).next_multiple_of(block); // An oversized node is split: an append would leave it so. - let append = !n.fresh && n.written > 0 && !st.cache.oversized(n) && usize::from(n.on_disk) * SECTOR + later <= node_bytes; - let sizes: Vec = n.keys.iter().map(|(_, v)| 8 * (KEY_U64S + v.len().div_ceil(8))).collect(); + let append = ids.len() == 1 && n.compact.is_none() && !n.fresh && n.written > 0 && !st.cache.oversized(n) && usize::from(n.on_disk) * SECTOR + later <= node_bytes; + let keys: Vec<_> = ids.iter().flat_map(|&id| st.cache.node(id).keys.iter().cloned()).collect(); + let changes: BTreeMap<_, _> = ids.iter().flat_map(|&id| st.cache.node(id).unwritten.iter().map(|(&p, &s)| (p, s))).collect(); + let sizes: Vec = keys.iter().map(|(_, v)| key_bytes(v)).collect(); // At two thirds, as upstream splits; an oversized node, too // full for two at that, in as few as each a node holds. let mut groups = mkfs::spread(&sizes, mkfs::capacity(node_bytes) - FIRST_KEYS); if groups.len() > 2 { groups = mkfs::spread(&sizes, node_bytes - FIRST_KEYS); } - (n.btree, n.level, root, append, groups) + (n.btree, n.level, root, st.cache.node(ids[0]).min, st.cache.node(*ids.last().unwrap()).max, keys, changes, append, groups) }; if groups.len() > MAX_GROUPS { // More than a node read holds once unpacked, and two @@ -292,11 +366,12 @@ impl Mount { return Err(Error::Unsupported("a btree node whose keys fill more than seven nodes")); } let written = groups.len() + usize::from(groups.len() > 1 && root); - let cost = if append { APPEND_COST } else { fresh_cost(written) }; + let cost = if append { APPEND_COST } else { fresh_cost(written) + (ids.len() - 1) as u64 * (ALLOC + 4 * SET + BUCKET_GENS) }; // A rewrite past the common one is a batch of its own, so the // nodes a batch's rewrites leave stay within reclaim's room. let alone = !append && written > COMMON_REWRITE; if spent + cost > K_TX || (alone && !plans.is_empty()) { + for &other in &ids { if other != id { self.st.borrow_mut().cache.change(other, |n| n.held -= 1); } } break; } spent += cost; @@ -318,15 +393,13 @@ impl Mount { for _ in 0..groups.len() + usize::from(splits && root) { sectors.push(self.alloc_sector().await?); } - let st = self.st.borrow(); - let n = st.cache.node(id); let mut nodes = Vec::new(); for (j, range) in groups.iter().enumerate() { - let min = if j == 0 { n.min } else { n.keys[range.start - 1].0.pos.successor().expect("a key below the last position") }; - let max = if j + 1 == groups.len() { n.max } else { n.keys[range.end - 1].0.pos }; - let keys = n.keys[range.clone()].to_vec(); - let journal_seq = n.unwritten.range(min..=max).map(|(_, s)| *s).max().unwrap_or(0); - nodes.push(self.fresh_node(btree, level, min, max, journal_seq, keys, sectors[j], magic)?); + let low = if j == 0 { min } else { keys[range.start - 1].0.pos.successor().expect("a key below the last position") }; + let high = if j + 1 == groups.len() { max } else { keys[range.end - 1].0.pos }; + let mine = keys[range.clone()].to_vec(); + let journal_seq = changes.range(low..=high).map(|(_, s)| *s).max().unwrap_or(0); + nodes.push(self.fresh_node(btree, level, low, high, journal_seq, mine, sectors[j], magic)?); } let above = (splits && root).then(|| { let keys = nodes.iter().map(|(p, _): &(Planned, Vec)| pointer(p.max, &p.ptr())).collect(); @@ -334,24 +407,62 @@ impl Mount { }); Write::Fresh { nodes, root: above.transpose()? } }; - plans.push(Plan { id, btree, level, root, write }); + { + let mut st = self.st.borrow_mut(); + let seq = st.journal.open.as_ref().map_or(st.journal.next_seq, |o| o.seq); + for &other in &ids { + st.cache.change(other, |n| { + if other != id { n.held -= 1; } + n.captured = Some(n.pin().unwrap_or(seq)); + n.unwritten.clear(); + n.compact = None; + n.writing = true; + }); + } + } + plans.push(Plan { ids, btree, level, root, write }); if alone { break; } } - // Captured only once the whole batch is planned: a plan that fails - // leaves every node as it was. - let mut st = self.st.borrow_mut(); - for p in &plans { - st.cache.change(p.id, |n| { - n.captured = n.unwritten.values().copied().min(); - n.unwritten.clear(); - n.writing = true; - }); - } Ok(plans) } + pub(super) fn minimum_fill(&self) -> usize { + (mkfs::capacity(self.sb.node_sectors() as usize * SECTOR) - FIRST_KEYS) / 2 - 255 * 8 + } + + /// An underfull node and one adjacent child of its parent. A busy + /// sibling leaves the node dirty until that rewrite has published. + async fn rebalance_pair(&self, id: NodeId) -> Result> { + let (btree, level, max) = { + let st = self.st.borrow(); + let n = st.cache.node(id); + let root = st.new_roots.get(&n.btree) == Some(&id) || st.roots.get(&n.btree).is_some_and(|r| r.ptr.ptr.sector == n.sector); + if root { + return Ok(if n.level > 0 && n.keys.len() == 1 { Vec::new() } else { alloc::vec![id] }); + } + if n.bytes >= self.minimum_fill() { return Ok(alloc::vec![id]); } + (n.btree, n.level, n.max) + }; + let parent = self.find(btree, level + 1, max).await?; + let adjacent = { + let st = self.st.borrow(); + let p = st.cache.node(parent); + let at = p.at(max); + if p.keys.len() == 1 { return Ok(Vec::new()); } + let index = if at > 0 { at - 1 } else { at + 1 }; + let (k, v) = p.keys.get(index).ok_or(Error::Corrupt("underfull node absent from its parent"))?; + (k.pos, NodePtr::decode(v)?) + }; + let other = self.load(btree, level, &adjacent.1, adjacent.0).await?; + let mut st = self.st.borrow_mut(); + let n = st.cache.node(other); + if n.writing || n.captured.is_some() || n.held > 0 || st.cache.oversized(n) { return Ok(Vec::new()); } + st.cache.change(other, |n| n.held += 1); + Ok(if adjacent.0 < max { alloc::vec![other, id] } else { alloc::vec![id, other] }) + } + /// A new node of `keys`, encoded as its one first record. #[allow(clippy::too_many_arguments)] fn fresh_node(&self, btree: Btree, level: u8, min: Bpos, max: Bpos, journal_seq: u64, keys: Vec<(Key, Vec)>, at: (u64, u8), magic: u64) -> Result<(Planned, Vec)> { @@ -511,7 +622,7 @@ impl Mount { } for p in plans { let writes: Vec<(u64, &[u8])> = match &p.write { - Write::Append { at, bytes } => alloc::vec![(self.st.borrow().cache.node(p.id).sector + at, &bytes[..])], + Write::Append { at, bytes } => alloc::vec![(self.st.borrow().cache.node(p.ids[0]).sector + at, &bytes[..])], Write::Fresh { nodes, root } => nodes.iter().chain(root).map(|(planned, bytes)| (planned.sector, &bytes[..])).collect(), }; for (sector, bytes) in writes { @@ -519,6 +630,13 @@ impl Mount { self.st.borrow_mut().clocks[1] += (bytes.len() / SECTOR) as u64; } } + { + let mut st = self.st.borrow_mut(); + for p in plans { + for &id in &p.ids { st.cache.change(id, |n| n.writing = false); } + } + } + self.written.wake(); Ok(()) } @@ -527,10 +645,10 @@ impl Mount { async fn point(&self, txn: &mut Txn, plans: Vec) -> Result<()> { let mut buckets: BTreeMap = BTreeMap::new(); for plan in plans { - let (min, max, seq, sector, gen, written) = { + let (min, max, seq, sector, gen) = { let st = self.st.borrow(); - let n = st.cache.node(plan.id); - (n.min, n.max, n.seq, n.sector, n.gen, n.written) + let n = st.cache.node(plan.ids[0]); + (n.min, n.max, n.seq, n.sector, n.gen) }; let (btree, level) = (plan.btree, plan.level); match plan.write { @@ -543,7 +661,7 @@ impl Mount { let (k, v) = pointer(max, &ptr); self.put(txn, btree, level + 1, max, Some((k, v))).await?; } - txn.installs.push(Install::Append { id: plan.id, sectors }); + txn.installs.push(Install::Append { id: plan.ids[0], sectors }); } Write::Fresh { nodes, root } => { let by: Vec = nodes.into_iter().map(|(p, _)| p).collect(); @@ -552,6 +670,10 @@ impl Mount { (Some(r), _) => txn.roots.push(Root { btree, level: r.level, key: Key::new(kind::BTREE_PTR_V2, Bpos::MAX), ptr: r.ptr() }), (None, true) => txn.roots.push(Root { btree, level, key: Key::new(kind::BTREE_PTR_V2, Bpos::MAX), ptr: by[0].ptr() }), (None, false) => { + for &id in &plan.ids { + let old_max = self.st.borrow().cache.node(id).max; + self.put(txn, btree, level + 1, old_max, None).await?; + } for p in &by { self.put(txn, btree, level + 1, p.max, Some(pointer(p.max, &p.ptr()))).await?; } @@ -560,10 +682,17 @@ impl Mount { for p in by.iter().chain(&root) { self.node_counted(txn, &mut buckets, btree, p.level, p.sector, p.gen, p.max, true).await?; } - if written > 0 { - self.node_counted(txn, &mut buckets, btree, level, sector, gen, max, false).await?; + for &id in &plan.ids { + let (sector, gen, max, written) = { + let st = self.st.borrow(); + let n = st.cache.node(id); + (n.sector, n.gen, n.max, n.written) + }; + if written > 0 { + self.node_counted(txn, &mut buckets, btree, level, sector, gen, max, false).await?; + } } - txn.installs.push(Install::Replace { id: plan.id, by, root }); + txn.installs.push(Install::Replace { ids: plan.ids, by, root }); } } } @@ -698,26 +827,39 @@ impl Mount { n.bsets += 1; n.captured = None; }), - Install::Replace { id, by, root } => { - let (btree, keys, unwritten) = st.cache.change(id, |n| { - n.captured = None; - (n.btree, Rc::clone(&n.keys), core::mem::take(&mut n.unwritten)) - }); - st.cache.remove(id); - if st.new_roots.get(&btree) == Some(&id) { - st.new_roots.remove(&btree); + Install::Replace { ids, by, root } => { + let mut keys = Vec::new(); + let mut unwritten = BTreeMap::new(); + for id in ids { + let btree = st.cache.change(id, |n| { + n.captured = None; + keys.extend(n.keys.iter().cloned()); + unwritten.append(&mut n.unwritten); + n.btree + }); + st.cache.remove(id); + if st.new_roots.get(&btree) == Some(&id) { st.new_roots.remove(&btree); } } for p in by { let mine: Vec<(Key, Vec)> = keys.iter().filter(|(k, _)| (p.min..=p.max).contains(&k.pos)).cloned().collect(); let changes: BTreeMap = unwritten.range(p.min..=p.max).map(|(k, v)| (*k, *v)).collect(); let id = st.cache.insert(fresh_cached(&p, mine)); - st.cache.change(id, |n| n.unwritten = changes); + let is_root = st.roots.get(&p.btree).is_some_and(|r| r.ptr.ptr.sector == p.sector); + let seq = st.journal.open.as_ref().map_or(st.journal.next_seq, |o| o.seq); + st.cache.change(id, |n| { + n.unwritten = changes; + if !is_root && n.bytes < self.minimum_fill() { n.compact = Some(seq); } + }); } if let Some(p) = root { let keys = p.keys.clone(); st.cache.insert(fresh_cached(&p, keys)); } } + Install::Collapse { id } => { + st.cache.change(id, |n| { n.unwritten.clear(); n.compact = None; n.captured = None; }); + st.cache.remove(id); + } } } @@ -731,6 +873,7 @@ impl Mount { let mut keys: Vec<_> = replayed.keys.into_iter().collect(); keys.sort_by_key(|((btree, level, pos), _)| (core::cmp::Reverse(*level), *btree, *pos)); for ((btree, level, pos), logged) in keys { + if self.st.borrow().roots.get(&btree).is_some_and(|root| level > root.level) { continue; } let id = self.find(btree, level, pos).await?; self.st.borrow_mut().cache.change(id, |n| n.put(pos, logged.key, logged.seq)); } @@ -803,6 +946,7 @@ fn fresh_cached(p: &Planned, keys: Vec<(Key, Vec)>) -> Cached { keys: Rc::new(keys), unwritten: BTreeMap::new(), captured: None, + compact: None, writing: false, fresh: false, held: 0, @@ -836,11 +980,19 @@ fn mark_bitmap(m: &mut Member, start: u64, end: u64) -> bool { #[cfg(test)] impl Plan { + pub(crate) fn merged_contains(&self, m: &Mount, btree: Btree, level: u8, pos: Bpos) -> bool { + let st = m.st.borrow(); + self.ids.len() > 1 && self.btree == btree && self.level == level && self.ids.iter().any(|&id| { + let n = st.cache.node(id); + (n.min..=n.max).contains(&pos) + }) + } + /// Where an appended bset went: the node's sector, the bset's byte /// offset in it and its length. pub(crate) fn appended(&self, m: &Mount) -> Option<(u64, usize, usize)> { match &self.write { - Write::Append { at, bytes } => Some((m.st.borrow().cache.node(self.id).sector, *at as usize * SECTOR, bytes.len())), + Write::Append { at, bytes } => Some((m.st.borrow().cache.node(self.ids[0]).sector, *at as usize * SECTOR, bytes.len())), Write::Fresh { .. } => None, } } @@ -848,7 +1000,7 @@ impl Plan { /// The sectors of every node it wrote. pub(crate) fn sectors(&self, m: &Mount) -> Vec { match &self.write { - Write::Append { .. } => alloc::vec![m.st.borrow().cache.node(self.id).sector], + Write::Append { .. } => alloc::vec![m.st.borrow().cache.node(self.ids[0]).sector], Write::Fresh { nodes, root } => nodes.iter().chain(root).map(|(p, _)| p.sector).collect(), } } diff --git a/lib/bcachefs/src/engine/tests/compact.rs b/lib/bcachefs/src/engine/tests/compact.rs new file mode 100644 index 00000000000..c4cc361e3bb --- /dev/null +++ b/lib/bcachefs/src/engine/tests/compact.rs @@ -0,0 +1,119 @@ +use super::mount::{mounted, random, MEMORY, NOW, OWNER}; +use super::*; + +#[test] +fn deleting_all_files_collapses_the_inode_tree_and_reuses_its_room() { + let dev = formatted(64 << 20, None); + for round in 0..3 { + mounted(&dev, async |m: &Mount<&Image>| { + let mut inodes = Vec::new(); + for i in 0..800 { + inodes.push(m.create(m.root(), format!("f{i}").as_bytes(), New::File, OWNER, NOW).await.unwrap().ino); + } + m.commit().await.unwrap(); + for (i, ino) in inodes.into_iter().enumerate() { + m.unlink(m.root(), format!("f{i}").as_bytes(), false, NOW).await.unwrap(); + m.release(ino).await.unwrap(); + } + }); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(v.roots().iter().find(|r| r.btree == Btree::Inodes).unwrap().level, 0, "round {round}"); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); + } +} + +#[test] +fn a_clean_node_relocation_keeps_its_keys_and_updates_its_pointer() { + let dev = formatted(64 << 20, None); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + let root = *v.roots().iter().find(|r| r.btree == Btree::Inodes).unwrap(); + drop(v); + let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); + assert!(block_on_ready(m.relocate_node(Btree::Inodes, root.level, Bpos::MAX, root.ptr.ptr.sector)).unwrap()); + block_on_ready(m.unmount()).unwrap(); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert_ne!(v.roots().iter().find(|r| r.btree == Btree::Inodes).unwrap().ptr.ptr.sector, root.ptr.ptr.sector); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} + +#[test] +fn a_change_after_sibling_bytes_are_written_survives_their_pointer_update() { + let mut tree = Tree::new(); + for i in 0..1200 { tree.file(Tree::ROOT, format!("f{i}").as_bytes(), &[], Meta { perm: 0o644, uid: 0, gid: 0 }).unwrap(); } + let dev = populated(64 << 20, &tree, 1); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + let inodes: Vec<_> = (0..1200).map(|i| path(&v, &format!("f{i}"))).collect(); + drop(v); + let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); + let mut changed = None; + for (i, &ino) in inodes.iter().enumerate().take(1100) { + block_on_ready(m.unlink(m.root(), format!("f{i}").as_bytes(), false, NOW)).unwrap(); + block_on_ready(m.release(ino)).unwrap(); + let plans = block_on_ready(m.write_batch(super::super::mount::Order::LeavesFirst)).unwrap(); + let target = inodes[i + 1..].iter().find(|&&ino| plans.iter().any(|p| p.merged_contains(&m, Btree::Inodes, 0, Bpos::new(0, ino, u32::MAX)))).copied(); + if let Some(ino) = target { + block_on_ready(m.set_mtime(ino, NOW + 123)).unwrap(); + changed = Some(ino); + } + block_on_ready(m.point_batch(plans)).unwrap(); + if changed.is_some() { break; } + } + let ino = changed.expect("a deletion merged inode siblings"); + assert_eq!(block_on_ready(m.inode(ino)).unwrap().mtime, NOW + 123); + block_on_ready(m.unmount()).unwrap(); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(block_on_ready(v.inode(ino)).unwrap().mtime, NOW + 123); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} + +#[test] +fn sibling_merges_and_root_collapse_recover_after_torn_reordered_writes() { + use core::cell::Cell; + use toyos_blockdev::model::{Model, Overlay}; + use super::crash::Watched; + + const FILES: usize = 600; + let mut tree = Tree::new(); + for i in 0..FILES { tree.file(Tree::ROOT, format!("f{i}").as_bytes(), &[], Meta { perm: 0o644, uid: 0, gid: 0 }).unwrap(); } + let dev = populated(64 << 20, &tree, 3); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert!(v.roots().iter().find(|r| r.btree == Btree::Inodes).unwrap().level > 0); + let inodes: Vec<_> = (0..FILES).map(|i| path(&v, &format!("f{i}"))).collect(); + drop(v); + let model = Model::new(dev.into_inner(), 512); + let applied = Cell::new(0); + let committed = Cell::new(0); + let judged = Cell::new(0u64); + let watched = Watched { + model: &model, + after: Box::new(|model| { + if applied.get() < FILES / 2 { return; } + judged.set(judged.get() + 1); + model.check(8, judged.get(), |bytes| { + let dev = Overlay::new(bytes); + let m = block_on_ready(Mount::open(&dev, MEMORY, random())).map_err(|e| format!("mount: {e:?}"))?; + let (entries, more) = block_on_ready(m.readdir(m.root(), 0, FILES + 1)).map_err(|e| format!("list: {e:?}"))?; + if more || !(FILES - applied.get()..=FILES - committed.get()).contains(&entries.len()) { + return Err(format!("{} names after {} applied/{} committed deletions", entries.len(), applied.get(), committed.get())); + } + block_on_ready(m.unmount()).map_err(|e| format!("unmount: {e:?}"))?; + let v = block_on_ready(Volume::open(&dev)).map_err(|e| format!("open: {e:?}"))?; + let found = block_on_ready(fsck(&v)).map_err(|e| format!("fsck: {e:?}"))?; + if !found.is_empty() { return Err(format!("fsck: {found:?}")); } + Ok(()) + }).unwrap_or_else(|e| panic!("{e}")); + }), + }; + let m = block_on_ready(Mount::open(&watched, MEMORY, random())).unwrap(); + for (i, ino) in inodes.into_iter().enumerate() { + applied.set(i + 1); + block_on_ready(m.unlink(m.root(), format!("f{i}").as_bytes(), false, NOW)).unwrap(); + block_on_ready(m.release(ino)).unwrap(); + if (i + 1) % 16 == 0 { + block_on_ready(m.commit()).unwrap(); + committed.set(i + 1); + } + } + block_on_ready(m.unmount()).unwrap(); + assert!(judged.get() > 10); +} diff --git a/lib/bcachefs/src/engine/tests/mod.rs b/lib/bcachefs/src/engine/tests/mod.rs index 08c527ffbf7..e945bcb5d98 100644 --- a/lib/bcachefs/src/engine/tests/mod.rs +++ b/lib/bcachefs/src/engine/tests/mod.rs @@ -1260,6 +1260,7 @@ mod admission; mod buckets; mod crash; mod data; +mod compact; mod interleave; mod journal; mod mount; From e3169f8e284b3ffc0c8e22351ff4c9550274be02 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 19:49:32 +0100 Subject: [PATCH 25/93] Track live metadata records and geometry-bounded allocation indexes Prepare shared capacity accounting by maintaining non-structural leaf bytes, BTREE bucket count and imported non-root node debt across applied changes. Allocation index and metadata-backpointer positions have finite bounds from device geometry; the other records are charged from their encoded lengths. Report space excluding outstanding metadata promises. This prepares the occupancy-based reservation ledger; admission still uses the earlier conservative per-token bound pending rebalance and bucket evacuation integration. no_std cargo check passes. --- lib/bcachefs/src/engine/mount/allocator.rs | 38 ++++++++++++++++++++ lib/bcachefs/src/engine/mount/data.rs | 30 +++++++++++++++- lib/bcachefs/src/engine/mount/mod.rs | 41 +++++++++++++++++++--- lib/bcachefs/src/engine/mount/reclaim.rs | 1 + lib/bcachefs/src/engine/tests/data.rs | 3 ++ 5 files changed, 107 insertions(+), 6 deletions(-) diff --git a/lib/bcachefs/src/engine/mount/allocator.rs b/lib/bcachefs/src/engine/mount/allocator.rs index 251eeb35e17..769f2fc15ed 100644 --- a/lib/bcachefs/src/engine/mount/allocator.rs +++ b/lib/bcachefs/src/engine/mount/allocator.rs @@ -20,6 +20,35 @@ use crate::engine::read::Source; use crate::engine::value::{accounting_pos, class, data_type}; use crate::engine::{journal, Btree, Error, Result}; +/// Records whose distinct positions are bounded by the device geometry. +pub(super) fn structural(btree: Btree, value: &[u8]) -> bool { + matches!(btree, Btree::Alloc | Btree::Freespace | Btree::Lru | Btree::NeedDiscard | Btree::BucketGens) + || (btree == Btree::Backpointers && value.get(2) == Some(&data_type::BTREE)) +} + +pub(super) async fn metadata_budget(source: &S) -> Result<(u64, u64, BTreeSet)> { + let mut bytes = 0u64; + let roots: BTreeSet = Btree::ALL.iter().filter_map(|&b| source.root_of(b).map(|r| r.ptr.ptr.sector)).collect(); + let mut baseline = BTreeSet::new(); + let shift = match source.superblock().flag(6, 40, 8) { 0 => 10, value => value as u32 }; + for btree in Btree::ALL { + source.walk(btree, Bpos::MIN, |key, value| { + if !structural(btree, value) { bytes += super::key_bytes(value) as u64; } + if btree == Btree::Backpointers && value.get(2) == Some(&data_type::BTREE) { + let sector = key.pos.offset >> shift; + if !roots.contains(§or) { baseline.insert(sector); } + } + Ok(true) + }).await?; + } + let pos = accounting_pos(&[class::DEV_DATA_TYPE, 0, data_type::BTREE]); + let buckets = match source.get(Btree::Accounting, pos).await? { + Some((_, value)) => journal::counters(&value)?.first().copied().unwrap_or(0), + None => 0, + }; + Ok((bytes, buckets, baseline)) +} + /// What a transaction's nodes may take of the room left. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Draw { @@ -55,6 +84,7 @@ pub(super) struct Allocator { /// Buckets taken whose filling is not yet applied, which the indexes /// still list as empty. taken: BTreeSet, + metadata_taken: BTreeSet, /// Empty buckets on the medium, not taken. free: u64, /// Buckets each sequence emptied, until it is durable. @@ -75,6 +105,7 @@ impl Allocator { holes: BTreeSet::new(), durable: 0, taken: BTreeSet::new(), + metadata_taken: BTreeSet::new(), free, returning: BTreeMap::new(), #[cfg(test)] @@ -91,6 +122,11 @@ impl Allocator { self.open_slots() + self.per_bucket * (self.free + self.returning.values().sum::() - self.pins.values().filter(|(_, empty)| empty.is_some()).count() as u64) } + pub(super) fn capacity(&self, metadata_buckets: u64) -> u64 { + let pinned = self.pins.values().filter(|(_, empty)| empty.is_some()).count() as u64; + (self.free + self.returning.values().sum::() - pinned + metadata_buckets + self.metadata_taken.len() as u64) * self.per_bucket + } + /// Node slots to be had now, past `blocked` buckets that a traversal /// may still lead into. pub(super) fn ready(&self, blocked: usize) -> u64 { @@ -131,6 +167,7 @@ impl Allocator { /// `b`, empty at generation `gen`, opened for nodes. pub(super) fn take(&mut self, b: u64, gen: u8) { self.taken.insert(b); + self.metadata_taken.insert(b); self.free = self.free.saturating_sub(1); self.open = Some((b, 0, gen)); } @@ -192,6 +229,7 @@ impl Allocator { /// `b`'s filling applied: the indexes no longer list it. pub(super) fn filled(&mut self, b: u64) { self.taken.remove(&b); + self.metadata_taken.remove(&b); if let Some((_, empty)) = self.pins.get_mut(&b) { if let Some(seq) = empty.take() { if seq <= self.durable { diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index 4c4843d314c..7dbdae54b34 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -208,8 +208,36 @@ impl Mount { /// Total volume bytes and bytes in currently free, reusable buckets. pub fn space(&self) -> (u64, u64) { + self.cancel_dropped(); let bucket = u64::from(self.sb.member.bucket_sectors) * SECTOR as u64; - (self.sb.member.nbuckets * bucket, self.st.borrow().alloc.free_bytes(bucket)) + let node = u64::from(self.sb.node_sectors()) * SECTOR as u64; + let st = self.st.borrow(); + let pending = self.reservations.borrow().active.len() as u64; + let logical = st.alloc.capacity(st.metadata_buckets).saturating_sub(self.metadata_nodes(&st, st.granted_bytes + pending * K_TX * 8)) * node; + let promised = pending * FOOTPRINT as u64 * reclaim::COMMON_REWRITE as u64 * node; + (self.sb.member.nbuckets * bucket, st.alloc.free_bytes(bucket).saturating_sub(promised).min(logical)) + } + + pub(super) fn metadata_nodes(&self, st: &State, pending_bytes: u64) -> u64 { + let buckets = u128::from(self.sb.member.nbuckets); + let slots = buckets * u128::from(self.sb.member.bucket_sectors) / u128::from(self.sb.node_sectors()); + // Each bucket has at most one alloc, freespace, discard and LRU + // record; each node slot one metadata backpointer. Generation + // records cover 256 buckets. Remaining fixed accounting records + // are bounded by the format's classes and btrees. + let structural = buckets * (104 + 3 * 40) + buckets.div_ceil(256) * 296 + slots * 72 + (Btree::ALL.len() * 9 * 64) as u128; + let bytes = u128::from(st.metadata_bytes) + structural + u128::from(pending_bytes); + let node_bytes = self.sb.node_sectors() as usize * SECTOR; + let minimum = (mkfs::capacity(node_bytes) - FIRST_KEYS) / 2 - MAX_KEY; + let stable = st.baseline_nodes.len() as u128 + 2 * (Btree::ALL.len() as u128 + bytes.div_ceil(minimum as u128)); + stable.min(u128::from(u64::MAX)) as u64 + } + + #[cfg(test)] + pub(crate) async fn check_metadata_budget(&self) { + let (bytes, buckets, _) = allocator::metadata_budget(self).await.unwrap(); + let st = self.st.borrow(); + assert_eq!((st.metadata_bytes, st.metadata_buckets), (bytes, buckets)); } /// The high-water mark is durable before any version beneath it is diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 0892a79c2c3..8a986458782 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -113,6 +113,10 @@ struct State { journal: Writer, key_version: u64, next_version: u64, + metadata_bytes: u64, + metadata_buckets: u64, + baseline_nodes: BTreeSet, + granted_bytes: u64, clocks: [u64; 2], /// Seconds since the epoch of the newest operation: each entry's /// datetime. @@ -206,6 +210,7 @@ struct Txn { /// Buckets taken that it fills. filled: Vec, allocated_data: Vec, + bucket_states: Vec<(u8, u8)>, /// New roots. roots: Vec, /// Written nodes made reachable, installed in the cache at apply. @@ -311,7 +316,8 @@ impl Mount { let volume = Volume::open(dev).await?; no_logged_ops(&volume).await?; let empty = allocator::empty_buckets(&volume).await?; - Self::from(volume.into_parts(), empty, options, random).await + let metadata = allocator::metadata_budget(&volume).await?; + Self::from(volume.into_parts(), empty, metadata, options, random).await } /// An encrypted volume, unlocked as [`Volume::unlock`] does, mounted @@ -321,11 +327,12 @@ impl Mount { let volume = Volume::unlock(dev, passphrase, limits).await?; no_logged_ops(&volume).await?; let empty = allocator::empty_buckets(&volume).await?; - Self::from(volume.into_parts(), empty, options, random).await + let metadata = allocator::metadata_budget(&volume).await?; + Self::from(volume.into_parts(), empty, metadata, options, random).await } /// `empty` is the volume's count of empty buckets. - async fn from(parts: Parts, empty: u64, options: MountOptions, random: Box) -> Result { + async fn from(parts: Parts, empty: u64, metadata: (u64, u64, BTreeSet), options: MountOptions, random: Box) -> Result { let Parts { dev, mut sb, key, roots, key_version, replay, root } = parts; let key_version = key_version.checked_add(if replay.is_some() && key.is_some() { 65_536 } else { 0 }).ok_or(Error::NoSpace("encrypted extent versions"))?; let ring: Vec = sb.journal.iter().flat_map(|&(first, count)| first..first + count).collect(); @@ -413,6 +420,10 @@ impl Mount { journal, key_version, next_version: key_version, + metadata_bytes: metadata.0, + metadata_buckets: metadata.1, + baseline_nodes: metadata.2, + granted_bytes: 0, clocks, now: sb.header.write_time, granted: 0, @@ -973,7 +984,7 @@ impl Mount { }; if room && st.journal.reserve(journal, class) { match class { - Class::Ordinary => st.granted += nodes, + Class::Ordinary => { st.granted += nodes; st.granted_bytes += journal * 8; } Class::Reclaim => st.reclaim_granted += nodes, } return Ok(true); @@ -1014,7 +1025,7 @@ impl Mount { fn ungrant(&self, st: &mut State, class: Class, journal: u64, nodes: usize) { st.journal.unreserve(journal); match class { - Class::Ordinary => st.granted -= nodes, + Class::Ordinary => { st.granted -= nodes; st.granted_bytes -= journal * 8; } Class::Reclaim => st.reclaim_granted -= nodes, } self.admit_reclaim.wake(); @@ -1080,6 +1091,26 @@ impl Mount { let stopped = st.read_only; let allocated = core::mem::take(&mut txn.allocated_data); if apply && stopped.is_none() { + for &(before, after) in &txn.bucket_states { + st.metadata_buckets = st.metadata_buckets + u64::from(after == data_type::BTREE) - u64::from(before == data_type::BTREE); + } + for (&(btree, level, pos), update) in &txn.updates { + if level != 0 { continue; } + let node = st.cache.node(held(st, btree, level, pos)); + let prior = node.keys.get(node.at(pos)).filter(|(k, _)| k.pos == pos); + let charge = |entry: Option<&(Key, Vec)>| entry.filter(|(_, v)| !allocator::structural(btree, v)).map_or(0, |(_, v)| key_bytes(v) as u64); + st.metadata_bytes = st.metadata_bytes + charge(update.as_ref()) - charge(prior); + if btree == Btree::Backpointers && update.is_none() && prior.is_some_and(|(_, value)| value.get(2) == Some(&data_type::BTREE)) { + st.baseline_nodes.remove(&(pos.offset >> self.bp_shift())); + } + } + for (&pos, delta) in &txn.accounting { + let node = st.cache.node(held(st, Btree::Accounting, 0, pos)); + let prior = node.keys.get(node.at(pos)).filter(|(k, _)| k.pos == pos); + let before = prior.map_or(0, |(_, v)| key_bytes(v) as u64); + let counters = prior.map_or(0, |(_, v)| v.len() / 8).max(delta.len()); + st.metadata_bytes = st.metadata_bytes + (KEY_U64S * 8 + counters * 8) as u64 - before; + } self.apply(st, txn, bound); for sector in allocated { st.alloc.unpin(sector / u64::from(self.sb.member.bucket_sectors)); } } else { diff --git a/lib/bcachefs/src/engine/mount/reclaim.rs b/lib/bcachefs/src/engine/mount/reclaim.rs index 96506a344bf..44f7ff528f8 100644 --- a/lib/bcachefs/src/engine/mount/reclaim.rs +++ b/lib/bcachefs/src/engine/mount/reclaim.rs @@ -748,6 +748,7 @@ impl Mount { /// accounting (A6.12–A6.24, A7.12). An emptied bucket's alloc key and /// need_discard entry wait for the transaction's sequence. pub(super) async fn bucket_changed(&self, txn: &mut Txn, b: u64, before: Bucket, after: Bucket) -> Result<()> { + txn.bucket_states.push((before.data_type, after.data_type)); let bucket_sectors = self.sb.member.bucket_sectors as u32; let alloc_pos = Bpos::new(0, b, 0); if after.data_type == data_type::NEED_DISCARD && before.data_type != data_type::NEED_DISCARD { diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index 1ee55fad109..4b47303cd9e 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -21,6 +21,7 @@ fn file_data_overwrites_sparse_ranges_and_truncation_survive_remount() { want[41_003..].fill(0); m.write(ino, 41_000, b"new tail", NOW).await.unwrap(); want[41_000..41_008].copy_from_slice(b"new tail"); + m.check_metadata_budget().await; m.commit().await.unwrap(); ino }); @@ -43,6 +44,7 @@ fn orphan_release_reclaims_file_data_and_keeps_other_files() { assert_eq!(bytes, [1; 4096]); m.release(a).await.unwrap(); assert_eq!(m.inode(a).await, Err(Error::NotFound)); + m.check_metadata_budget().await; }); let v = block_on_ready(Volume::open(&dev)).unwrap(); assert_eq!(read_all(&v, path(&v, "b")).unwrap(), [2; 8192]); @@ -200,6 +202,7 @@ fn a_reserved_slot_survives_its_bucket_emptying_and_generation_change() { m.release(a).await.unwrap(); m.commit().await.unwrap(); m.write_reserved(&mut tb, 0, &[2; BLOCK], NOW).await.unwrap(); + m.check_metadata_budget().await; }); let v = block_on_ready(Volume::open(&dev)).unwrap(); assert_eq!(read_all(&v, path(&v, "b")).unwrap(), [2; BLOCK]); From c83997681e675310691f31eccd196aa09b352129 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:01:29 +0100 Subject: [PATCH 26/93] Reserve metadata growth separately from shared rewrite capacity Hold exact data sectors while pending pages charge their enforced journal growth bound. Prepay geometry-bounded allocator records and account live non-structural records; include imported underfull nodes until rewritten. Sparse closed metadata buckets can move through the existing node writer. Keep a reservation until transaction application succeeds so failed preparation retains the accepted page's capacity. Authenticated version reservation needs no additional leaf-key growth. The 64 MiB pressure test accepts 1200 pending pages and writes all of them after allocation refusal. Twelve data tests and no_std clippy pass. Worst-case shared reclaim sizing and GC pressure integration remain under active review; this checkpoint does not claim that physical proof complete. --- lib/bcachefs/src/engine/mount/allocator.rs | 58 ++++++++++++++++++---- lib/bcachefs/src/engine/mount/data.rs | 32 ++++++------ lib/bcachefs/src/engine/mount/gc.rs | 31 ++++++++++++ lib/bcachefs/src/engine/mount/mod.rs | 40 ++++++++++----- lib/bcachefs/src/engine/mount/reclaim.rs | 2 +- lib/bcachefs/src/engine/tests/data.rs | 6 ++- 6 files changed, 129 insertions(+), 40 deletions(-) create mode 100644 lib/bcachefs/src/engine/mount/gc.rs diff --git a/lib/bcachefs/src/engine/mount/allocator.rs b/lib/bcachefs/src/engine/mount/allocator.rs index 769f2fc15ed..cb0e9b65db9 100644 --- a/lib/bcachefs/src/engine/mount/allocator.rs +++ b/lib/bcachefs/src/engine/mount/allocator.rs @@ -20,24 +20,44 @@ use crate::engine::read::Source; use crate::engine::value::{accounting_pos, class, data_type}; use crate::engine::{journal, Btree, Error, Result}; -/// Records whose distinct positions are bounded by the device geometry. -pub(super) fn structural(btree: Btree, value: &[u8]) -> bool { - matches!(btree, Btree::Alloc | Btree::Freespace | Btree::Lru | Btree::NeedDiscard | Btree::BucketGens) - || (btree == Btree::Backpointers && value.get(2) == Some(&data_type::BTREE)) +/// Bytes beyond the records prepaid from the device geometry. +pub(super) fn charge(btree: Btree, value: &[u8]) -> u64 { + let prepaid = match btree { + Btree::Alloc => 104, + Btree::Freespace | Btree::Lru | Btree::NeedDiscard => 40, + Btree::BucketGens => 296, + Btree::Backpointers if value.get(2) == Some(&data_type::BTREE) => 72, + _ => 0, + }; + super::key_bytes(value).saturating_sub(prepaid) as u64 +} + +pub(super) struct Metadata { + pub bytes: u64, + pub buckets: u64, + pub baseline: BTreeSet, + pub sparse: BTreeSet, } -pub(super) async fn metadata_budget(source: &S) -> Result<(u64, u64, BTreeSet)> { +pub(super) async fn metadata_budget(source: &S) -> Result { let mut bytes = 0u64; let roots: BTreeSet = Btree::ALL.iter().filter_map(|&b| source.root_of(b).map(|r| r.ptr.ptr.sector)).collect(); let mut baseline = BTreeSet::new(); + let mut sparse = BTreeSet::new(); let shift = match source.superblock().flag(6, 40, 8) { 0 => 10, value => value as u32 }; for btree in Btree::ALL { source.walk(btree, Bpos::MIN, |key, value| { - if !structural(btree, value) { bytes += super::key_bytes(value) as u64; } + bytes += charge(btree, value); if btree == Btree::Backpointers && value.get(2) == Some(&data_type::BTREE) { let sector = key.pos.offset >> shift; if !roots.contains(§or) { baseline.insert(sector); } } + if btree == Btree::Alloc { + let bucket = super::AllocV4::decode(value)?; + if bucket.data_type == data_type::BTREE && u64::from(bucket.dirty) <= u64::from(source.superblock().member.bucket_sectors) / 2 { + sparse.insert(key.pos.offset); + } + } Ok(true) }).await?; } @@ -46,7 +66,7 @@ pub(super) async fn metadata_budget(source: &S) -> Result<(u64, u64, Some((_, value)) => journal::counters(&value)?.first().copied().unwrap_or(0), None => 0, }; - Ok((bytes, buckets, baseline)) + Ok(Metadata { bytes, buckets, baseline, sparse }) } /// What a transaction's nodes may take of the room left. @@ -127,6 +147,16 @@ impl Allocator { (self.free + self.returning.values().sum::() - pinned + metadata_buckets + self.metadata_taken.len() as u64) * self.per_bucket } + pub(super) fn metadata_open(&self, bucket: u64) -> bool { + self.open.is_some_and(|(b, _, _)| b == bucket) + } + + pub(super) fn stable_slots(&self, nodes: u64) -> u64 { + // A closed bucket that cannot be evacuated is more than half + // full. One open bucket may be arbitrarily empty. + nodes * if self.per_bucket == 1 { 1 } else { 2 } + self.per_bucket + } + /// Node slots to be had now, past `blocked` buckets that a traversal /// may still lead into. pub(super) fn ready(&self, blocked: usize) -> u64 { @@ -145,12 +175,20 @@ impl Allocator { /// them, and [`OVERSIZED`] covers nodes read packed. A write that /// replaces a node gives its bucket back only once that is durable, so /// nothing here counts on it. - pub(super) fn fits(&self, nodes: usize, reclaim_room: usize, draw: Draw) -> bool { + pub(super) fn transient(nodes: usize, reclaim_room: usize, draw: Draw) -> u64 { let reserve = match draw { Draw::Ordinary | Draw::Reserved(_) => FREEING_RESERVE, Draw::Freeing => 0, }; - (COMMON_REWRITE * (nodes + reclaim_room)) as u64 + OVERSIZED + reserve <= self.slots() + (COMMON_REWRITE * (nodes + reclaim_room)) as u64 + OVERSIZED + reserve + } + + pub(super) fn fits(&self, nodes: usize, reclaim_room: usize, draw: Draw) -> bool { + Self::transient(nodes, reclaim_room, draw) <= self.slots() + } + + pub(super) fn keeps(&self, metadata_buckets: u64, stable: u64, nodes: usize, reclaim_room: usize, draw: Draw, data: bool) -> bool { + self.stable_slots(stable) + Self::transient(nodes, reclaim_room, draw) + if data { self.per_bucket } else { 0 } <= self.capacity(metadata_buckets) } /// The open bucket's next slot: its sector and generation. @@ -192,7 +230,7 @@ impl Allocator { } pub(super) fn data_fits(&self, nodes: usize, reclaim_room: usize) -> bool { - let need = (COMMON_REWRITE * (nodes + reclaim_room)) as u64 + OVERSIZED + FREEING_RESERVE; + let need = Self::transient(nodes, reclaim_room, Draw::Ordinary); need + self.per_bucket <= self.slots() } diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index 7dbdae54b34..e6fba2ac714 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -21,12 +21,12 @@ pub(super) struct Reservations { } impl Reservations { - pub(super) fn pending(&self, draw: Draw) -> usize { + pub(super) fn pending(&self, draw: Draw) -> u64 { let own = match draw { Draw::Reserved(id) => usize::from(self.active.contains_key(&id)), _ => 0, }; - (self.active.len() - own) * FOOTPRINT + (self.active.len() - own) as u64 * K_TX * 8 } } @@ -94,7 +94,7 @@ impl Mount { } let _data = self.data_slot().await; let id = self - .transact(1, FOOTPRINT, Draw::Ordinary, async |txn: &mut Txn| { + .transact(K_TX, FOOTPRINT, Draw::Ordinary, async |txn: &mut Txn| { regular(&self.inode_in(txn, ino).await?)?; self.block_extents(ino, first, first + bytes as u64).await?; let (sector, _) = self.data_sector(self.sb.block_bytes() as u64 / SECTOR as u64).await?; @@ -206,16 +206,17 @@ impl Mount { .await } - /// Total volume bytes and bytes in currently free, reusable buckets. + /// Total bytes and capacity left after pending data and metadata claims. pub fn space(&self) -> (u64, u64) { self.cancel_dropped(); let bucket = u64::from(self.sb.member.bucket_sectors) * SECTOR as u64; let node = u64::from(self.sb.node_sectors()) * SECTOR as u64; let st = self.st.borrow(); - let pending = self.reservations.borrow().active.len() as u64; - let logical = st.alloc.capacity(st.metadata_buckets).saturating_sub(self.metadata_nodes(&st, st.granted_bytes + pending * K_TX * 8)) * node; - let promised = pending * FOOTPRINT as u64 * reclaim::COMMON_REWRITE as u64 * node; - (self.sb.member.nbuckets * bucket, st.alloc.free_bytes(bucket).saturating_sub(promised).min(logical)) + let pending = self.reservations.borrow().pending(Draw::Ordinary); + let stable = st.alloc.stable_slots(self.metadata_nodes(&st, st.granted_bytes + pending)); + let transient = Allocator::transient(st.cache.dirty + st.cache.unpointed + st.granted + FOOTPRINT, st.reclaim_room, Draw::Ordinary); + let logical = st.alloc.capacity(st.metadata_buckets).saturating_sub(stable + transient) * node; + (self.sb.member.nbuckets * bucket, st.alloc.free_bytes(bucket).min(logical)) } pub(super) fn metadata_nodes(&self, st: &State, pending_bytes: u64) -> u64 { @@ -227,17 +228,17 @@ impl Mount { // are bounded by the format's classes and btrees. let structural = buckets * (104 + 3 * 40) + buckets.div_ceil(256) * 296 + slots * 72 + (Btree::ALL.len() * 9 * 64) as u128; let bytes = u128::from(st.metadata_bytes) + structural + u128::from(pending_bytes); - let node_bytes = self.sb.node_sectors() as usize * SECTOR; - let minimum = (mkfs::capacity(node_bytes) - FIRST_KEYS) / 2 - MAX_KEY; + let minimum = self.minimum_fill(); let stable = st.baseline_nodes.len() as u128 + 2 * (Btree::ALL.len() as u128 + bytes.div_ceil(minimum as u128)); stable.min(u128::from(u64::MAX)) as u64 } #[cfg(test)] pub(crate) async fn check_metadata_budget(&self) { - let (bytes, buckets, _) = allocator::metadata_budget(self).await.unwrap(); + let metadata = allocator::metadata_budget(self).await.unwrap(); let st = self.st.borrow(); - assert_eq!((st.metadata_bytes, st.metadata_buckets), (bytes, buckets)); + assert_eq!((st.metadata_bytes, st.metadata_buckets), (metadata.bytes, metadata.buckets)); + assert_eq!(st.sparse_metadata, metadata.sparse); } /// The high-water mark is durable before any version beneath it is @@ -247,7 +248,7 @@ impl Mount { return Ok(Version::default()); } if self.st.borrow().next_version == self.st.borrow().key_version { - self.transact(1, 0, Draw::Ordinary, async |_txn: &mut Txn| { + self.transact(0, 0, Draw::Freeing, async |_txn: &mut Txn| { let mut st = self.st.borrow_mut(); st.key_version = st.key_version.checked_add(VERSIONS).ok_or(Error::NoSpace("encrypted extent versions"))?; Ok(()) @@ -306,8 +307,9 @@ impl Mount { } { let st = self.st.borrow(); - let held = st.cache.dirty + st.cache.unpointed + st.granted + self.reservations.borrow().pending(Draw::Ordinary); - if !st.alloc.data_fits(held, st.reclaim_room) { + let held = st.cache.dirty + st.cache.unpointed + st.granted; + let stable = self.metadata_nodes(&st, st.granted_bytes + self.reservations.borrow().pending(Draw::Ordinary)); + if !st.alloc.data_fits(held, st.reclaim_room) || !st.alloc.keeps(st.metadata_buckets, stable, held, st.reclaim_room, Draw::Ordinary, true) { return Err(Error::NoSpace("no data bucket beyond metadata and freeing reserves")); } } diff --git a/lib/bcachefs/src/engine/mount/gc.rs b/lib/bcachefs/src/engine/mount/gc.rs new file mode 100644 index 00000000000..334bfaae5d5 --- /dev/null +++ b/lib/bcachefs/src/engine/mount/gc.rs @@ -0,0 +1,31 @@ +//! Evacuate sparse closed metadata buckets through ordinary node publication. + +use super::*; + +impl Mount { + pub(crate) async fn compact_metadata(&self) -> Result { + let candidates: Vec<_> = { + let st = self.st.borrow(); + st.sparse_metadata.iter().copied().filter(|&b| !st.alloc.metadata_open(b)).collect() + }; + let bucket = u64::from(self.sb.member.bucket_sectors); + for b in candidates { + let first = b * bucket; + let end = (first + bucket) << self.bp_shift(); + let mut pointers = Vec::new(); + Source::walk(self, Btree::Backpointers, Bpos::new(0, first << self.bp_shift(), 0), |key, value| { + if key.pos.inode != 0 || key.pos.offset >= end { return Ok(false); } + if value.len() != value::BACKPOINTER_BYTES || value[2] != data_type::BTREE || value[1] == 0 { + return Err(Error::Corrupt("metadata bucket holds a non-metadata backpointer")); + } + let btree = Btree::from_id(u32::from(value[0])).ok_or(Error::Corrupt("backpointer btree"))?; + pointers.push((btree, value[1] - 1, Bpos::decode(&value[12..32]), key.pos.offset >> self.bp_shift())); + Ok(true) + }).await?; + for (btree, level, pos, sector) in pointers { + if self.relocate_node(btree, level, pos, sector).await? { return Ok(true); } + } + } + Ok(false) + } +} diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 8a986458782..36ef3e5fbc8 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -39,6 +39,7 @@ use super::{kind, Btree, Error, Result, SECTOR}; mod allocator; mod data; +mod gc; mod ops; mod reclaim; @@ -115,6 +116,7 @@ struct State { next_version: u64, metadata_bytes: u64, metadata_buckets: u64, + sparse_metadata: BTreeSet, baseline_nodes: BTreeSet, granted_bytes: u64, clocks: [u64; 2], @@ -210,7 +212,7 @@ struct Txn { /// Buckets taken that it fills. filled: Vec, allocated_data: Vec, - bucket_states: Vec<(u8, u8)>, + bucket_states: Vec<(u64, Bucket, Bucket)>, /// New roots. roots: Vec, /// Written nodes made reachable, installed in the cache at apply. @@ -332,7 +334,7 @@ impl Mount { } /// `empty` is the volume's count of empty buckets. - async fn from(parts: Parts, empty: u64, metadata: (u64, u64, BTreeSet), options: MountOptions, random: Box) -> Result { + async fn from(parts: Parts, empty: u64, metadata: allocator::Metadata, options: MountOptions, random: Box) -> Result { let Parts { dev, mut sb, key, roots, key_version, replay, root } = parts; let key_version = key_version.checked_add(if replay.is_some() && key.is_some() { 65_536 } else { 0 }).ok_or(Error::NoSpace("encrypted extent versions"))?; let ring: Vec = sb.journal.iter().flat_map(|&(first, count)| first..first + count).collect(); @@ -420,9 +422,10 @@ impl Mount { journal, key_version, next_version: key_version, - metadata_bytes: metadata.0, - metadata_buckets: metadata.1, - baseline_nodes: metadata.2, + metadata_bytes: metadata.bytes, + metadata_buckets: metadata.buckets, + sparse_metadata: metadata.sparse, + baseline_nodes: metadata.baseline, granted_bytes: 0, clocks, now: sb.header.write_time, @@ -971,14 +974,19 @@ impl Mount { let room = match class { Class::Ordinary => { let held = st.cache.dirty + st.cache.unpointed + st.granted; - if !st.alloc.fits(nodes + pending, st.reclaim_room, draw) && held == 0 && st.reclaim_granted == 0 && !st.alloc.returning() { + let stable = self.metadata_nodes(&st, pending + st.granted_bytes + journal * 8); + let keeps = st.alloc.keeps(st.metadata_buckets, stable, nodes, st.reclaim_room, draw, false); + let physical = st.alloc.fits(nodes, st.reclaim_room, draw); + let can_compact = st.sparse_metadata.iter().any(|&b| !st.alloc.metadata_open(b)); + if (!keeps || (!physical && !can_compact)) && held == 0 && st.reclaim_granted == 0 && !st.alloc.returning() { return Err(Error::NoSpace(match draw { Draw::Ordinary | Draw::Reserved(_) => "no room left for the btree nodes a change may write", Draw::Freeing => "no room left for the btree nodes a removal may write", })); } // Nothing more lands in a node reclaim has yet to split. - held + nodes <= st.limit && st.cache.oversized == 0 && st.alloc.fits(held + nodes + pending, st.reclaim_room, draw) + held + nodes <= st.limit && st.cache.oversized == 0 && st.alloc.fits(held + nodes, st.reclaim_room, draw) + && st.alloc.keeps(st.metadata_buckets, stable, held + nodes, st.reclaim_room, draw, false) } Class::Reclaim => st.reclaim_granted + nodes <= st.reclaim_room, }; @@ -1002,7 +1010,7 @@ impl Mount { fn can_free(&self) -> bool { let st = self.st.borrow(); let flush = st.journal_frees() || st.alloc.returning(); - !st.journal.closed.is_empty() || (flush && st.journal.may_close()) || st.cache.ids().into_iter().any(|id| { + !st.journal.closed.is_empty() || (flush && st.journal.may_close()) || st.sparse_metadata.iter().any(|&b| !st.alloc.metadata_open(b)) || st.cache.ids().into_iter().any(|id| { let n = st.cache.node(id); n.dirty() && !n.writing && n.captured.is_none() && n.held == 0 }) @@ -1018,6 +1026,7 @@ impl Mount { if self.reclaim_round(Order::OldestPin).await? { return Ok(()); } + if self.compact_metadata().await? { return Ok(()); } self.flush_open().await.map(|_| ()) } @@ -1068,10 +1077,12 @@ impl Mount { } self.ungrant(&mut st, Class::Ordinary, bound, nodes); }; - if let Draw::Reserved(id) = draw { self.reservations.borrow_mut().active.remove(&id).expect("admission consumes a reservation"); } let mut txn = Txn::default(); let out = prepare(&mut txn).await; self.finish(txn, out.is_ok(), Class::Ordinary, bound, nodes)?; + if out.is_ok() { + if let Draw::Reserved(id) = draw { self.reservations.borrow_mut().active.remove(&id).expect("a completed reserved write consumes its reservation"); } + } out } @@ -1091,14 +1102,19 @@ impl Mount { let stopped = st.read_only; let allocated = core::mem::take(&mut txn.allocated_data); if apply && stopped.is_none() { - for &(before, after) in &txn.bucket_states { - st.metadata_buckets = st.metadata_buckets + u64::from(after == data_type::BTREE) - u64::from(before == data_type::BTREE); + for &(b, before, after) in &txn.bucket_states { + st.metadata_buckets = st.metadata_buckets + u64::from(after.data_type == data_type::BTREE) - u64::from(before.data_type == data_type::BTREE); + if after.data_type == data_type::BTREE && u64::from(after.dirty) <= u64::from(self.sb.member.bucket_sectors) / 2 { + st.sparse_metadata.insert(b); + } else { + st.sparse_metadata.remove(&b); + } } for (&(btree, level, pos), update) in &txn.updates { if level != 0 { continue; } let node = st.cache.node(held(st, btree, level, pos)); let prior = node.keys.get(node.at(pos)).filter(|(k, _)| k.pos == pos); - let charge = |entry: Option<&(Key, Vec)>| entry.filter(|(_, v)| !allocator::structural(btree, v)).map_or(0, |(_, v)| key_bytes(v) as u64); + let charge = |entry: Option<&(Key, Vec)>| entry.map_or(0, |(_, v)| allocator::charge(btree, v)); st.metadata_bytes = st.metadata_bytes + charge(update.as_ref()) - charge(prior); if btree == Btree::Backpointers && update.is_none() && prior.is_some_and(|(_, value)| value.get(2) == Some(&data_type::BTREE)) { st.baseline_nodes.remove(&(pos.offset >> self.bp_shift())); diff --git a/lib/bcachefs/src/engine/mount/reclaim.rs b/lib/bcachefs/src/engine/mount/reclaim.rs index 44f7ff528f8..eedb4dc2d92 100644 --- a/lib/bcachefs/src/engine/mount/reclaim.rs +++ b/lib/bcachefs/src/engine/mount/reclaim.rs @@ -748,7 +748,7 @@ impl Mount { /// accounting (A6.12–A6.24, A7.12). An emptied bucket's alloc key and /// need_discard entry wait for the transaction's sequence. pub(super) async fn bucket_changed(&self, txn: &mut Txn, b: u64, before: Bucket, after: Bucket) -> Result<()> { - txn.bucket_states.push((before.data_type, after.data_type)); + txn.bucket_states.push((b, before, after)); let bucket_sectors = self.sb.member.bucket_sectors as u32; let alloc_pos = Bpos::new(0, b, 0); if after.data_type == data_type::NEED_DISCARD && before.data_type != data_type::NEED_DISCARD { diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index 4b47303cd9e..735998d66c8 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -166,10 +166,11 @@ fn reserved_blocks_write_after_capacity_refusal_and_dropped_tokens_return_capaci Err(Error::NoSpace(_)) => break, Err(e) => panic!("reserve: {e:?}"), } - assert!(tokens.len() < 1000); + assert!(tokens.len() < dev.blocks() as usize); } - assert!(tokens.len() > 1); + assert!(tokens.len() >= 64, "an ordinary 256 KiB buffered write fits a 64 MiB volume"); let count = tokens.len(); + std::println!("64 MiB volume accepts {count} pending 4 KiB blocks before refusal"); drop(tokens); let mut tokens = Vec::new(); for i in 0..count { @@ -292,6 +293,7 @@ fn a_failed_reserved_data_write_keeps_the_old_contents_and_reports_failure() { let mut token = block_on_ready(m.reserve(ino, 0, BLOCK)).unwrap(); assert_eq!(block_on_ready(m.write_reserved(&mut token, 0, &[0x44; BLOCK], NOW)), Err(Error::Device(toyos_blockdev::Error::Device))); assert_eq!(failing.attempts.get(), 3); + assert!(matches!(block_on_ready(m.write_reserved(&mut token, 0, &[0x44; BLOCK], NOW)), Err(Error::ReadOnly(_)))); assert!(matches!(block_on_ready(m.commit()), Err(Error::ReadOnly(_)))); let mut got = [0; BLOCK]; block_on_ready(m.read(ino, 0, &mut got)).unwrap(); From 3a47539ba52ca8394936dcb86bc8e96e3a4bcffd Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:06:44 +0100 Subject: [PATCH 27/93] Align filesystem guest probes with served namespace contracts Judge fstat after unlink, list only granted ancestors under /state, and atomically retarget symlinks using rename. Register readdir_bound only under its explicit metal row and remove a stale syscall import. Validation: all guest binaries compile and selected images stage successfully; no bench run claimed. --- .../a_held_files_fstat_is_its_servers_stat.rs | 16 ++++++---------- tests/guest/rust-tests/src/bin/launch_toctou.rs | 12 ++++++++---- .../rust-tests/src/bin/pipe_other_end_gone.rs | 2 +- tests/guest/rust-tests/src/bin/state_view.rs | 14 +++++--------- tests/toyos.rs | 2 ++ 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/tests/guest/rust-tests/src/bin/a_held_files_fstat_is_its_servers_stat.rs b/tests/guest/rust-tests/src/bin/a_held_files_fstat_is_its_servers_stat.rs index 5e760d8b63a..15e5b1a674a 100644 --- a/tests/guest/rust-tests/src/bin/a_held_files_fstat_is_its_servers_stat.rs +++ b/tests/guest/rust-tests/src/bin/a_held_files_fstat_is_its_servers_stat.rs @@ -1,12 +1,6 @@ -//! A served file's `fstat` through a descriptor holding its paged object is -//! answered by the kernel from what the object publishes, without a request. -//! After a write, a truncate and a set time, it agrees with the server's -//! `stat` of the file's path on size and mtime, and on every field the open -//! answered. -//! -//! A guest test because the two answers come from two components, the -//! kernel's object and the file server's volume, which only a guest runs -//! together; each side's rule is host-tested (`fileserver` pages, `fat`). +//! A served descriptor and its path agree on metadata after writes, +//! truncation and explicit timestamps. An unlinked descriptor retains its +//! identity while its link count changes. use std::fs::{self, File, OpenOptions}; use std::io::Write; @@ -35,7 +29,9 @@ fn main() { f.set_modified(then).expect("set the mtime"); agree(&f, "a set time"); assert_eq!(f.metadata().unwrap().modified().unwrap(), then, "the time set"); - drop(f); fs::remove_file(FILE).unwrap(); + assert_eq!(f.metadata().unwrap().nlink(), 0, "unlink updates an already open descriptor"); + assert_eq!(f.metadata().unwrap().len(), 100); + drop(f); println!("a_held_files_fstat_is_its_servers_stat: passed"); } diff --git a/tests/guest/rust-tests/src/bin/launch_toctou.rs b/tests/guest/rust-tests/src/bin/launch_toctou.rs index e2859c1cedc..73cfc894461 100644 --- a/tests/guest/rust-tests/src/bin/launch_toctou.rs +++ b/tests/guest/rust-tests/src/bin/launch_toctou.rs @@ -67,11 +67,14 @@ fn attacker() { let flipper = std::thread::spawn({ let stop = Arc::clone(&stop); move || { - // tmpfs displaces the existing entry, so each call is one swap of - // what the link names. + // Rename atomically replaces the link; symlink itself never + // overwrites an existing name. + const NEXT: &str = "/tmp/ps-next"; while !stop.load(Ordering::Relaxed) { - symlink(SELF_PATH, LINK).expect("re-point the link at this binary"); - symlink(DECLARED, LINK).expect("re-point the link at the declared row"); + for target in [SELF_PATH, DECLARED] { + symlink(target, NEXT).expect("prepare the next link"); + std::fs::rename(NEXT, LINK).expect("replace the link atomically"); + } } } }); @@ -100,6 +103,7 @@ fn attacker() { stop.store(true, Ordering::Relaxed); flipper.join().expect("the re-point thread failed"); + std::fs::remove_file(LINK).expect("remove the race link"); assert!(under_row > 0, "none of {ATTEMPTS} launches ran the `toybox` row's program under its row"); assert!(refused > 0, "the link never named this binary when the launcher asked in {ATTEMPTS} launches"); println!("launch_toctou: PASS ({under_row} of {ATTEMPTS} launches ran the row's program under it, none the caller's)"); diff --git a/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs b/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs index f391cf87719..6fc279a8c88 100644 --- a/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs +++ b/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs @@ -32,7 +32,7 @@ use toyos::ipc::Connection; use toyos::poller::{Poller, OTHER_END_GONE, READABLE}; use toyos::syscap::SysCap; use toyos::{namespace, pipe_pair, port, AsHandle, BorrowedHandle, OwnedHandle, Pipe}; -use toyos_abi::syscall::{self, SyscallError}; +use toyos_abi::syscall::SyscallError; #[path = "../roster.rs"] mod roster; diff --git a/tests/guest/rust-tests/src/bin/state_view.rs b/tests/guest/rust-tests/src/bin/state_view.rs index 9d5bcea0d9f..72e97917893 100644 --- a/tests/guest/rust-tests/src/bin/state_view.rs +++ b/tests/guest/rust-tests/src/bin/state_view.rs @@ -35,8 +35,7 @@ fn main() { } let _ = fs::remove_file(&file); - // Another row's state and the supervisor's store, by name: no capability - // names either, so each reaches only the kernel's bare mount point. + // Another row's state and the supervisor's store have no capability. for path in [SERVICE, "/state/supervisor/grants"] { if let Ok(meta) = fs::metadata(path) { red.push(format!("{path} answered metadata ({} bytes)", meta.len())); @@ -54,13 +53,10 @@ fn main() { red.push("a grant line was appended to /state/supervisor/grants".to_string()); } } - // `/state` with no capability is a mount point of the kernel's, and lists - // nothing of what the file server holds under it. - if let Ok(listing) = fs::read_dir("/state") { - let found: Vec<_> = listing.filter_map(Result::ok).map(|e| e.file_name()).collect(); - if !found.is_empty() { - red.push(format!("/state lists {found:?}")); - } + // The synthetic ancestor exposes the granted child's name only. + match fs::read_dir("/state").and_then(|entries| entries.map(|entry| entry.map(|e| e.file_name())).collect::>>()) { + Ok(found) if found == [std::ffi::OsString::from("test-runner")] => {} + other => red.push(format!("/state did not list exactly its granted child: {other:?}")), } if !red.is_empty() { diff --git a/tests/toyos.rs b/tests/toyos.rs index 4308b7aeaaf..f14e739e54a 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -68,6 +68,8 @@ const ACTUATOR_KERNEL: &[&str] = toyos_build::build::TEST_KERNEL; // Rust helper binaries that are spawned by tests, not tests themselves. const RUST_SKIP: &[&str] = &[ + // Its explicit metal registration owns the directory listing workload. + "readdir_bound", // It writes the log disk for ever, so that its boot's deadline resets the // machine under it: `the_disk_survives_a_reset_under_load` runs it alone. "stick_load", From c432f641eee2a15b852bd85f43fba4087323314a Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:04:33 +0100 Subject: [PATCH 28/93] Keep buffered write reservations until their bytes are copied and written Give an incoming write its own capacity until each page is copied, so pressure can flush an older dirty version without consuming the new write's reservation. Cancellation preserves the older version's token. Report the completed prefix when volume writeback fails. End that prefix successfully and retain the suffix for retry, using bounded leading-part completion of generic paged writeback ranges. Dirty supply now stops at the first present or refused page so its count identifies copied bytes. Cover overlapping writes under dirty pressure, cancelled overwrites, partial COW preparation failures and retry, and paged range refusal. Add the actual batched dirty-supply contract to the existing pager probe. --- kernel/pure/paged.rs | 28 +++++-- kernel/src/object/paged.rs | 15 +++- sdk/abi/src/syscall.rs | 8 +- sdk/toyos/src/paged.rs | 9 ++- system/fileserver/src/absent.rs | 4 +- system/fileserver/src/data.rs | 55 +++++++------ system/fileserver/src/fat.rs | 10 +-- system/fileserver/src/memory.rs | 2 +- system/fileserver/src/pages.rs | 64 ++++++++------- system/fileserver/src/pages/tests.rs | 114 ++++++++++++++++++++++++++- system/fileserver/src/root.rs | 4 +- system/fileserver/src/volume.rs | 22 ++++-- tests/guest/pagerprobe/src/main.rs | 20 +++++ 13 files changed, 267 insertions(+), 88 deletions(-) diff --git a/kernel/pure/paged.rs b/kernel/pure/paged.rs index 210649580ca..f60832144ba 100644 --- a/kernel/pure/paged.rs +++ b/kernel/pure/paged.rs @@ -602,11 +602,13 @@ impl Pages { Ok(begun) } - /// End the write-back of exactly a range begun: a bit per page still - /// dirty, holding its slot. A page removed meanwhile is clear. + /// End a range begun, or its leading part, retaining the unended suffix: + /// a bit per page still dirty. A page removed meanwhile is clear. pub fn writeback_end(&mut self, o: &mut Object, first: u64, count: u64, written: bool) -> Result<[u64; 8], Refused> { - let at = self.open.iter().position(|r| *r == Some((o.id, first, count))).ok_or(Refused::InvalidArgument)?; - self.open[at] = None; + let at = self.open.iter().position(|r| r.is_some_and(|(id, a, n)| id == o.id && a == first && count > 0 && count <= n)) + .ok_or(Refused::InvalidArgument)?; + let (_, _, total) = self.open[at].expect("the range found"); + self.open[at] = (count < total).then_some((o.id, first + count, total - count)); let mut kept = [0u64; 8]; for k in 0..count { let Some(i) = self.find(o.id, first + k) else { continue }; @@ -902,13 +904,25 @@ mod tests { } #[test] - fn a_range_must_be_ended_exactly_as_begun_and_once() { + fn a_range_ends_once_in_leading_parts_and_keeps_its_unwritten_suffix() { let mut p: Pages = Pages::new(8, 8).unwrap(); let mut o = p.object(8, PAGE * 1024); + for page in 0..4 { + p.supply(&mut o, page, page, true, (page + 1) * PAGE).unwrap(); + } p.writeback_begin(&mut o, 0, 4).unwrap(); assert_eq!(p.writeback_begin(&mut o, 3, 2), Err(Refused::InvalidArgument), "overlapping"); - assert_eq!(p.writeback_end(&mut o, 0, 3, true), Err(Refused::InvalidArgument), "not the range begun"); - assert!(p.writeback_end(&mut o, 0, 4, true).is_ok()); + assert_eq!(p.writeback_end(&mut o, 1, 3, true), Err(Refused::InvalidArgument), "not a leading part"); + assert_eq!(p.writeback_end(&mut o, 0, 0, true), Err(Refused::InvalidArgument), "empty"); + assert_eq!(p.writeback_end(&mut o, 0, 5, true), Err(Refused::InvalidArgument), "past the range"); + assert_eq!(p.writeback_end(&mut o, 0, 2, true), Ok([0; 8])); + assert_eq!(p.dirty_held(), 2); + assert_eq!(p.writeback_begin(&mut o, 2, 2), Err(Refused::InvalidArgument), "the suffix remains held"); + assert_eq!(p.writeback_end(&mut o, 0, 2, true), Err(Refused::InvalidArgument), "the prefix already ended"); + assert_eq!(p.writeback_end(&mut o, 2, 2, false).unwrap()[0], 0b11); + assert_eq!(p.writeback_begin(&mut o, 0, 4).unwrap()[0], 0b1100, "only the unwritten suffix is retried"); + assert_eq!(p.writeback_end(&mut o, 0, 4, true), Ok([0; 8])); + assert_eq!(p.dirty_held(), 0); assert_eq!(p.writeback_end(&mut o, 0, 4, true), Err(Refused::InvalidArgument), "twice"); assert_eq!(p.writeback_begin(&mut o, 0, RANGE_PAGES + 1), Err(Refused::InvalidArgument)); assert_eq!(p.writeback_begin(&mut o, 0, 0), Err(Refused::InvalidArgument)); diff --git a/kernel/src/object/paged.rs b/kernel/src/object/paged.rs index 833046731a5..fb5693a90e3 100644 --- a/kernel/src/object/paged.rs +++ b/kernel/src/object/paged.rs @@ -635,8 +635,8 @@ impl PagerObject { } /// Copy `src` into fresh pages of object `id` from page-aligned `offset`, - /// clean or dirty: the pages installed. A dirty supply is a write. The - /// caller checked the span is in reach. + /// clean or dirty: the pages installed. A dirty supply is a write and + /// stops at the first present or refused page. The span is checked. pub fn supply(&self, id: u64, offset: u64, src: &UserBytes, dirty: bool) -> u64 { if !offset.is_multiple_of(PAGE) || src.len() == 0 { return SyscallError::InvalidArgument.to_u64(); @@ -709,9 +709,15 @@ impl PagerObject { ends[k] = page_at + (src.len() - from).min(PAGE as usize) as u64; } let mut refused = None; + let mut stopped = false; { let mut state = self.state.lock(); for k in 0..reserved { + if stopped { + let gone = state.gone; + state.held.give_back(self.core.job(), gone, frames[k], false); + continue; + } let answer = match state.live(id) { Ok(_) => { let (pages, object) = state.split(slot); @@ -727,11 +733,13 @@ impl PagerObject { Ok(Supplied::Present) => { let gone = state.gone; state.held.give_back(self.core.job(), gone, frames[k], false); + stopped = dirty; } Err(why) => { let gone = state.gone; state.held.give_back(self.core.job(), gone, frames[k], false); refused.get_or_insert(why); + stopped = true; } } } @@ -740,6 +748,9 @@ impl PagerObject { if let Some(why) = refused { return refusal(installed, word(why)); } + if stopped { + return installed; + } done += reserved * PAGE as usize; } installed diff --git a/sdk/abi/src/syscall.rs b/sdk/abi/src/syscall.rs index 650e96d34d0..1069779f591 100644 --- a/sdk/abi/src/syscall.rs +++ b/sdk/abi/src/syscall.rs @@ -3101,7 +3101,9 @@ pub enum PagedOp { /// Copy `len` bytes from `buf` into fresh pages at `offset`, a whole page /// in: clean, or dirty with [`PAGED_DIRTY`], which also grows the size to /// the end of the bytes and is a write ([`PagedOp::Write`]). A page - /// already present is left as it is. Answers the pages installed; + /// already present is left as it is. Clean supply skips it; dirty supply + /// stops there, so its installed count describes a copied prefix. + /// Answers the pages installed; /// `ResourceExhausted` where no page could be had, `WouldBlock` at the /// dirty ceiling, `InvalidArgument` for a clean page past the size or a /// dirty one past its end. @@ -3127,9 +3129,9 @@ pub enum PagedOp { /// [`PAGED_RANGE_PAGES`]: each dirty page is cleaned and held until the /// end. `buf` receives a 64-byte bitmap of the pages begun. WritebackBegin = 5, - /// End exactly a range begun, written to the device or not + /// End a range begun or its leading part, written to the device or not /// ([`PAGED_WRITTEN`]): `buf` receives a 64-byte bitmap of the pages - /// still dirty, dirtied again since the begin or not written. + /// still dirty. Any remaining suffix stays under write-back. WritebackEnd = 6, /// Read `len` bytes at `offset` into `buf`, never waiting: `WouldBlock` /// where the first page is absent, a short count where a later one is. diff --git a/sdk/toyos/src/paged.rs b/sdk/toyos/src/paged.rs index 2c0a7c05546..f09b6e76b42 100644 --- a/sdk/toyos/src/paged.rs +++ b/sdk/toyos/src/paged.rs @@ -119,8 +119,9 @@ impl Pager { /// `bytes` into fresh pages of object `id` from page-aligned `offset`, at /// most [`PAGED_RANGE_PAGES`] of them: clean, as the file holds them, or - /// dirty, which grows the size to their end. A page already present is - /// kept. The pages installed. + /// dirty, which grows the size to their end. Returns the pages installed. + /// Clean supply skips present pages; dirty supply stops at the first + /// present or refused page, so its count is the copied prefix. pub fn supply(&self, id: u64, offset: u64, bytes: &[u8], dirty: bool) -> Result { let flags = if dirty { PAGED_DIRTY } else { 0 }; let args = PagedArgs { object: id, offset, len: bytes.len() as u64, buf: bytes.as_ptr() as u64, flags }; @@ -167,8 +168,8 @@ impl Pager { Ok(bits) } - /// End the write-back of exactly a range begun, `written` to the device - /// or not: a bit per page still dirty. + /// End a range begun or its leading part, `written` to the device or not: + /// a bit per page still dirty. Any remaining suffix stays under write-back. pub fn writeback_end(&self, id: u64, first: u64, count: u64, written: bool) -> Result<[u64; 8], Refused> { let mut bits = [0u64; 8]; let flags = if written { PAGED_WRITTEN } else { 0 }; diff --git a/system/fileserver/src/absent.rs b/system/fileserver/src/absent.rs index 04f5df12417..efd0fbe05b7 100644 --- a/system/fileserver/src/absent.rs +++ b/system/fileserver/src/absent.rs @@ -74,8 +74,8 @@ impl Volume for Absent { Err(Error::NotFound) } - fn write_back(&mut self, _node: Node, _offset: u64, _data: &[u8]) -> Result<(), Error> { - Err(Error::NotFound) + fn write_back(&mut self, _node: Node, _offset: u64, _data: &[u8]) -> crate::volume::WriteBack { + Err((0, Error::NotFound)) } fn truncate(&mut self, _node: Node, _size: u64) -> Result<(), Error> { diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index e5b497760b8..ace348eb973 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -2,7 +2,7 @@ //! sizes and capacity reservations stay here until their pages are written. use crate::cache::{Cache, Disk}; -use crate::volume::{Kind, Listed, Meta, Node, OpenHow, Out, Reserved, Volume, parent}; +use crate::volume::{Kind, Listed, Meta, Node, OpenHow, Out, Reserved, Volume, WriteBack, parent}; use bcachefs::engine::{self, DirentType, Mount, MountOptions, New, Owner, Reservation}; use std::collections::BTreeMap; use std::rc::Rc; @@ -28,6 +28,7 @@ struct Open { mtime: u64, changed: bool, pages: BTreeMap, + incoming: BTreeMap, } pub struct DataVolume { fs: Mount>>, @@ -198,7 +199,7 @@ impl DataVolume { } fn persist(&mut self, node: Node) -> Result<(), Error> { let open = self.open.get(&node).ok_or(Error::Stale)?; - if !open.pages.is_empty() { + if !open.pages.is_empty() || !open.incoming.is_empty() { return Err(Error::Io); } if !open.changed { @@ -299,6 +300,7 @@ impl Volume for DataVolume { mtime: inode.mtime.max(0) as u64, changed: false, pages: BTreeMap::new(), + incoming: BTreeMap::new(), }) .holders += 1; if how.truncate @@ -317,7 +319,9 @@ impl Volume for DataVolume { } let inode = mapped("close", self.cache.block_on(self.fs.inode(node)))?; if inode.nlink == 0 { - self.open.get_mut(&node).unwrap().pages.clear(); + let open = self.open.get_mut(&node).unwrap(); + open.pages.clear(); + open.incoming.clear(); mapped("orphan close", self.cache.block_on(self.fs.release(node)))?; } else { self.persist(node)?; @@ -352,6 +356,7 @@ impl Volume for DataVolume { fn reserve(&mut self, node: Node, offset: u64, len: u64) -> Result { let end = offset.checked_add(len).ok_or(Error::FileTooLarge)?; let open = self.open.get(&node).ok_or(Error::Stale)?; + assert!(open.incoming.is_empty(), "a write completes or cancels before the next reserve"); let ticket = self.next_ticket; self.next_ticket = ticket.checked_add(1).expect("reservation generation exhausted"); let reserved = Reserved { size: open.size, contents: open.size, ticket }; @@ -359,12 +364,9 @@ impl Volume for DataVolume { return Ok(reserved); } for page in offset / BLOCK as u64..end.div_ceil(BLOCK as u64) { - if self.open[&node].pages.contains_key(&page) { - continue; - } match self.cache.block_on(self.fs.reserve(node, page * BLOCK as u64, BLOCK)) { Ok(reservation) => { - self.open.get_mut(&node).unwrap().pages.insert(page, Pending { ticket, reservation }); + self.open.get_mut(&node).unwrap().incoming.insert(page, Pending { ticket, reservation }); } Err(error) => { self.unreserve(node, reserved, 0)?; @@ -378,31 +380,33 @@ impl Volume for DataVolume { open.changed = true; Ok(reserved) } + fn landed(&mut self, node: Node, reserved: Reserved, offset: u64, len: usize) { + if len == 0 { return; } + let open = self.open.get_mut(&node).expect("a reserved inode"); + for page in offset / BLOCK as u64..(offset + len as u64).div_ceil(BLOCK as u64) { + let pending = open.incoming.remove(&page).expect("each copied page has incoming capacity"); + assert_eq!(pending.ticket, reserved.ticket, "the write's incoming capacity"); + open.pages.insert(page, pending); + } + } fn unreserve(&mut self, node: Node, reserved: Reserved, landed: u64) -> Result<(), Error> { let open = self.open.get_mut(&node).ok_or(Error::Stale)?; - let cancelled: Vec<_> = open - .pages - .iter() - .filter_map(|(&page, pending)| { - (pending.ticket == reserved.ticket && page * BLOCK as u64 >= landed).then_some(page) - }) - .collect(); - for page in cancelled { - open.pages.remove(&page); - } + open.incoming.retain(|_, pending| pending.ticket != reserved.ticket); + open.pages.retain(|&page, pending| pending.ticket != reserved.ticket || page * (BLOCK as u64) < landed); open.size = reserved.size.max(landed); Ok(()) } - fn write_back(&mut self, node: Node, offset: u64, data: &[u8]) -> Result<(), Error> { - let open = self.open.get_mut(&node).ok_or(Error::Stale)?; + fn write_back(&mut self, node: Node, offset: u64, data: &[u8]) -> WriteBack { + let open = self.open.get_mut(&node).ok_or((0, Error::Stale))?; if !offset.is_multiple_of(BLOCK as u64) || offset.checked_add(data.len() as u64).is_none_or(|end| end > open.size) { - return Err(Error::InvalidRequest); + return Err((0, Error::InvalidRequest)); } for (i, bytes) in data.chunks(BLOCK).enumerate() { let page = offset / BLOCK as u64 + i as u64; - let pending = open.pages.get_mut(&page).ok_or(Error::Io)?; + let done = i * BLOCK; + let pending = open.pages.get_mut(&page).ok_or((done, Error::Io))?; let written = mapped( "writeback", self.cache.block_on(self.fs.write_reserved( @@ -411,9 +415,9 @@ impl Volume for DataVolume { bytes, time(open.mtime), )), - )?; + ).map_err(|error| (done, error))?; if written != bytes.len() { - return Err(Error::Io); + return Err((done, Error::Io)); } open.pages.remove(&page); } @@ -517,7 +521,7 @@ impl Volume for DataVolume { Ok(refused) } fn owes(&self) -> bool { - self.open.values().any(|open| open.changed || !open.pages.is_empty()) + self.open.values().any(|open| open.changed || !open.pages.is_empty() || !open.incoming.is_empty()) } fn commit(&mut self, node: Option) -> Result<(), Error> { if let Some(node) = node { @@ -618,7 +622,8 @@ mod tests { fn cancelling_one_reservation_keeps_an_earlier_dirty_page() { let mut v = volume(); let f = v.open("home/f", CREATE).unwrap(); - v.reserve(f, 0, 4).unwrap(); + let first = v.reserve(f, 0, 4).unwrap(); + v.landed(f, first, 0, 4); let before = v.room().unwrap().free; let later = v.reserve(f, BLOCK as u64, 4).unwrap(); v.unreserve(f, later, 0).unwrap(); diff --git a/system/fileserver/src/fat.rs b/system/fileserver/src/fat.rs index 576a87fc592..d0ee179c2f6 100644 --- a/system/fileserver/src/fat.rs +++ b/system/fileserver/src/fat.rs @@ -28,7 +28,7 @@ use toyos_blockdev::BlockDev; use toyos_fat32::{Error, Fat32, FatTime}; use crate::cache::{Cache, Disk}; -use crate::volume::{parent, Identities, Kind, Listed, Meta, Node, OpenHow, Out, Reserved, Volume, NANOS_PER_SEC}; +use crate::volume::{parent, Identities, Kind, Listed, Meta, Node, OpenHow, Out, Reserved, Volume, WriteBack, NANOS_PER_SEC}; /// The most entries one directory listing materialises. const MAX_LIST: usize = 16_384; @@ -331,13 +331,13 @@ impl Volume for FatVolume { Ok(reserved) } - fn write_back(&mut self, node: Node, offset: u64, data: &[u8]) -> Result<(), fs::Error> { - let open = self.open.get_mut(&node).ok_or(fs::Error::Stale)?; + fn write_back(&mut self, node: Node, offset: u64, data: &[u8]) -> WriteBack { + let open = self.open.get_mut(&node).ok_or((0, fs::Error::Stale))?; if open.gone { - return Err(fs::Error::Stale); + return Err((0, fs::Error::Stale)); } open.dirty = true; - self.cache.block_on(self.fs.write_back(&mut open.file, offset, data)).map_err(|e| logged("write-back", &open.path, e)) + self.cache.block_on(self.fs.write_back(&mut open.file, offset, data)).map_err(|e| (0, logged("write-back", &open.path, e))) } fn truncate(&mut self, node: Node, size: u64) -> Result<(), fs::Error> { diff --git a/system/fileserver/src/memory.rs b/system/fileserver/src/memory.rs index cb2b0b03976..788eb450805 100644 --- a/system/fileserver/src/memory.rs +++ b/system/fileserver/src/memory.rs @@ -156,7 +156,7 @@ impl Volume for MemoryVolume { Ok(Reserved { size, contents: size, ticket: 0 }) } - fn write_back(&mut self, _: Node, _: u64, _: &[u8]) -> Result<(), Error> { + fn write_back(&mut self, _: Node, _: u64, _: &[u8]) -> crate::volume::WriteBack { unreachable!("memory file bytes stay in their paged object") } diff --git a/system/fileserver/src/pages.rs b/system/fileserver/src/pages.rs index 9151e318186..c663992624d 100644 --- a/system/fileserver/src/pages.rs +++ b/system/fileserver/src/pages.rs @@ -30,7 +30,7 @@ use toyos::fs::Error; use toyos::paged::{Notice, Refused, PAGE, PAGED_RANGE_PAGES}; use toyos::shm::Bytes; -use crate::volume::{Buf, Node, Volume, Window, Writes}; +use crate::volume::{Buf, Node, Reserved, Volume, Window, Writes}; /// The most pages one supply or write-back carries. const RANGE: u64 = PAGED_RANGE_PAGES; @@ -422,7 +422,7 @@ impl Files

{ self.touch(identity); let id = self.files[&identity].object; let mut put = Put { done: 0, held: false }; - let writing = Writing { identity, node, old, contents: reserved.contents(), offset, data }; + let writing = Writing { identity, node, reserved, offset, data }; let written = self.put(volume, &writing, &mut put); if let Err(e) = &written { // What was reserved and not put would read as the blocks' old bytes. @@ -450,7 +450,7 @@ impl Files

{ /// counting into `put` what landed and whether a call left the write /// under way. fn put(&mut self, volume: &mut dyn Volume, w: &Writing<'_>, put: &mut Put) -> Result<(), Error> { - let Writing { identity, node, old, contents, offset, data } = *w; + let Writing { identity, node, reserved, offset, data } = *w; let end = offset + data.len() as u64; let done = &mut put.done; while *done < data.len() { @@ -462,7 +462,7 @@ impl Files

{ put.held |= wrote != Ok(data.len() - *done); match wrote { Ok(n) => { - self.landed(identity, at, n); + self.landed(volume, w, at, n); *done += n; continue; } @@ -474,20 +474,19 @@ impl Files

{ let whole = ((data.len() - *done) as u64 / PAGE).min(RANGE); if at.is_multiple_of(PAGE) && whole > 0 { let bytes = &data[*done..*done + (whole * PAGE) as usize]; - // Each page taken is written; one already present was not - // taken, and the write above puts it on the next turn. + // Dirty supply stops at the first present page. The next + // write handles it, so every reported byte is a prefix. let taken = self.supplied(volume, identity, page, whole, bytes, true)?; - self.landed(identity, at, (taken * PAGE) as usize); - if taken == whole { - *done += bytes.len(); - } + let len = (taken * PAGE) as usize; + self.landed(volume, w, at, len); + *done += len; continue; } // A page in part: what it held below the old end, zeros past the // old contents, then the write's. let start = page * PAGE; - let below = old.saturating_sub(start).min(PAGE) as usize; - let held = contents.saturating_sub(start).min(PAGE) as usize; + let below = reserved.size.saturating_sub(start).min(PAGE) as usize; + let held = reserved.contents().saturating_sub(start).min(PAGE) as usize; let upto = end.min(start + PAGE); let mut buf = std::mem::take(&mut self.buf); buf.clear(); @@ -500,8 +499,9 @@ impl Files

{ }); self.buf = buf; if supplied? == 1 { - self.landed(identity, start, PAGE as usize); - *done += (upto - at) as usize; + let len = (upto - at) as usize; + self.landed(volume, w, at, len); + *done += len; } } Ok(()) @@ -509,9 +509,10 @@ impl Files

{ /// `len` bytes at `at` landed dirty: the next write-back takes them, a /// write-back the same write makes room with included. - fn landed(&mut self, identity: u64, at: u64, len: usize) { + fn landed(&mut self, volume: &mut dyn Volume, w: &Writing<'_>, at: u64, len: usize) { if len > 0 { - let f = self.files.get_mut(&identity).expect("a file being written"); + volume.landed(w.node, w.reserved, at, len); + let f = self.files.get_mut(&w.identity).expect("a file being written"); f.dirtied(at / PAGE, (at + len as u64).div_ceil(PAGE)); } } @@ -551,6 +552,7 @@ impl Files

{ let pager = self.pager.as_ref().expect("paged"); let begun = pager.writeback_begin(object, first, count).map_err(word)?; let mut written = Ok(()); + let mut complete = count; for (start, pages) in runs(&begun, count) { let at = (first + start) * PAGE; let len = (pages * PAGE).min(size - at) as usize; @@ -559,23 +561,29 @@ impl Files

{ // SAFETY: `buf` is this process's, reached only through these // bytes while they live. let out = unsafe { Bytes::from_raw(buf.as_mut_ptr(), len) }; - written = match pager.read(object, out, at) { + let result = match pager.read(object, out, at) { Ok(n) if n == len => volume.write_back(node, at, &buf), Ok(n) => { println!("fileserver: file {identity}'s pages under write-back at {at} read {n} of {len} bytes"); - Err(Error::Io) + Err((0, Error::Io)) } - Err(why) => Err(word(why)), + Err(why) => Err((0, word(why))), }; - if written.is_err() { + if let Err((done, error)) = result { + assert!(done < len && (done as u64).is_multiple_of(PAGE), "write-back refusal reports a whole-page prefix"); + complete = start + done as u64 / PAGE; + written = Err(error); break; } } - let still = pager.writeback_end(object, first, count, written.is_ok()).map_err(word)?; - for (start, pages) in runs(&still, count) { - let (a, b) = (first + start, first + start + pages); - let (x, y) = kept.unwrap_or((a, b)); - kept = Some((x.min(a), y.max(b))); + for (at, pages, accepted) in [(first, complete, true), (first + complete, count - complete, false)] { + if pages == 0 { continue; } + let still = pager.writeback_end(object, at, pages, accepted).map_err(word)?; + for (start, pages) in runs(&still, pages) { + let (a, b) = (at + start, at + start + pages); + let (x, y) = kept.unwrap_or((a, b)); + kept = Some((x.min(a), y.max(b))); + } } failed = failed.and(written); first += count; @@ -671,14 +679,12 @@ struct Put { held: bool, } -/// A write under way: its file, at `offset`, which was `old` bytes long. +/// A write under way and the capacity held until its bytes land. #[derive(Clone, Copy)] struct Writing<'a> { identity: u64, node: Node, - old: u64, - /// The end of the old bytes the volume holds ([`crate::volume::Reserved::contents`]). - contents: u64, + reserved: Reserved, offset: u64, data: &'a [u8], } diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index b4ccc410dc6..bbb00ce9365 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -1,4 +1,4 @@ -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::collections::{BTreeMap, BTreeSet}; use std::rc::Rc; @@ -8,6 +8,29 @@ use super::*; use crate::data::DataVolume; use crate::ram::Ram; use crate::volume::OpenHow; +use toyos_blockdev::{Block, BlockDev}; + +struct RefusingRead { + ram: Ram, + refusing: Rc>, +} + +impl BlockDev for RefusingRead { + fn blocks(&self) -> u64 { self.ram.blocks() } + async fn read(&self, lba: u64, out: &mut [Block]) -> Result<(), toyos_blockdev::Error> { + self.ram.read(lba, out).await?; + if self.refusing.get() && out.contains(&[0xb2; toyos_blockdev::BLOCK]) { + return Err(toyos_blockdev::Error::Device); + } + Ok(()) + } + async fn write(&self, lba: u64, data: &[Block]) -> Result<(), toyos_blockdev::Error> { + self.ram.write(lba, data).await + } + async fn flush(&self) -> Result<(), toyos_blockdev::Error> { self.ram.flush().await } +} + +impl crate::cache::Disk for RefusingRead {} /// The kernel's paged objects as far as the server can tell them apart: a /// table of `capacity` pages, `ceiling` of them dirty or under write-back, @@ -83,6 +106,7 @@ impl Fake { let why = if !dirty && page >= size.div_ceil(PAGE) { Some(Refused::Kernel(SyscallError::InvalidArgument)) } else if m.pages.contains_key(&(id, page)) { + if dirty { return Ok(installed); } continue; } else if m.pages.len() >= m.capacity { Some(Refused::NoRoom) @@ -476,6 +500,94 @@ fn a_write_refused_part_way_is_cut_back_to_what_it_put() { assert_eq!(on_volume(&mut v, "home/f"), b"kept"); } +#[test] +fn an_overwrite_keeps_incoming_capacity_while_its_older_dirty_page_is_flushed() { + let mut v = volume(); + let fake = Fake::new(4, 1); + let mut files = Files::new(Some(fake.clone())); + let (node, id) = open(&mut files, &mut v, "home/f"); + files.write(&mut v, id, node, 0, &pattern(2 * PAGE as usize, 1)).unwrap(); + files.fsync(&mut v, Some((id, node))).unwrap(); + files.write(&mut v, id, node, PAGE, &pattern(PAGE as usize, 2)).unwrap(); + let object = files.object(id).unwrap(); + assert!(!fake.0.borrow().pages[&(object, 0)].dirty); + assert!(fake.0.borrow().pages[&(object, 1)].dirty); + + let replacement = pattern(2 * PAGE as usize, 3); + files.write(&mut v, id, node, 0, &replacement).unwrap(); + files.fsync(&mut v, Some((id, node))).unwrap(); + let mut v = after_a_stop(v); + assert_eq!(on_volume(&mut v, "home/f"), replacement); +} + +#[test] +fn a_partial_writeback_cleans_only_its_prefix_and_retries_the_same_reserved_suffix() { + let refusing = Rc::new(Cell::new(false)); + let disk = RefusingRead { ram: Ram::new(16384), refusing: Rc::clone(&refusing) }; + let mut v = DataVolume::format(disk, 256, &["home"], clock).unwrap(); + let node = v.open("home/f", CREATE).unwrap(); + let mut old = vec![0xa1; PAGE as usize]; + old.extend(vec![0xb2; PAGE as usize]); + v.write(node, 0, &old).unwrap(); + v.close(node).unwrap(); + v.sync().unwrap(); + + let fake = Fake::new(4, 4); + let mut files = Files::new(Some(fake.clone())); + let (node, id) = open(&mut files, &mut v, "home/f"); + let new = pattern(2 * PAGE as usize, 8); + files.write(&mut v, id, node, 0, &new).unwrap(); + refusing.set(true); + assert_eq!(files.fsync(&mut v, Some((id, node))), Err(Error::Io)); + let object = files.object(id).unwrap(); + let m = fake.0.borrow(); + assert!(!m.pages[&(object, 0)].dirty, "the successful prefix must not consume its reservation twice"); + assert!(m.pages[&(object, 1)].dirty, "the refused suffix still owes its bytes"); + drop(m); + refusing.set(false); + files.fsync(&mut v, Some((id, node))).unwrap(); + let mut v = match DataVolume::probe(v.stopped(), 256, &["home"], clock) { + crate::data::Probed::Mounted(v) => *v, + _ => panic!("the committed file must mount"), + }; + let node = v.open("home/f", CREATE).unwrap(); + let mut bytes = vec![0; new.len()]; + assert_eq!(v.read(node, 0, &mut Buf(&mut bytes)), Ok(bytes.len())); + assert_eq!(bytes, new); +} + +#[test] +fn cancelling_an_overwrite_before_copy_keeps_the_older_dirty_reservation() { + let mut v = volume(); + let fake = Fake::new(4, 4); + let mut files = Files::new(Some(fake.clone())); + let (node, id) = open(&mut files, &mut v, "home/f"); + let old = pattern(PAGE as usize, 4); + files.write(&mut v, id, node, PAGE, &old).unwrap(); + fake.0.borrow_mut().refusing = true; + assert_eq!(files.write(&mut v, id, node, 0, &pattern(2 * PAGE as usize, 5)), Err(Error::Io)); + fake.0.borrow_mut().refusing = false; + files.fsync(&mut v, Some((id, node))).unwrap(); + let mut v = after_a_stop(v); + let bytes = on_volume(&mut v, "home/f"); + assert!(bytes[..PAGE as usize].iter().all(|&b| b == 0)); + assert_eq!(bytes[PAGE as usize..], old); +} + +#[test] +fn a_supply_stops_before_a_present_page_and_resumes_after_the_write_copies_it() { + let mut v = volume(); + let fake = Fake::new(4, 2); + let mut files = Files::new(Some(fake)); + let (node, id) = open(&mut files, &mut v, "home/f"); + files.write(&mut v, id, node, PAGE, &pattern(PAGE as usize, 6)).unwrap(); + let replacement = pattern(3 * PAGE as usize, 7); + files.write(&mut v, id, node, 0, &replacement).unwrap(); + files.fsync(&mut v, Some((id, node))).unwrap(); + let mut v = after_a_stop(v); + assert_eq!(on_volume(&mut v, "home/f"), replacement); +} + /// A write into a hole the file was extended over, refused before any byte /// landed: the block its reserve took there goes back, though the hole lies /// inside the old size, and the hole still reads zeros. diff --git a/system/fileserver/src/root.rs b/system/fileserver/src/root.rs index 1fba2ea1604..eb11f851b64 100644 --- a/system/fileserver/src/root.rs +++ b/system/fileserver/src/root.rs @@ -214,8 +214,8 @@ impl Volume for RootVolume { Err(Error::ReadOnly) } - fn write_back(&mut self, _: Node, _: u64, _: &[u8]) -> Result<(), Error> { - Err(Error::ReadOnly) + fn write_back(&mut self, _: Node, _: u64, _: &[u8]) -> crate::volume::WriteBack { + Err((0, Error::ReadOnly)) } fn truncate(&mut self, _: Node, _: u64) -> Result<(), Error> { diff --git a/system/fileserver/src/volume.rs b/system/fileserver/src/volume.rs index e17e3105a8f..1804d4269e3 100644 --- a/system/fileserver/src/volume.rs +++ b/system/fileserver/src/volume.rs @@ -58,6 +58,10 @@ pub struct OpenHow { /// An open file, while any client holds it. pub type Node = u64; +/// A write-back is whole, or reports the byte count of the complete-page +/// prefix written before its refusal. The suffix keeps its capacity for retry. +pub type WriteBack = Result<(), (usize, Error)>; + /// Where a read's bytes go, `len()` of them at most: the client's window, which /// is shared with the client and so is written through a copy and never /// through a reference, or a buffer of this process's own. @@ -159,19 +163,21 @@ pub trait Volume { /// its reserve gives back its room through [`Self::unreserve`]. fn reserve(&mut self, node: Node, offset: u64, len: u64) -> Result; - /// A write refused after its reserve: the size back to the reserve's old - /// one, or to `landed` where the write's bytes reached past it, and every - /// block the reserve took past both given back, since it holds what the - /// file that last had it left there. `landed` is 0 where none landed. - /// Called before any other call on the file, so nothing has recorded the - /// reserve's blocks. + /// Bytes of this reservation copied into the file's buffer. Called once + /// for each touched page, before any write-back can take the new bytes. + /// An older dirty version may be written back before this call. + fn landed(&mut self, _node: Node, _reserved: Reserved, _offset: u64, _len: usize) {} + + /// Cancel the capacity for bytes this write did not copy, restoring the + /// size to the old one or `landed`, whichever is larger. `landed` is 0 + /// where none landed. Older dirty bytes retain their capacity. fn unreserve(&mut self, node: Node, reserved: Reserved, landed: u64) -> Result<(), Error> { self.truncate(node, reserved.size.max(landed)) } /// `data` at the whole-page `offset`, inside the size, into the room /// [`Self::reserve`] made; a last page in part is zeros past the size. - fn write_back(&mut self, node: Node, offset: u64, data: &[u8]) -> Result<(), Error>; + fn write_back(&mut self, node: Node, offset: u64, data: &[u8]) -> WriteBack; fn truncate(&mut self, node: Node, size: u64) -> Result<(), Error>; @@ -269,7 +275,9 @@ pub trait Writes: Volume { let written = self.read(node, start, &mut Buf(&mut page[..below])).and_then(|_| { page[(from - start) as usize..(to - start) as usize] .copy_from_slice(&data[(from - offset) as usize..(to - offset) as usize]); + self.landed(node, reserved, from, (to - from) as usize); self.write_back(node, start, &page[..(size - start).min(PAGE) as usize]) + .map_err(|(_, error)| error) }); if let Err(e) = written { if let Err(again) = self.unreserve(node, reserved, landed) { diff --git a/tests/guest/pagerprobe/src/main.rs b/tests/guest/pagerprobe/src/main.rs index 058e3b87ca7..ecde923fe28 100644 --- a/tests/guest/pagerprobe/src/main.rs +++ b/tests/guest/pagerprobe/src/main.rs @@ -40,6 +40,7 @@ fn main() { return hold(); } let pager = Pager::create(LARGE + 16, 0, 4, Duration::from_secs(1)).expect("pagerprobe: its row starts it as a pager"); + dirty_supply_prefix(&pager); let large = pager.object(LARGE * PAGE).expect("pagerprobe: the large object"); fill(&pager, large); assert_eq!(pager.withdraw(large, 0, u64::MAX), Ok(LARGE), "pagerprobe: the whole object withdrawn at once"); @@ -102,6 +103,25 @@ fn main() { println!("pagerprobe: ok"); } +fn dirty_supply_prefix(pager: &Pager) { + let object = pager.anonymous(4 * PAGE).unwrap(); + let old = vec![0x42; PAGE as usize]; + assert_eq!(pager.supply(object, PAGE, &old, true), Ok(1)); + assert_eq!(pager.supply(object, 3 * PAGE, &old, true), Ok(1)); + let new = vec![0x75; 4 * PAGE as usize]; + assert_eq!(pager.supply(object, 0, &new, true), Ok(1), "dirty supply stops at the first present page"); + pager.end_write(object).unwrap(); + let mut bytes = vec![0; new.len()]; + // SAFETY: the local buffer is borrowed only by this read. + let out = unsafe { toyos::shm::Bytes::from_raw(bytes.as_mut_ptr(), bytes.len()) }; + assert_eq!(pager.read(object, out, 0), Ok(bytes.len())); + assert_eq!(&bytes[..PAGE as usize], &new[..PAGE as usize]); + assert_eq!(&bytes[PAGE as usize..2 * PAGE as usize], &old); + assert!(bytes[2 * PAGE as usize..3 * PAGE as usize].iter().all(|&b| b == 0), "nothing past the returned prefix was installed"); + assert_eq!(&bytes[3 * PAGE as usize..], &old); + pager.release(object).unwrap(); +} + /// Every page of `object`, clean. fn fill(pager: &Pager, object: u64) { let run = vec![0x5a_u8; (PAGED_RANGE_PAGES * PAGE) as usize]; From 9ff9d05f3ed27a4ded93a6aeb6e0534fde13e7a8 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:07:14 +0100 Subject: [PATCH 29/93] Exercise dirty supply prefix without a capacity limit masking the gap Give the model enough dirty capacity to install pages on both sides of an existing page. A supply that skips the existing page would then report a noncontiguous count and silently leave the overwritten middle page old. The checked negative control reproduces that data mismatch. --- system/fileserver/src/pages/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index bbb00ce9365..130893e490c 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -577,7 +577,7 @@ fn cancelling_an_overwrite_before_copy_keeps_the_older_dirty_reservation() { #[test] fn a_supply_stops_before_a_present_page_and_resumes_after_the_write_copies_it() { let mut v = volume(); - let fake = Fake::new(4, 2); + let fake = Fake::new(4, 4); let mut files = Files::new(Some(fake)); let (node, id) = open(&mut files, &mut v, "home/f"); files.write(&mut v, id, node, PAGE, &pattern(PAGE as usize, 6)).unwrap(); From 336ad3060ea31135a46cbd1d980a64ef4ba02584 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:13:41 +0100 Subject: [PATCH 30/93] Start the userland temporary filesystem in console and test images --- images/console.toml | 2 +- tests/images/acpicase.toml | 2 +- tests/images/benchcase.toml | 2 +- tests/images/designatecase.toml | 2 +- tests/images/floorcase.toml | 2 +- tests/images/hostedclangcase.toml | 2 +- tests/images/jobcase.toml | 2 +- tests/images/latencycase.toml | 2 +- tests/images/linkcase.toml | 2 +- tests/images/metalcase.toml | 2 +- tests/images/metaldevicecase.toml | 2 +- tests/images/netcase.toml | 2 +- tests/images/proctreecase.toml | 2 +- tests/images/testcases.toml | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/images/console.toml b/images/console.toml index 5237fe1f6cb..edf9f6a30a2 100644 --- a/images/console.toml +++ b/images/console.toml @@ -125,5 +125,5 @@ receives = ["disk", "input"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] diff --git a/tests/images/acpicase.toml b/tests/images/acpicase.toml index 067863b199f..00e3d5cbb90 100644 --- a/tests/images/acpicase.toml +++ b/tests/images/acpicase.toml @@ -58,7 +58,7 @@ receives = ["disk"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] # The virtio-serial driver, for the VM's console alone: this image runs no agent. diff --git a/tests/images/benchcase.toml b/tests/images/benchcase.toml index b3714fc441e..63146bf9bba 100644 --- a/tests/images/benchcase.toml +++ b/tests/images/benchcase.toml @@ -137,7 +137,7 @@ receives = ["disk"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] # The virtio-serial driver, for the VM's console alone: `osbench_capability_groups_exit_0` diff --git a/tests/images/designatecase.toml b/tests/images/designatecase.toml index 688978a2b77..cdb2a044127 100644 --- a/tests/images/designatecase.toml +++ b/tests/images/designatecase.toml @@ -46,5 +46,5 @@ receives = ["disk"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] diff --git a/tests/images/floorcase.toml b/tests/images/floorcase.toml index 6f5a3dee4ba..4f4ccb2b343 100644 --- a/tests/images/floorcase.toml +++ b/tests/images/floorcase.toml @@ -80,5 +80,5 @@ receives = ["disk"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] diff --git a/tests/images/hostedclangcase.toml b/tests/images/hostedclangcase.toml index 77cd1fe468d..3806e6ea700 100644 --- a/tests/images/hostedclangcase.toml +++ b/tests/images/hostedclangcase.toml @@ -28,7 +28,7 @@ receives = ["disk"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] # The virtio-serial driver, for the VM's console alone: this image runs no agent. diff --git a/tests/images/jobcase.toml b/tests/images/jobcase.toml index 167037bb88c..fe6d279f7d7 100644 --- a/tests/images/jobcase.toml +++ b/tests/images/jobcase.toml @@ -49,7 +49,7 @@ receives = ["disk"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] # The virtio-serial driver, for the VM's console alone: this image runs no agent. diff --git a/tests/images/latencycase.toml b/tests/images/latencycase.toml index 96c790ca88c..a5934e8d545 100644 --- a/tests/images/latencycase.toml +++ b/tests/images/latencycase.toml @@ -65,5 +65,5 @@ receives = ["disk"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] diff --git a/tests/images/linkcase.toml b/tests/images/linkcase.toml index 0a0e535fa81..1a3cbd93802 100644 --- a/tests/images/linkcase.toml +++ b/tests/images/linkcase.toml @@ -42,7 +42,7 @@ receives = ["disk"] [programs.fileserver] pager = true restart = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] # The virtio-serial driver, for the VM's console alone: this image runs no agent. diff --git a/tests/images/metalcase.toml b/tests/images/metalcase.toml index 2fff1232785..3ca598acf9a 100644 --- a/tests/images/metalcase.toml +++ b/tests/images/metalcase.toml @@ -113,5 +113,5 @@ receives = ["disk", "input"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] diff --git a/tests/images/metaldevicecase.toml b/tests/images/metaldevicecase.toml index 7b5ae347971..010b24758e3 100644 --- a/tests/images/metaldevicecase.toml +++ b/tests/images/metaldevicecase.toml @@ -89,5 +89,5 @@ receives = ["disk"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] diff --git a/tests/images/netcase.toml b/tests/images/netcase.toml index 9526c488eda..171f8a6ee37 100644 --- a/tests/images/netcase.toml +++ b/tests/images/netcase.toml @@ -76,7 +76,7 @@ receives = ["disk"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] # The virtio-serial driver, for the VM's console alone: this image runs no agent. diff --git a/tests/images/proctreecase.toml b/tests/images/proctreecase.toml index edfe77f4644..0cd030ff652 100644 --- a/tests/images/proctreecase.toml +++ b/tests/images/proctreecase.toml @@ -81,7 +81,7 @@ receives = ["disk"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] # The virtio-serial driver, for the VM's console alone: this image runs no agent. diff --git a/tests/images/testcases.toml b/tests/images/testcases.toml index 5c6bee0cfe6..dc2cc99444f 100644 --- a/tests/images/testcases.toml +++ b/tests/images/testcases.toml @@ -128,7 +128,7 @@ receives = ["disk"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] # The virtio-serial driver, for the VM's console alone: this image runs no agent. From f9a97293a366cd8638fcb03a2bebc6fe560427f6 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:13:14 +0100 Subject: [PATCH 31/93] Keep merged btrees at their journaled depth and size a reclaim batch honestly Retain the bounded single-child root spine instead of publishing a lower root while older journal entries still contain higher-level keys. Remove the reader-only obsolete-level filter that hid the incompatible journal. Only full-range nodes are exempt from sibling occupancy; the stable-node ledger must charge at most four such nodes per btree. Charge each pin for the largest supported rewrite, and cache each distinct position a full batch can dirty. Increase native minimum journal geometry accordingly and refuse an inadequate imported ring for read-write access. These bounds cover one batch and its pins; they do NOT prove a bound on the recursive derived-tree frontier. That remaining gap needs a separate representation/admission change, not an assumed common-case cascade. Fix admission spinning on an absent journal entry while another writer holds the available grant. Make reuse tests drive reclaim explicitly. Remove the obsolete 60-create cache-pressure case: it cannot pressure the new footprint; the yielding 32-directory concurrent test checks actual cache waits, completion, the dirty peak, and final filesystem consistency. Validation: cargo test -p bcachefs --lib, 141 passed and four failed. Three failures start with a nearly-full fixture below the new physical reserve; the allocator/capacity work is owned by the engine branch. The fourth is its already-obsolete logged-truncate refusal expectation. All compact, admission, explicit reuse, and encrypted reuse crash cases pass. A compiled negative control dropping post-capture deltas failed the mtime-after-remount oracle, and the correct implementation was restored. Full logs: /private/tmp/toyos-bcachefs-reclaim/spine-checkpoint.log and captured-delta-negative.log. No whole-cascade safety claim is made. --- lib/bcachefs/src/engine/mkfs.rs | 10 +-- lib/bcachefs/src/engine/mount/mod.rs | 41 +++++++--- lib/bcachefs/src/engine/mount/reclaim.rs | 94 +++++----------------- lib/bcachefs/src/engine/tests/admission.rs | 16 +--- lib/bcachefs/src/engine/tests/buckets.rs | 2 + lib/bcachefs/src/engine/tests/compact.rs | 9 ++- lib/bcachefs/src/engine/tests/crash.rs | 6 +- lib/bcachefs/src/engine/tests/mod.rs | 7 +- lib/bcachefs/src/engine/tests/mount.rs | 24 ++---- 9 files changed, 76 insertions(+), 133 deletions(-) diff --git a/lib/bcachefs/src/engine/mkfs.rs b/lib/bcachefs/src/engine/mkfs.rs index 510c9591ffa..34b87aa9867 100644 --- a/lib/bcachefs/src/engine/mkfs.rs +++ b/lib/bcachefs/src/engine/mkfs.rs @@ -375,7 +375,7 @@ impl Geometry { } let layout = Layout { max_bits: SB_MAX_BITS, offsets: alloc::vec![PRIMARY, second, end] }; let front = (second + SB_SECTORS).div_ceil(bucket); - let journal = (front, (nbuckets / 128).max(8)); + let journal = (front, (nbuckets / 128).max(8).max(super::mount::journal_buckets(bucket, BLOCK))); Ok(Geometry { bucket, nbuckets, node: bucket.min(MAX_NODE_SECTORS), layout, journal, data_buckets: 0 }) } @@ -707,10 +707,7 @@ mod tests { use super::*; #[test] - fn a_small_device_lays_out_as_format_does() { - // Upstream's own 32 MiB layout: 65 536 sectors in 32-sector buckets, copies at 8, - // 2 056 and 63 488; buckets 0–127 hold 32 superblock sectors, 128 - // holds 8, 1 984–2 047 hold 32. + fn a_small_device_keeps_superblock_regions_and_sizes_the_reclaim_journal() { let g = Geometry::new(65536).unwrap(); assert_eq!((g.bucket, g.nbuckets, g.node), (32, 2048, 32)); assert_eq!(g.layout.offsets, [8, 2056, 63488]); @@ -720,7 +717,8 @@ mod tests { assert_eq!(sb.len(), 129 + 64); assert_eq!(sb[128], (128, 8)); assert!(sb.iter().filter(|&&(b, _)| b != 128).all(|&(_, s)| s == 32)); - assert_eq!(g.journal, (129, 16)); + assert_eq!(g.journal, (129, super::super::mount::journal_buckets(g.bucket, BLOCK))); + assert!(g.journal.1 > g.nbuckets / 128); } #[test] diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 36ef3e5fbc8..2d00acac610 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -49,7 +49,7 @@ pub use ops::{Link, Removed, Rename}; pub(crate) use reclaim::Order; #[cfg(test)] pub(crate) use reclaim::Plan; -use reclaim::{fresh_cost, node_ptr, MAX_GROUPS, RECLAIM_ROOM, REWRITTEN}; +use reclaim::{fresh_cost, node_ptr, MAX_GROUPS, MAX_REWRITE, RECLAIM_ROOM, REWRITTEN}; /// How much a mount may hold in memory. #[derive(Clone, Copy, Debug)] @@ -65,13 +65,13 @@ pub(crate) const K_TX: u64 = 512; pub(crate) const FOOTPRINT: usize = 28; /// Nodes one reclaim batch writes. const BATCH: usize = 2; -/// u64s a reclaim share holds for each node that can pin the journal: -/// sized for a rewrite to one node, the common case. A share too short for -/// the splits a batch needs stops the mount by name ([`Mount::admit`]). -const PER_NODE: u64 = fresh_cost(1); +/// A pinning node may require the largest supported split. This also +/// exceeds a two-source sibling rewrite's cost, even with one clean source. +const PER_NODE: u64 = fresh_cost(MAX_REWRITE); /// Cache room past the limit that only reclaim takes: what its transaction /// dirties, and the nodes a batch's rewrites replace its nodes with. const RECLAIM: usize = RECLAIM_ROOM + REWRITTEN; +pub(crate) const MIN_CACHE: usize = RECLAIM + 3 * FOOTPRINT; /// The smallest btree node this writer writes. Past its first bset a /// smaller one has room for about two whole-block appends, and reclaim's own /// changes to the allocator's leaves then rewrite more nodes than they @@ -84,6 +84,15 @@ const RW_FEATURES: u64 = 1 << 8 | 1 << 9 | 1 << 12 | 1 << 13 | 1 << 17 | 1 << 18 /// Feature bit 5: the superblock holds a blacklist. const BLACKLIST_FEATURE: u64 = 1 << 5; +/// Ring geometry that leaves two worst-case reclaim shares and an ordinary +/// quarter, with room for at least one maximum-size ordinary transaction. +pub(crate) fn journal_buckets(bucket_sectors: u64, block: usize) -> u64 { + let journal = Writer::new(alloc::vec![0], bucket_sectors, block, K_TX); + let per_entry = journal.body() - K_TX; + let share = ((RECLAIM + FOOTPRINT) as u64 * PER_NODE).div_ceil(per_entry) + 1; + (share * 8).div_ceil(3).div_ceil(journal.slots()) +} + /// What a new node is. pub enum New { File, @@ -227,10 +236,9 @@ struct Txn { enum Install { /// `id`'s appended bset, now `sectors` long in all. Append { id: NodeId, sectors: u16 }, - /// `id` replaced by the nodes written for it, and a root above them when - /// a root split. + /// Sources replaced by nodes covering their combined range, with a new + /// root when the old root split. Replace { ids: Vec, by: Vec, root: Option }, - Collapse { id: NodeId }, } /// A new node as planned and written. @@ -397,11 +405,15 @@ impl Mount { // every node that can pin it, and leave transactions a quarter. let per_entry = journal.body() - journal.max_tx; let share = journal.slots() * 3 / 8; - let by_journal = ((share - 1) * per_entry / PER_NODE) as usize; - let limit = options.cache_nodes.saturating_sub(reclaim_room + 2 * FOOTPRINT).min(by_journal.saturating_sub(reclaim_room)); - if limit < FOOTPRINT { - return Err(Error::Invalid("a cache or journal too small for a transaction and its reclaim")); + let by_journal = (share.saturating_sub(1) * per_entry / PER_NODE) as usize; + if options.cache_nodes < MIN_CACHE { + return Err(Error::Invalid("cache too small for a transaction and worst-case reclaim")); } + if by_journal < reclaim_room + FOOTPRINT { + return Err(Error::Unsupported("journal too small for a transaction and worst-case reclaim")); + } + let limit = options.cache_nodes.saturating_sub(reclaim_room + 2 * FOOTPRINT).min(by_journal.saturating_sub(reclaim_room)); + assert!(limit >= FOOTPRINT, "validated cache and journal admit one transaction"); journal.reclaim_entries = ((limit + reclaim_room) as u64 * PER_NODE).div_ceil(per_entry) + 1; let journal_next = journal.next_seq; @@ -1010,7 +1022,10 @@ impl Mount { fn can_free(&self) -> bool { let st = self.st.borrow(); let flush = st.journal_frees() || st.alloc.returning(); - !st.journal.closed.is_empty() || (flush && st.journal.may_close()) || st.sparse_metadata.iter().any(|&b| !st.alloc.metadata_open(b)) || st.cache.ids().into_iter().any(|id| { + !st.journal.closed.is_empty() + || (flush && st.journal.open.is_some() && st.journal.may_close()) + || st.sparse_metadata.iter().any(|&b| !st.alloc.metadata_open(b)) + || st.cache.ids().into_iter().any(|id| { let n = st.cache.node(id); n.dirty() && !n.writing && n.captured.is_none() && n.held == 0 }) diff --git a/lib/bcachefs/src/engine/mount/reclaim.rs b/lib/bcachefs/src/engine/mount/reclaim.rs index eedb4dc2d92..e7b0965519b 100644 --- a/lib/bcachefs/src/engine/mount/reclaim.rs +++ b/lib/bcachefs/src/engine/mount/reclaim.rs @@ -52,7 +52,7 @@ impl Plan { fn cost(&self) -> u64 { match self.written() { 0 => APPEND_COST, - k => fresh_cost(k) + (self.ids.len() - 1) as u64 * (ALLOC + 4 * SET + BUCKET_GENS), + k => fresh_cost(k) + (self.ids.len() - 1) as u64 * MERGE_EXTRA, } } @@ -89,6 +89,7 @@ const HEADERS: u64 = 8; /// An append's update: the pointer's written sectors, in its record. const APPEND_COST: u64 = 1 + POINTER; +const MERGE_EXTRA: u64 = ALLOC + 4 * SET + BUCKET_GENS; /// u64s the pointer update of a node rewritten to `nodes` new ones /// journals at most. Each new node: its pointer, its bucket's change and its @@ -141,12 +142,15 @@ pub(super) const COMMON_REWRITE: usize = 3; /// Node slots one batch can take: its common rewrites, or one alone. const BATCH_SLOTS: u64 = (if BATCH * COMMON_REWRITE > MAX_REWRITE { BATCH * COMMON_REWRITE } else { MAX_REWRITE }) as u64; -/// Cache room reclaim's transaction takes past the limit, for the leaves -/// it dirties: every position of a rewrite to one node in a leaf of its -/// own. A batch whose positions could need more is granted this, as they -/// share leaves; one that dirties more is refused and stops the mount, never -/// overfilling the cache. -pub(super) const RECLAIM_ROOM: usize = fresh_nodes(1); +/// Every changed position may occupy a separate leaf. A common batch may +/// include two sibling rewrites; an oversized rewrite runs alone. +pub(super) const RECLAIM_ROOM: usize = { + let pairs = BATCH * (fresh_nodes(2) + 7); + let common = BATCH * fresh_nodes(COMMON_REWRITE); + let oversized = fresh_nodes(MAX_REWRITE); + let batch = if pairs > common { pairs } else { common }; + if batch > oversized { batch } else { oversized } +}; /// Nodes past the limit a batch's rewrites can leave dirty: their new /// nodes, each taking the changes since its node was captured, beside the @@ -157,6 +161,7 @@ pub(super) const REWRITTEN: usize = { }; const _: () = assert!(fresh_cost(MAX_REWRITE) <= K_TX, "the largest rewrite fits one reclaim transaction"); +const _: () = assert!(fresh_cost(2) + MERGE_EXTRA <= fresh_cost(MAX_REWRITE), "a sibling rewrite fits a pin's journal debt"); impl Mount { /// One batch of dirty nodes chosen by `order`, planned under the slot @@ -184,7 +189,6 @@ impl Mount { } async fn write_batch_once(&self, order: Order) -> Result> { - while self.collapse_root().await? {} self.make_room().await?; let plans = { let _slot = self.slot(Class::Reclaim).await; @@ -201,7 +205,6 @@ impl Mount { /// evacuation uses the same publication and reader barriers as reclaim. pub(crate) async fn relocate_node(&self, btree: Btree, level: u8, pos: Bpos, expected_sector: u64) -> Result { let out = async { - while self.collapse_root().await? {} self.make_room().await?; let plans = { let _slot = self.slot(Class::Reclaim).await; @@ -224,54 +227,6 @@ impl Mount { self.stopped_by(out) } - async fn collapse_root(&self) -> Result { - let candidate = { - let st = self.st.borrow(); - st.roots.values().find_map(|r| { - let id = st.cache.id_at(r.ptr.ptr.sector)?; - let n = st.cache.node(id); - (n.level > 0 && n.keys.len() == 1 && n.dirty() && !n.writing && n.captured.is_none() && n.held == 0).then_some((r.btree, n.sector)) - }) - }; - let Some((btree, expected)) = candidate else { return Ok(false) }; - let bound = fresh_cost(1); - let room = RECLAIM_ROOM; - self.flush_closed().await?; - self.admit(Class::Reclaim, Draw::Freeing, bound, room).await?; - let _slot = self.slot(Class::Reclaim).await; - let mut txn = Txn::default(); - let out = async { - let id = { - let st = self.st.borrow(); - if st.roots.get(&btree).is_none_or(|r| r.ptr.ptr.sector != expected) { return Ok(false); } - let Some(id) = st.cache.id_at(expected) else { return Ok(false) }; - id - }; - let (level, sector, gen, key, ptr) = { - let st = self.st.borrow(); - let n = st.cache.node(id); - if n.keys.len() != 1 || n.writing || n.captured.is_some() || n.held != 0 { return Ok(false); } - let (key, value) = &n.keys[0]; - (n.level, n.sector, n.gen, *key, NodePtr::decode(value)?) - }; - let child = self.load(btree, level - 1, &ptr, key.pos).await?; - { - let st = self.st.borrow(); - let n = st.cache.node(child); - if n.writing || n.captured.is_some() { return Ok(false); } - } - if ptr.min != Bpos::MIN || key.pos != Bpos::MAX { return Err(Error::Corrupt("only root child does not span the btree")); } - txn.roots.push(Root { btree, level: level - 1, key, ptr }); - let mut buckets = BTreeMap::new(); - self.node_counted(&mut txn, &mut buckets, btree, level, sector, gen, Bpos::MAX, false).await?; - for (b, (before, after)) in buckets { self.bucket_changed(&mut txn, b, before, after).await?; } - txn.installs.push(Install::Collapse { id }); - Ok(true) - }.await; - self.finish(txn, out.is_ok(), Class::Reclaim, bound, room)?; - out - } - /// Reclaim's second half: the written batch's pointer updates /// journaled, on reclaim's reserves and ahead in the slot. pub(crate) async fn point_batch(&self, plans: Vec) -> Result<()> { @@ -294,7 +249,8 @@ impl Mount { async fn point_batch_once(&self, plans: Vec) -> Result<()> { let bound = plans.iter().map(Plan::cost).sum(); - let nodes = plans.iter().map(Plan::nodes).sum::().min(RECLAIM_ROOM); + let nodes = plans.iter().map(Plan::nodes).sum::(); + assert!(nodes <= RECLAIM_ROOM, "a reclaim batch exceeds its derived node bound"); // Closed entries hold pipeline depth that only writing them returns: // reclaim never waits on what it would itself have to do. self.flush_closed().await?; @@ -366,7 +322,7 @@ impl Mount { return Err(Error::Unsupported("a btree node whose keys fill more than seven nodes")); } let written = groups.len() + usize::from(groups.len() > 1 && root); - let cost = if append { APPEND_COST } else { fresh_cost(written) + (ids.len() - 1) as u64 * (ALLOC + 4 * SET + BUCKET_GENS) }; + let cost = if append { APPEND_COST } else { fresh_cost(written) + (ids.len() - 1) as u64 * MERGE_EXTRA }; // A rewrite past the common one is a batch of its own, so the // nodes a batch's rewrites leave stay within reclaim's room. let alone = !append && written > COMMON_REWRITE; @@ -438,10 +394,9 @@ impl Mount { let (btree, level, max) = { let st = self.st.borrow(); let n = st.cache.node(id); - let root = st.new_roots.get(&n.btree) == Some(&id) || st.roots.get(&n.btree).is_some_and(|r| r.ptr.ptr.sector == n.sector); - if root { - return Ok(if n.level > 0 && n.keys.len() == 1 { Vec::new() } else { alloc::vec![id] }); - } + // Root depth stays: older journal entries can still hold keys + // at every former level. The sole-child spine is bounded by it. + if n.min == Bpos::MIN && n.max == Bpos::MAX { return Ok(alloc::vec![id]); } if n.bytes >= self.minimum_fill() { return Ok(alloc::vec![id]); } (n.btree, n.level, n.max) }; @@ -817,9 +772,8 @@ impl Mount { Ok(()) } - /// A reclaimed node made reachable: an appended bset now counted in its - /// node, or the node replaced by those written for it, each holding its - /// share of the node's keys and of the changes since it was captured. + /// Published replacements take the sources' current keys, including + /// every change since capture, partitioned by their new ranges. pub(super) fn install(&self, st: &mut State, install: Install) { match install { Install::Append { id, sectors } => st.cache.change(id, |n| { @@ -845,11 +799,10 @@ impl Mount { let mine: Vec<(Key, Vec)> = keys.iter().filter(|(k, _)| (p.min..=p.max).contains(&k.pos)).cloned().collect(); let changes: BTreeMap = unwritten.range(p.min..=p.max).map(|(k, v)| (*k, *v)).collect(); let id = st.cache.insert(fresh_cached(&p, mine)); - let is_root = st.roots.get(&p.btree).is_some_and(|r| r.ptr.ptr.sector == p.sector); let seq = st.journal.open.as_ref().map_or(st.journal.next_seq, |o| o.seq); st.cache.change(id, |n| { n.unwritten = changes; - if !is_root && n.bytes < self.minimum_fill() { n.compact = Some(seq); } + if (n.min != Bpos::MIN || n.max != Bpos::MAX) && n.bytes < self.minimum_fill() { n.compact = Some(seq); } }); } if let Some(p) = root { @@ -857,10 +810,6 @@ impl Mount { st.cache.insert(fresh_cached(&p, keys)); } } - Install::Collapse { id } => { - st.cache.change(id, |n| { n.unwritten.clear(); n.compact = None; n.captured = None; }); - st.cache.remove(id); - } } } @@ -874,7 +823,6 @@ impl Mount { let mut keys: Vec<_> = replayed.keys.into_iter().collect(); keys.sort_by_key(|((btree, level, pos), _)| (core::cmp::Reverse(*level), *btree, *pos)); for ((btree, level, pos), logged) in keys { - if self.st.borrow().roots.get(&btree).is_some_and(|root| level > root.level) { continue; } let id = self.find(btree, level, pos).await?; self.st.borrow_mut().cache.change(id, |n| n.put(pos, logged.key, logged.seq)); } diff --git a/lib/bcachefs/src/engine/tests/admission.rs b/lib/bcachefs/src/engine/tests/admission.rs index 3cb9728c5a0..f9d0fe0e4a3 100644 --- a/lib/bcachefs/src/engine/tests/admission.rs +++ b/lib/bcachefs/src/engine/tests/admission.rs @@ -92,10 +92,8 @@ fn writers(dev: &Image, cache_nodes: usize, dirs: &[u64], writers: usize, per: u Pressure { limit, room, peak, waits } } -/// The least cache a mount takes: room for one transaction's footprint -/// beside reclaim's, so each writer waits for reclaim to clean what the -/// last one dirtied. -const SMALL_CACHE: usize = 54; +/// The least cache a mount takes: one maximum transaction beside reclaim. +const SMALL_CACHE: usize = super::super::mount::MIN_CACHE; /// Creates past what the journal holds, with no commit: entries fill and are /// written, the ring fills with entries the dirty nodes pin, and a writer @@ -109,16 +107,6 @@ fn a_writer_waiting_on_a_full_journal_completes_while_reclaim_runs() { assert!(p.waits[0] > 0, "no writer waited for the journal: {:?}", p.waits); } -/// A writer whose every transaction fills the dirty-node limit waits for -/// cache room until reclaim cleans the last one's nodes. Red when cache room -/// is taken after the slot, or when reclaim draws on ordinary admission. -#[test] -fn a_writer_filling_the_cache_with_dirty_nodes_completes_while_reclaim_runs() { - let dev = formatted(64 << 20, None); - let p = writers(&dev, SMALL_CACHE, &[mkfs::ROOT_INODE], 1, 60); - assert!(p.waits[1] > 0, "no writer waited for cache room: {:?}", p.waits); -} - /// Thirty-two writers at once, each in a directory whose names live in a /// leaf of their own: admission counts what it has granted and not /// yet applied against the limit, so their dirty nodes and reclaim's diff --git a/lib/bcachefs/src/engine/tests/buckets.rs b/lib/bcachefs/src/engine/tests/buckets.rs index 851df957b49..39f1f8b24ae 100644 --- a/lib/bcachefs/src/engine/tests/buckets.rs +++ b/lib/bcachefs/src/engine/tests/buckets.rs @@ -80,6 +80,8 @@ fn churn(dev: &D, options: MountOptions, files: usize, reuses: usiz let (i, round) = (n % files, n / files); let (from, to) = if round % 2 == 0 { ("a", "b") } else { ("b", "a") }; m.rename((r, format!("{from}{i}").as_bytes()), (r, format!("{to}{i}").as_bytes()), Rename::NoReplace, NOW).await.unwrap(); + let plans = m.write_batch(super::super::mount::Order::OldestPin).await.unwrap(); + m.point_batch(plans).await.unwrap(); n += 1; if n % every == 0 { m.commit().await.unwrap(); diff --git a/lib/bcachefs/src/engine/tests/compact.rs b/lib/bcachefs/src/engine/tests/compact.rs index c4cc361e3bb..0e5269bf162 100644 --- a/lib/bcachefs/src/engine/tests/compact.rs +++ b/lib/bcachefs/src/engine/tests/compact.rs @@ -2,7 +2,7 @@ use super::mount::{mounted, random, MEMORY, NOW, OWNER}; use super::*; #[test] -fn deleting_all_files_collapses_the_inode_tree_and_reuses_its_room() { +fn deleting_all_files_leaves_only_the_inode_spine_and_reuses_its_room() { let dev = formatted(64 << 20, None); for round in 0..3 { mounted(&dev, async |m: &Mount<&Image>| { @@ -17,7 +17,10 @@ fn deleting_all_files_collapses_the_inode_tree_and_reuses_its_room() { } }); let v = block_on_ready(Volume::open(&dev)).unwrap(); - assert_eq!(v.roots().iter().find(|r| r.btree == Btree::Inodes).unwrap().level, 0, "round {round}"); + let root = v.roots().iter().find(|r| r.btree == Btree::Inodes).unwrap(); + assert_eq!(root.level, 1, "round {round}"); + let node = block_on_ready(super::super::read::read_node(&dev, v.superblock(), None, Btree::Inodes, root.level, &root.ptr, Bpos::MAX)).unwrap(); + assert_eq!(node.keys.len(), 1, "all inode siblings merged in round {round}"); assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } } @@ -67,7 +70,7 @@ fn a_change_after_sibling_bytes_are_written_survives_their_pointer_update() { } #[test] -fn sibling_merges_and_root_collapse_recover_after_torn_reordered_writes() { +fn sibling_merges_recover_after_torn_reordered_writes() { use core::cell::Cell; use toyos_blockdev::model::{Model, Overlay}; use super::crash::Watched; diff --git a/lib/bcachefs/src/engine/tests/crash.rs b/lib/bcachefs/src/engine/tests/crash.rs index 80362b37eee..1d73bfe756f 100644 --- a/lib/bcachefs/src/engine/tests/crash.rs +++ b/lib/bcachefs/src/engine/tests/crash.rs @@ -222,6 +222,10 @@ fn ops_survive_a_crash_on(image: Vec, passphrase: Option<&[u8]>, options: Mo for (i, op) in ops.iter().enumerate() { applied.set(i + 1); apply(&mount, op).await; + if matches!(op, Op::Rename(..)) { + let plans = mount.write_batch(super::super::mount::Order::OldestPin).await.unwrap(); + mount.point_batch(plans).await.unwrap(); + } if (i + 1) % every == 0 { mount.commit().await.unwrap(); committed.set(i + 1); @@ -274,7 +278,7 @@ fn encrypted_namespace_operations_survive_a_crash_at_every_request() { /// A cache whose dirty-node limit is one transaction's footprint: reclaim /// writes before every operation, so leaves fill and rewrite. -pub(super) const PRESSED: MountOptions = MountOptions { cache_nodes: 54 }; +pub(super) const PRESSED: MountOptions = MountOptions { cache_nodes: super::super::mount::MIN_CACHE }; /// `files` files made, then each renamed back and forth `rounds` times: /// the same leaves rewritten again and again, each rewrite emptying the diff --git a/lib/bcachefs/src/engine/tests/mod.rs b/lib/bcachefs/src/engine/tests/mod.rs index e945bcb5d98..3eab3bbd230 100644 --- a/lib/bcachefs/src/engine/tests/mod.rs +++ b/lib/bcachefs/src/engine/tests/mod.rs @@ -839,10 +839,9 @@ fn a_populated_volume_is_a_function_of_its_tree_random_source_and_time() { #[test] fn a_tree_past_the_device_is_refused() { - // 8 MiB: data from bucket 137, the end copy from bucket 448, one node - // per bucket. 512 buckets of data put the first node at 649; 310 put - // the 15 roots at 447–461. Each needs those and the end copy's 64. - for (bytes, blocks) in [(8 << 20, (649 + 1 + 64) * 4), (310 << 14, (461 + 1 + 64) * 4)] { + // The enlarged journal puts data at bucket 239. A rejected layout + // still reports room for its nodes and the end superblock's 64 buckets. + for (bytes, blocks) in [(8 << 20, (751 + 1 + 64) * 4), (310 << 14, (549 + 1 + 64) * 4)] { let data = vec![1u8; bytes]; let mut tree = Tree::new(); tree.file(Tree::ROOT, b"big", &data, Meta { perm: 0o644, uid: 0, gid: 0 }).unwrap(); diff --git a/lib/bcachefs/src/engine/tests/mount.rs b/lib/bcachefs/src/engine/tests/mount.rs index 5dbf0ed00a5..bc7ba017ffd 100644 --- a/lib/bcachefs/src/engine/tests/mount.rs +++ b/lib/bcachefs/src/engine/tests/mount.rs @@ -85,35 +85,21 @@ fn leaves_and_roots_split_and_their_buckets_and_accounting_derive() { assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } -/// The upstream volume left mid-write (BB-A2-1) on 64 MiB, of 32 KiB nodes, -/// mounted read-write: its window replayed past the unflushed entries after -/// it, the journal moved on by the post-crash skip with the skipped -/// sequences blacklisted, every node it writes a fresh one, then unmounted -/// clean. +/// The imported ring cannot hold the writer's worst-case reclaim shares; +/// its committed contents remain available through read-only replay. #[test] -fn the_unclean_fixture_mounts_read_write_past_the_skip_and_unmounts_clean() { +fn an_imported_journal_too_small_for_reclaim_is_read_and_refused_read_write() { let dev = fixture("unclean-32k-nodes.img.gz"); - mounted(&dev, async |m: &Mount<&Image>| { - for i in 0..20 { - m.create(m.root(), format!("toyos{i}").as_bytes(), New::File, OWNER, NOW).await.unwrap(); - } - m.commit().await.unwrap(); - }); + assert_eq!(block_on_ready(Mount::open(&dev, MEMORY, random())).err(), Some(Error::Unsupported("journal too small for a transaction and worst-case reclaim"))); let v = block_on_ready(Volume::open(&dev)).unwrap(); - let sb = v.superblock(); - assert!(sb.clean_flag()); - assert_eq!(sb.blacklist, [(19, 23 + 4096)], "F was 18, the newest entry found 22"); - assert!(sb.clean.as_ref().unwrap().journal_seq > 23 + 4096); // As upstream's own FUSE mount of the fixture reads it: f1 to f139 // whole, f140 made but empty; the rest were in the unflushed entries. let (entries, _) = block_on_ready(v.readdir(v.root(), 0, 1000)).unwrap(); - assert_eq!(entries.len(), 140 + 20 + 1, "the fixture's 140, these 20 and lost+found"); + assert_eq!(entries.len(), 140 + 1, "the fixture's 140 and lost+found"); for i in 1..=139 { assert_eq!(read_all(&v, path(&v, &format!("f{i}"))).unwrap(), format!("{i}\n").into_bytes(), "f{i}"); } assert_eq!(read_all(&v, path(&v, "f140")).unwrap(), b""); - let found = block_on_ready(fsck(&v)).unwrap(); - assert_eq!(found, UPSTREAM_ONLY, "only what upstream itself writes otherwise"); } /// The same on 32 MiB, of 16 KiB nodes, is read, its window replayed, and From 23097b9b7807a3a9230f0ba1fe6d25359c1d580d Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:15:45 +0100 Subject: [PATCH 32/93] Account retained root spines and exercise data capacity reclamation Allow four whole-range root-spine nodes per tree without assuming root height can decrease while older journal records remain replayable. Keep the non-spine occupancy bound and imported-node debt separate. Validate 64-page buffering on 64 MiB and 4 GiB geometries, sparse metadata bucket evacuation, and filling then releasing and reusing a large orphan. All 15 focused data tests pass with the corrected journal geometry. --- lib/bcachefs/src/engine/mount/data.rs | 4 +- lib/bcachefs/src/engine/tests/data.rs | 89 +++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index e6fba2ac714..f0e3e6af35c 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -229,7 +229,9 @@ impl Mount { let structural = buckets * (104 + 3 * 40) + buckets.div_ceil(256) * 296 + slots * 72 + (Btree::ALL.len() * 9 * 64) as u128; let bytes = u128::from(st.metadata_bytes) + structural + u128::from(pending_bytes); let minimum = self.minimum_fill(); - let stable = st.baseline_nodes.len() as u128 + 2 * (Btree::ALL.len() as u128 + bytes.div_ceil(minimum as u128)); + // A tree may retain its at-most-four-node whole-range root spine. + // Other interior nodes have at least two children after rebalance. + let stable = st.baseline_nodes.len() as u128 + 4 * Btree::ALL.len() as u128 + 2 * bytes.div_ceil(minimum as u128); stable.min(u128::from(u64::MAX)) as u64 } diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index 735998d66c8..e6dc057392c 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -1,5 +1,6 @@ use super::mount::{mounted, random, MEMORY, NOW, OWNER}; use super::*; +use super::super::read::Source; #[test] fn file_data_overwrites_sparse_ranges_and_truncation_survive_remount() { @@ -318,3 +319,91 @@ fn a_large_imported_extent_is_sliced_without_rewriting_its_neighbors() { assert_eq!(read_all(&v, ino).unwrap(), want); assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } + +#[test] +fn small_and_large_geometries_reserve_an_ordinary_dirty_window() { + use super::buckets::Sparse; + for bytes in [64 << 20, 4u64 << 30] { + let dev = Sparse::new(bytes); + block_on_ready(mkfs(&dev, &Options { label: b"data", time: NOW as u64, encryption: None }, &Tree::new(), &mut Seeded(7))).unwrap(); + let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); + let ino = block_on_ready(m.create(m.root(), b"f", New::File, OWNER, NOW)).unwrap().ino; + let before = m.space().1; + let mut tokens = Vec::new(); + for i in 0..64 { tokens.push(block_on_ready(m.reserve(ino, i * BLOCK as u64, BLOCK)).unwrap()); } + let reserved = before - m.space().1; + std::println!("{} MiB: pending 256 KiB consumes {reserved} reported available bytes", bytes >> 20); + assert!(reserved < 16 << 20); + for (i, token) in tokens.iter_mut().enumerate() { + block_on_ready(m.write_reserved(token, (i * BLOCK) as u64, &[i as u8; BLOCK], NOW)).unwrap(); + } + block_on_ready(m.check_metadata_budget()); + block_on_ready(m.unmount()).unwrap(); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + let got = read_all(&v, ino).unwrap(); + assert_eq!(got.len(), 64 * BLOCK); + for (i, block) in got.chunks_exact(BLOCK).enumerate() { assert_eq!(block, &[i as u8; BLOCK]); } + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); + } +} + +#[test] +fn metadata_bucket_evacuation_returns_sparse_bucket_capacity() { + use super::buckets::Sparse; + let dev = Sparse::new(4 << 30); + block_on_ready(mkfs(&dev, &Options { label: b"gc", time: NOW as u64, encryption: None }, &Tree::new(), &mut Seeded(7))).unwrap(); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + let bucket = u64::from(v.superblock().member.bucket_sectors); + let mut roots = std::collections::BTreeMap::>::new(); + for root in v.roots() { roots.entry(root.ptr.ptr.sector / bucket).or_default().push(*root); } + let (victim, sources) = roots.into_iter().max_by_key(|(_, roots)| roots.len()).unwrap(); + assert!(sources.len() >= 4); + drop(v); + let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); + for root in sources.iter().take(sources.len() / 2) { + assert!(block_on_ready(m.relocate_node(root.btree, root.level, Bpos::MAX, root.ptr.ptr.sector)).unwrap()); + } + let pos = Bpos::new(0, victim, 0); + let mut moved = 0; + while block_on_ready(m.get(Btree::Alloc, pos)).unwrap().is_some_and(|(_, v)| value::AllocV4::decode(&v).unwrap().data_type == value::data_type::BTREE) { + assert!(block_on_ready(m.compact_metadata()).unwrap(), "a sparse closed bucket has a reachable victim"); + moved += 1; + assert!(moved < 64); + } + assert!(moved > 0); + block_on_ready(m.check_metadata_budget()); + block_on_ready(m.unmount()).unwrap(); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} + +#[test] +fn a_full_data_volume_can_release_a_large_orphan_and_use_its_capacity_again() { + let dev = formatted(64 << 20, None); + let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); + let ino = block_on_ready(m.create(m.root(), b"full", New::File, OWNER, NOW)).unwrap().ino; + let start = m.space().1; + let bytes = vec![0x62; 1 << 20]; + let mut written = 0; + loop { + match block_on_ready(m.write(ino, written, &bytes, NOW)) { + Ok(n) => written += n as u64, + Err(Error::NoSpace(_)) => break, + Err(error) => panic!("fill: {error:?}"), + } + assert!(written <= 64 << 20); + } + assert!(written > 16 << 20); + block_on_ready(m.commit()).unwrap(); + block_on_ready(m.unlink(m.root(), b"full", false, NOW)).unwrap(); + block_on_ready(m.release(ino)).unwrap(); + block_on_ready(m.commit()).unwrap(); + assert!(m.space().1 > start / 2); + let ino = block_on_ready(m.create(m.root(), b"again", New::File, OWNER, NOW)).unwrap().ino; + assert_eq!(block_on_ready(m.write(ino, 0, &bytes, NOW)).unwrap(), bytes.len()); + block_on_ready(m.check_metadata_budget()); + block_on_ready(m.unmount()).unwrap(); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(read_all(&v, ino).unwrap(), bytes); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} From 8b73ea07d592a454d4613965af73ca27d93ccb36 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:19:29 +0100 Subject: [PATCH 33/93] Check fixture validation at the writer admission boundary --- lib/bcachefs/src/engine/tests/mod.rs | 5 ++--- lib/bcachefs/src/engine/tests/mount.rs | 18 +++++++----------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/lib/bcachefs/src/engine/tests/mod.rs b/lib/bcachefs/src/engine/tests/mod.rs index 3eab3bbd230..327612c7614 100644 --- a/lib/bcachefs/src/engine/tests/mod.rs +++ b/lib/bcachefs/src/engine/tests/mod.rs @@ -870,10 +870,9 @@ fn an_encrypted_volume_takes_directories_but_no_data() { } /// The superblock rules ToyOS writes by and upstream's volumes do not: -/// upstream sets inline data, discard and rotational, names the device +/// upstream sets discard and rotational, names the device /// and keeps its counters, errors and recovery-pass fields. -const UPSTREAM_ONLY: [Finding; 7] = [ - Finding::Superblock("format's feature set"), +const UPSTREAM_ONLY: [Finding; 6] = [ Finding::Superblock("member discard off"), Finding::Superblock("member not rotational, and known"), Finding::Superblock("member device name, model and serial zero"), diff --git a/lib/bcachefs/src/engine/tests/mount.rs b/lib/bcachefs/src/engine/tests/mount.rs index bc7ba017ffd..9bb4f3ca7fe 100644 --- a/lib/bcachefs/src/engine/tests/mount.rs +++ b/lib/bcachefs/src/engine/tests/mount.rs @@ -114,7 +114,7 @@ fn the_unclean_fixture_of_16_kib_nodes_is_read_and_refused_read_write() { /// What this writer does not handle in full is refused read-write by name /// (§3.1): a superblock field of a type v1.39.4 does not define, and a -/// logged operation it cannot resume. +/// malformed logged truncate it cannot safely resume. #[test] fn a_volume_this_writer_cannot_keep_is_refused_read_write() { let open = |dev: &Image| block_on_ready(Mount::open(dev, MEMORY, random())).err(); @@ -126,21 +126,19 @@ fn a_volume_this_writer_cannot_keep_is_refused_read_write() { let pos = Bpos::new(0, 0, 0); keys.push((Key::new(kind::LOGGED_OP_TRUNCATE, pos), vec![0; 24])); }); - assert_eq!(open(&dev), Some(Error::Unsupported("writing a volume with a logged operation to resume"))); + assert_eq!(open(&dev), Some(Error::Corrupt("logged truncate value"))); assert!(block_on_ready(Volume::open(&dev)).is_ok(), "still read"); } -/// A key of a type v1.39.4 does not define, met by a read while mounted, is -/// refused by name, and the mount stops writing (§3.1). +/// The mount inventories keys for its capacity ledger before writing. An +/// unknown type is refused during that walk, before the volume is writable. #[test] -fn a_key_of_an_unknown_type_met_while_mounted_stops_writing() { +fn a_key_of_an_unknown_type_is_refused_before_writing() { let dev = formatted(64 << 20, None); rewrite_leaf(&dev, Btree::Dirents, Bpos::MIN, 0, |keys| { keys.push((Key::new(kind::UNKNOWN, Bpos::new(mkfs::ROOT_INODE, 77, u32::MAX)), vec![0; 8])); }); - let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); - assert_eq!(block_on_ready(m.readdir(m.root(), 0, 10)).err(), Some(Error::Unsupported("key type v1.39.4 does not define"))); - assert_eq!(block_on_ready(m.create(m.root(), b"f", New::File, OWNER, NOW)).err(), Some(Error::ReadOnly("key type v1.39.4 does not define"))); + assert_eq!(block_on_ready(Mount::open(&dev, MEMORY, random())).err(), Some(Error::Unsupported("key type v1.39.4 does not define"))); } /// One crash state of going read-write: it opens, never as a volume marked @@ -270,9 +268,7 @@ fn a_pointer_to_a_cached_node_of_another_btree_is_refused() { let inodes = clean.body[from + 8..from + len].to_vec(); clean.body[to + 8..to + len].copy_from_slice(&inodes); }); - let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); - let made = Executor::new().block_on(async { m.create_unnamed(m.root(), OWNER, NOW).await.map(|i| i.ino) }, || panic!("stalled")); - assert_eq!(made, Err(Error::Corrupt("two btree pointers to one node"))); + assert_eq!(block_on_ready(Mount::open(&dev, MEMORY, random())).err(), Some(Error::Corrupt("btree node of another btree or level than its pointer's"))); } /// Rewrites the accounting root leaf, on an unencrypted volume, as upstream From 8655a2f4266c6e29554d6d5d8cac844a6b4ca132 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:22:12 +0100 Subject: [PATCH 34/93] Keep storage bounds and tests clean under workspace lint checks --- build/build.rs | 8 ++++---- lib/bcachefs/src/engine/mount/data.rs | 2 +- lib/bcachefs/src/engine/tests/data.rs | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build/build.rs b/build/build.rs index 9333e1cc691..a38b345e6c0 100644 --- a/build/build.rs +++ b/build/build.rs @@ -2052,7 +2052,7 @@ fn shipped_parts(root: &Path, boot: &Boot, plan: &Plan) -> (Vec, Vec, Us /// a real one. /// /// Designates the result, because every caller here is making a scratch disk -/// for a guest that expects a working `/apps` and `/home`, and the kernel will +/// for a guest that expects a working `/apps` and `/home`, and the file server will /// not format an undesignated one. Leaving it to the call sites would mean two /// places to forget; forgetting is not silent (the boot says so and both paths /// are volatile) but it is not worth the chance. @@ -2065,12 +2065,12 @@ pub fn create_sparse(path: &Path, len: u64) { } /// Partition this image as a DATA disk and stamp the designation on that -/// partition, so the kernel is allowed to format it. +/// partition, so the file server is allowed to format it. /// -/// The kernel never formats a volume that does not carry the stamp, which is +/// The file server never formats a volume that does not carry the stamp, which is /// what stops it taking the disk of any machine it is booted on. So a throwaway /// image has to say so, and this is the whole of the test harness's opt-in: -/// **data on a scratch file, not a build flag.** The kernel binary and the +/// **data on a scratch file, not a build flag.** The file-server binary and the /// code path are identical either way — `probe` runs the same match /// on metal as it does here — so the configuration under test is the /// configuration that ships, which a `#[cfg]` could not have given us. diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index f0e3e6af35c..bdda48f04e5 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -177,7 +177,7 @@ impl Mount { /// Writes as many bytes as fit. A later chunk's failure returns the /// completed byte count; `commit` is the durability barrier. pub async fn write(&self, ino: u64, offset: u64, data: &[u8], now: i64) -> Result { - offset.checked_add(data.len() as u64).filter(|n| *n <= i64::MAX as u64).ok_or(Error::Invalid("file size past 2^63-1"))?; + offset.checked_add(data.len() as u64).filter(|n| i64::try_from(*n).is_ok()).ok_or(Error::Invalid("file size past 2^63-1"))?; let _slot = self.data_slot().await; self.observe(now); let mut done = 0; diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index e6dc057392c..cf76cf4ad37 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -185,7 +185,7 @@ fn reserved_blocks_write_after_capacity_refusal_and_dropped_tokens_return_capaci }); let v = block_on_ready(Volume::open(&dev)).unwrap(); let bytes = read_all(&v, ino).unwrap(); - for (i, block) in bytes.chunks_exact(BLOCK).enumerate() { + for (i, block) in bytes.as_chunks::().0.iter().enumerate() { assert_eq!(block, &[i as u8; BLOCK]); } assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); @@ -342,7 +342,7 @@ fn small_and_large_geometries_reserve_an_ordinary_dirty_window() { let v = block_on_ready(Volume::open(&dev)).unwrap(); let got = read_all(&v, ino).unwrap(); assert_eq!(got.len(), 64 * BLOCK); - for (i, block) in got.chunks_exact(BLOCK).enumerate() { assert_eq!(block, &[i as u8; BLOCK]); } + for (i, block) in got.as_chunks::().0.iter().enumerate() { assert_eq!(block, &[i as u8; BLOCK]); } assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } } From a4e94cfc75ec0958eec794d0b657f29ca63d28d8 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:26:45 +0100 Subject: [PATCH 35/93] Exercise both bucket sizes with the reclaim journal included in ROOT size --- build/image.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/image.rs b/build/image.rs index 19e6a2fdac3..58b4727a609 100644 --- a/build/image.rs +++ b/build/image.rs @@ -999,7 +999,9 @@ mod tests { #[test] fn file_sets_across_a_bucket_size_edge_fit_and_read_back() { let mut sides = std::collections::BTreeSet::new(); - for kib in (58880..59904).step_by(256) { + // Leave room for format bookkeeping on either side; the journal's + // reclaim reserve is part of the image, not of these file bytes. + for kib in [48 << 10, 56 << 10, 60 << 10, 64 << 10] { let big: Vec = (0..kib << 10).map(|i| (i / 4096 + kib) as u8).collect(); let files: Vec<(String, Vec)> = std::iter::once(("bin/big".to_string(), big)) .chain((0..200u32).map(|i| (format!("share/s{i:03}"), vec![i as u8; 3000 + i as usize]))) From 7f52fb7347e2c2b9a6fb30a07cc776d019b6f822 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:27:16 +0100 Subject: [PATCH 36/93] Fix exported file spawning and exclusive symlink creation Keep an exported connection's ordinary DUP right so the spawn API can copy the borrowed handle before transferring it. Each standard Command spawn still exports a new connection; dup2 aliases adopt one connection once. The filesystem grant and exported descriptions do not change. Resolve the final component without following it for create_new, refusing every existing kind before opening the volume. This preserves dangling symlinks and leaves their targets absent. Update app_view to require only the capability-derived ancestor names. Extend served_file_offset with a refused kernel spawn that preserves the original export and shared offset. Existing repeated-spawn and dup2 cases cover successful transfer. --- system/fileserver/src/main.rs | 25 ++++++------------- tests/guest/rust-tests/src/bin/app_view.rs | 18 +++++++------ .../rust-tests/src/bin/served_file_offset.rs | 20 +++++++++++++++ tests/guest/rust-tests/src/bin/std_fs.rs | 2 +- 4 files changed, 39 insertions(+), 26 deletions(-) diff --git a/system/fileserver/src/main.rs b/system/fileserver/src/main.rs index 8037035b471..67dc895e9da 100644 --- a/system/fileserver/src/main.rs +++ b/system/fileserver/src/main.rs @@ -30,9 +30,9 @@ //! //! **An export is a connection this server makes to itself** through a private //! port only it holds a connector to, for its life: connect, then accept, which -//! answers exactly that connection since this loop is single-threaded. The end -//! handed out cannot be duplicated, so it has one holder. A restart takes the -//! port with it, so nothing pending reaches a new instance. +//! answers exactly that connection since this loop is single-threaded. Each +//! child gets a fresh exported connection and adopts its aliases together. A +//! restart takes the port with it, so nothing pending reaches a new instance. //! //! **A server never blocks on a client.** Accept and the first frame are two //! events; a request is buffered until whole; every reply is one `try_send`, @@ -74,7 +74,6 @@ use toyos::poller::{Poller, READABLE}; use toyos::port::{self, Acceptor}; use toyos::shm::{Bytes, SharedMemory}; use toyos::Pipe; -use toyos_abi::handle::Rights; use toyos_abi::syscall::{SyscallError, MAX_BADGE, PIPE_CAPACITY, SERVE_PREFIX}; /// What one turn of a stream appends at most. @@ -88,11 +87,6 @@ const STREAM_PRESENT: usize = PIPE_CAPACITY / STREAM_READ; /// The name the private export port has in this server's own namespace. const EXPORT_PORT: &str = "export"; -/// What the end an export hands out carries: a connection's rights without -/// `DUP`, so whoever holds it is its one holder, and may pass it on. -const EXPORTED: Rights = - Rights::TRANSFER.union(Rights::WAIT).union(Rights::READ).union(Rights::WRITE).union(Rights::MAP); - // One wait watches the acceptor, every connection and every stream at once. const _: () = assert!(1 + MAX_SERVED + MAX_HANDSHAKES + MAX_STREAMS <= Poller::MAX_HANDLES as usize); @@ -216,15 +210,12 @@ impl Exports { Self { acceptor, names } } - /// A connection to this server: the end to hand away, narrowed to - /// [`EXPORTED`], and this server's. + /// A connection to this server: the end to hand away and this server's. + /// The exported end keeps `DUP` for spawning from a borrowed handle. fn connect(&self) -> Result<(Connection, Connection), Error> { let theirs = self.names.open(EXPORT_PORT).map_err(|e| Error::kernel("an export's connection", e))?; let ours = self.acceptor.accept().unwrap_or_else(|e| panic!("fileserver: the export port refused an accept ({e:?})")); - let narrowed = toyos::AsHandle::as_handle(&theirs) - .try_clone_narrowed(EXPORTED) - .map_err(|e| Error::kernel("an export's narrowed end", e))?; - Ok((Connection::received(narrowed), ours)) + Ok((theirs, ours)) } } @@ -1105,7 +1096,7 @@ impl Server { return Err(Error::InvalidRequest); } let rel = self.path(id, r)?; - let path = match self.resolve(id, r.fid, &rel, true)? { + let path = match self.resolve(id, r.fid, &rel, r.flags & O_CREATE_NEW == 0)? { Resolved::Absolute(p) => return self.follow(id, r, &p), Resolved::Path(p) => p, }; @@ -1119,7 +1110,7 @@ impl Server { Err(e) => return Err(e), }; let (held, meta) = match found { - Some(Kind::Dir) if r.flags & O_CREATE_NEW != 0 => return Err(Error::AlreadyExists), + Some(_) if r.flags & O_CREATE_NEW != 0 => return Err(Error::AlreadyExists), Some(Kind::Dir) if writes => return Err(Error::IsADirectory), Some(Kind::Dir) => (Held::Dir { path: path.clone(), removed: false }, self.volume.lstat(&path)?), _ if r.flags & O_DIRECTORY != 0 => return Err(if found.is_some() { Error::NotADirectory } else { Error::NotFound }), diff --git a/tests/guest/rust-tests/src/bin/app_view.rs b/tests/guest/rust-tests/src/bin/app_view.rs index 02e2080dcef..9524e78fe23 100644 --- a/tests/guest/rust-tests/src/bin/app_view.rs +++ b/tests/guest/rust-tests/src/bin/app_view.rs @@ -230,14 +230,16 @@ fn app() { Ok(bytes) => red.push(format!("{file} read {} bytes", bytes.len())), } } - // A directory no capability names is a mount point of the kernel's, and - // lists nothing of what the file server holds under it. - for dir in ["/apps", "/home", "/home/toy", "/home/toy/Apps", "/config", "/state", "/log"] { - if let Ok(listing) = fs::read_dir(dir) { - let names: Vec<_> = listing.filter_map(Result::ok).map(|e| e.file_name()).collect(); - if !names.is_empty() { - red.push(format!("{dir} lists {names:?}")); - } + // Synthetic ancestors expose only paths to the capabilities held here. + for (dir, expected) in [("/apps", "appview"), ("/home", "toy"), ("/home/toy", "Apps"), ("/home/toy/Apps", "appview")] { + match fs::read_dir(dir).and_then(|listing| listing.map(|entry| entry.map(|e| e.file_name())).collect::, _>>()) { + Ok(names) if names == [std::ffi::OsStr::new(expected)] => {} + other => red.push(format!("{dir} did not list only {expected}: {other:?}")), + } + } + for dir in ["/config", "/state", "/log"] { + if fs::read_dir(dir).is_ok() { + red.push(format!("{dir} is visible without a granted descendant")); } } diff --git a/tests/guest/rust-tests/src/bin/served_file_offset.rs b/tests/guest/rust-tests/src/bin/served_file_offset.rs index 319b8e5b64c..c3370b9d541 100644 --- a/tests/guest/rust-tests/src/bin/served_file_offset.rs +++ b/tests/guest/rust-tests/src/bin/served_file_offset.rs @@ -69,6 +69,7 @@ fn main() { assert_eq!(appender.stream_position().unwrap(), 12, "an append leaves the offset at the end"); drop(appender); assert_eq!(fs::read(FILE).unwrap(), b"01ab456789xy"); + refused_spawn_keeps_export(f.as_raw_fd()); // The child writes `CHILD\n` at the parent's offset, 4, and the parent // its `!` after it, over the `x`. @@ -108,6 +109,25 @@ fn main() { println!("served_file_offset: passed"); } +fn refused_spawn_keeps_export(fd: i32) { + use toyos::AsHandle; + let exported = toyos::files::export(fd).expect("export the shared description"); + let fds = [(1, exported.as_handle(), toyos::fd::Kind::File)]; + let request = toyos::spawn::Spawn { + program: "/system/bin/toybox", image: None, image_held: false, + argv: b"toybox\0", env: &[], cwd: "/", fds: &fds, + endow: Vec::new(), provided: Vec::new(), parent: toyos::spawn::Parent::Caller, + independent: false, launcher: None, pager: None, + }; + let toyos::spawn::Start::Helper(helper) = request.start().expect("a helper") else { panic!("a helper became a launch") }; + // No image: the kernel refuses after the SDK duplicated the endowments. + assert!(matches!(helper.spawn(None), Err(toyos::spawn::Refused::Syscall(toyos_abi::syscall::SyscallError::InvalidArgument)))); + let mut dir = toyos::fs::Dir::adopt(toyos::ipc::Connection::received(exported)).expect("refusal kept the original export"); + let fid = toyos::fs::Fid { id: 1, generation: dir.generation() }; + assert_eq!(dir.seek(fid, toyos::fs::SEEK_CURRENT, 0).unwrap(), 4, "refusal kept the shared offset"); + dir.close(fid); +} + #[repr(C)] struct FileActions { actions: usize, reserved: [usize; 7] } diff --git a/tests/guest/rust-tests/src/bin/std_fs.rs b/tests/guest/rust-tests/src/bin/std_fs.rs index 80f2faf1b90..4570e715ef0 100644 --- a/tests/guest/rust-tests/src/bin/std_fs.rs +++ b/tests/guest/rust-tests/src/bin/std_fs.rs @@ -48,7 +48,7 @@ fn the_empty_path_names_nothing() { } } -/// `create_new` on the kernel's `/tmp` makes a file once and refuses the name +/// `create_new` on `/tmp` makes a file once and refuses the name /// after, as a lock file needs. fn create_new_is_exclusive_on_tmp() { let path = "/tmp/std_fs_create_new"; From 30be18ad5b1870337de7b492a45eb3db0e47e55d Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:35:22 +0100 Subject: [PATCH 37/93] Test cancelled writes without assuming metadata reclaim leaves space unchanged --- system/fileserver/src/pages/tests.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index 130893e490c..aa0f13ce5e5 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -606,8 +606,13 @@ fn a_write_refused_in_a_hole_gives_back_the_block_it_took() { files.sync(&mut v).unwrap(); let room = v.room().unwrap().free; fake.0.borrow_mut().refusing = true; - assert!(files.write(&mut v, id, node, PAGE, &pattern(PAGE as usize, 5)).is_err()); - assert_eq!(v.room().unwrap().free, room, "the refused write kept the hole's block, or gave back more"); + for _ in 0..2 { + assert_eq!(files.write(&mut v, id, node, PAGE, &pattern(PAGE as usize, 5)), Err(Error::Io)); + assert_eq!(files.sync(&mut v).unwrap(), [], "a cancelled write left capacity awaiting bytes"); + } + // Reservation admission may compact older metadata, increasing free + // space. It must not keep capacity for either refused write. + assert!(v.room().unwrap().free >= room, "the refused writes kept capacity"); assert_eq!(v.node_meta(node).unwrap().size, 3 * PAGE); fake.0.borrow_mut().refusing = false; let mut out = vec![0xEEu8; PAGE as usize]; From 3b5b1e018b8e0e33bcf67fd800917d4097bf9e95 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:42:19 +0100 Subject: [PATCH 38/93] Evacuate sparse data buckets without changing logical extent ranges Repeated 32 KiB writes followed by 4 KiB truncation stranded USER buckets and exhausted a 64 MiB volume with only 4.3 MiB live. Pack live sectors into shared checksum ranges, keeping each extent's range and encoded value no larger. Publish replacements in bounded freeing transactions after data completion; preserve reader epochs, pinned unpublished ranges, and fresh durable encryption versions. Run sparse metadata evacuation at the physical low-water mark without recursing into itself. Count live btree nodes from backpointers and original transaction deltas for the pending reclaim-generation admission work. Derive near-full fixtures from native reserve requirements. All 203 local library tests and no_std clippy pass. Disabling USER evacuation makes the retention regression fail; restoring it passes. Torn/reordered plaintext and authenticated packing tests permit only torn-copy findings before recovery and require a clean check afterward. The shared recursive metadata frontier remains a separate integration dependency; this commit does not establish that final bound. --- lib/bcachefs/src/engine/mount/allocator.rs | 30 ++++-- lib/bcachefs/src/engine/mount/data.rs | 82 ++++++++++---- lib/bcachefs/src/engine/mount/gc.rs | 118 +++++++++++++++++++++ lib/bcachefs/src/engine/mount/mod.rs | 18 +++- lib/bcachefs/src/engine/mount/reclaim.rs | 8 +- lib/bcachefs/src/engine/tests/buckets.rs | 18 ++-- lib/bcachefs/src/engine/tests/crash.rs | 6 +- lib/bcachefs/src/engine/tests/data.rs | 74 +++++++++++++ 8 files changed, 311 insertions(+), 43 deletions(-) diff --git a/lib/bcachefs/src/engine/mount/allocator.rs b/lib/bcachefs/src/engine/mount/allocator.rs index cb0e9b65db9..7f1bfa46648 100644 --- a/lib/bcachefs/src/engine/mount/allocator.rs +++ b/lib/bcachefs/src/engine/mount/allocator.rs @@ -35,20 +35,25 @@ pub(super) fn charge(btree: Btree, value: &[u8]) -> u64 { pub(super) struct Metadata { pub bytes: u64, pub buckets: u64, + pub nodes: u64, pub baseline: BTreeSet, pub sparse: BTreeSet, + pub sparse_data: BTreeSet, } pub(super) async fn metadata_budget(source: &S) -> Result { let mut bytes = 0u64; let roots: BTreeSet = Btree::ALL.iter().filter_map(|&b| source.root_of(b).map(|r| r.ptr.ptr.sector)).collect(); let mut baseline = BTreeSet::new(); + let mut nodes = 0; let mut sparse = BTreeSet::new(); + let mut sparse_data = BTreeSet::new(); let shift = match source.superblock().flag(6, 40, 8) { 0 => 10, value => value as u32 }; for btree in Btree::ALL { source.walk(btree, Bpos::MIN, |key, value| { bytes += charge(btree, value); if btree == Btree::Backpointers && value.get(2) == Some(&data_type::BTREE) { + nodes += 1; let sector = key.pos.offset >> shift; if !roots.contains(§or) { baseline.insert(sector); } } @@ -57,6 +62,9 @@ pub(super) async fn metadata_budget(source: &S) -> Result { if bucket.data_type == data_type::BTREE && u64::from(bucket.dirty) <= u64::from(source.superblock().member.bucket_sectors) / 2 { sparse.insert(key.pos.offset); } + if bucket.data_type == data_type::USER && u64::from(bucket.dirty) <= u64::from(source.superblock().member.bucket_sectors) / 2 { + sparse_data.insert(key.pos.offset); + } } Ok(true) }).await?; @@ -66,7 +74,7 @@ pub(super) async fn metadata_budget(source: &S) -> Result { Some((_, value)) => journal::counters(&value)?.first().copied().unwrap_or(0), None => 0, }; - Ok(Metadata { bytes, buckets, baseline, sparse }) + Ok(Metadata { bytes, buckets, nodes, baseline, sparse, sparse_data }) } /// What a transaction's nodes may take of the room left. @@ -144,13 +152,21 @@ impl Allocator { pub(super) fn capacity(&self, metadata_buckets: u64) -> u64 { let pinned = self.pins.values().filter(|(_, empty)| empty.is_some()).count() as u64; - (self.free + self.returning.values().sum::() - pinned + metadata_buckets + self.metadata_taken.len() as u64) * self.per_bucket + (self.free + self.returning.values().sum::() - pinned) * self.per_bucket + self.occupied_metadata(metadata_buckets) + } + + pub(super) fn occupied_metadata(&self, metadata_buckets: u64) -> u64 { + (metadata_buckets + self.metadata_taken.len() as u64) * self.per_bucket } pub(super) fn metadata_open(&self, bucket: u64) -> bool { self.open.is_some_and(|(b, _, _)| b == bucket) } + pub(super) fn movable_data(&self, bucket: u64) -> bool { + !self.pins.contains_key(&bucket) && self.data.is_none_or(|(b, _, _)| b != bucket) + } + pub(super) fn stable_slots(&self, nodes: u64) -> u64 { // A closed bucket that cannot be evacuated is more than half // full. One open bucket may be arbitrarily empty. @@ -216,9 +232,9 @@ impl Allocator { self.data = Some((b, 0, gen)); } - pub(super) fn next_data(&mut self, bucket_sectors: u64, sectors: u64) -> Option<(u64, u8)> { - if let Some((_, sector)) = self.holes.pop_first() { - return Some((sector, 0)); + pub(super) fn next_data(&mut self, bucket_sectors: u64, sectors: u64, fixed: bool) -> Option<(u64, u8)> { + if fixed { + if let Some((_, sector)) = self.holes.pop_first() { return Some((sector, 0)); } } let (b, next, gen) = self.data?; if sectors > bucket_sectors - next { @@ -229,8 +245,8 @@ impl Allocator { Some((b * bucket_sectors + next, gen)) } - pub(super) fn data_fits(&self, nodes: usize, reclaim_room: usize) -> bool { - let need = Self::transient(nodes, reclaim_room, Draw::Ordinary); + pub(super) fn data_fits(&self, nodes: usize, reclaim_room: usize, draw: Draw) -> bool { + let need = Self::transient(nodes, reclaim_room, draw); need + self.per_bucket <= self.slots() } diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index bdda48f04e5..785f1d52621 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -93,11 +93,12 @@ impl Mount { return Err(Error::Invalid("reservation outside one filesystem block")); } let _data = self.data_slot().await; - let id = self + let id = loop { + let out = self .transact(K_TX, FOOTPRINT, Draw::Ordinary, async |txn: &mut Txn| { regular(&self.inode_in(txn, ino).await?)?; self.block_extents(ino, first, first + bytes as u64).await?; - let (sector, _) = self.data_sector(self.sb.block_bytes() as u64 / SECTOR as u64).await?; + let (sector, _) = self.data_sector(self.sb.block_bytes() as u64 / SECTOR as u64, Draw::Ordinary, true).await?; self.st.borrow_mut().alloc.pin(sector / u64::from(self.sb.member.bucket_sectors)); let mut pool = self.reservations.borrow_mut(); let id = pool.next; @@ -105,7 +106,12 @@ impl Mount { pool.active.insert(id, ReservedBlock { ino, sector }); Ok(id) }) - .await?; + .await; + match out { + Err(Error::NoSpace(_)) if self.compact_user().await? => continue, + other => break other?, + } + }; Ok(Reservation { owner: Rc::clone(&self.reservations), id, first, bytes }) } @@ -187,6 +193,14 @@ impl Mount { Ok(version) => self.write_chunk(ino, offset + done as u64, &data[done..], now, version, None).await, Err(e) => Err(e), }; + let out = match out { + Err(error @ Error::NoSpace(_)) => match self.compact_user().await { + Ok(true) => continue, + Ok(false) => Err(error), + Err(error) => Err(error), + }, + other => other, + }; match out { Ok(n) => done += n, Err(_) if done > 0 => return Ok(done), @@ -239,13 +253,30 @@ impl Mount { pub(crate) async fn check_metadata_budget(&self) { let metadata = allocator::metadata_budget(self).await.unwrap(); let st = self.st.borrow(); - assert_eq!((st.metadata_bytes, st.metadata_buckets), (metadata.bytes, metadata.buckets)); + assert_eq!((st.metadata_bytes, st.metadata_buckets, st.btree_nodes), (metadata.bytes, metadata.buckets, metadata.nodes)); + assert!(st.btree_nodes <= st.alloc.occupied_metadata(st.metadata_buckets)); assert_eq!(st.sparse_metadata, metadata.sparse); + assert_eq!(st.sparse_data, metadata.sparse_data); + } + + #[cfg(test)] + pub(crate) fn test_reserve_buckets(&self, extra_metadata: u64) -> u64 { + let st = self.st.borrow(); + let stable = st.alloc.stable_slots(self.metadata_nodes(&st, extra_metadata + K_TX * 8)); + let transient = Allocator::transient(FOOTPRINT, st.reclaim_room, Draw::Ordinary); + let per_bucket = u64::from(self.sb.member.bucket_sectors) / u64::from(self.sb.node_sectors()); + (stable + transient + (reclaim::COMMON_REWRITE * FOOTPRINT) as u64).div_ceil(per_bucket) + } + + #[cfg(test)] + pub(crate) async fn test_compact_user(&self) -> Result { + let _data = self.data_slot().await; + self.compact_user().await } /// The high-water mark is durable before any version beneath it is /// used, including a ciphertext write whose metadata never commits. - async fn data_version(&self) -> Result { + pub(super) async fn data_version(&self) -> Result { if self.key.is_none() { return Ok(Version::default()); } @@ -299,24 +330,24 @@ impl Mount { .await } - async fn data_sector(&self, sectors: u64) -> Result<(u64, u8)> { + pub(super) async fn data_sector(&self, sectors: u64, draw: Draw, fixed: bool) -> Result<(u64, u8)> { let bucket = u64::from(self.sb.member.bucket_sectors); if sectors > bucket { return Err(Error::Unsupported("data extent larger than a bucket")); } - if let Some(at) = self.st.borrow_mut().alloc.next_data(bucket, sectors) { + if let Some(at) = self.st.borrow_mut().alloc.next_data(bucket, sectors, fixed) { return Ok(at); } { let st = self.st.borrow(); let held = st.cache.dirty + st.cache.unpointed + st.granted; let stable = self.metadata_nodes(&st, st.granted_bytes + self.reservations.borrow().pending(Draw::Ordinary)); - if !st.alloc.data_fits(held, st.reclaim_room) || !st.alloc.keeps(st.metadata_buckets, stable, held, st.reclaim_room, Draw::Ordinary, true) { + if !st.alloc.data_fits(held, st.reclaim_room, draw) || !st.alloc.keeps(st.metadata_buckets, stable, held, st.reclaim_room, draw, true) { return Err(Error::NoSpace("no data bucket beyond metadata and freeing reserves")); } } self.take_bucket(true).await?; - Ok(self.st.borrow_mut().alloc.next_data(bucket, sectors).expect("a fresh bucket holds the extent")) + Ok(self.st.borrow_mut().alloc.next_data(bucket, sectors, fixed).expect("a fresh bucket holds the extent")) } async fn block_extents(&self, ino: u64, first: u64, end: u64) -> Result)>> { @@ -367,7 +398,7 @@ impl Mount { let physical = (blocks * BLOCK / SECTOR) as u32; let sector = match reserved { Some(sector) => sector, - None => self.data_sector(u64::from(physical)).await?.0, + None => self.data_sector(u64::from(physical), Draw::Ordinary, true).await?.0, }; if reserved.is_none() { self.st.borrow_mut().alloc.pin(sector / u64::from(self.sb.member.bucket_sectors)); @@ -375,17 +406,7 @@ impl Mount { } let gen = self.bucket(txn, sector / u64::from(self.sb.member.bucket_sectors)).await?.reused_gen(); let ptr = Ptr { sector, dev: 0, gen, cached: false, unwritten: false }; - let value = if let Some(master) = &self.key { - let nonce = Nonce::extent(version.hi, version.lo, 0); - master.xor(nonce, data.as_flattened_mut())?; - let tag = master.mac(nonce, data.as_flattened())?; - Extent::authenticated(physical, tag, self.sb.flag(1, 9, 1) != 0, ptr) - } else { - if physical > 512 { - return Err(Error::Unsupported("unencrypted extent past crc64 maximum")); - } - Extent::encode(physical, csum_type::CRC32C, crate::crc32c::crc32c_zero(data.as_flattened()), ptr) - }; + let value = self.encode_data(&mut data, version, ptr, self.sb.flag(1, 9, 1) != 0)?; let pos = Bpos::new(ino, (first + bytes.len() as u64) / SECTOR as u64, ROOT_SNAPSHOT); let key = Key { size: (bytes.len() / SECTOR) as u32, version, ..Key::new(kind::EXTENT, pos) }; self.reissue("file data write failed", async || { @@ -398,7 +419,22 @@ impl Mount { self.replace_data_keys(txn, inode, old, retained).await } - async fn replace_data_keys(&self, txn: &mut Txn, inode: &mut InodeValue, old: Vec<(Key, Vec)>, new: Vec<(Key, Vec)>) -> Result<()> { + pub(super) fn encode_data(&self, data: &mut [[u8; BLOCK]], version: Version, ptr: Ptr, wide: bool) -> Result> { + let physical = (data.len() * BLOCK / SECTOR) as u32; + Ok(if let Some(master) = &self.key { + let nonce = Nonce::extent(version.hi, version.lo, 0); + master.xor(nonce, data.as_flattened_mut())?; + let tag = master.mac(nonce, data.as_flattened())?; + Extent::authenticated(physical, tag, wide, ptr) + } else { + if physical > 512 { + return Err(Error::Unsupported("unencrypted extent past crc64 maximum")); + } + Extent::encode(physical, csum_type::CRC32C, crate::crc32c::crc32c_zero(data.as_flattened()), ptr) + }) + } + + pub(super) async fn replace_data_keys(&self, txn: &mut Txn, inode: &mut InodeValue, old: Vec<(Key, Vec)>, new: Vec<(Key, Vec)>) -> Result<()> { let mut buckets = BTreeMap::new(); for (k, v) in old { self.data_counted(txn, &mut buckets, &k, &v, false).await?; @@ -575,7 +611,7 @@ impl Mount { } } -fn checked_extent(v: &[u8]) -> Result { +pub(super) fn checked_extent(v: &[u8]) -> Result { let extent = Extent::decode(v)?; let crc = extent.crc.ok_or(Error::Unsupported("modifying data without a checksum entry"))?; if extent.ptr.dev != 0 || extent.ptr.cached || extent.ptr.unwritten || crc.compression != 0 || crc.compressed != crc.uncompressed { diff --git a/lib/bcachefs/src/engine/mount/gc.rs b/lib/bcachefs/src/engine/mount/gc.rs index 334bfaae5d5..6d6f0b2a9a3 100644 --- a/lib/bcachefs/src/engine/mount/gc.rs +++ b/lib/bcachefs/src/engine/mount/gc.rs @@ -1,9 +1,127 @@ //! Evacuate sparse closed metadata buckets through ordinary node publication. use super::*; +use super::super::value::{csum_type, Extent}; + +struct Evacuating<'a>(&'a Cell); +impl Drop for Evacuating<'_> { + fn drop(&mut self) { self.0.set(false); } +} + +struct DataRange<'a, D> { + mount: &'a Mount, + sector: u64, + published: bool, +} + +impl Drop for DataRange<'_, D> { + fn drop(&mut self) { + let bucket = u64::from(self.mount.sb.member.bucket_sectors); + let mut st = self.mount.st.borrow_mut(); + if self.published { st.alloc.unpin(self.sector / bucket); } else { st.alloc.cancel_data(self.sector, bucket); } + } +} impl Mount { + /// Called with the file-data slot held, before an ordinary allocation + /// is accepted. Live bytes are packed; key ranges and counts stay fixed. + pub(super) async fn compact_user(&self) -> Result { + let candidate = { + let st = self.st.borrow(); + st.sparse_data.iter().copied().find(|&b| st.alloc.movable_data(b)) + }; + let Some(victim) = candidate else { return Ok(false); }; + let bucket = u64::from(self.sb.member.bucket_sectors); + let first = (victim * bucket) << self.bp_shift(); + let end = ((victim + 1) * bucket) << self.bp_shift(); + let mut positions = Vec::new(); + Source::walk(self, Btree::Backpointers, Bpos::new(0, first, 0), |key, value| { + if key.pos.inode != 0 || key.pos.offset >= end { return Ok(false); } + if value.len() != value::BACKPOINTER_BYTES || value[0] != Btree::Extents as u8 || value[1] != 0 || value[2] != data_type::USER { + return Err(Error::Unsupported("evacuating data other than regular extents")); + } + positions.push((Bpos::decode(&value[12..32]), key.pos.offset, value[3], super::super::le32(value, 8))); + Ok(true) + }).await?; + let mut entries = Vec::new(); + for (pos, bp, gen, size) in positions { + let (key, value) = Source::get(self, Btree::Extents, pos).await?.ok_or(Error::Corrupt("data backpointer has no extent"))?; + if key.kind != kind::EXTENT || key.size == 0 || pos.snapshot != ROOT_SNAPSHOT { return Err(Error::Unsupported("evacuating a non-live extent")); } + let extent = data::checked_extent(&value)?; + if extent.ptr.sector / bucket != victim || (extent.ptr.sector << self.bp_shift()) + u64::from(extent.crc.unwrap().offset) != bp || extent.ptr.gen != gen || key.size != size { + return Err(Error::Corrupt("data backpointer disagrees with its extent")); + } + let bytes = key.size as usize * SECTOR; + if bytes > 256 << 10 { return Err(Error::Unsupported("evacuating an extent beyond the encoded-size limit")); } + let mut live = alloc::vec![0u8; bytes]; + // The data slot excludes writes, truncation and final release + // until these bytes have been copied and their pointers moved. + Source::extent(self, &key, &value, 0, &mut live).await?; + entries.push((key, value, live)); + } + if entries.is_empty() { return Err(Error::Corrupt("nonempty data bucket has no backpointers")); } + let chunk = (64 << 10).max(self.sb.block_bytes()); + let mut begin = 0; + while begin < entries.len() { + let mut end = begin + 1; + let mut bytes = entries[begin].2.len(); + while end < entries.len() && bytes + entries[end].2.len() <= chunk { + bytes += entries[end].2.len(); + end += 1; + } + let version = self.data_version().await?; + let physical = bytes.div_ceil(self.sb.block_bytes()) * self.sb.block_bytes(); + let mut blocks = alloc::vec![[0u8; BLOCK]; physical / BLOCK]; + let mut at = 0; + for (_, _, live) in &entries[begin..end] { + blocks.as_flattened_mut()[at..at + live.len()].copy_from_slice(live); + at += live.len(); + } + let (sector, gen) = self.transact(K_TX, FOOTPRINT, Draw::Freeing, async |txn: &mut Txn| { + let sector = self.data_sector((physical / SECTOR) as u64, Draw::Freeing, false).await?.0; + let gen = self.bucket(txn, sector / bucket).await?.reused_gen(); + self.st.borrow_mut().alloc.pin(sector / bucket); + Ok((sector, gen)) + }).await?; + let mut range = DataRange { mount: self, sector, published: false }; + let ptr = Ptr { sector, gen, dev: 0, cached: false, unwritten: false }; + let encoded = self.encode_data(&mut blocks, version, ptr, true)?; + let crc = Extent::decode(&encoded)?.crc.unwrap(); + let mut replacements = Vec::new(); + let mut offset = 0; + for (old, value, live) in &entries[begin..end] { + let mut crc = crc; + crc.offset = offset; + if self.key.is_some() && data::checked_extent(value)?.crc.unwrap().csum_type == csum_type::CHACHA_POLY_80 { + crc.csum_type = csum_type::CHACHA_POLY_80; + crc.csum_len = 10; + } + let value_new = Extent::with_crc(&crc, ptr); + if value_new.len() > value.len() { return Err(Error::Unsupported("evacuation would enlarge an extent record")); } + replacements.push((Key { version, ..*old }, value_new)); + offset += (live.len() / SECTOR) as u32; + } + self.reissue("data evacuation write failed", async || { + self.dev.write_data(sector * SECTOR as u64 / BLOCK as u64, &blocks).await?; + Ok(()) + }).await?; + self.st.borrow_mut().clocks[1] += (physical / SECTOR) as u64; + for ((old, value, _), new) in entries[begin..end].iter().zip(replacements) { + self.transact(K_TX, FOOTPRINT, Draw::Freeing, async |txn: &mut Txn| { + let mut inode = self.inode_in(txn, old.pos.inode).await?; + self.replace_data_keys(txn, &mut inode, alloc::vec![(*old, value.clone())], alloc::vec![new]).await + }).await?; + range.published = true; + } + begin = end; + } + self.commit().await?; + Ok(true) + } + pub(crate) async fn compact_metadata(&self) -> Result { + if self.metadata_gc.replace(true) { return Ok(false); } + let _evacuating = Evacuating(&self.metadata_gc); let candidates: Vec<_> = { let st = self.st.borrow(); st.sparse_metadata.iter().copied().filter(|&b| !st.alloc.metadata_open(b)).collect() diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 2d00acac610..a1c47d6fb67 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -125,7 +125,9 @@ struct State { next_version: u64, metadata_bytes: u64, metadata_buckets: u64, + btree_nodes: u64, sparse_metadata: BTreeSet, + sparse_data: BTreeSet, baseline_nodes: BTreeSet, granted_bytes: u64, clocks: [u64; 2], @@ -183,6 +185,7 @@ pub struct Mount { data_held: Cell, data_queue: Queue, reservations: Rc>, + metadata_gc: Cell, root: u64, } @@ -436,7 +439,9 @@ impl Mount { next_version: key_version, metadata_bytes: metadata.bytes, metadata_buckets: metadata.buckets, + btree_nodes: metadata.nodes, sparse_metadata: metadata.sparse, + sparse_data: metadata.sparse_data, baseline_nodes: metadata.baseline, granted_bytes: 0, clocks, @@ -472,6 +477,7 @@ impl Mount { data_held: Cell::new(false), data_queue: Queue::default(), reservations: Rc::default(), + metadata_gc: Cell::new(false), root, }; mount.go_read_write(restate).await?; @@ -1124,6 +1130,11 @@ impl Mount { } else { st.sparse_metadata.remove(&b); } + if after.data_type == data_type::USER && u64::from(after.dirty) <= u64::from(self.sb.member.bucket_sectors) / 2 { + st.sparse_data.insert(b); + } else { + st.sparse_data.remove(&b); + } } for (&(btree, level, pos), update) in &txn.updates { if level != 0 { continue; } @@ -1131,8 +1142,11 @@ impl Mount { let prior = node.keys.get(node.at(pos)).filter(|(k, _)| k.pos == pos); let charge = |entry: Option<&(Key, Vec)>| entry.map_or(0, |(_, v)| allocator::charge(btree, v)); st.metadata_bytes = st.metadata_bytes + charge(update.as_ref()) - charge(prior); - if btree == Btree::Backpointers && update.is_none() && prior.is_some_and(|(_, value)| value.get(2) == Some(&data_type::BTREE)) { - st.baseline_nodes.remove(&(pos.offset >> self.bp_shift())); + if btree == Btree::Backpointers { + let before = prior.is_some_and(|(_, value)| value.get(2) == Some(&data_type::BTREE)); + let after = update.as_ref().is_some_and(|(_, value)| value.get(2) == Some(&data_type::BTREE)); + st.btree_nodes = st.btree_nodes + u64::from(after) - u64::from(before); + if before && !after { st.baseline_nodes.remove(&(pos.offset >> self.bp_shift())); } } } for (&pos, delta) in &txn.accounting { diff --git a/lib/bcachefs/src/engine/mount/reclaim.rs b/lib/bcachefs/src/engine/mount/reclaim.rs index e7b0965519b..04d38904c7d 100644 --- a/lib/bcachefs/src/engine/mount/reclaim.rs +++ b/lib/bcachefs/src/engine/mount/reclaim.rs @@ -436,10 +436,14 @@ impl Mount { async fn make_room(&self) -> Result<()> { loop { let blocked = self.readers.borrow_mut().blocked(); - let (ready, returning) = { + let (ready, returning, low) = { let st = self.st.borrow(); - (st.alloc.ready(blocked), st.alloc.returning()) + (st.alloc.ready(blocked), st.alloc.returning(), Allocator::transient(st.cache.dirty + st.cache.unpointed + st.granted, st.reclaim_room, Draw::Freeing)) }; + if !self.metadata_gc.get() && ready < low && Box::pin(self.compact_metadata()).await? { + self.flush_open().await?; + continue; + } if ready >= BATCH_SLOTS { return Ok(()); } diff --git a/lib/bcachefs/src/engine/tests/buckets.rs b/lib/bcachefs/src/engine/tests/buckets.rs index 39f1f8b24ae..3d2bc2ab0e3 100644 --- a/lib/bcachefs/src/engine/tests/buckets.rs +++ b/lib/bcachefs/src/engine/tests/buckets.rs @@ -149,9 +149,15 @@ fn empty_buckets(v: &Volume) -> u64 { .sum() } -/// A 64 MiB volume whose one file leaves about `free` buckets empty. -pub(super) fn nearly_full(data: &mut Vec, free: u64) -> Image { - let v = block_on_ready(Volume::open(formatted(64 << 20, None))).unwrap(); +/// The native admission reserve, plus one transaction's rewrite footprint. +/// The populated file needs at most one extent and backpointer per bucket; +/// count their bytes twice to cover both occupancy and imported-node debt. +pub(super) fn nearly_full(data: &mut Vec) -> Image { + let blank = formatted(64 << 20, None); + let m = block_on_ready(Mount::open(&blank, MEMORY, random())).unwrap(); + let free = m.test_reserve_buckets(super::read::Source::superblock(&m).member.nbuckets * 2 * 128); + block_on_ready(m.unmount()).unwrap(); + let v = block_on_ready(Volume::open(blank)).unwrap(); let (empty, bucket) = (empty_buckets(&v), u64::from(v.superblock().member.bucket_sectors) * SECTOR as u64); data.resize(((empty - free) * bucket) as usize * 63 / 64, 7); let mut tree = Tree::new(); @@ -171,7 +177,7 @@ pub(super) fn nearly_full(data: &mut Vec, free: u64) -> Image { #[test] fn a_full_volume_refuses_changes_by_name_and_still_removes_names_and_unmounts_clean() { let mut data = Vec::new(); - let dev = nearly_full(&mut data, 400); + let dev = nearly_full(&mut data); let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); let stalled = || panic!("stalled; {}", m.dump()); let made = Executor::new().block_on( @@ -201,7 +207,7 @@ fn a_full_volume_refuses_changes_by_name_and_still_removes_names_and_unmounts_cl }, stalled, ); - assert!(made > 1_000, "{made} creates filled the volume"); + assert!(made > 21, "the capacity refusal follows enough creates to exercise removals: {made}"); block_on_ready(m.unmount()).unwrap(); let v = block_on_ready(Volume::open(&dev)).unwrap(); let (entries, more) = block_on_ready(v.readdir(v.root(), 0, usize::MAX)).unwrap(); @@ -242,7 +248,7 @@ pub(super) fn reclaim_all(m: &Mount) -> Vec { #[test] fn a_full_volume_crashed_mounts_read_write_and_still_removes_names() { let mut data = Vec::new(); - let dev = nearly_full(&mut data, 400); + let dev = nearly_full(&mut data); let made = { let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); let made = Executor::new().block_on( diff --git a/lib/bcachefs/src/engine/tests/crash.rs b/lib/bcachefs/src/engine/tests/crash.rs index 1d73bfe756f..df14162ba86 100644 --- a/lib/bcachefs/src/engine/tests/crash.rs +++ b/lib/bcachefs/src/engine/tests/crash.rs @@ -292,15 +292,15 @@ fn renames(files: usize, rounds: usize) -> Vec { ops } -/// Allocation pressure: a volume a file has filled to a few hundred empty -/// buckets, its leaves rewritten until emptied buckets are taken again, a +/// Allocation pressure: a file leaves the native rewrite reserve and one +/// transaction's headroom, then leaves rewrite until emptied buckets are taken again, a /// crash at every request. Every crash state replays to a prefix holding /// every commit and checks clean: a bucket is written again only once its /// emptying is durable. Red when one is reused before. #[test] fn renames_reusing_buckets_on_a_nearly_full_volume_survive_a_crash_at_every_request() { let mut data = Vec::new(); - let image = super::buckets::nearly_full(&mut data, 400).into_inner(); + let image = super::buckets::nearly_full(&mut data).into_inner(); let reused = ops_survive_a_crash_on(image, None, PRESSED, &renames(8, 8), 4, 8); assert!(reused > 4, "{reused} buckets taken again"); } diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index cf76cf4ad37..4b918c8a8ce 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -407,3 +407,77 @@ fn a_full_data_volume_can_release_a_large_orphan_and_use_its_capacity_again() { assert_eq!(read_all(&v, ino).unwrap(), bytes); assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } + +#[test] +fn retained_small_files_reclaim_their_truncated_data_buckets() { + let dev = formatted(64 << 20, None); + let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); + let mut inodes = Vec::new(); + for i in 0..1600 { + let ino = block_on_ready(m.create(m.root(), format!("f{i}").as_bytes(), New::File, OWNER, NOW)).unwrap().ino; + assert_eq!(block_on_ready(m.write(ino, 0, &[i as u8; 32_768], NOW)).unwrap(), 32_768); + block_on_ready(m.truncate(ino, 4096, NOW)).unwrap(); + inodes.push(ino); + if i % 32 == 0 { block_on_ready(m.commit()).unwrap(); } + } + block_on_ready(m.check_metadata_budget()); + block_on_ready(m.unmount()).unwrap(); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + for (i, ino) in inodes.into_iter().enumerate() { assert_eq!(read_all(&v, ino).unwrap(), [i as u8; 4096]); } + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} + +fn interrupted_user_evacuation(encrypted: bool) { + use core::cell::Cell; + use toyos_blockdev::model::{Model, Overlay}; + use super::crash::Watched; + let dev = formatted(64 << 20, encrypted.then_some(Encryption { passphrase: b"test", kdf: MIN_KDF })); + let m = if encrypted { block_on_ready(Mount::unlock(&dev, b"test", LIMITS, MEMORY, random())).unwrap() } else { block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap() }; + let mut inodes = Vec::new(); + for i in 0..16 { + let ino = block_on_ready(m.create(m.root(), format!("f{i}").as_bytes(), New::File, OWNER, NOW)).unwrap().ino; + block_on_ready(m.write(ino, 0, &[i as u8; BLOCK], NOW)).unwrap(); + inodes.push(ino); + } + for &ino in &inodes { block_on_ready(m.truncate(ino, 512, NOW)).unwrap(); } + block_on_ready(m.unmount()).unwrap(); + let model = Model::new(dev.into_inner(), 512); + let requests = Cell::new(0u64); + let watched = Watched { model: &model, after: Box::new(|model| { + requests.set(requests.get() + 1); + model.check(16, requests.get(), |bytes| { + let dev = Overlay::new(bytes); + let raw = if encrypted { block_on_ready(Volume::unlock(&dev, b"test", LIMITS)) } else { block_on_ready(Volume::open(&dev)) }.map_err(|e| format!("raw open: {e:?}"))?; + let findings = block_on_ready(fsck(&raw)).map_err(|e| format!("raw fsck: {e:?}"))?; + if findings.iter().any(|finding| !matches!(finding, Finding::Copy(_))) { return Err(format!("raw fsck: {findings:?}")); } + drop(raw); + let m = if encrypted { block_on_ready(Mount::unlock(&dev, b"test", LIMITS, MEMORY, random())) } else { block_on_ready(Mount::open(&dev, MEMORY, random())) }.map_err(|e| format!("open: {e:?}"))?; + for (i, &ino) in inodes.iter().enumerate() { + let mut got = [0; 512]; + let n = block_on_ready(m.read(ino, 0, &mut got)).map_err(|e| format!("read: {e:?}"))?; + if n != got.len() || got != [i as u8; 512] { return Err(format!("changed file {i}")); } + } + block_on_ready(m.unmount()).map_err(|e| format!("recover: {e:?}"))?; + let v = if encrypted { block_on_ready(Volume::unlock(&dev, b"test", LIMITS)) } else { block_on_ready(Volume::open(&dev)) }.map_err(|e| format!("open: {e:?}"))?; + let findings = block_on_ready(fsck(&v)).map_err(|e| format!("fsck: {e:?}"))?; + if !findings.is_empty() { return Err(format!("fsck: {findings:?}")); } + Ok(()) + }).unwrap_or_else(|e| panic!("{e}")); + }) }; + let m = if encrypted { block_on_ready(Mount::unlock(&watched, b"test", LIMITS, MEMORY, random())).unwrap() } else { block_on_ready(Mount::open(&watched, MEMORY, random())).unwrap() }; + assert!(block_on_ready(m.test_compact_user()).unwrap()); + assert!(block_on_ready(m.test_compact_user()).unwrap()); + block_on_ready(m.check_metadata_budget()); + block_on_ready(m.unmount()).unwrap(); + assert!(requests.get() > 10); +} + +#[test] +fn packed_live_sectors_survive_torn_reordered_evacuation() { + interrupted_user_evacuation(false); +} + +#[test] +fn authenticated_live_sectors_survive_torn_reordered_evacuation() { + interrupted_user_evacuation(true); +} From 40724bfea603f09f29629fbc7381f996c255ae9c Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:44:49 +0100 Subject: [PATCH 39/93] Refuse unreadable data checksum layouts before reserving a page Reservation validated compression and pointer shape but could accept an extent whose checksum the COW read path does not support. Match the reader's checksum and MAC-layout capabilities before accepting buffered capacity, including the encrypted-volume authentication requirement. Native CRC32C and authenticated formats are unchanged. no_std clippy passes. --- lib/bcachefs/src/engine/mount/data.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index 785f1d52621..367d500a9ca 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -368,7 +368,10 @@ impl Mount { for (k, v) in &old { match k.kind { kind::EXTENT => { - checked_extent(v)?; + let extent = checked_extent(v)?; + let authenticated = matches!(extent.crc.unwrap().csum_type, csum_type::CHACHA_POLY_80 | csum_type::CHACHA_POLY_128); + if self.key.is_some() && !authenticated { return Err(Error::Corrupt("unauthenticated extent on an encrypted volume")); } + if self.key.is_none() && authenticated { return Err(Error::Locked); } } kind::INLINE_DATA | kind::WHITEOUT | kind::EXTENT_WHITEOUT => {} _ => return Err(Error::Unsupported("modifying an extent type this writer cannot account")), @@ -617,6 +620,11 @@ pub(super) fn checked_extent(v: &[u8]) -> Result { if extent.ptr.dev != 0 || extent.ptr.cached || extent.ptr.unwritten || crc.compression != 0 || crc.compressed != crc.uncompressed { return Err(Error::Unsupported("modifying cached, unwritten, compressed or other-device data")); } + match crc.csum_type { + csum_type::NONE | csum_type::CRC32C_NONZERO | csum_type::CRC32C => {} + csum_type::CHACHA_POLY_80 | csum_type::CHACHA_POLY_128 if crc.csum_len == csum_type::bytes(crc.csum_type) => {} + _ => return Err(Error::Unsupported("modifying data with a checksum this reader cannot verify")), + } Ok(extent) } From 51b18b8a2f9f6281fea03cf74ba8f5fa8553ca85 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:43:18 +0100 Subject: [PATCH 40/93] Schedule a sole-child parent when its underfull child cannot merge A clean imported non-root parent can have only one child. Deferring that child without scheduling its parent leaves no operation able to provide a sibling. Mark the parent for compaction so reclaim can work upward until the child gains a neighbor. Full-range root spines remain exempt. Validation: cargo test -p bcachefs --lib engine::tests::compact, exit 0, four passed. Log: /private/tmp/toyos-bcachefs-reclaim/unary-parent.log. The imported sole-child case is established by inspection; the existing merge, relocation, captured-delta and crash tests are regression coverage. --- lib/bcachefs/src/engine/mount/reclaim.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/bcachefs/src/engine/mount/reclaim.rs b/lib/bcachefs/src/engine/mount/reclaim.rs index 04d38904c7d..b471ecbe44f 100644 --- a/lib/bcachefs/src/engine/mount/reclaim.rs +++ b/lib/bcachefs/src/engine/mount/reclaim.rs @@ -402,10 +402,14 @@ impl Mount { }; let parent = self.find(btree, level + 1, max).await?; let adjacent = { - let st = self.st.borrow(); + let mut st = self.st.borrow_mut(); let p = st.cache.node(parent); let at = p.at(max); - if p.keys.len() == 1 { return Ok(Vec::new()); } + if p.keys.len() == 1 { + let seq = st.journal.open.as_ref().map_or(st.journal.next_seq, |o| o.seq); + st.cache.change(parent, |n| { n.compact.get_or_insert(seq); }); + return Ok(Vec::new()); + } let index = if at > 0 { at - 1 } else { at + 1 }; let (k, v) = p.keys.get(index).ok_or(Error::Corrupt("underfull node absent from its parent"))?; (k.pos, NodePtr::decode(v)?) From 63783da5d39ca87235e2638951271a5c79947065 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:56:15 +0100 Subject: [PATCH 41/93] Check native and retired filesystem signatures before designation --- lib/blockserve/src/designate.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/blockserve/src/designate.rs b/lib/blockserve/src/designate.rs index ead324504c2..a911397971a 100644 --- a/lib/blockserve/src/designate.rs +++ b/lib/blockserve/src/designate.rs @@ -259,13 +259,19 @@ mod tests { #[test] fn a_recognised_or_judged_partition_is_never_written() { let mut ours = Disk::new(300, |_| 0); - ours.blocks[0][..4].copy_from_slice(b"BCFS"); + ours.blocks[1][24..40].copy_from_slice(&[ + 0xc6, 0x85, 0x73, 0xf6, 0x66, 0xce, 0x90, 0xa9, + 0xd9, 0x6a, 0x60, 0xcf, 0x80, 0x3d, 0xf7, 0xef, + ]); + let mut retired = Disk::new(300, |_| 0); + retired.blocks[0][..4].copy_from_slice(b"BCFS"); let mut stamped = Disk::new(300, |_| 0); stamped.blocks[0] = bcachefs::designation(300); let mut ext4 = Disk::new(300, |_| 0); ext4.blocks[0][1080..1082].copy_from_slice(&[0x53, 0xEF]); let cases = [ (ours, true, Designated::Volume), + (retired, true, Designated::Foreign("the retired ToyOS provisional file system".into())), (stamped, true, Designated::Already), (ext4, true, Designated::Foreign("an ext2/3/4 file system".into())), (Disk::new(300, |_| 7), false, Designated::WouldWrite), From a2ce051f8ebc7cb20e9aa41ff7de0bb424ecd099 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 20:58:41 +0100 Subject: [PATCH 42/93] Keep evacuated small-file checksums within one filesystem block Packing unrelated files into a 64 KiB checksum range made a small read verify that whole range. Combine live slices only within one filesystem block; keep an existing larger extent alone. Sort by live size for packing and skip a candidate whose packed allocation cannot release bucket space. A device wrapper verifies that both 4 KiB and 512-byte files read exactly one physical block after plaintext and authenticated evacuation. All 19 data tests, including retention pressure, cancellation and torn writes, pass; no_std clippy passes. Clarify that successful application consumes a token while failed preparation preserves it, without promising that all metadata admission errors are impossible at writeback. --- lib/bcachefs/src/engine/mount/data.rs | 4 +-- lib/bcachefs/src/engine/mount/gc.rs | 29 +++++++++++++++----- lib/bcachefs/src/engine/tests/data.rs | 38 +++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index 367d500a9ca..b2480140f1d 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -115,8 +115,8 @@ impl Mount { Ok(Reservation { owner: Rc::clone(&self.reservations), id, first, bytes }) } - /// Consumes a reservation with one write inside its block. Capacity - /// cannot fail after acceptance; device failures remain errors. + /// Uses the reserved data block. Successful application consumes the + /// token; failed preparation leaves it available for retry. pub async fn write_reserved(&self, reservation: &mut Reservation, offset: u64, data: &[u8], now: i64) -> Result { if !Rc::ptr_eq(&reservation.owner, &self.reservations) { return Err(Error::Invalid("reservation belongs to another mount")); diff --git a/lib/bcachefs/src/engine/mount/gc.rs b/lib/bcachefs/src/engine/mount/gc.rs index 6d6f0b2a9a3..96f394b366e 100644 --- a/lib/bcachefs/src/engine/mount/gc.rs +++ b/lib/bcachefs/src/engine/mount/gc.rs @@ -26,11 +26,17 @@ impl Mount { /// Called with the file-data slot held, before an ordinary allocation /// is accepted. Live bytes are packed; key ranges and counts stay fixed. pub(super) async fn compact_user(&self) -> Result { - let candidate = { + let candidates: Vec<_> = { let st = self.st.borrow(); - st.sparse_data.iter().copied().find(|&b| st.alloc.movable_data(b)) + st.sparse_data.iter().copied().filter(|&b| st.alloc.movable_data(b)).collect() }; - let Some(victim) = candidate else { return Ok(false); }; + for victim in candidates { + if self.compact_user_bucket(victim).await? { return Ok(true); } + } + Ok(false) + } + + async fn compact_user_bucket(&self, victim: u64) -> Result { let bucket = u64::from(self.sb.member.bucket_sectors); let first = (victim * bucket) << self.bp_shift(); let end = ((victim + 1) * bucket) << self.bp_shift(); @@ -60,7 +66,12 @@ impl Mount { entries.push((key, value, live)); } if entries.is_empty() { return Err(Error::Corrupt("nonempty data bucket has no backpointers")); } - let chunk = (64 << 10).max(self.sb.block_bytes()); + // A small read verifies no more than one filesystem block. An + // existing larger extent keeps its own checksum range. + let chunk = self.sb.block_bytes(); + entries.sort_unstable_by_key(|(_, _, live)| core::cmp::Reverse(live.len())); + let mut groups = Vec::new(); + let mut packed = 0; let mut begin = 0; while begin < entries.len() { let mut end = begin + 1; @@ -69,8 +80,15 @@ impl Mount { bytes += entries[end].2.len(); end += 1; } - let version = self.data_version().await?; let physical = bytes.div_ceil(self.sb.block_bytes()) * self.sb.block_bytes(); + groups.push((begin, end, physical)); + packed += physical; + begin = end; + } + // Do not cycle an unhelpful packing through successive buckets. + if packed >= bucket as usize * SECTOR { return Ok(false); } + for (begin, end, physical) in groups { + let version = self.data_version().await?; let mut blocks = alloc::vec![[0u8; BLOCK]; physical / BLOCK]; let mut at = 0; for (_, _, live) in &entries[begin..end] { @@ -113,7 +131,6 @@ impl Mount { }).await?; range.published = true; } - begin = end; } self.commit().await?; Ok(true) diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index 4b918c8a8ce..f47431ba0a8 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -481,3 +481,41 @@ fn packed_live_sectors_survive_torn_reordered_evacuation() { fn authenticated_live_sectors_survive_torn_reordered_evacuation() { interrupted_user_evacuation(true); } + +#[test] +fn evacuation_keeps_small_reads_within_one_physical_block() { + use core::cell::Cell; + use toyos_blockdev::Block; + struct Reads<'a> { dev: &'a Image, largest: Cell } + impl BlockDev for Reads<'_> { + fn blocks(&self) -> u64 { self.dev.blocks() } + async fn read(&self, lba: u64, out: &mut [Block]) -> core::result::Result<(), toyos_blockdev::Error> { self.dev.read(lba, out).await } + async fn write(&self, lba: u64, bytes: &[Block]) -> core::result::Result<(), toyos_blockdev::Error> { self.dev.write(lba, bytes).await } + async fn flush(&self) -> core::result::Result<(), toyos_blockdev::Error> { self.dev.flush().await } + async fn read_data(&self, lba: u64, out: &mut [Block]) -> core::result::Result<(), toyos_blockdev::Error> { + self.largest.set(self.largest.get().max(out.len())); + self.dev.read_data(lba, out).await + } + } + for encrypted in [false, true] { + let dev = formatted(64 << 20, encrypted.then_some(Encryption { passphrase: b"test", kdf: MIN_KDF })); + let counted = Reads { dev: &dev, largest: Cell::new(0) }; + let m = if encrypted { block_on_ready(Mount::unlock(&counted, b"test", LIMITS, MEMORY, random())).unwrap() } else { block_on_ready(Mount::open(&counted, MEMORY, random())).unwrap() }; + let mut inodes = Vec::new(); + for i in 0..8 { + let ino = block_on_ready(m.create(m.root(), format!("f{i}").as_bytes(), New::File, OWNER, NOW)).unwrap().ino; + block_on_ready(m.write(ino, 0, &[i as u8; BLOCK], NOW)).unwrap(); + inodes.push(ino); + } + for &ino in &inodes[1..] { block_on_ready(m.truncate(ino, 512, NOW)).unwrap(); } + assert!(block_on_ready(m.test_compact_user()).unwrap()); + for (i, &ino) in inodes.iter().enumerate() { + let mut got = vec![0; if i == 0 { BLOCK } else { 512 }]; + counted.largest.set(0); + assert_eq!(block_on_ready(m.read(ino, 0, &mut got)).unwrap(), got.len()); + assert!(got.iter().all(|&byte| byte == i as u8)); + assert_eq!(counted.largest.get(), 1); + } + block_on_ready(m.unmount()).unwrap(); + } +} From f8cf9852d400b95c15a9d42f11055fcf283fc2e9 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 21:09:31 +0100 Subject: [PATCH 43/93] Add explicit native DATA formatting and reboot persistence evidence Give signed-image preparation rows raw DATA access through the existing partition grant. disks format data reports its unique target and contents, requires --commit to overwrite them, and verifies then cleanly unmounts the new bcachefs volume. Ordinary images receive no formatting authority. Add an explicitly selected preparation boot and two mounted-DATA boots that check Rust file operations across a real reboot using a fresh shared token. Align DATA evidence with native bcachefs and stage argument-bearing Rust test commands by their executable name. --- Cargo.lock | 5 + build/build.rs | 14 +- lib/manifest/src/authorize.rs | 13 +- lib/manifest/src/lib.rs | 2 +- system/disks/Cargo.toml | 9 +- system/disks/src/format.rs | 136 ++++++++++++ system/disks/src/main.rs | 204 ++++++++++++++++-- system/fileserver/src/data.rs | 9 +- system/supervisor/src/main.rs | 4 + tests/checks.rs | 11 +- tests/common/hostdisk.rs | 6 +- tests/common/metal.rs | 9 +- .../rust-tests/src/bin/data_roundtrip.rs | 118 ++++++++++ tests/images/datacase.toml | 40 ++++ tests/images/formatcase.toml | 44 ++++ tests/toyos.rs | 54 ++++- 16 files changed, 637 insertions(+), 41 deletions(-) create mode 100644 system/disks/src/format.rs create mode 100644 tests/guest/rust-tests/src/bin/data_roundtrip.rs create mode 100644 tests/images/datacase.toml create mode 100644 tests/images/formatcase.toml diff --git a/Cargo.lock b/Cargo.lock index 01ce56ed62e..d20737adf1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,9 +767,14 @@ dependencies = [ name = "disks" version = "0.1.0" dependencies = [ + "bcachefs", "diskserver", + "getrandom 0.3.4", "toyos", + "toyos-blockdev", "toyos-blockring", + "toyos-blockserve", + "toyos-crypto", "toyos-gpt", "toyos-manifest", ] diff --git a/build/build.rs b/build/build.rs index a38b345e6c0..9e0ef899225 100644 --- a/build/build.rs +++ b/build/build.rs @@ -3047,12 +3047,14 @@ mod tests { #[test] fn a_disk_changing_authorization_goes_to_no_row_a_login_can_start() { let gate = |text: &str| held_by_their_holders_alone(&toml::from_str(text).unwrap()); - assert!(gate("[programs.runner]\nauthorize = [\"designate:data\"]\n").is_ok()); + for word in ["designate:data", "format:data"] { + assert!(gate(&format!("[programs.runner]\nauthorize = [\"{word}\"]\n")).is_ok()); + let launched = gate(&format!("[programs.disks]\nauthorize = [\"{word}\"]\n[programs.shell]\nstarts = [\"disks\"]\n")); + assert!(launched.unwrap_err().contains("may start it")); + let login = gate(&format!("[boot]\nstart = [\"sshserver\"]\n[programs.sshserver]\nlogin = true\nauthorize = [\"{word}\"]\n")); + assert!(login.unwrap_err().contains("is a login authority")); + } assert!(gate("[programs.disks]\nauthorize = [\"tables\"]\n[programs.shell]\nstarts = [\"disks\"]\n").is_ok()); - let launched = gate("[programs.disks]\nauthorize = [\"designate:data\"]\n[programs.shell]\nstarts = [\"disks\"]\n"); - assert!(launched.unwrap_err().contains("may start it")); - let login = gate("[boot]\nstart = [\"sshserver\"]\n[programs.sshserver]\nlogin = true\nauthorize = [\"designate:data\"]\n"); - assert!(login.unwrap_err().contains("is a login authority")); assert!(gate("[programs.sshserver]\nlogin = true\nauthorize = [\"tables\"]\n").is_ok(), "reading the tables is no change"); assert!(gate("[programs.disks]\nauthorize = [\"format:everything\"]\n").unwrap_err().contains("no disk authorization")); } @@ -3291,6 +3293,8 @@ mod tests { "tests/images/benchcase.toml", "tests/images/clipboardcase.toml", "tests/images/designatecase.toml", + "tests/images/formatcase.toml", + "tests/images/datacase.toml", "tests/images/drivescase.toml", "tests/images/hostedclangcase.toml", "tests/images/jobcase.toml", diff --git a/lib/manifest/src/authorize.rs b/lib/manifest/src/authorize.rs index a7968ae4be9..e9c9d5bc3e5 100644 --- a/lib/manifest/src/authorize.rs +++ b/lib/manifest/src/authorize.rs @@ -1,7 +1,6 @@ //! What a row may do to the machine's disks beyond its own partitions, which //! only the signed image grants: `authorize = [...]` in its row. The -//! supervisor mints each at every start of the row, one action on one target -//! that the block service resolves each time it is used, and puts it in the +//! supervisor mints each at every start of the row and puts it in the //! row's namespace under [`Authorization::name`]. The build refuses one that //! changes a disk to a login authority or a row a login can start, and no //! `[apps]` row can spell one. @@ -14,16 +13,21 @@ pub enum Authorization { /// Designations of the machine's one DATA partition, resolved each time /// one is asked for, for as long as the row runs. DesignateData, + /// Raw read/write access to DATA partitions, including existing volumes. + /// The trusted formatter selects one; this is not a format-only or + /// single-use grant. It grants neither the disk table nor other types. + FormatData, } impl Authorization { - pub const ALL: [Self; 2] = [Self::Tables, Self::DesignateData]; + pub const ALL: [Self; 3] = [Self::Tables, Self::DesignateData, Self::FormatData]; /// How a row spells it. pub const fn word(self) -> &'static str { match self { Self::Tables => "tables", Self::DesignateData => "designate:data", + Self::FormatData => "format:data", } } @@ -36,11 +40,12 @@ impl Authorization { match self { Self::Tables => "block-tables", Self::DesignateData => "block-designate-data", + Self::FormatData => "block-format-data", } } /// Whether it changes a disk, rather than reading one. pub const fn changes(self) -> bool { - matches!(self, Self::DesignateData) + matches!(self, Self::DesignateData | Self::FormatData) } } diff --git a/lib/manifest/src/lib.rs b/lib/manifest/src/lib.rs index 90a42dd4209..c23c739c93c 100644 --- a/lib/manifest/src/lib.rs +++ b/lib/manifest/src/lib.rs @@ -829,7 +829,7 @@ mod tests { source: Source::Path("/system/bin/update".into()), slots: true, state: true, - authorize: vec!["tables".into(), "designate:data".into()], + authorize: vec!["tables".into(), "designate:data".into(), "format:data".into()], ..Program::default() }, Program { diff --git a/system/disks/Cargo.toml b/system/disks/Cargo.toml index 97529e1cd65..d3c868e3316 100644 --- a/system/disks/Cargo.toml +++ b/system/disks/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "disks" -description = "disks [list | designate data [--commit]]: the machine's disks and their tables, and the designation of its DATA partition, through the block service under the authorizations its row is minted." +description = "List disks and explicitly initialize DATA through the block service under signed-image authorizations." version = "0.1.0" edition = "2024" license = "MIT OR Apache-2.0" @@ -11,6 +11,11 @@ toyos-blockring = { path = "../../lib/blockring" } toyos-gpt = { path = "../../lib/gpt" } toyos-manifest = { path = "../../lib/manifest" } diskserver = { path = "../diskserver" } +bcachefs = { path = "../../lib/bcachefs" } +toyos-blockdev = { path = "../../lib/blockdev" } +toyos-blockserve = { path = "../../lib/blockserve" } +toyos-crypto = { path = "../../lib/crypto" } +getrandom = "0.3" [package.metadata.toyos.host] -exempt.manages = "ToyOS's disks, through its block service's tables and designation badges" +exempt.manages = "ToyOS's disks, through the block capabilities its signed-image row receives" diff --git a/system/disks/src/format.rs b/system/disks/src/format.rs new file mode 100644 index 00000000000..f5bf38763a1 --- /dev/null +++ b/system/disks/src/format.rs @@ -0,0 +1,136 @@ +//! Explicit DATA initialization. The caller holds the selected partition +//! exclusively and prints its contents before invoking `create`. + +use bcachefs::engine::{self, Mount, MountOptions, Options, Tree}; +use toyos_blockdev::{BLOCK, BlockDev}; +use toyos_blockring::wire::Listed; +use toyos_blockserve::content::{self, Content}; +use toyos_crypto::Random; +use toyos_gpt::Guid; + +pub fn select(listed: &[Listed]) -> Result { + let data: Vec<_> = listed + .iter() + .filter(|p| p.kind == Guid::TOYOS_DATA.0) + .copied() + .collect(); + match data[..] { + [one] => Ok(one), + [] => Err("no DATA partition is available".into()), + _ => Err(format!( + "{} DATA partitions are available; none was selected", + data.len() + )), + } +} + +pub async fn contents(disk: &impl BlockDev) -> Result { + let blocks = disk.blocks().min((content::READ_BYTES / BLOCK) as u64) as usize; + let mut front = vec![[0; BLOCK]; blocks]; + disk.read(0, &mut front).await?; + Ok(match content::judge(front.as_flattened(), disk.blocks()) { + Content::Volume => "bcachefs".into(), + Content::Designated(_) => "a ToyOS designation stamp".into(), + Content::Foreign(what) => what.into(), + Content::Unrecognised => "unrecognised bytes (not evidence of an empty partition)".into(), + }) +} + +pub async fn create( + disk: &impl BlockDev, + time: u64, + mut random: impl Random + 'static, +) -> engine::Result<()> { + let options = Options { + label: b"ToyOS DATA", + time, + encryption: None, + }; + engine::mkfs(disk, &options, &Tree::new(), &mut random).await?; + let mounted = Mount::open(disk, MountOptions { cache_nodes: 512 }, Box::new(random)).await?; + if !mounted.inode(mounted.root()).await?.is_dir() { + return Err(engine::Error::Invalid( + "the formatted root is not a directory", + )); + } + mounted.unmount().await +} + +#[cfg(test)] +mod tests { + use super::*; + use toyos_blockdev::{Bytes, ReadOnlyBytes, block_on_ready}; + + struct Seeded(u64); + impl Random for Seeded { + fn fill(&mut self, out: &mut [u8]) -> Result<(), toyos_crypto::Error> { + for byte in out { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + *byte = self.0 as u8; + } + Ok(()) + } + } + + #[test] + fn selection_refuses_absence_and_ambiguity_including_duplicate_guids() { + let data = Listed { + disk: 0, + unique: [7; 16], + kind: Guid::TOYOS_DATA.0, + }; + let foreign = Listed { + disk: 0, + unique: [8; 16], + kind: Guid::MICROSOFT_BASIC.0, + }; + assert!(select(&[]).is_err()); + assert!(select(&[foreign]).is_err()); + assert_eq!(select(&[foreign, data]), Ok(data)); + assert!(select(&[data, Listed { disk: 1, ..data }]).is_err()); + assert!( + select(&[ + data, + Listed { + unique: [9; 16], + ..data + } + ]) + .is_err() + ); + } + + #[test] + fn inspection_never_changes_a_recognized_volume_and_explicit_creation_replaces_it() { + let mut old = vec![0xa5; 64 << 20]; + old[..4].copy_from_slice(b"BCFS"); + let disk = Bytes::new(old); + assert_eq!( + block_on_ready(contents(&disk)).unwrap(), + "the retired ToyOS provisional file system" + ); + assert_eq!(&disk.get()[..4], b"BCFS"); + assert!(disk.get()[4..].iter().all(|&b| b == 0xa5)); + block_on_ready(create(&disk, 1_700_000_000_000_000_000, Seeded(7))).unwrap(); + assert_eq!(block_on_ready(contents(&disk)).unwrap(), "bcachefs"); + let volume = block_on_ready(engine::Volume::open(&disk)).unwrap(); + assert!( + block_on_ready(volume.inode(volume.root())) + .unwrap() + .is_dir() + ); + } + + #[test] + fn a_device_that_refuses_writes_cannot_report_success() { + let bytes = vec![0; 64 << 20]; + let disk = ReadOnlyBytes::new(&bytes).unwrap(); + assert!(matches!( + block_on_ready(create(&disk, 1, Seeded(7))), + Err(engine::Error::Device(toyos_blockdev::Error::ReadOnly)) + )); + assert!(bytes.iter().all(|&b| b == 0)); + } +} diff --git a/system/disks/src/main.rs b/system/disks/src/main.rs index 37cbea3f25a..07d86055b12 100644 --- a/system/disks/src/main.rs +++ b/system/disks/src/main.rs @@ -5,11 +5,13 @@ //! - `disks designate data`: judge the machine's one DATA partition for //! designation and write nothing; `--commit` writes the stamp, which the //! DATA file server formats at its next start. +//! - `disks format data`: describe the unique DATA partition without writing; +//! `--commit` destroys its contents and creates native, unencrypted bcachefs. //! //! **It holds no authority of its own.** Each command needs the //! authorization its row is minted under its name //! (`toyos_manifest::authorize`): `tables` to list, `designate:data` to -//! designate. A run without one is refused, whoever runs it and from where; +//! designate, `format:data` for raw DATA access. A run without one is refused; //! a login grants none. Exit status 0 where it was done, 1 where it was //! refused or failed, 2 for a command line it does not take. @@ -17,16 +19,36 @@ use toyos_blockring::wire::{Designated, DiskHead, SlotLine}; use toyos_gpt::Guid; use toyos_manifest::authorize::Authorization; -const USAGE: &str = "usage: disks [list | designate data [--commit]]"; +mod format; + +const USAGE: &str = "usage: disks [list | designate data [--commit] | format data [--commit]]"; + +#[derive(Debug, PartialEq, Eq)] +enum Command { + List, + Designate { commit: bool }, + Format { commit: bool }, +} + +fn command(args: &[&str]) -> Option { + match args { + [] | ["list"] => Some(Command::List), + ["designate", "data"] => Some(Command::Designate { commit: false }), + ["designate", "data", "--commit"] => Some(Command::Designate { commit: true }), + ["format", "data"] => Some(Command::Format { commit: false }), + ["format", "data", "--commit"] => Some(Command::Format { commit: true }), + _ => None, + } +} fn main() { let args: Vec = std::env::args().skip(1).collect(); let args: Vec<&str> = args.iter().map(String::as_str).collect(); - let code = match args[..] { - [] | ["list"] => list(), - ["designate", "data"] => designate(false), - ["designate", "data", "--commit"] => designate(true), - _ => { + let code = match command(&args) { + Some(Command::List) => list(), + Some(Command::Designate { commit }) => designate(commit), + Some(Command::Format { commit }) => format(commit), + None => { eprintln!("{USAGE}"); 2 } @@ -43,12 +65,18 @@ fn unauthorized(authorization: Authorization) -> i32 { } fn list() -> i32 { - let Some(names) = toyos::endow::namespace() else { return unauthorized(Authorization::Tables) }; + let Some(names) = toyos::endow::namespace() else { + return unauthorized(Authorization::Tables); + }; let service = Authorization::Tables.name(); let listed = match diskserver::list(names, service) { Ok(listed) => listed, Err(diskserver::Error::NotHeld) => return unauthorized(Authorization::Tables), - Err(why) => return refuse(&format!("disks: the block service would not list its partitions: {why:?}")), + Err(why) => { + return refuse(&format!( + "disks: the block service would not list its partitions: {why:?}" + )); + } }; let mut disks: Vec = listed.iter().map(|l| l.disk).collect(); disks.sort_unstable(); @@ -70,14 +98,106 @@ fn list() -> i32 { } fn designate(commit: bool) -> i32 { - let Some(names) = toyos::endow::namespace() else { return unauthorized(Authorization::DesignateData) }; + let Some(names) = toyos::endow::namespace() else { + return unauthorized(Authorization::DesignateData); + }; match diskserver::tables::designate(names, Authorization::DesignateData.name(), commit) { Err(diskserver::Error::NotHeld) => unauthorized(Authorization::DesignateData), Ok(done) => { println!("{}", designated(&done)); - i32::from(!matches!(done, Designated::Written | Designated::WouldWrite | Designated::Already)) + i32::from(!matches!( + done, + Designated::Written | Designated::WouldWrite | Designated::Already + )) } - Err(why) => refuse(&format!("disks: the DATA partition was not designated: {why:?}")), + Err(why) => refuse(&format!( + "disks: the DATA partition was not designated: {why:?}" + )), + } +} + +struct OsRandom; +impl toyos_crypto::Random for OsRandom { + fn fill(&mut self, out: &mut [u8]) -> Result<(), toyos_crypto::Error> { + getrandom::fill(out).map_err(|_| toyos_crypto::Error::Entropy) + } +} + +fn format(commit: bool) -> i32 { + use toyos_blockdev::{BlockDev, block_on}; + let authorization = Authorization::FormatData; + let Some(names) = toyos::endow::namespace() else { + return unauthorized(authorization); + }; + let service = authorization.name(); + let listed = match diskserver::list(names, service) { + Ok(listed) => listed, + Err(diskserver::Error::NotHeld) => return unauthorized(authorization), + Err(why) => return refuse(&format!("disks: DATA was not listed: {why:?}")), + }; + let target = match format::select(&listed) { + Ok(target) => target, + Err(why) => return refuse(&format!("disks: {why}; nothing was written")), + }; + let own = match toyos::namespace::build().keep(names, &[service]).finish() { + Ok(own) => own, + Err(why) => { + return refuse(&format!( + "disks: the formatting capability could not be kept: {why:?}" + )); + } + }; + // The block service holds the partition exclusively for this session. + let session = match diskserver::Session::open(own, service, target.unique) { + Ok(session) => session, + Err(why) => { + return refuse(&format!( + "disks: DATA {} could not be opened exclusively: {why:?}; nothing was written", + Guid(target.unique) + )); + } + }; + let disk = diskserver::disk::Served::new(session); + let contents = match block_on(format::contents(&disk), || disk.wait()) { + Ok(contents) => contents, + Err(why) => { + return refuse(&format!( + "disks: DATA could not be inspected: {why:?}; nothing was written" + )); + } + }; + println!( + "disks: target disk {} DATA {}: {} bytes; contents: {contents}", + target.disk, + Guid(target.unique), + disk.blocks() * toyos_blockdev::BLOCK as u64 + ); + if !commit { + println!( + "disks: would destroy all contents and create unencrypted bcachefs; nothing was written (--commit writes)" + ); + return 0; + } + println!("disks: destroying DATA contents and creating unencrypted bcachefs"); + let now = match std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) { + Ok(now) => now.as_nanos().min(u64::MAX as u128) as u64, + Err(why) => { + return refuse(&format!( + "disks: no filesystem timestamp: {why}; nothing was written" + )); + } + }; + match block_on(format::create(&disk, now, OsRandom), || disk.wait()) { + Ok(()) => { + println!( + "disks: DATA {} formatted as bcachefs; flushed and mounted for verification", + Guid(target.unique) + ); + 0 + } + Err(why) => refuse(&format!( + "disks: formatting failed: {why:?}; DATA may be incomplete" + )), } } @@ -108,7 +228,10 @@ fn kind(guid: [u8; 16]) -> String { (Guid::TOYOS_SLOTS, "ToyOS slot table"), (Guid::TOYOS_SCRATCH, "ToyOS scratch"), ]; - named.iter().find(|(g, _)| g.0 == guid).map_or_else(|| Guid(guid).to_string(), |(_, name)| (*name).to_string()) + named + .iter() + .find(|(g, _)| g.0 == guid) + .map_or_else(|| Guid(guid).to_string(), |(_, name)| (*name).to_string()) } /// `bytes` in the largest binary unit that leaves at least 10 of it. @@ -161,6 +284,27 @@ fn describe(disk: u32, head: &DiskHead, slots: &[SlotLine]) -> String { mod tests { use super::*; + #[test] + fn formatting_requires_the_exact_explicit_commit_command() { + assert_eq!( + command(&["format", "data"]), + Some(Command::Format { commit: false }) + ); + assert_eq!( + command(&["format", "data", "--commit"]), + Some(Command::Format { commit: true }) + ); + for args in [ + vec!["format"], + vec!["format", "root", "--commit"], + vec!["format", "data", "--force"], + vec!["format", "data", "--commit", "extra"], + vec!["--commit", "format", "data"], + ] { + assert_eq!(command(&args), None, "{args:?}"); + } + } + /// A disk prints its size, its table's state and each partition by its /// one-based number, type, size, unique GUID, name and use; copies that /// disagree say what each is. @@ -190,14 +334,35 @@ mod tests { }; let text = describe(1, &head, &[data]); let lines: Vec<&str> = text.lines().collect(); - assert_eq!(lines[0], "disk 1: 238 GiB (500118192 sectors of 512 bytes), a GPT, both copies agree"); + assert_eq!( + lines[0], + "disk 1: 238 GiB (500118192 sectors of 512 bytes), a GPT, both copies agree" + ); assert_eq!( lines[1].split_whitespace().collect::>(), - ["4", "ToyOS", "DATA", "170", "GiB", "11111111-1111-1111-1111-111111111111", "\"ToyOS", "data\"", "free"] + [ + "4", + "ToyOS", + "DATA", + "170", + "GiB", + "11111111-1111-1111-1111-111111111111", + "\"ToyOS", + "data\"", + "free" + ] ); assert_eq!(lines.len(), 2); - let torn = DiskHead { agree: false, backup: "Gpt(HeaderCrc { stored: 1, computed: 2 })".into(), ..head }; - assert!(describe(0, &torn, &[]).contains("primary valid, backup Gpt(HeaderCrc"), "{}", describe(0, &torn, &[])); + let torn = DiskHead { + agree: false, + backup: "Gpt(HeaderCrc { stored: 1, computed: 2 })".into(), + ..head + }; + assert!( + describe(0, &torn, &[]).contains("primary valid, backup Gpt(HeaderCrc"), + "{}", + describe(0, &torn, &[]) + ); assert_eq!(size(4096), "4096 B"); assert_eq!(size(10 << 30), "10 GiB"); } @@ -205,6 +370,9 @@ mod tests { #[test] fn a_designation_is_said_as_what_it_came_to() { assert!(designated(&Designated::Written).contains("read back")); - assert!(designated(&Designated::Foreign("an NTFS file system".into())).contains("carries an NTFS file system")); + assert!( + designated(&Designated::Foreign("an NTFS file system".into())) + .contains("carries an NTFS file system") + ); } } diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index ace348eb973..d4abb21c33e 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -94,6 +94,7 @@ impl DataVolume { let cache = Rc::new(Cache::new(disk, clean_limit)); let first = cache.block_on(Mount::open(Rc::clone(&cache), MountOptions { cache_nodes: NODES }, Box::new(OsRandom))); + let mut formatted = false; let fs = match first { Ok(fs) => fs, Err(error) => { @@ -114,6 +115,7 @@ impl DataVolume { { return Probed::Unmountable(format!("format: {error:?}")); } + formatted = true; match cache.block_on(Mount::open( Rc::clone(&cache), MountOptions { cache_nodes: NODES }, @@ -124,7 +126,12 @@ impl DataVolume { } } }; - Self::over(fs, cache, roots, clock).map_or_else(Probed::Unmountable, |volume| Probed::Mounted(Box::new(volume))) + Self::over(fs, cache, roots, clock).map_or_else(Probed::Unmountable, |volume| { + if !formatted { + println!("fileserver: mounted the DATA volume"); + } + Probed::Mounted(Box::new(volume)) + }) } fn over(fs: Mount>>, cache: Rc>, roots: &[&str], clock: fn() -> u64) -> Result { // Mount recovery precedes every live description. diff --git a/system/supervisor/src/main.rs b/system/supervisor/src/main.rs index 1c3394d9cde..02320576aa2 100644 --- a/system/supervisor/src/main.rs +++ b/system/supervisor/src/main.rs @@ -3362,6 +3362,10 @@ impl Grants<'_> { let badge = match authorization { Authorization::Tables => toyos_blockring::wire::Badge::Tables, Authorization::DesignateData => toyos_blockring::wire::Badge::DesignateData, + Authorization::FormatData => toyos_blockring::wire::Badge::Partitions(toyos_blockring::wire::Grant { + scope: toyos_blockring::wire::Scope::Kind(toyos_gpt::Guid::TOYOS_DATA.0), + writes: true, + }), }; let Some(ports) = &self.block else { return Err(format!("`{word}` is authorized and no block service runs")) }; let ports = ports.borrow(); diff --git a/tests/checks.rs b/tests/checks.rs index 389441b4b8d..dc20a381839 100644 --- a/tests/checks.rs +++ b/tests/checks.rs @@ -10,6 +10,15 @@ mod checks { // outside that text, so each of those names the printer's `eprintln!`. use toyos_build::eprintln; + #[test] + fn a_data_round_trip_needs_one_complete_token_per_boot() { + let token = "0123456789abcdef0123456789abcdef"; + assert_eq!(data_roundtrip_token(&format!("[job] DATA_ROUNDTRIP written {token}\n"), "written").unwrap(), token); + for text in [String::new(), format!("DATA_ROUNDTRIP verified {token}"), "DATA_ROUNDTRIP written 123".into(), format!("DATA_ROUNDTRIP written {token}junk"), format!("DATA_ROUNDTRIP written {token}\nDATA_ROUNDTRIP written {token}\n")] { + assert!(data_roundtrip_token(&text, "written").is_err(), "{text}"); + } + } + #[path = "audio.rs"] mod audio_checks; #[path = "claims.rs"] @@ -1222,7 +1231,7 @@ mod checks { .iter() .map(|name| ((*name).to_string(), Vec::new())) .collect(); - let jobs = vec!["test_rs_parent".to_string()]; + let jobs = vec!["test_rs_parent --write".to_string()]; let staged = |text: &str| -> Vec { metal::reached(text, &jobs, &bins).into_iter().map(|(path, _)| path).collect() }; diff --git a/tests/common/hostdisk.rs b/tests/common/hostdisk.rs index 3ee276184c2..d0c3c9a8091 100644 --- a/tests/common/hostdisk.rs +++ b/tests/common/hostdisk.rs @@ -128,16 +128,14 @@ pub fn left_alone(log: &Serial, ours: &[Guid]) -> Result<(usize, Data), String> let said = |line: &str| log.text().lines().any(|l| l.contains(line)); let fates = [ (Data::Foreign, said("fileserver: the DATA partition holds no volume of ours and no designation stamp; DATA is absent this boot")), - (Data::Formatted, said("designates this partition for ToyOS; formatting it")), + (Data::Formatted, said("fileserver: formatting the explicitly designated DATA partition as bcachefs")), (Data::Mounted, said("fileserver: mounted the DATA volume")), ]; let fate = match fates.iter().filter(|(_, said)| *said).map(|(fate, _)| *fate).collect::>()[..] { [one] => one, ref many => return Err(format!("DATA's file server said {many:?} of foreign, formatted and mounted, and one is its answer")), }; - if fate == Data::Foreign { - log.must_say("fileserver: no volume of ours and no designation stamp; nothing will be written to this partition")?; - } else { + if fate != Data::Foreign { let serving = log.must_say("fileserver: Data serving ")?; if !serving.contains(" — bcachefs") { return Err(format!("DATA is {fate:?} and served by no bcachefs volume: {serving:?}")); diff --git a/tests/common/metal.rs b/tests/common/metal.rs index 126c8741e82..a11f61c8e86 100644 --- a/tests/common/metal.rs +++ b/tests/common/metal.rs @@ -744,7 +744,7 @@ pub fn reached( // together are a fraction of one helper binary. if name.ends_with(".so") { out.push((format!("lib/{name}"), data.clone())); - } else if !jobs.contains(&staged) && names(&staged) { + } else if !jobs.iter().any(|job| binary(job) == staged) && names(&staged) { out.push((format!("bin/{staged}"), data.clone())); } } @@ -780,12 +780,13 @@ fn build( // carrying two hundred binaries it never runs is a minute of flash. let mut extra: Vec<(String, Vec)> = Vec::new(); for job in &batch.jobs { - let Some(name) = job.strip_prefix("test_rs_") else { continue }; + let bin = binary(job); + let Some(name) = bin.strip_prefix("test_rs_") else { continue }; let (_, data) = rust_bins .iter() .find(|(n, _)| n == name) .ok_or_else(|| format!("the job {job:?} names no binary under tests/guest/rust-tests"))?; - extra.push((format!("bin/{job}"), data.clone())); + extra.push((format!("bin/{bin}"), data.clone())); } extra.extend(batch.files.iter().cloned()); // **What a job spawns comes with it, and it is not optional.** A binary that @@ -812,7 +813,7 @@ fn build( reachable.push_str(to); } for job in &batch.jobs { - let Some(name) = job.strip_prefix("test_rs_") else { continue }; + let Some(name) = binary(job).strip_prefix("test_rs_") else { continue }; if let Ok(source) = std::fs::read_to_string(bin.join(format!("{name}.rs"))) { reachable.push('\n'); reachable.push_str(&source); diff --git a/tests/guest/rust-tests/src/bin/data_roundtrip.rs b/tests/guest/rust-tests/src/bin/data_roundtrip.rs new file mode 100644 index 00000000000..50ea2fb4285 --- /dev/null +++ b/tests/guest/rust-tests/src/bin/data_roundtrip.rs @@ -0,0 +1,118 @@ +//! Two real boots: write and sync through std, then verify after DATA remounts. +//! The host compares the random token from both boots, rejecting stale evidence. + +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::os::toyos::fs::MetadataExt; +use std::path::Path; +use std::time::{Duration, SystemTime}; + +const ROOT: &str = "/home/toyos-data-roundtrip"; +const LENGTH: usize = (256 << 10) + 123; +const MTIME: u64 = 1_700_000_000; + +fn bytes(token: &[u8; 16], length: usize) -> Vec { + let mut out: Vec<_> = (0..length) + .map(|i| (i as u8).wrapping_mul(73) ^ token[i % token.len()]) + .collect(); + out[..16].copy_from_slice(token); + out +} + +fn file(root: &Path, name: &str, data: &[u8]) -> File { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(root.join(name)) + .expect("create a new round-trip file"); + file.write_all(data).expect("write"); + file.sync_all().expect("sync contents and metadata"); + file +} + +fn report(phase: &str, token: &[u8; 16]) { + let token: String = token.iter().map(|b| format!("{b:02x}")).collect(); + println!("DATA_ROUNDTRIP {phase} {token}"); +} + +fn write(root: &Path) { + // Refuse a previous incomplete run rather than erasing its evidence. + fs::create_dir(root) + .expect("create round-trip directory (remove a previous failed run explicitly)"); + let mut token = [0; 16]; + getrandom::getrandom(&mut token).expect("random round-trip token"); + let data = file(root, "before-rename", &bytes(&token, 1 << 20)); + data.set_len(LENGTH as u64) + .expect("truncate an allocated file"); + data.set_modified(SystemTime::UNIX_EPOCH + Duration::from_secs(MTIME)) + .expect("set mtime"); + data.sync_all().expect("sync truncation and mtime"); + let ino = data.metadata().expect("metadata").ino(); + drop(data); + fs::rename(root.join("before-rename"), root.join("payload")).expect("rename"); + drop(file(root, "identity", &ino.to_le_bytes())); + drop(file( + root, + "removed", + b"this name must not return after reboot", + )); + fs::remove_file(root.join("removed")).expect("unlink"); + fs::create_dir(root.join("empty")).expect("mkdir"); + File::open(root) + .expect("open directory") + .sync_all() + .expect("sync directory"); + File::open("/home") + .expect("open parent") + .sync_all() + .expect("sync parent"); + report("written", &token); +} + +fn read(root: &Path) { + let payload = fs::read(root.join("payload")).expect("read payload after reboot"); + assert_eq!(payload.len(), LENGTH, "the truncated length persisted"); + let token: [u8; 16] = payload[..16].try_into().unwrap(); + assert_eq!(payload, bytes(&token, LENGTH), "every byte persisted"); + let identity = fs::read(root.join("identity")).expect("read inode witness"); + let ino = u64::from_le_bytes(identity.try_into().expect("one inode number")); + let meta = fs::metadata(root.join("payload")).expect("metadata after reboot"); + assert_eq!( + meta.ino(), + ino, + "rename and remount preserve inode identity" + ); + assert_eq!(meta.nlink(), 1); + assert_eq!( + meta.modified().unwrap(), + SystemTime::UNIX_EPOCH + Duration::from_secs(MTIME) + ); + for missing in ["before-rename", "removed"] { + assert_eq!( + fs::metadata(root.join(missing)).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + } + assert!(fs::metadata(root.join("empty")).unwrap().is_dir()); + let mut names: Vec<_> = fs::read_dir(root) + .expect("readdir") + .map(|e| e.unwrap().file_name()) + .collect(); + names.sort(); + assert_eq!(names, ["empty", "identity", "payload"]); + fs::remove_dir_all(root).expect("remove verified fixture"); + File::open("/home") + .expect("open parent") + .sync_all() + .expect("sync cleanup"); + report("verified", &token); +} + +fn main() { + let root = Path::new(ROOT); + match std::env::args().skip(1).collect::>().as_slice() { + [phase] if phase == "write" => write(root), + [phase] if phase == "read" => read(root), + _ => panic!("expected data_roundtrip write or read"), + } +} diff --git a/tests/images/datacase.toml b/tests/images/datacase.toml new file mode 100644 index 00000000000..b0e925456d7 --- /dev/null +++ b/tests/images/datacase.toml @@ -0,0 +1,40 @@ +# Mounted DATA tests on both benches. No formatting or designation authority. +[boot] +start = ["logkeeper", "diskserver", "fileserver", "acpiserver", "test-runner"] + +[programs.logkeeper] +service = true +syscap = ["logread"] +serves = ["log"] + +[programs.acpiserver] +service = true +devices = ["acpi"] +receives = ["power"] + +[programs.test-runner] +receives = ["power"] + +[programs.toybox] + +[symlinks] +"bin/reboot" = "/system/bin/toybox" + +[programs.diskserver] +service = true +restart = true +serves = ["block", "disk"] + +[programs.nvme] +drives = ["pci:01:08:02"] +receives = ["disk"] + +[programs.usbd] +drives = ["pci:0c:03:30/1033:0194", "pci:0c:03:30@boot"] +receives = ["disk"] + +[programs.fileserver] +restart = true +pager = true +roles = ["data", "log", "boot", "memory"] +receives = ["block"] diff --git a/tests/images/formatcase.toml b/tests/images/formatcase.toml new file mode 100644 index 00000000000..6f752174fe4 --- /dev/null +++ b/tests/images/formatcase.toml @@ -0,0 +1,44 @@ +# Explicit destructive DATA preparation only. No DATA file server holds the +# partition; only this image's boot-started runner gets raw DATA authority. +[boot] +start = ["logkeeper", "diskserver", "fileserver", "acpiserver", "test-runner"] + +[programs.logkeeper] +service = true +syscap = ["logread"] +serves = ["log"] + +[programs.acpiserver] +service = true +devices = ["acpi"] +receives = ["power"] + +[programs.test-runner] +receives = ["power"] +authorize = ["format:data"] + +[programs.disks] + +[programs.toybox] + +[symlinks] +"bin/reboot" = "/system/bin/toybox" + +[programs.diskserver] +service = true +restart = true +serves = ["block", "disk"] + +[programs.nvme] +drives = ["pci:01:08:02"] +receives = ["disk"] + +[programs.usbd] +drives = ["pci:0c:03:30/1033:0194", "pci:0c:03:30@boot"] +receives = ["disk"] + +[programs.fileserver] +restart = true +pager = true +roles = ["log", "boot", "memory"] +receives = ["block"] diff --git a/tests/toyos.rs b/tests/toyos.rs index f14e739e54a..13e682cf7ce 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -68,6 +68,8 @@ const ACTUATOR_KERNEL: &[&str] = toyos_build::build::TEST_KERNEL; // Rust helper binaries that are spawned by tests, not tests themselves. const RUST_SKIP: &[&str] = &[ + // The DATA round trip has separate write and read boots, not a no-argument run. + "data_roundtrip", // Its explicit metal registration owns the directory listing workload. "readdir_bound", // It writes the log disk for ever, so that its boot's deadline resets the @@ -1593,6 +1595,47 @@ const METAL: &[(&str, metal::Metal)] = &[ "bench_data_designated", metal::Metal { arms: DESIGNATE, judge: |b| bench_data_designated(&b[0].log(), &b[1].log()) }, ), + ( + // Explicitly destroys the unique DATA partition on either bench. + // Never part of an unfiltered suite; no production image holds this grant. + "bench_data_formatted", + metal::Metal { + arms: &[metal::Arm { + named_only: true, + ..metal::once("data-format", "tests/images/formatcase.toml", &[], &["disks format data --commit"]) + }], + judge: |b| { + b[0].job_passed("disks")?; + b[0].log().must_say("supervisor: test-runner is authorized for `format:data`")?; + b[0].log().must_say("formatted as bcachefs; flushed and mounted for verification")?; + Ok(()) + }, + }, + ), + ( + // Both benches: ordinary Rust writes, flushes and namespace changes + // survive a real reboot and a fresh native DATA mount. + "bench_data_survives_reboot", + metal::Metal { + arms: &[ + metal::Arm { named_only: true, ..metal::once("data-roundtrip-1-write", "tests/images/datacase.toml", &[], &["test_rs_data_roundtrip write"]) }, + metal::Arm { named_only: true, ..metal::once("data-roundtrip-2-read", "tests/images/datacase.toml", &[], &["test_rs_data_roundtrip read"]) }, + ], + judge: |b| { + for boot in b { + boot.job_passed("test_rs_data_roundtrip")?; + let (_, fate) = common::hostdisk::left_alone(&boot.log(), &common::hostdisk::loader_named(&boot.loader())?)?; + if fate != common::hostdisk::Data::Mounted { + return Err("the round trip requires an already formatted native DATA volume".into()); + } + } + let written = data_roundtrip_token(b[0].log().text(), "written")?; + let read = data_roundtrip_token(b[1].log().text(), "verified")?; + if written != read { return Err("the second boot did not verify the first boot's DATA token".into()); } + Ok(()) + }, + }, + ), ( // The log disk written while the boot deadline resets the machine: the // pass after the reset reads the deadline's record and the host boots @@ -5053,7 +5096,7 @@ fn bench_data_designated(first: &serial::Serial, second: &serial::Serial) -> Res nvme_wrote_the_designation(first.text())?; first.must_say("the DATA partition is designated: the stamp is on the disk and read back")?; first.must_not_say("are in memory and will not survive a reboot")?; - second.must_say("fileserver: block 0 designates this partition for ToyOS; formatting it")?; + second.must_say("fileserver: formatting the explicitly designated DATA partition as bcachefs")?; let serving = second.must_say("fileserver: Data serving ")?; if !serving.contains(" — bcachefs") { return Err(format!("DATA is served by no bcachefs volume: {serving:?}")); @@ -5063,6 +5106,15 @@ fn bench_data_designated(first: &serial::Serial, second: &serial::Serial) -> Res Ok(()) } +fn data_roundtrip_token(text: &str, phase: &str) -> Result { + let marker = format!("DATA_ROUNDTRIP {phase} "); + let tokens: Vec<_> = text.lines().filter_map(|line| line.split_once(&marker).map(|(_, token)| token.trim())).collect(); + match tokens[..] { + [token] if token.len() == 32 && token.bytes().all(|b| b.is_ascii_hexdigit()) => Ok(token.to_string()), + _ => Err(format!("the DATA round trip has no unique valid {phase} token: {tokens:?}")), + } +} + /// A boot of the designation image beside [`common::hostdisk`]'s disk: a /// judge-only `disks designate data` writes nothing and says it would, the /// committed one is written by the NVMe driver and read back, and once the From 8bcffd21ccf599cf6184edbc398f49b6a24e305c Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 21:30:39 +0100 Subject: [PATCH 44/93] Count all data blocks read after sparse-file evacuation --- lib/bcachefs/src/engine/tests/data.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index f47431ba0a8..a5ed43839db 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -486,20 +486,20 @@ fn authenticated_live_sectors_survive_torn_reordered_evacuation() { fn evacuation_keeps_small_reads_within_one_physical_block() { use core::cell::Cell; use toyos_blockdev::Block; - struct Reads<'a> { dev: &'a Image, largest: Cell } + struct Reads<'a> { dev: &'a Image, blocks: Cell } impl BlockDev for Reads<'_> { fn blocks(&self) -> u64 { self.dev.blocks() } async fn read(&self, lba: u64, out: &mut [Block]) -> core::result::Result<(), toyos_blockdev::Error> { self.dev.read(lba, out).await } async fn write(&self, lba: u64, bytes: &[Block]) -> core::result::Result<(), toyos_blockdev::Error> { self.dev.write(lba, bytes).await } async fn flush(&self) -> core::result::Result<(), toyos_blockdev::Error> { self.dev.flush().await } async fn read_data(&self, lba: u64, out: &mut [Block]) -> core::result::Result<(), toyos_blockdev::Error> { - self.largest.set(self.largest.get().max(out.len())); + self.blocks.set(self.blocks.get() + out.len()); self.dev.read_data(lba, out).await } } for encrypted in [false, true] { let dev = formatted(64 << 20, encrypted.then_some(Encryption { passphrase: b"test", kdf: MIN_KDF })); - let counted = Reads { dev: &dev, largest: Cell::new(0) }; + let counted = Reads { dev: &dev, blocks: Cell::new(0) }; let m = if encrypted { block_on_ready(Mount::unlock(&counted, b"test", LIMITS, MEMORY, random())).unwrap() } else { block_on_ready(Mount::open(&counted, MEMORY, random())).unwrap() }; let mut inodes = Vec::new(); for i in 0..8 { @@ -511,10 +511,10 @@ fn evacuation_keeps_small_reads_within_one_physical_block() { assert!(block_on_ready(m.test_compact_user()).unwrap()); for (i, &ino) in inodes.iter().enumerate() { let mut got = vec![0; if i == 0 { BLOCK } else { 512 }]; - counted.largest.set(0); + counted.blocks.set(0); assert_eq!(block_on_ready(m.read(ino, 0, &mut got)).unwrap(), got.len()); assert!(got.iter().all(|&byte| byte == i as u8)); - assert_eq!(counted.largest.get(), 1); + assert_eq!(counted.blocks.get(), 1); } block_on_ready(m.unmount()).unwrap(); } From 605c7c3cb8fdf489011a7a21e3ad36b46ec468c1 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 21:23:31 +0100 Subject: [PATCH 45/93] Own replay keys in fallibly prepared sorted runs Replace the separate replay key and accounting maps with one pending owner. Insertion identity selects the newest ordinary value while the earliest original journal sequence retains its pin; accounting deltas remain ordered and versioned. Batch preflight funds incoming and merge storage, then moves owned values without allocating during apply. Replay overlays use the same ordered view. Captured transfers establish the destination pin before retiring their source and leave newer entries intact. Run searches avoid scanning the full pending set for each transfer. This is a representation checkpoint. Runtime transaction staging and reclaim liveness are not changed by it. Validation: engine library suite passed 161 tests; the subsequent test-only allocation fault injector passed all seven pending tests, including separate incoming and merge allocation failures. no_std library clippy passed with warnings denied. The inherited integration test target still references the interim filesystem API being removed by the consolidation branch. --- lib/bcachefs/src/engine/journal.rs | 52 ++- lib/bcachefs/src/engine/mod.rs | 3 + lib/bcachefs/src/engine/mount/reclaim.rs | 55 ++-- lib/bcachefs/src/engine/pending.rs | 394 +++++++++++++++++++++++ lib/bcachefs/src/engine/tests/journal.rs | 5 +- 5 files changed, 449 insertions(+), 60 deletions(-) create mode 100644 lib/bcachefs/src/engine/pending.rs diff --git a/lib/bcachefs/src/engine/journal.rs b/lib/bcachefs/src/engine/journal.rs index 90f3c2bf14d..90947ad8d4e 100644 --- a/lib/bcachefs/src/engine/journal.rs +++ b/lib/bcachefs/src/engine/journal.rs @@ -12,7 +12,7 @@ use alloc::vec::Vec; use toyos_blockdev::BlockDev; -use super::bkey::{Bpos, Format, Key, Version, KEY_U64S}; +use super::bkey::{Bpos, Format, Key, KEY_U64S}; use super::crypt::{MasterKey, Nonce}; use super::node::check_key; use super::read::read_bytes; @@ -374,21 +374,11 @@ fn keys(btree: Btree, level: u8, mut b: &[u8]) -> Result)>> { /// A key replay lays over a node's, or its deletion. pub(crate) type Replayed = Option<(Key, Vec)>; -/// A replayed key and the sequence of the entry it came from. -pub(crate) struct Logged { - pub seq: u64, - pub key: Replayed, -} - /// The window's records laid over the btrees: what a reader of an unclean /// volume sees in place of the nodes' keys. #[derive(Default)] pub(crate) struct Replay { - /// Every replayed key but accounting's, by btree, level and position, the - /// latest winning; a deletion as `None`. - pub keys: BTreeMap<(Btree, u8, Bpos), Logged>, - /// Accounting deltas by position, in journal order. - pub accounting: BTreeMap)>>, + pub pending: super::pending::Pending, /// The latest root record of each btree. pub roots: Vec, pub key_version: Option, @@ -409,17 +399,14 @@ pub(crate) fn replay(scan: &Scan, sb: &Superblock, w: &Window) -> Result match record { Record::Keys { btree, level, keys } => { r.read += keys.len(); - for (key, value) in keys { - if btree == Btree::Accounting { - if key.kind != kind::ACCOUNTING { - return Err(Error::Corrupt("accounting record of another type")); - } - r.accounting.entry(key.pos).or_default().push((key.version, counters(&value)?)); - continue; + if btree == Btree::Accounting && level == 0 { + for (key, value) in &keys { + if key.kind != kind::ACCOUNTING { return Err(Error::Corrupt("accounting record of another type")); } + counters(value)?; } - let replayed = (key.kind != kind::DELETED).then_some((key, value)); - r.keys.insert((btree, level, key.pos), Logged { seq: found.seq, key: replayed }); } + r.pending.reserve(keys.len())?; + r.pending.apply(found.seq, keys.into_iter().map(|(key, value)| (btree, level, key, value))); } Record::Root(root) => { r.roots.retain(|old| old.btree != root.btree); @@ -451,33 +438,38 @@ impl Replay { /// (A7.8), dropped when its counters come to zero. pub(crate) fn over(&self, btree: Btree, level: u8, min: Bpos, max: Bpos, keys: Vec<(Key, Vec)>) -> Result)>> { let mut merged: BTreeMap)> = keys.into_iter().map(|(k, v)| (k.pos, (k, v))).collect(); + let mut pending = self.pending.range(btree, level, min, max).peekable(); if btree == Btree::Accounting && level == 0 { - for (&pos, deltas) in self.accounting.range(min..=max) { + while let Some(first) = pending.peek() { + let pos = first.key().pos; let stored = merged.remove(&pos); let (mut key, mut sums) = match stored { Some((key, value)) => (key, counters(&value)?), None => (Key::new(kind::ACCOUNTING, pos), Vec::new()), }; let since = key.version; - for (version, delta) in deltas.iter().filter(|(v, _)| *v > since) { + while pending.peek().is_some_and(|entry| entry.key().pos == pos) { + let entry = pending.next().unwrap(); + let version = entry.key().version; + if version <= since { continue; } + let delta = counters(entry.value())?; if sums.len() < delta.len() { sums.resize(delta.len(), 0); } for (sum, d) in sums.iter_mut().zip(delta) { - *sum = sum.wrapping_add(*d); + *sum = sum.wrapping_add(d); } - key.version = key.version.max(*version); + key.version = key.version.max(version); } if sums.iter().any(|&c| c != 0) { merged.insert(pos, (key, sums.iter().flat_map(|c| c.to_le_bytes()).collect())); } } } else { - for ((_, _, pos), logged) in self.keys.range((btree, level, min)..=(btree, level, max)) { - match &logged.key { - Some(kv) => merged.insert(*pos, kv.clone()), - None => merged.remove(pos), - }; + for logged in pending { + let key = *logged.key(); + if key.kind == kind::DELETED { merged.remove(&key.pos); } + else { merged.insert(key.pos, (key, logged.value().to_vec())); } } } Ok(merged.into_values().collect()) diff --git a/lib/bcachefs/src/engine/mod.rs b/lib/bcachefs/src/engine/mod.rs index 0c6ddaab777..f261aa9603c 100644 --- a/lib/bcachefs/src/engine/mod.rs +++ b/lib/bcachefs/src/engine/mod.rs @@ -19,6 +19,7 @@ mod journal; mod mkfs; mod node; mod mount; +mod pending; mod read; mod superblock; mod sync; @@ -39,6 +40,8 @@ pub use value::DirentType; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Error { Device(toyos_blockdev::Error), + /// Memory for an operation could not be obtained before it applied. + NoMemory, /// A checksum or MAC disagreed with the bytes it covers. Checksum(&'static str), /// Bytes that are not the format, and what was wrong with them. diff --git a/lib/bcachefs/src/engine/mount/reclaim.rs b/lib/bcachefs/src/engine/mount/reclaim.rs index b471ecbe44f..74c9a7d2523 100644 --- a/lib/bcachefs/src/engine/mount/reclaim.rs +++ b/lib/bcachefs/src/engine/mount/reclaim.rs @@ -828,36 +828,37 @@ impl Mount { /// nodes are within the limit again. pub(super) async fn install_replay(&self, replayed: journal::Replay) -> Result<()> { self.st.borrow_mut().journal.replaying = true; - let mut keys: Vec<_> = replayed.keys.into_iter().collect(); - keys.sort_by_key(|((btree, level, pos), _)| (core::cmp::Reverse(*level), *btree, *pos)); - for ((btree, level, pos), logged) in keys { + let mut pending = replayed.pending; + let mut captured = Vec::new(); + captured.try_reserve(pending.iter().count()).map_err(|_| Error::NoMemory)?; + captured.extend(pending.iter().map(|entry| entry.capture())); + captured.sort_unstable_by_key(|c| (core::cmp::Reverse(c.position.1), c.position.0, c.position.2, c.order)); + let mut accounting = None; + for capture in captured { + let (btree, level, pos) = capture.position; let id = self.find(btree, level, pos).await?; - self.st.borrow_mut().cache.change(id, |n| n.put(pos, logged.key, logged.seq)); - } - for (pos, deltas) in replayed.accounting { - let id = self.find(Btree::Accounting, 0, pos).await?; - self.st.borrow_mut().cache.change(id, |n| { - let i = n.at(pos); - let (mut key, mut sums) = match n.keys.get(i) { - Some((k, v)) if k.pos == pos => (*k, journal::counters(v).expect("a cached accounting key decodes")), - _ => (Key::new(kind::ACCOUNTING, pos), Vec::new()), - }; - let since = key.version; - let mut first = None; - for (version, delta) in deltas.iter().filter(|(v, _)| *v > since) { - sums.resize(sums.len().max(delta.len()), 0); - for (sum, d) in sums.iter_mut().zip(delta) { - *sum = sum.wrapping_add(*d); + assert!(pending.transfer(capture, |entry| { + self.st.borrow_mut().cache.change(id, |n| { + if btree != Btree::Accounting || level != 0 { + let key = (entry.key.kind != kind::DELETED).then_some((entry.key, entry.value)); + n.put(pos, key, entry.first); + return; } - key.version = *version; - let seq = u64::from(version.hi) << 32 | version.lo >> 32; - first = Some(first.map_or(seq, |f: u64| f.min(seq))); - } - if let Some(seq) = first { - n.put(pos, Some((key, value::counters(&sums))), seq); - } - }); + let (mut key, mut sums) = match n.keys.get(n.at(pos)) { + Some((k, v)) if k.pos == pos => (*k, journal::counters(v).expect("a cached accounting key decodes")), + _ => (Key::new(kind::ACCOUNTING, pos), Vec::new()), + }; + if accounting.is_none_or(|(previous, _)| previous != pos) { accounting = Some((pos, key.version)); } + if entry.key.version <= accounting.unwrap().1 { return; } + let delta = journal::counters(&entry.value).expect("a replayed accounting delta decodes"); + sums.resize(sums.len().max(delta.len()), 0); + for (sum, d) in sums.iter_mut().zip(delta) { *sum = sum.wrapping_add(d); } + key.version = key.version.max(entry.key.version); + n.put(pos, Some((key, value::counters(&sums))), entry.first); + }); + }), "a replay capture has no concurrent replacement"); } + assert!(pending.oldest_pin().is_none(), "all replay pins transferred"); { let st = self.st.borrow(); let need = ((st.cache.dirty + st.cache.unpointed) as u64 * PER_NODE).div_ceil(st.journal.body() - st.journal.max_tx) + 1; diff --git a/lib/bcachefs/src/engine/pending.rs b/lib/bcachefs/src/engine/pending.rs new file mode 100644 index 00000000000..d27fb947148 --- /dev/null +++ b/lib/bcachefs/src/engine/pending.rs @@ -0,0 +1,394 @@ +//! Journal-owned keys, independent of the leaves that will materialize them. + +use alloc::vec::Vec; + +use super::bkey::{Bpos, Key}; +use super::{Btree, Error, Result}; + +pub(crate) type Position = (Btree, u8, Bpos); +pub(crate) type Update = (Btree, u8, Key, Vec); +const LEVELS: usize = u64::BITS as usize; + +struct Entry { + btree: Btree, + level: u8, + key: Key, + value: Vec, + first: u64, + order: u64, + live: bool, +} + +impl Entry { + fn position(&self) -> Position { (self.btree, self.level, self.key.pos) } + fn ordering(&self) -> (Position, u64) { (self.position(), self.order) } + fn accounting(&self) -> bool { self.btree == Btree::Accounting && self.level == 0 } +} + +/// An identity survives run merges; a newer update at its position does not +/// become part of a transfer already captured. +#[derive(Clone, Copy)] +pub(crate) struct Capture { + pub position: Position, + pub order: u64, + pub first: u64, +} + +pub(crate) struct View<'a> { + entry: &'a Entry, + first: u64, +} + +impl View<'_> { + pub fn position(&self) -> Position { self.entry.position() } + pub fn key(&self) -> &Key { &self.entry.key } + pub fn value(&self) -> &[u8] { &self.entry.value } + pub fn capture(&self) -> Capture { Capture { position: self.position(), order: self.entry.order, first: self.first } } +} + +pub(crate) struct Transferred { + pub key: Key, + pub value: Vec, + pub first: u64, +} + +pub(crate) struct Pending { + runs: [Option>; LEVELS], + incoming: Vec, + merged: Vec, + next: u64, + prepared: usize, +} + +impl Default for Pending { + fn default() -> Self { + Self { runs: core::array::from_fn(|_| None), incoming: Vec::new(), merged: Vec::new(), next: 0, prepared: 0 } + } +} + +impl Pending { + /// Reserve all workspace before making a logical batch visible. + pub fn reserve(&mut self, additional: usize) -> Result<()> { + assert!(self.incoming.is_empty() && self.merged.is_empty()); + let mut total = additional; + for run in &self.runs { + let Some(run) = run else { break }; + total = total.checked_add(run.iter().filter(|e| e.live).count()).ok_or(Error::NoMemory)?; + } + if self.runs.iter().all(Option::is_some) || u64::try_from(additional).ok().and_then(|n| self.next.checked_add(n)).is_none() { + return Err(Error::NoMemory); + } + self.incoming.try_reserve(additional).map_err(|_| Error::NoMemory)?; + self.merged.try_reserve(total).map_err(|_| Error::NoMemory)?; + self.prepared = additional; + Ok(()) + } + + /// `reserve` funded this batch. Values move; sorting and merging allocate + /// nothing. Equal positions retain the latest value and earliest pin. + pub fn apply(&mut self, seq: u64, updates: impl IntoIterator) { + for (btree, level, key, value) in updates { + assert!(self.incoming.len() < self.prepared, "pending batch exceeded its reservation"); + self.incoming.push(Entry { btree, level, key, value, first: seq, order: self.next, live: true }); + self.next += 1; + } + self.prepared = 0; + if self.incoming.is_empty() { return; } + self.incoming.sort_unstable_by_key(Entry::ordering); + let level = self.runs.iter().position(Option::is_none).expect("a prepared run slot"); + let mut at = [0usize; LEVELS]; + let mut incoming_at = 0; + loop { + let mut chosen = None; + let mut best = self.incoming.get(incoming_at).map(Entry::ordering); + for (i, cursor) in at.iter_mut().enumerate().take(level) { + let run = self.runs[i].as_ref().expect("preceding run occupied"); + while run.get(*cursor).is_some_and(|e| !e.live) { *cursor += 1; } + if let Some(e) = run.get(*cursor) { + if best.is_none_or(|old| e.ordering() < old) { + best = Some(e.ordering()); + chosen = Some(i); + } + } + } + if best.is_none() { break; } + let source = match chosen { + Some(i) => { + let e = &mut self.runs[i].as_mut().unwrap()[at[i]]; + at[i] += 1; + e + } + None => { + let e = &mut self.incoming[incoming_at]; + incoming_at += 1; + e + } + }; + let mut entry = Entry { value: core::mem::take(&mut source.value), ..*source }; + source.live = false; + if !entry.accounting() && self.merged.last().is_some_and(|old| old.position() == entry.position()) { + entry.first = entry.first.min(self.merged.last().unwrap().first); + *self.merged.last_mut().unwrap() = entry; + } else { + self.merged.push(entry); + } + } + self.incoming.clear(); + for run in &mut self.runs[..level] { *run = None; } + self.runs[level] = Some(core::mem::take(&mut self.merged)); + } + + pub fn oldest_pin(&self) -> Option { + self.runs.iter().flatten().flat_map(|r| r.iter()).filter(|e| e.live).map(|e| e.first).min() + } + + pub fn iter(&self) -> Iter<'_> { self.between(None) } + + pub fn range(&self, btree: Btree, level: u8, min: Bpos, max: Bpos) -> Iter<'_> { + self.between(Some(((btree, level, min), (btree, level, max)))) + } + + fn between(&self, bounds: Option<(Position, Position)>) -> Iter<'_> { + let mut at = [0; LEVELS]; + if let Some((min, _)) = bounds { + for (i, run) in self.runs.iter().enumerate() { + if let Some(run) = run { at[i] = run.partition_point(|e| e.position() < min); } + } + } + Iter { pending: self, at, end: bounds.map(|(_, max)| max), saved: None } + } + + /// The callback installs the value and its pin before the source retires. + /// It is infallible and cannot yield. `false` means capture was superseded. + pub fn transfer(&mut self, capture: Capture, install: impl FnOnce(Transferred)) -> bool { + let wanted = (capture.position, capture.order); + let source = self.runs.iter_mut().flatten().find_map(|run| { + let at = run.partition_point(|e| e.ordering() < wanted); + run.get_mut(at).filter(|e| e.live && e.ordering() == wanted) + }); + let Some(source) = source else { return false }; + let accounting = source.accounting(); + let value = core::mem::take(&mut source.value); + install(Transferred { key: source.key, value, first: capture.first }); + for run in self.runs.iter_mut().flatten() { + let at = run.partition_point(|e| e.position() < capture.position); + for entry in run[at..].iter_mut().take_while(|e| e.position() == capture.position && e.order <= capture.order) { + if entry.live && (entry.order == capture.order || !accounting) { + entry.live = false; + entry.value = Vec::new(); + } + } + } + true + } +} + +pub(crate) struct Iter<'a> { + pending: &'a Pending, + at: [usize; LEVELS], + end: Option, + saved: Option<&'a Entry>, +} + +impl<'a> Iter<'a> { + fn raw(&mut self) -> Option<&'a Entry> { + if let Some(saved) = self.saved.take() { return Some(saved); } + let mut chosen = None; + let mut best = None; + for (i, run) in self.pending.runs.iter().enumerate() { + let Some(run) = run else { continue }; + while run.get(self.at[i]).is_some_and(|e| !e.live) { self.at[i] += 1; } + if let Some(entry) = run.get(self.at[i]) { + if self.end.is_none_or(|end| entry.position() <= end) && best.is_none_or(|old| entry.ordering() < old) { + chosen = Some(i); + best = Some(entry.ordering()); + } + } + } + let i = chosen?; + let entry = &self.pending.runs[i].as_ref().unwrap()[self.at[i]]; + self.at[i] += 1; + Some(entry) + } +} + +impl<'a> Iterator for Iter<'a> { + type Item = View<'a>; + fn next(&mut self) -> Option { + let mut entry = self.raw()?; + let mut first = entry.first; + if !entry.accounting() { + while let Some(next) = self.raw() { + if next.position() != entry.position() { + self.saved = Some(next); + break; + } + first = first.min(next.first); + entry = next; + } + } + Some(View { entry, first }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use super::super::{journal, value, Version}; + use alloc::vec; + use std::collections::BTreeMap; + + mod allocation { + use core::cell::Cell; + use std::alloc::{GlobalAlloc, Layout, System}; + + std::thread_local! { static FAIL: Cell> = const { Cell::new(None) }; } + + struct Allocator; + #[global_allocator] + static ALLOCATOR: Allocator = Allocator; + + fn fail() -> bool { + FAIL.try_with(|f| match f.get() { + None => false, + Some(0) => { f.set(None); true } + Some(n) => { f.set(Some(n - 1)); false } + }).unwrap_or(false) + } + + // Successful operations and every deallocation use System's + // matching layout and pointer; failure returns its null sentinel. + unsafe impl GlobalAlloc for Allocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + if fail() { core::ptr::null_mut() } else { unsafe { System.alloc(layout) } } + } + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + if fail() { core::ptr::null_mut() } else { unsafe { System.alloc_zeroed(layout) } } + } + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, size: usize) -> *mut u8 { + if fail() { core::ptr::null_mut() } else { unsafe { System.realloc(ptr, layout, size) } } + } + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { unsafe { System.dealloc(ptr, layout) } } + } + + pub fn after(successful: usize, f: impl FnOnce() -> T) -> T { + struct Reset; + impl Drop for Reset { fn drop(&mut self) { FAIL.set(None); } } + FAIL.set(Some(successful)); + let _reset = Reset; + f() + } + } + + fn update(pos: u64, value: u64) -> Update { + (Btree::Inodes, 0, Key::new(super::super::kind::INODE_V3, Bpos::new(0, pos, 0)), value.to_le_bytes().to_vec()) + } + + fn add(pending: &mut Pending, seq: u64, updates: Vec) { + pending.reserve(updates.len()).unwrap(); + pending.apply(seq, updates); + } + + #[test] + fn batches_keep_latest_values_and_original_pins_without_cloning_values() { + let mut pending = Pending::default(); + let mut wanted = BTreeMap::new(); + for batch in 0..257u64 { + let updates: Vec<_> = (0..7).map(|i| update((batch * 7 + i) % 61, batch * 7 + i)).collect(); + for (_, _, key, value) in &updates { + let old = wanted.get(&key.pos).map_or(batch + 10, |(_, first, _)| *first); + wanted.insert(key.pos, (value.as_ptr(), old, value.clone())); + } + add(&mut pending, batch + 10, updates); + let got: Vec<_> = pending.iter().collect(); + assert_eq!(got.len(), wanted.len()); + for (entry, (pos, (ptr, first, bytes))) in got.iter().zip(&wanted) { + assert_eq!((entry.key().pos, entry.value(), entry.capture().first), (*pos, &bytes[..], *first)); + assert_eq!(entry.value().as_ptr(), *ptr, "owned values move through run merges"); + } + } + assert_eq!(pending.oldest_pin(), Some(10)); + } + + #[test] + fn a_captured_transfer_leaves_a_concurrent_newer_value_pending() { + let mut pending = Pending::default(); + add(&mut pending, 3, vec![update(1, 11)]); + add(&mut pending, 4, vec![update(2, 22)]); + let capture = pending.range(Btree::Inodes, 0, Bpos::new(0, 1, 0), Bpos::new(0, 1, 0)).next().unwrap().capture(); + add(&mut pending, 7, vec![update(1, 33)]); + let mut destination = None; + assert!(pending.transfer(capture, |entry| destination = Some((entry.first, entry.value)))); + assert_eq!(destination, Some((3, 11u64.to_le_bytes().to_vec()))); + let latest = pending.range(Btree::Inodes, 0, Bpos::new(0, 1, 0), Bpos::new(0, 1, 0)).next().unwrap(); + assert_eq!((latest.value(), latest.capture().first), (&33u64.to_le_bytes()[..], 7)); + assert_eq!(destination.unwrap().0.min(pending.oldest_pin().unwrap()), 3, "destination owns the transferred dependency"); + } + + #[test] + fn coalescing_a_captured_value_requires_a_new_capture() { + let mut pending = Pending::default(); + add(&mut pending, 3, vec![update(1, 11)]); + let capture = pending.iter().next().unwrap().capture(); + add(&mut pending, 7, vec![update(1, 33)]); + assert!(!pending.transfer(capture, |_| panic!("superseded capture installed"))); + let latest = pending.iter().next().unwrap(); + assert_eq!((latest.value(), latest.capture().first), (&33u64.to_le_bytes()[..], 3)); + } + + #[test] + fn failed_capacity_preflight_preserves_the_visible_batch_and_pin() { + let mut pending = Pending::default(); + add(&mut pending, 3, vec![update(1, 11)]); + assert_eq!(pending.reserve(usize::MAX), Err(Error::NoMemory)); + assert_eq!(pending.oldest_pin(), Some(3)); + assert_eq!(pending.iter().next().unwrap().value(), &11u64.to_le_bytes()); + add(&mut pending, 7, vec![update(1, 33)]); + assert_eq!(pending.iter().next().unwrap().value(), &33u64.to_le_bytes()); + } + + #[test] + fn either_workspace_allocation_can_fail_before_any_key_or_pin_changes() { + for successful in 0..2 { + let mut pending = Pending::default(); + add(&mut pending, 3, vec![update(1, 11)]); + let extra = pending.incoming.capacity() + 1; + assert_eq!(allocation::after(successful, || pending.reserve(extra)), Err(Error::NoMemory)); + assert_eq!(pending.oldest_pin(), Some(3)); + let before: Vec<_> = pending.iter().collect(); + assert_eq!(before.len(), 1); + assert_eq!(before[0].value(), &11u64.to_le_bytes()); + add(&mut pending, 7, vec![update(1, 33)]); + assert_eq!(pending.iter().next().unwrap().value(), &33u64.to_le_bytes()); + } + } + + #[test] + fn accounting_versions_survive_run_merges_and_overlay_only_once() { + let pos = Bpos::new(0, 9, 0); + let mut replay = journal::Replay::default(); + for (version, delta) in [(2, 2), (3, 3), (4, 13u64.wrapping_neg()), (5, 7)] { + let key = Key { version: Version { hi: 0, lo: version }, ..Key::new(super::super::kind::ACCOUNTING, pos) }; + add(&mut replay.pending, version, vec![(Btree::Accounting, 0, key, value::counters(&[delta]))]); + } + assert_eq!(replay.pending.iter().count(), 4); + let stored = Key { version: Version { hi: 0, lo: 2 }, ..Key::new(super::super::kind::ACCOUNTING, pos) }; + let once = replay.over(Btree::Accounting, 0, Bpos::MIN, Bpos::MAX, vec![(stored, value::counters(&[10]))]).unwrap(); + assert_eq!(journal::counters(&once[0].1).unwrap(), [7]); + assert_eq!(once[0].0.version.lo, 5); + let twice = replay.over(Btree::Accounting, 0, Bpos::MIN, Bpos::MAX, once.clone()).unwrap(); + assert_eq!(once, twice); + } + + #[test] + fn deletions_overlay_stored_keys_and_ranges_exclude_neighboring_positions() { + let mut replay = journal::Replay::default(); + add(&mut replay.pending, 1, vec![update(1, 11), update(2, 22), update(3, 33)]); + let erased = Key::new(super::super::kind::DELETED, Bpos::new(0, 2, 0)); + add(&mut replay.pending, 2, vec![(Btree::Inodes, 0, erased, Vec::new())]); + let (_, _, key, value) = update(2, 99); + assert_eq!(replay.over(Btree::Inodes, 0, key.pos, key.pos, vec![(key, value)]).unwrap(), []); + let full = replay.over(Btree::Inodes, 0, Bpos::MIN, Bpos::MAX, Vec::new()).unwrap(); + assert_eq!(full.iter().map(|(k, _)| k.pos.offset).collect::>(), [1, 3]); + } +} diff --git a/lib/bcachefs/src/engine/tests/journal.rs b/lib/bcachefs/src/engine/tests/journal.rs index 1a903a1963e..46da1392d53 100644 --- a/lib/bcachefs/src/engine/tests/journal.rs +++ b/lib/bcachefs/src/engine/tests/journal.rs @@ -20,8 +20,7 @@ fn the_unclean_fixture_is_read_through_its_journal() { let (window, replay) = v.replayed().expect("an unclean volume replays"); assert_eq!((window.first, window.last, window.next), (61, 62, 63)); // Accounting deltas add rather than replace, so each is kept. - let deltas: usize = replay.accounting.values().map(Vec::len).sum(); - assert_eq!((replay.read, replay.keys.len() + deltas), (115, 90)); + assert_eq!((replay.read, replay.pending.iter().count()), (115, 90)); let mut want: Vec = (1..=200).map(|i| format!("f{i}")).collect(); want.push("lost+found".into()); want.sort(); @@ -133,7 +132,7 @@ fn a_torn_entry_is_refused(encryption: Option) { if window.last != last { return Err(format!("replay ended at {} with the fourth entry {}", window.last, if whole { "whole" } else { "torn or lost" })); } - let inodes = replay.keys.keys().filter(|(b, _, _)| *b == Btree::Inodes).count() as u64; + let inodes = replay.pending.iter().filter(|entry| entry.position().0 == Btree::Inodes).count() as u64; if inodes != last { return Err(format!("{inodes} inodes replayed through entry {last}")); } From 0507cbe36243df1a75332e5a396fa5e3263f1238 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 22:08:08 +0100 Subject: [PATCH 46/93] Keep post-admission spawn failure coverage after removing kernel paths A missing executable now fails in userland before the spawn syscall. Use an existing malformed ELF so both process-tree arms still exercise kernel loader refusal after admission, and retain the host check for both refusal records. Remove the obsolete claim that the kernel opens the temporary executable. The T14 hardware run exposed the stale oracle: the guest passed, while the host correctly refused to credit a loader path that had not run. Independent review approved this replacement; both benches will rerun the affected boot. --- tests/guest/rust-tests/src/bin/process_tree.rs | 13 ++++++------- tests/toyos.rs | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/guest/rust-tests/src/bin/process_tree.rs b/tests/guest/rust-tests/src/bin/process_tree.rs index 29c6ebc21d0..c05784201d9 100644 --- a/tests/guest/rust-tests/src/bin/process_tree.rs +++ b/tests/guest/rust-tests/src/bin/process_tree.rs @@ -44,11 +44,9 @@ use std::os::toyos::process::ExitStatusExt; use toyos_abi::end::{End, Fault}; const SELF_PATH: &str = "/system/bin/test_rs_process_tree"; -/// This program copied to `/tmp`, which the kernel opens itself, under the -/// same name: a chain's links spawn it with no file server's connection -/// each, so a file server's bound on one share's clients is not what stops -/// the chain before its depth does. +/// The chain loads its executable through the temporary file server. const LINK_PATH: &str = "/tmp/test_rs_process_tree"; +const INVALID_PATH: &str = "/tmp/process_tree_invalid_elf"; /// `/system/bin/toybox` under the name of an applet that reads its input until /// it ends: a declared program, so a launch, and one that runs until killed /// while A holds its input open. @@ -97,9 +95,11 @@ fn main() { } fn test() { + std::fs::write(INVALID_PATH, [0; 64]).expect("write an invalid executable"); for end in [Ending::Killed, Ending::CpuFault] { an_end_takes_its_subtree(end); } + std::fs::remove_file(INVALID_PATH).expect("remove the invalid executable"); a_manage_only_handle_is_no_place(); a_pipe_is_no_place(); a_chain_stops_at_max_depth_and_dies_whole(); @@ -394,9 +394,8 @@ fn park() -> ! { fn b() -> ! { let conn = endow::service(BACK).expect("A endowed a port back"); - // Admitted under B, then refused by the loader: its hold on B goes. On - // `/tmp`, which the kernel opens itself, so its loader is what refuses it. - assert!(Command::new("/tmp/no_such_program").spawn().is_err(), "a program that is not there started"); + // The supplied image reaches ELF validation after admission under B. + assert!(Command::new(INVALID_PATH).spawn().is_err(), "an invalid executable started"); let c = Command::new(SELF_PATH).arg("c").stdin(Stdio::null()).spawn().expect("B spawns C"); let d = Command::new(HELD).independent().spawn().expect("B launches D"); let e = Command::new(HELD).independent().detached().spawn().expect("B launches E detached"); diff --git a/tests/toyos.rs b/tests/toyos.rs index 13e682cf7ce..1a2e1fd137f 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -6477,10 +6477,10 @@ fn process_tree(back: &metal::Readback) -> Result<(), String> { back.job_passed("test_rs_process_tree")?; let kernel = back.kernel(); let log = kernel.text(); - let unloaded = log.lines().filter(|l| l.contains("spawn: /tmp/no_such_program: ")).count(); + let unloaded = log.lines().filter(|l| l.contains("spawn: /tmp/process_tree_invalid_elf: ")).count(); if unloaded != 2 { return Err(format!( - "the loader refused /tmp/no_such_program {unloaded} times, not once per B" + "the loader refused the invalid executable {unloaded} times, not once per B" )); } let refused: Vec<&str> = log.lines().filter(|l| l.contains("spawn: refused under pid ")).collect(); From a879ad4f53378a06b714bcfa7ac2014f5f20f097 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 21:41:03 +0100 Subject: [PATCH 47/93] Record the engine's preexisting in-flight cancellation gap Ordinary returned errors and unused reservation drops are in the current consolidation scope. Arbitrary future cancellation is a distinct finding: the fileserver currently runs engine operations to completion, and no new cancellation support is claimed by the pending-key representation. --- ...s-operation-does-not-release-its-grants.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 issues/dropping-an-in-flight-bcachefs-operation-does-not-release-its-grants.md diff --git a/issues/dropping-an-in-flight-bcachefs-operation-does-not-release-its-grants.md b/issues/dropping-an-in-flight-bcachefs-operation-does-not-release-its-grants.md new file mode 100644 index 00000000000..2802acc2c78 --- /dev/null +++ b/issues/dropping-an-in-flight-bcachefs-operation-does-not-release-its-grants.md @@ -0,0 +1,25 @@ +--- +status: open +kind: finding +opened: 2026-11-02 +--- + +# Dropping an in-flight bcachefs operation does not release its grants + +`Mount::transact` in `lib/bcachefs/src/engine/mount/mod.rs` acquires a +journal/cache grant and then awaits `prepare`. Dropping that future during +device I/O drops the writer slot, but does not call `finish`: the grant, +cached-node holds, and any `Txn::allocated_data` cleanup remain outstanding. +The reclaim path also retains captured or writing nodes if its future is +dropped between planning, node I/O and pointer publication. + +This is a code-inspection finding; cancellation has not been injected. The +fileserver currently runs these operations to completion with `block_on`. +Unused `Reservation` drop and returned operation errors have separate +cleanup paths and are not this finding. + +The owning component is the bcachefs engine. A future cancellation change +must test dropping at each awaited boundary, including already completed +node writes, and establish either retryable ownership or an explicit stopped +mount without leaving admissions waiting forever. It must preserve reserved +DATA slots, reader barriers and the original journal dependencies. From 11632e8b6876e5c93910b62bb5d1f2c2476c62cb Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 22:21:24 +0100 Subject: [PATCH 48/93] Stage runtime btree changes in journal-owned pending runs Move logical apply and replay to one pending-key owner, merging its overlay into every cached tree read. Materialize bounded ranges on reclaim pressure, transfer original pins before source retirement, and run allocator/accounting triggers only at original transaction apply. Fund pending workspace before prepare so allocation refusal preserves DATA reservation retry state. Track latest represented journal sequence separately from the oldest lifetime pin and preserve source bset dependencies through fresh rewrites and clean relocation. Permit finite freeing operations past the ordinary stable-growth estimate while retaining physical and journal admission checks. This is an isolated review checkpoint, not a completed reclaim solution. Full-volume reboot/removal tests still exhaust the live journal reserve at removal 865. Both negative controls for bset provenance build and fail at their intended assertion; the fixes are restored. All 163 other library tests pass, and no-default-features library clippy passes. Keep the expanded all-removal, metadata-only churn and reuse tests as the remaining liveness gate. --- lib/bcachefs/src/engine/cache.rs | 27 ++- lib/bcachefs/src/engine/journal.rs | 46 +---- lib/bcachefs/src/engine/mount/allocator.rs | 4 +- lib/bcachefs/src/engine/mount/data.rs | 4 +- lib/bcachefs/src/engine/mount/mod.rs | 140 +++++++++------ lib/bcachefs/src/engine/mount/reclaim.rs | 180 ++++++++++++-------- lib/bcachefs/src/engine/pending.rs | 127 +++++++++++--- lib/bcachefs/src/engine/tests/admission.rs | 31 ++++ lib/bcachefs/src/engine/tests/buckets.rs | 49 ++++-- lib/bcachefs/src/engine/tests/compact.rs | 16 +- lib/bcachefs/src/engine/tests/crash.rs | 2 +- lib/bcachefs/src/engine/tests/data.rs | 29 +++- lib/bcachefs/src/engine/tests/interleave.rs | 25 ++- lib/bcachefs/src/engine/tests/journal.rs | 19 +-- 14 files changed, 472 insertions(+), 227 deletions(-) diff --git a/lib/bcachefs/src/engine/cache.rs b/lib/bcachefs/src/engine/cache.rs index 69545bb93dc..add03c9de2d 100644 --- a/lib/bcachefs/src/engine/cache.rs +++ b/lib/bcachefs/src/engine/cache.rs @@ -23,6 +23,12 @@ pub(crate) type Keys = Rc)>>; #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub(crate) struct NodeId(usize); +#[derive(Clone, Copy)] +pub(crate) struct Change { + pub first: u64, + pub last: u64, +} + /// One node in memory. pub(crate) struct Cached { pub btree: Btree, @@ -40,12 +46,14 @@ pub(crate) struct Cached { pub on_disk: u16, /// Bsets on the medium. pub bsets: u16, + /// Maximum dependency of the materialized bsets, not a replay watermark. + pub journal_seq: u64, pub keys: Keys, /// The bytes its keys take in a bset. pub bytes: usize, - /// Positions changed since the last capture, each with the sequence of - /// its first change since. - pub unwritten: BTreeMap, + /// Positions changed since capture, with their oldest lifetime pin and + /// latest represented journal dependency. + pub unwritten: BTreeMap, /// The oldest sequence a written bset holds whose pointer update is not /// applied. pub captured: Option, @@ -64,7 +72,7 @@ pub(crate) struct Cached { impl Cached { pub(crate) fn pin(&self) -> Option { - let oldest = self.unwritten.values().copied().chain(self.compact).min(); + let oldest = self.unwritten.values().map(|c| c.first).chain(self.compact).min(); match (oldest, self.captured) { (Some(a), Some(b)) => Some(a.min(b)), (a, b) => a.or(b), @@ -86,8 +94,8 @@ impl Cached { self.keys.partition_point(|(k, _)| k.pos < pos) } - /// Sets the key at `pos`, or removes it; `seq` is the change's sequence. - pub(crate) fn put(&mut self, pos: Bpos, kv: Option<(Key, Vec)>, seq: u64) { + /// `first` owns the original pin; `last` is the represented dependency. + pub(crate) fn put(&mut self, pos: Bpos, kv: Option<(Key, Vec)>, first: u64, last: u64) { let keys = Rc::make_mut(&mut self.keys); let i = keys.partition_point(|(k, _)| k.pos < pos); let here = keys.get(i).is_some_and(|(k, _)| k.pos == pos); @@ -105,7 +113,7 @@ impl Cached { } (None, false) => {} } - self.unwritten.entry(pos).or_insert(seq); + self.unwritten.entry(pos).and_modify(|c| { c.first = c.first.min(first); c.last = c.last.max(last); }).or_insert(Change { first, last }); } } @@ -286,6 +294,7 @@ mod tests { written: 1, on_disk: 1, bsets: 1, + journal_seq: 1, keys: Rc::new(Vec::new()), bytes: 0, unwritten: BTreeMap::new(), @@ -302,9 +311,9 @@ mod tests { fn a_node_just_inserted_outlives_its_own_insert_when_only_it_is_clean() { let mut cache = Cache::new(1, usize::MAX); let dirty = cache.insert(clean(8)); - cache.change(dirty, |n| n.put(Bpos::MAX, Some((Key::new(kind::SET, Bpos::MAX), Vec::new())), 3)); + cache.change(dirty, |n| n.put(Bpos::MAX, Some((Key::new(kind::SET, Bpos::MAX), Vec::new())), 3, 3)); let new = cache.insert(clean(16)); - cache.change(new, |n| n.put(Bpos::MAX, None, 4)); + cache.change(new, |n| n.put(Bpos::MAX, None, 4, 4)); assert_eq!((cache.id_at(16), cache.dirty, cache.oldest_pin()), (Some(new), 2, Some(3))); } } diff --git a/lib/bcachefs/src/engine/journal.rs b/lib/bcachefs/src/engine/journal.rs index 90947ad8d4e..2ee5d743a73 100644 --- a/lib/bcachefs/src/engine/journal.rs +++ b/lib/bcachefs/src/engine/journal.rs @@ -437,42 +437,12 @@ impl Replay { /// and an accounting key adds every delta newer than its own version /// (A7.8), dropped when its counters come to zero. pub(crate) fn over(&self, btree: Btree, level: u8, min: Bpos, max: Bpos, keys: Vec<(Key, Vec)>) -> Result)>> { - let mut merged: BTreeMap)> = keys.into_iter().map(|(k, v)| (k.pos, (k, v))).collect(); - let mut pending = self.pending.range(btree, level, min, max).peekable(); + let mut keys = self.pending.over(btree, level, min, max, keys)?; if btree == Btree::Accounting && level == 0 { - while let Some(first) = pending.peek() { - let pos = first.key().pos; - let stored = merged.remove(&pos); - let (mut key, mut sums) = match stored { - Some((key, value)) => (key, counters(&value)?), - None => (Key::new(kind::ACCOUNTING, pos), Vec::new()), - }; - let since = key.version; - while pending.peek().is_some_and(|entry| entry.key().pos == pos) { - let entry = pending.next().unwrap(); - let version = entry.key().version; - if version <= since { continue; } - let delta = counters(entry.value())?; - if sums.len() < delta.len() { - sums.resize(delta.len(), 0); - } - for (sum, d) in sums.iter_mut().zip(delta) { - *sum = sum.wrapping_add(d); - } - key.version = key.version.max(version); - } - if sums.iter().any(|&c| c != 0) { - merged.insert(pos, (key, sums.iter().flat_map(|c| c.to_le_bytes()).collect())); - } - } - } else { - for logged in pending { - let key = *logged.key(); - if key.kind == kind::DELETED { merged.remove(&key.pos); } - else { merged.insert(key.pos, (key, logged.value().to_vec())); } - } + for (_, value) in &keys { counters(value)?; } + keys.retain(|(_, value)| value.iter().any(|&b| b != 0)); } - Ok(merged.into_values().collect()) + Ok(keys) } } @@ -542,13 +512,9 @@ pub(crate) struct Writer { pub closing: bool, /// The largest reservation any transaction makes, in u64s. pub max_tx: u64, - /// Entries of the ring reclaim needs to journal the pointer update of - /// every node that can pin it. Ordinary reservations leave twice this: - /// one share for reclaim as it runs, and one for the reclaim that writes - /// out a replayed window after a crash, when the first may be spent. + /// Entries left for a reclaim share. Ordinary admission leaves two; + /// live reclaim leaves the second for recovery after a crash. pub reclaim_entries: u64, - /// The mount is writing out a replayed window, and reclaim may take - /// both shares. pub replaying: bool, } diff --git a/lib/bcachefs/src/engine/mount/allocator.rs b/lib/bcachefs/src/engine/mount/allocator.rs index 7f1bfa46648..507eab01de7 100644 --- a/lib/bcachefs/src/engine/mount/allocator.rs +++ b/lib/bcachefs/src/engine/mount/allocator.rs @@ -204,7 +204,9 @@ impl Allocator { } pub(super) fn keeps(&self, metadata_buckets: u64, stable: u64, nodes: usize, reclaim_room: usize, draw: Draw, data: bool) -> bool { - self.stable_slots(stable) + Self::transient(nodes, reclaim_room, draw) + if data { self.per_bucket } else { 0 } <= self.capacity(metadata_buckets) + // Imported packing debt must not prevent its own removal. Freeing + // still draws on the physical reserve checked by fits/data_fits. + matches!(draw, Draw::Freeing) || self.stable_slots(stable) + Self::transient(nodes, reclaim_room, draw) + if data { self.per_bucket } else { 0 } <= self.capacity(metadata_buckets) } /// The open bucket's next slot: its sector and generation. diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index b2480140f1d..be4083f2654 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -115,8 +115,8 @@ impl Mount { Ok(Reservation { owner: Rc::clone(&self.reservations), id, first, bytes }) } - /// Uses the reserved data block. Successful application consumes the - /// token; failed preparation leaves it available for retry. + /// Writes inside the reserved block. Successful application consumes the + /// reservation; a failed write leaves it reserved for retry until dropped. pub async fn write_reserved(&self, reservation: &mut Reservation, offset: u64, data: &[u8], now: i64) -> Result { if !Rc::ptr_eq(&reservation.owner, &self.reservations) { return Err(Error::Invalid("reservation belongs to another mount")); diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index a1c47d6fb67..9ea53f7796f 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -1,7 +1,6 @@ //! A volume mounted read-write: one transaction at a time under the writer -//! slot (§3.3), the btree nodes it changes held in memory, the journal that -//! makes each transaction durable, and the reclaim that writes the nodes so -//! the journal can move on. +//! slot (§3.3), journal-owned logical changes, and bounded materialization +//! into btree nodes so the journal can move on. //! //! **Order of waits.** A transaction takes admission first — journal space //! and cache room at its bound — and the slot second. Nothing waits under the @@ -26,11 +25,12 @@ use toyos_crypto::{scrypt, Random}; use super::bkey::{Bpos, Key, Version, KEY_U64S}; use super::bucket::Bucket; -use super::cache::{key_bytes, Cache, Cached, Keys, NodeId}; +use super::cache::{key_bytes, Cache, Cached, Change, Keys, NodeId}; use super::crypt::MasterKey; use super::journal::{self, Class, Closed, Jset, Replayed, Writer, SKIP}; use super::mkfs::{self, COMPAT_CLEAN, ROOT_SNAPSHOT}; use super::node::{self, Build, FIRST_KEYS}; +use super::pending::Pending; use super::read::{self, Inode, Parts, Source, Volume}; use super::superblock::{field_type, Field, Header, Member, Root, Superblock}; use super::sync::{Broadcast, Queue}; @@ -51,7 +51,7 @@ pub(crate) use reclaim::Order; pub(crate) use reclaim::Plan; use reclaim::{fresh_cost, node_ptr, MAX_GROUPS, MAX_REWRITE, RECLAIM_ROOM, REWRITTEN}; -/// How much a mount may hold in memory. +/// The mount's node cache capacity. Pending journal keys are held separately. #[derive(Clone, Copy, Debug)] pub struct MountOptions { /// Btree nodes cached. Dirty and unpointed nodes are bounded well below @@ -117,6 +117,7 @@ struct State { member: Member, blacklist: Vec<(u64, u64)>, cache: Cache, + pending: Pending, roots: BTreeMap, /// Btrees given a root in memory, not yet written. new_roots: BTreeMap, @@ -160,6 +161,8 @@ struct State { /// room, as tests construct them. #[cfg(test)] waits: [usize; 2], + #[cfg(test)] + journal_bytes: [usize; 3], } /// A volume mounted read-write. @@ -238,7 +241,7 @@ struct Txn { /// A written node, made reachable by the transaction that holds it. enum Install { /// `id`'s appended bset, now `sectors` long in all. - Append { id: NodeId, sectors: u16 }, + Append { id: NodeId, sectors: u16, journal_seq: u64 }, /// Sources replaced by nodes covering their combined range, with a new /// root when the old root split. Replace { ids: Vec, by: Vec, root: Option }, @@ -255,6 +258,7 @@ struct Planned { sector: u64, gen: u8, sectors: u16, + journal_seq: u64, keys: Vec<(Key, Vec)>, } @@ -432,6 +436,7 @@ impl Mount { member: sb.member.clone(), blacklist, cache: Cache::new(options.cache_nodes, sb.node_sectors() as usize * SECTOR - FIRST_KEYS), + pending: Pending::default(), roots: roots.into_iter().map(|r| (r.btree, r)).collect(), new_roots: BTreeMap::new(), journal, @@ -458,6 +463,8 @@ impl Mount { peak: 0, #[cfg(test)] waits: [0; 2], + #[cfg(test)] + journal_bytes: [0; 3], }; let mount = Mount { dev, @@ -677,6 +684,7 @@ impl Mount { written: ptr.written, on_disk: ptr.written, bsets: node.bset_seqs.len() as u16, + journal_seq: node.bset_seqs.iter().copied().max().unwrap_or(0), bytes: node.keys.iter().map(|(_, v)| key_bytes(v)).sum(), keys: Rc::new(node.keys), unwritten: BTreeMap::new(), @@ -722,7 +730,8 @@ impl Mount { if n.level < level { return Err(Error::Corrupt("a key above its btree's root")); } - let (key, value) = n.keys.get(n.at(pos)).ok_or(Error::Corrupt("interior node short of its range"))?; + let keys = st.keys(id)?; + let (key, value) = keys.get(keys.partition_point(|(k, _)| k.pos < pos)).ok_or(Error::Corrupt("interior node short of its range"))?; (n.level - 1, NodePtr::decode(value)?, key.pos) }; id = self.load(btree, child, &ptr, max).await?; @@ -746,6 +755,7 @@ impl Mount { written: 0, on_disk: 0, bsets: 0, + journal_seq: 0, bytes: 0, keys: Rc::new(Vec::new()), unwritten: BTreeMap::new(), @@ -779,7 +789,8 @@ impl Mount { } else if n.level < level { return Err(Error::Corrupt("a key above its btree's root")); } else { - let (key, value) = n.keys.get(n.at(pos)).ok_or(Error::Corrupt("interior node short of its range"))?; + let keys = st.keys(id)?; + let (key, value) = keys.get(keys.partition_point(|(k, _)| k.pos < pos)).ok_or(Error::Corrupt("interior node short of its range"))?; Some((n.level - 1, NodePtr::decode(value)?, key.pos)) } }; @@ -853,12 +864,22 @@ impl Mount { } impl State { + fn keys(&self, id: NodeId) -> Result { + let n = self.cache.node(id); + if self.pending.range(n.btree, n.level, n.min, n.max).next().is_none() { return Ok(Rc::clone(&n.keys)); } + Ok(Rc::new(self.pending.over(n.btree, n.level, n.min, n.max, n.keys.to_vec())?)) + } + + fn oldest_pin(&self) -> Option { + self.cache.oldest_pin().into_iter().chain(self.pending.oldest_pin()).min() + } + /// Whether writing the journal now would free a bucket of its ring: the /// last sequence an entry closed now carries passes some bucket's newest. fn journal_frees(&self) -> bool { let j = &self.journal; let next = j.open.as_ref().map_or(j.next_seq, |o| o.seq); - let last = self.cache.oldest_pin().map_or(next, |p| p.min(next)); + let last = self.oldest_pin().map_or(next, |p| p.min(next)); j.newest.iter().enumerate().any(|(i, n)| i != j.head && n.is_some_and(|n| n < last)) } } @@ -882,7 +903,8 @@ fn held(st: &State, btree: Btree, level: u8, pos: Bpos) -> NodeId { if n.level == level { return id; } - let (_, value) = &n.keys[n.at(pos)]; + let keys = st.keys(id).expect("a held logical node decodes"); + let (_, value) = &keys[keys.partition_point(|(k, _)| k.pos < pos)]; let ptr = NodePtr::decode(value).expect("a pointer the cache holds decodes"); id = st.cache.id_at(ptr.ptr.sector).expect("a held node in the cache"); } @@ -917,12 +939,12 @@ impl Source for Mount { fn rootless(&self, btree: Btree) -> Result)>> { let st = self.st.borrow(); - Ok(st.new_roots.get(&btree).map_or_else(Vec::new, |&id| st.cache.node(id).keys.to_vec())) + st.new_roots.get(&btree).map_or_else(|| Ok(Vec::new()), |&id| st.keys(id).map(|keys| keys.to_vec())) } async fn keys(&self, btree: Btree, level: u8, ptr: &NodePtr, max: Bpos) -> Result { let id = self.load(btree, level, ptr, max).await?; - Ok(Rc::clone(&self.st.borrow().cache.node(id).keys)) + self.st.borrow().keys(id) } fn begin_read(&self) -> u64 { @@ -996,7 +1018,7 @@ impl Mount { let keeps = st.alloc.keeps(st.metadata_buckets, stable, nodes, st.reclaim_room, draw, false); let physical = st.alloc.fits(nodes, st.reclaim_room, draw); let can_compact = st.sparse_metadata.iter().any(|&b| !st.alloc.metadata_open(b)); - if (!keeps || (!physical && !can_compact)) && held == 0 && st.reclaim_granted == 0 && !st.alloc.returning() { + if (!keeps || (!physical && !can_compact)) && held == 0 && st.pending.oldest_pin().is_none() && st.reclaim_granted == 0 && !st.alloc.returning() { return Err(Error::NoSpace(match draw { Draw::Ordinary | Draw::Reserved(_) => "no room left for the btree nodes a change may write", Draw::Freeing => "no room left for the btree nodes a removal may write", @@ -1022,13 +1044,15 @@ impl Mount { Ok(false) } - /// Whether anything can be freed now: entries to write, a dirty node to + /// Whether anything can be freed now: pending runs to materialize, + /// entries to write, a dirty node to /// reclaim, or a journal whose last sequence can move on or that holds /// an emptying. fn can_free(&self) -> bool { let st = self.st.borrow(); let flush = st.journal_frees() || st.alloc.returning(); - !st.journal.closed.is_empty() + st.pending.oldest_pin().is_some() + || !st.journal.closed.is_empty() || (flush && st.journal.open.is_some() && st.journal.may_close()) || st.sparse_metadata.iter().any(|&b| !st.alloc.metadata_open(b)) || st.cache.ids().into_iter().any(|id| { @@ -1098,6 +1122,7 @@ impl Mount { } self.ungrant(&mut st, Class::Ordinary, bound, nodes); }; + self.prepare_pending(Class::Ordinary, bound, nodes)?; let mut txn = Txn::default(); let out = prepare(&mut txn).await; self.finish(txn, out.is_ok(), Class::Ordinary, bound, nodes)?; @@ -1107,6 +1132,16 @@ impl Mount { out } + fn prepare_pending(&self, class: Class, bound: u64, nodes: usize) -> Result<()> { + let mut st = self.st.borrow_mut(); + if let Err(error) = st.pending.reserve(bound as usize / KEY_U64S) { + self.ungrant(&mut st, class, bound, nodes); + self.written.wake(); + return Err(error); + } + Ok(()) + } + /// Applies `txn` when `apply`, else drops it; either way releases its /// holds and its grant. `ReadOnly` when the mount stopped writing while /// it was prepared. @@ -1138,8 +1173,8 @@ impl Mount { } for (&(btree, level, pos), update) in &txn.updates { if level != 0 { continue; } - let node = st.cache.node(held(st, btree, level, pos)); - let prior = node.keys.get(node.at(pos)).filter(|(k, _)| k.pos == pos); + let keys = st.keys(held(st, btree, level, pos)).expect("a prepared logical node decodes"); + let prior = keys.get(keys.partition_point(|(k, _)| k.pos < pos)).filter(|(k, _)| k.pos == pos); let charge = |entry: Option<&(Key, Vec)>| entry.map_or(0, |(_, v)| allocator::charge(btree, v)); st.metadata_bytes = st.metadata_bytes + charge(update.as_ref()) - charge(prior); if btree == Btree::Backpointers { @@ -1150,8 +1185,8 @@ impl Mount { } } for (&pos, delta) in &txn.accounting { - let node = st.cache.node(held(st, Btree::Accounting, 0, pos)); - let prior = node.keys.get(node.at(pos)).filter(|(k, _)| k.pos == pos); + let keys = st.keys(held(st, Btree::Accounting, 0, pos)).expect("a prepared accounting node decodes"); + let prior = keys.get(keys.partition_point(|(k, _)| k.pos < pos)).filter(|(k, _)| k.pos == pos); let before = prior.map_or(0, |(_, v)| key_bytes(v) as u64); let counters = prior.map_or(0, |(_, v)| v.len() / 8).max(delta.len()); st.metadata_bytes = st.metadata_bytes + (KEY_U64S * 8 + counters * 8) as u64 - before; @@ -1175,10 +1210,9 @@ impl Mount { } } - /// Installs `txn`: its records appended to the open entry at the sequence - /// that entry has, then its keys in their nodes, its roots and its - /// written nodes. No await: what the slot's holder prepared, the cache - /// holds. + /// Appends the transaction's records, transfers its keys to pending + /// ownership, and publishes roots and completed node writes. No await + /// separates these logical changes. fn apply(&self, st: &mut State, mut txn: Txn, bound: u64) { let installs = core::mem::take(&mut txn.installs); let holds = core::mem::take(&mut txn.held); @@ -1206,32 +1240,15 @@ impl Mount { self.readers.borrow_mut().emptied(b); } - // Every change's node is found before any changes: a pointer update - // this transaction makes would lead the descent to a node not yet - // installed. - let updates: Vec<_> = core::mem::take(&mut txn.updates).into_iter().map(|((btree, level, pos), kv)| (held(st, btree, level, pos), pos, kv)).collect(); - let accounting: Vec<_> = core::mem::take(&mut txn.accounting).into_iter().map(|(pos, delta)| (held(st, Btree::Accounting, 0, pos), pos, delta)).collect(); - for (id, pos, kv) in updates { - st.cache.change(id, |n| n.put(pos, kv, seq)); - } - for (id, pos, delta) in accounting { - let version = versions[&pos]; - st.cache.change(id, |n| { - let i = n.at(pos); - let mut sums = match n.keys.get(i) { - Some((k, v)) if k.pos == pos => journal::counters(v).expect("a cached accounting key decodes"), - _ => Vec::new(), - }; - sums.resize(sums.len().max(delta.len()), 0); - for (sum, d) in sums.iter_mut().zip(&delta) { - *sum = sum.wrapping_add(*d); - } - let key = Key { version, ..Key::new(kind::ACCOUNTING, pos) }; - // A total of zero stays, its version with it, so a replay - // never adds the delta that made it zero again (A7.23, A7.24). - n.put(pos, Some((key, value::counters(&sums))), seq); - }); - } + let updates = core::mem::take(&mut txn.updates).into_iter().map(|((btree, level, pos), kv)| { + let (key, value) = kv.unwrap_or_else(|| (Key::new(kind::DELETED, pos), Vec::new())); + (btree, level, key, value) + }); + let accounting = core::mem::take(&mut txn.accounting).into_iter().map(|(pos, delta)| { + let key = Key { version: versions[&pos], ..Key::new(kind::ACCOUNTING, pos) }; + (Btree::Accounting, 0, key, value::counters(&delta)) + }); + st.pending.apply(seq, updates.chain(accounting)); for root in core::mem::take(&mut txn.roots) { st.roots.insert(root.btree, root); } @@ -1256,13 +1273,19 @@ impl Mount { /// a clean unmount's last (A1.42). fn close(&self, st: &mut State, own: bool) { let mut open = st.journal.open.take().expect("an entry open"); - let pinned = st.cache.oldest_pin().map_or(open.seq, |p| p.min(open.seq)); + let pinned = st.oldest_pin().map_or(open.seq, |p| p.min(open.seq)); let last_seq = if own { pinned } else { pinned.min(st.journal.durable) }; assert!(last_seq >= st.journal.durable_last, "a journal entry's last sequence moved back"); let roots = encoded_roots(&st.roots); journal::tail(&mut open.area, roots.iter().map(|(b, l, k)| (*b, *l, &k[..])), Some(st.now), st.key_version, st.clocks); let entry = Jset { seq: open.seq, last_seq, area: &open.area }; let bytes = journal::encode(&st.header.internal_uuid, self.sb.block_bytes(), self.key.as_ref(), &entry).expect("a journal entry encrypts"); + #[cfg(test)] + { + st.journal_bytes[0] += 1; + st.journal_bytes[1] += open.area.len(); + st.journal_bytes[2] += bytes.len(); + } st.journal.closed.push_back(Closed { seq: open.seq, last_seq, bytes }); } @@ -1489,10 +1512,10 @@ impl Mount { self.write_superblock(Some(clean)).await } - /// Whether any node is dirty or unpointed. + /// Whether journal-owned keys or unfinished node writes remain. fn busy(&self) -> bool { let st = self.st.borrow(); - st.cache.dirty > 0 || st.cache.unpointed > 0 + st.pending.oldest_pin().is_some() || st.cache.dirty > 0 || st.cache.unpointed > 0 } } @@ -1511,12 +1534,23 @@ impl Mount { self.st.borrow().alloc.reused } + pub(crate) fn pending_pressure(&self) -> (usize, usize, usize) { + let st = self.st.borrow(); + (st.pending.iter().count(), st.pending.allocated_bytes(), st.cache.dirty + st.cache.unpointed) + } + /// What a stalled test reports: the counts admission and reclaim look /// at, and every node not clean. pub(crate) fn dump(&self) -> alloc::string::String { let st = self.st.borrow(); let j = &st.journal; let dirty: Vec<_> = st.cache.ids().into_iter().map(|id| st.cache.node(id)).filter(|n| n.dirty() || n.captured.is_some() || n.writing || n.held > 0).map(|n| (n.btree, n.level, n.dirty(), n.pin(), n.captured, n.writing, n.held, n.bytes)).collect(); - alloc::format!("dirty={} unp={} over={} granted={} rg={} limit={} room={} open={:?} closed={} ring_free={} openable={} reclaim_entries={} durable_last={} newest={:?} head={} hs={} pins={:?} slot={} admitq_empty={} reclq={} nodes={:?}", st.cache.dirty, st.cache.unpointed, st.cache.oversized, st.granted, st.reclaim_granted, st.limit, st.reclaim_room, j.open.as_ref().map(|o| (o.seq, o.area.len()/8)), j.closed.len(), j.ring_free(), j.openable(), j.reclaim_entries, j.durable_last, j.newest, j.head, j.head_sectors, st.cache.oldest_pin(), self.slot_held.get(), self.admit.is_empty(), self.admit_reclaim.is_empty(), dirty) + let mut owners = BTreeMap::new(); + for entry in st.pending.iter().filter(|e| Some(e.capture().first) == st.pending.oldest_pin()) { + let position = entry.position(); + *owners.entry((position.0, position.1)).or_insert(0) += 1; + } + let geometry = (st.alloc.capacity(st.metadata_buckets), st.alloc.ready(0), st.btree_nodes, st.alloc.occupied_metadata(st.metadata_buckets), st.metadata_bytes, st.baseline_nodes.len(), self.metadata_nodes(&st, K_TX * 8)); + alloc::format!("stopped={:?} capacity_free_live_occupied_bytes_baseline_stable={geometry:?} pending={} pending_bytes={} pending_pin={:?} pending_owners={owners:?} journal_entries_payload_encoded={:?} dirty={} unp={} over={} granted={} rg={} limit={} room={} open={:?} closed={} ring_free={} openable={} reclaim_entries={} durable_last={} newest={:?} head={} hs={} pins={:?} slot={} admitq_empty={} reclq={} nodes={:?}", st.read_only, st.pending.iter().count(), st.pending.allocated_bytes(), st.pending.oldest_pin(), st.journal_bytes, st.cache.dirty, st.cache.unpointed, st.cache.oversized, st.granted, st.reclaim_granted, st.limit, st.reclaim_room, j.open.as_ref().map(|o| (o.seq, o.area.len()/8)), j.closed.len(), j.ring_free(), j.openable(), j.reclaim_entries, j.durable_last, j.newest, j.head, j.head_sectors, st.oldest_pin(), self.slot_held.get(), self.admit.is_empty(), self.admit_reclaim.is_empty(), dirty) } } diff --git a/lib/bcachefs/src/engine/mount/reclaim.rs b/lib/bcachefs/src/engine/mount/reclaim.rs index 74c9a7d2523..2561eab665f 100644 --- a/lib/bcachefs/src/engine/mount/reclaim.rs +++ b/lib/bcachefs/src/engine/mount/reclaim.rs @@ -33,7 +33,7 @@ pub(crate) struct Plan { enum Write { /// A bset appended at sector `at` of the node. - Append { at: u64, bytes: Vec }, + Append { at: u64, bytes: Vec, journal_seq: u64 }, /// New nodes, each with its bytes, and a new root above them when a /// root split. Fresh { nodes: Vec<(Planned, Vec)>, root: Option<(Planned, Vec)> }, @@ -125,9 +125,9 @@ pub(super) const fn fresh_nodes(nodes: usize) -> usize { /// the slot, as no transaction enters it while a changed node is oversized /// ([`Mount::transact`]), and the pointer update of the one reclaim round in /// flight, each at most `K_TX` u64s of keys. Spread evenly, all that fills -/// this many nodes of at least [`MIN_NODE`], each a key short of full. A -/// replay installs its window's keys with no such bound -/// (`issues/a-replayed-bcachefs-node-past-seven-nodes-is-refused.md`). +/// this many nodes of at least [`MIN_NODE`], each a key short of full. +/// Materialization takes at most one transaction of distinct key positions, +/// including during replay. pub(super) const MAX_GROUPS: usize = 7; /// New nodes one rewrite writes at most: a split's, and a root above them. @@ -257,6 +257,7 @@ impl Mount { // Reclaim's room for nodes is its own share; no draw limits it. self.admit(Class::Reclaim, Draw::Freeing, bound, nodes).await?; let _slot = self.slot(Class::Reclaim).await; + self.prepare_pending(Class::Reclaim, bound, nodes)?; let mut txn = Txn::default(); let out = self.point(&mut txn, plans).await; self.finish(txn, out.is_ok(), Class::Reclaim, bound, nodes)?; @@ -268,6 +269,9 @@ impl Mount { /// A node is appended to when it may be and has room; otherwise it is /// rewritten to fresh nodes, split past two thirds full. async fn plan(&self, order: Order, only: Option) -> Result> { + if only.is_none() { + self.materialize(order).await?; + } let chosen: Vec = if let Some(id) = only { alloc::vec![id] } else { let st = self.st.borrow(); // An oversized node first; then by `order`. @@ -298,7 +302,7 @@ impl Mount { if self.st.borrow().cache.node(id).captured.is_some() { continue; } let ids = self.rebalance_pair(id).await?; if ids.is_empty() { continue; } - let (btree, level, root, min, max, keys, changes, append, groups) = { + let (btree, level, root, min, max, keys, changes, dependency, append, groups) = { let st = self.st.borrow(); let n = st.cache.node(id); let root = st.new_roots.get(&n.btree) == Some(&id) || st.roots.get(&n.btree).is_some_and(|r| r.ptr.ptr.sector == n.sector); @@ -307,6 +311,7 @@ impl Mount { let append = ids.len() == 1 && n.compact.is_none() && !n.fresh && n.written > 0 && !st.cache.oversized(n) && usize::from(n.on_disk) * SECTOR + later <= node_bytes; let keys: Vec<_> = ids.iter().flat_map(|&id| st.cache.node(id).keys.iter().cloned()).collect(); let changes: BTreeMap<_, _> = ids.iter().flat_map(|&id| st.cache.node(id).unwritten.iter().map(|(&p, &s)| (p, s))).collect(); + let dependency = ids.iter().map(|&id| st.cache.node(id).journal_seq).max().unwrap(); let sizes: Vec = keys.iter().map(|(_, v)| key_bytes(v)).collect(); // At two thirds, as upstream splits; an oversized node, too // full for two at that, in as few as each a node holds. @@ -314,11 +319,9 @@ impl Mount { if groups.len() > 2 { groups = mkfs::spread(&sizes, node_bytes - FIRST_KEYS); } - (n.btree, n.level, root, st.cache.node(ids[0]).min, st.cache.node(*ids.last().unwrap()).max, keys, changes, append, groups) + (n.btree, n.level, root, st.cache.node(ids[0]).min, st.cache.node(*ids.last().unwrap()).max, keys, changes, dependency, append, groups) }; if groups.len() > MAX_GROUPS { - // More than a node read holds once unpacked, and two - // transactions ([`MAX_GROUPS`]): a replay's leaf. return Err(Error::Unsupported("a btree node whose keys fill more than seven nodes")); } let written = groups.len() + usize::from(groups.len() > 1 && root); @@ -335,11 +338,11 @@ impl Mount { let st = self.st.borrow(); let n = st.cache.node(id); let keys: Vec> = n.unwritten.keys().map(|pos| changed(n, *pos)).collect(); - let journal_seq = n.unwritten.values().copied().max().expect("a dirty node has changes"); + let journal_seq = n.unwritten.values().map(|c| c.last).max().expect("a dirty node has changes").max(n.journal_seq); let at = u64::from(n.on_disk); let later = node::Later { btree, level, seq: n.seq, offset: at as usize * SECTOR, journal_seq, keys: &keys }; let bytes = node::encode_later(block, self.key.as_ref(), &later)?; - Write::Append { at, bytes } + Write::Append { at, bytes, journal_seq } } else { let splits = groups.len() > 1; if splits && root && level == 3 { @@ -354,12 +357,13 @@ impl Mount { let low = if j == 0 { min } else { keys[range.start - 1].0.pos.successor().expect("a key below the last position") }; let high = if j + 1 == groups.len() { max } else { keys[range.end - 1].0.pos }; let mine = keys[range.clone()].to_vec(); - let journal_seq = changes.range(low..=high).map(|(_, s)| *s).max().unwrap_or(0); + let journal_seq = changes.range(low..=high).map(|(_, c)| c.last).max().unwrap_or(0).max(dependency); nodes.push(self.fresh_node(btree, level, low, high, journal_seq, mine, sectors[j], magic)?); } let above = (splits && root).then(|| { let keys = nodes.iter().map(|(p, _): &(Planned, Vec)| pointer(p.max, &p.ptr())).collect(); - self.fresh_node(btree, level + 1, Bpos::MIN, Bpos::MAX, 0, keys, sectors[groups.len()], magic) + let dependency = nodes.iter().map(|(p, _)| p.journal_seq).max().unwrap(); + self.fresh_node(btree, level + 1, Bpos::MIN, Bpos::MAX, dependency, keys, sectors[groups.len()], magic) }); Write::Fresh { nodes, root: above.transpose()? } }; @@ -384,6 +388,70 @@ impl Mount { Ok(plans) } + /// Transfers one bounded range from journal ownership to a cached + /// node. No transaction is replayed and no allocation triggers repeat. + async fn materialize(&self, order: Order) -> Result<()> { + let next = { + let st = self.st.borrow(); + match order { + Order::OldestPin => st.pending.oldest().filter(|c| st.cache.oldest_pin().is_none_or(|p| c.first < p)), + Order::LeavesFirst => (0..4).find_map(|level| Btree::ALL.iter().find_map(|&btree| { + st.pending.range(btree, level, Bpos::MIN, Bpos::MAX).next().map(|e| e.capture()) + })), + } + }; + let Some(next) = next else { return Ok(()) }; + let (btree, level, pos) = next.position; + let id = self.find(btree, level, pos).await?; + let mut st = self.st.borrow_mut(); + let n = st.cache.node(id); + if n.writing || n.captured.is_some() || n.held != 0 { return Ok(()); } + let mut captures = Vec::new(); + let mut bytes = 0; + let mut pending = st.pending.range(btree, level, pos, n.max).peekable(); + while let Some(first) = pending.peek() { + let at = first.key().pos; + let start = captures.len(); + let mut size = if btree == Btree::Accounting && level == 0 { + n.keys.get(n.at(at)).filter(|(k, _)| k.pos == at).map_or(0, |(_, v)| key_bytes(v)) + } else { 0 }; + while pending.peek().is_some_and(|e| e.key().pos == at) { + let entry = pending.next().unwrap(); + size = size.max(key_bytes(entry.value())); + captures.try_reserve(1).map_err(|_| Error::NoMemory)?; + captures.push(entry.capture()); + } + if bytes + size > K_TX as usize * 8 && start != 0 { captures.truncate(start); break; } + bytes += size; + } + let st = &mut *st; + let mut accounting = None; + for capture in captures { + let pos = capture.position.2; + assert!(st.pending.transfer(capture, |entry| { + st.cache.change(id, |n| { + if btree != Btree::Accounting || level != 0 { + let kv = (entry.key.kind != kind::DELETED).then_some((entry.key, entry.value)); + n.put(pos, kv, entry.first, entry.last); + return; + } + let (mut key, mut sums) = match n.keys.get(n.at(pos)) { + Some((key, value)) if key.pos == pos => (*key, journal::counters(value).expect("cached accounting decodes")), + _ => (Key::new(kind::ACCOUNTING, pos), Vec::new()), + }; + if accounting.is_none_or(|(previous, _)| previous != pos) { accounting = Some((pos, key.version)); } + if entry.key.version <= accounting.unwrap().1 { return; } + let delta = journal::counters(&entry.value).expect("pending accounting decodes"); + sums.resize(sums.len().max(delta.len()), 0); + for (sum, d) in sums.iter_mut().zip(delta) { *sum = sum.wrapping_add(d); } + key.version = key.version.max(entry.key.version); + n.put(pos, Some((key, value::counters(&sums))), entry.first, entry.last); + }); + }), "materialization cannot race a pending replacement under the slot"); + } + Ok(()) + } + pub(super) fn minimum_fill(&self) -> usize { (mkfs::capacity(self.sb.node_sectors() as usize * SECTOR) - FIRST_KEYS) / 2 - 255 * 8 } @@ -403,15 +471,15 @@ impl Mount { let parent = self.find(btree, level + 1, max).await?; let adjacent = { let mut st = self.st.borrow_mut(); - let p = st.cache.node(parent); - let at = p.at(max); - if p.keys.len() == 1 { + let keys = st.keys(parent)?; + let at = keys.partition_point(|(k, _)| k.pos < max); + if keys.len() == 1 { let seq = st.journal.open.as_ref().map_or(st.journal.next_seq, |o| o.seq); st.cache.change(parent, |n| { n.compact.get_or_insert(seq); }); return Ok(Vec::new()); } let index = if at > 0 { at - 1 } else { at + 1 }; - let (k, v) = p.keys.get(index).ok_or(Error::Corrupt("underfull node absent from its parent"))?; + let (k, v) = keys.get(index).ok_or(Error::Corrupt("underfull node absent from its parent"))?; (k.pos, NodePtr::decode(v)?) }; let other = self.load(btree, level, &adjacent.1, adjacent.0).await?; @@ -430,7 +498,7 @@ impl Mount { let build = Build { btree, level, seq, min, max, journal_seq, keys: &encoded }; let bytes = node::encode(magic, self.sb.block_bytes(), self.key.as_ref(), &build)?; let sectors = (bytes.len() / SECTOR) as u16; - Ok((Planned { btree, level, min, max, seq, sector: at.0, gen: at.1, sectors, keys }, bytes)) + Ok((Planned { btree, level, min, max, seq, sector: at.0, gen: at.1, sectors, journal_seq, keys }, bytes)) } /// Room for a batch's new nodes now, outside the slot: buckets emptied @@ -585,7 +653,7 @@ impl Mount { } for p in plans { let writes: Vec<(u64, &[u8])> = match &p.write { - Write::Append { at, bytes } => alloc::vec![(self.st.borrow().cache.node(p.ids[0]).sector + at, &bytes[..])], + Write::Append { at, bytes, .. } => alloc::vec![(self.st.borrow().cache.node(p.ids[0]).sector + at, &bytes[..])], Write::Fresh { nodes, root } => nodes.iter().chain(root).map(|(planned, bytes)| (planned.sector, &bytes[..])).collect(), }; for (sector, bytes) in writes { @@ -615,7 +683,7 @@ impl Mount { }; let (btree, level) = (plan.btree, plan.level); match plan.write { - Write::Append { at, bytes } => { + Write::Append { at, bytes, journal_seq } => { let sectors = (at + (bytes.len() / SECTOR) as u64) as u16; let ptr = NodePtr { seq, written: sectors, min, ptr: node_ptr(sector, gen) }; if plan.root { @@ -624,7 +692,7 @@ impl Mount { let (k, v) = pointer(max, &ptr); self.put(txn, btree, level + 1, max, Some((k, v))).await?; } - txn.installs.push(Install::Append { id: plan.ids[0], sectors }); + txn.installs.push(Install::Append { id: plan.ids[0], sectors, journal_seq }); } Write::Fresh { nodes, root } => { let by: Vec = nodes.into_iter().map(|(p, _)| p).collect(); @@ -784,7 +852,8 @@ impl Mount { /// every change since capture, partitioned by their new ranges. pub(super) fn install(&self, st: &mut State, install: Install) { match install { - Install::Append { id, sectors } => st.cache.change(id, |n| { + Install::Append { id, sectors, journal_seq } => st.cache.change(id, |n| { + n.journal_seq = n.journal_seq.max(journal_seq); n.written = sectors; n.on_disk = sectors; n.bsets += 1; @@ -805,7 +874,7 @@ impl Mount { } for p in by { let mine: Vec<(Key, Vec)> = keys.iter().filter(|(k, _)| (p.min..=p.max).contains(&k.pos)).cloned().collect(); - let changes: BTreeMap = unwritten.range(p.min..=p.max).map(|(k, v)| (*k, *v)).collect(); + let changes: BTreeMap = unwritten.range(p.min..=p.max).map(|(k, v)| (*k, *v)).collect(); let id = st.cache.insert(fresh_cached(&p, mine)); let seq = st.journal.open.as_ref().map_or(st.journal.next_seq, |o| o.seq); st.cache.change(id, |n| { @@ -821,56 +890,26 @@ impl Mount { } } - /// Lays the replayed window over the nodes: each key installed in its - /// node as a change of the entry it came from, from the top level down so - /// each level's pointers lead to the next; each accounting delta newer - /// than its stored key added (A7.8). Then reclaim writes until the dirty - /// nodes are within the limit again. + /// Replay has the same pending owner as runtime changes. Recover enough + /// journal headroom for ordinary admission without loading every leaf. pub(super) async fn install_replay(&self, replayed: journal::Replay) -> Result<()> { - self.st.borrow_mut().journal.replaying = true; - let mut pending = replayed.pending; - let mut captured = Vec::new(); - captured.try_reserve(pending.iter().count()).map_err(|_| Error::NoMemory)?; - captured.extend(pending.iter().map(|entry| entry.capture())); - captured.sort_unstable_by_key(|c| (core::cmp::Reverse(c.position.1), c.position.0, c.position.2, c.order)); - let mut accounting = None; - for capture in captured { - let (btree, level, pos) = capture.position; - let id = self.find(btree, level, pos).await?; - assert!(pending.transfer(capture, |entry| { - self.st.borrow_mut().cache.change(id, |n| { - if btree != Btree::Accounting || level != 0 { - let key = (entry.key.kind != kind::DELETED).then_some((entry.key, entry.value)); - n.put(pos, key, entry.first); - return; - } - let (mut key, mut sums) = match n.keys.get(n.at(pos)) { - Some((k, v)) if k.pos == pos => (*k, journal::counters(v).expect("a cached accounting key decodes")), - _ => (Key::new(kind::ACCOUNTING, pos), Vec::new()), - }; - if accounting.is_none_or(|(previous, _)| previous != pos) { accounting = Some((pos, key.version)); } - if entry.key.version <= accounting.unwrap().1 { return; } - let delta = journal::counters(&entry.value).expect("a replayed accounting delta decodes"); - sums.resize(sums.len().max(delta.len()), 0); - for (sum, d) in sums.iter_mut().zip(delta) { *sum = sum.wrapping_add(d); } - key.version = key.version.max(entry.key.version); - n.put(pos, Some((key, value::counters(&sums))), entry.first); - }); - }), "a replay capture has no concurrent replacement"); - } - assert!(pending.oldest_pin().is_none(), "all replay pins transferred"); { - let st = self.st.borrow(); - let need = ((st.cache.dirty + st.cache.unpointed) as u64 * PER_NODE).div_ceil(st.journal.body() - st.journal.max_tx) + 1; - if need > st.journal.ring_free() { - return Err(Error::NoSpace("a journal too full to write out its replay")); - } + let mut st = self.st.borrow_mut(); + st.pending = replayed.pending; + st.journal.replaying = true; } - while { - let st = self.st.borrow(); - st.cache.dirty + st.cache.unpointed > st.limit - } { - self.reclaim_round(Order::OldestPin).await?; + loop { + let room = { + let mut st = self.st.borrow_mut(); + if st.journal.reserve(K_TX, Class::Ordinary) { + st.journal.unreserve(K_TX); + true + } else { false } + }; + if room { break; } + if !self.reclaim_round(Order::OldestPin).await? { + return Err(Error::NoSpace("a journal too full to materialize its replay")); + } } self.flush_open().await?; self.st.borrow_mut().journal.replaying = false; @@ -900,6 +939,7 @@ fn fresh_cached(p: &Planned, keys: Vec<(Key, Vec)>) -> Cached { written: p.sectors, on_disk: p.sectors, bsets: 1, + journal_seq: p.journal_seq, bytes: keys.iter().map(|(_, v)| key_bytes(v)).sum(), keys: Rc::new(keys), unwritten: BTreeMap::new(), @@ -950,7 +990,7 @@ impl Plan { /// offset in it and its length. pub(crate) fn appended(&self, m: &Mount) -> Option<(u64, usize, usize)> { match &self.write { - Write::Append { at, bytes } => Some((m.st.borrow().cache.node(self.ids[0]).sector, *at as usize * SECTOR, bytes.len())), + Write::Append { at, bytes, .. } => Some((m.st.borrow().cache.node(self.ids[0]).sector, *at as usize * SECTOR, bytes.len())), Write::Fresh { .. } => None, } } diff --git a/lib/bcachefs/src/engine/pending.rs b/lib/bcachefs/src/engine/pending.rs index d27fb947148..53a86725ff7 100644 --- a/lib/bcachefs/src/engine/pending.rs +++ b/lib/bcachefs/src/engine/pending.rs @@ -1,9 +1,11 @@ //! Journal-owned keys, independent of the leaves that will materialize them. use alloc::vec::Vec; +use alloc::collections::BTreeMap; use super::bkey::{Bpos, Key}; use super::{Btree, Error, Result}; +use super::{journal, kind}; pub(crate) type Position = (Btree, u8, Bpos); pub(crate) type Update = (Btree, u8, Key, Vec); @@ -15,10 +17,27 @@ struct Entry { key: Key, value: Vec, first: u64, + last: u64, order: u64, live: bool, } +struct Run { + entries: Vec, + pins: Vec, + oldest: usize, +} + +impl Run { + fn oldest(&self) -> Option<&Entry> { + self.pins.get(self.oldest).map(|&i| &self.entries[i]) + } + + fn retire(&mut self) { + while self.pins.get(self.oldest).is_some_and(|&i| !self.entries[i].live) { self.oldest += 1; } + } +} + impl Entry { fn position(&self) -> Position { (self.btree, self.level, self.key.pos) } fn ordering(&self) -> (Position, u64) { (self.position(), self.order) } @@ -50,36 +69,41 @@ pub(crate) struct Transferred { pub key: Key, pub value: Vec, pub first: u64, + pub last: u64, } pub(crate) struct Pending { - runs: [Option>; LEVELS], + runs: [Option; LEVELS], incoming: Vec, merged: Vec, + pins: Vec, next: u64, prepared: usize, } impl Default for Pending { fn default() -> Self { - Self { runs: core::array::from_fn(|_| None), incoming: Vec::new(), merged: Vec::new(), next: 0, prepared: 0 } + Self { runs: core::array::from_fn(|_| None), incoming: Vec::new(), merged: Vec::new(), pins: Vec::new(), next: 0, prepared: 0 } } } impl Pending { /// Reserve all workspace before making a logical batch visible. pub fn reserve(&mut self, additional: usize) -> Result<()> { + #[cfg(test)] + let _allocations = tests::allocation::Scope::enter(); assert!(self.incoming.is_empty() && self.merged.is_empty()); let mut total = additional; for run in &self.runs { let Some(run) = run else { break }; - total = total.checked_add(run.iter().filter(|e| e.live).count()).ok_or(Error::NoMemory)?; + total = total.checked_add(run.entries.iter().filter(|e| e.live).count()).ok_or(Error::NoMemory)?; } if self.runs.iter().all(Option::is_some) || u64::try_from(additional).ok().and_then(|n| self.next.checked_add(n)).is_none() { return Err(Error::NoMemory); } self.incoming.try_reserve(additional).map_err(|_| Error::NoMemory)?; self.merged.try_reserve(total).map_err(|_| Error::NoMemory)?; + self.pins.try_reserve(total).map_err(|_| Error::NoMemory)?; self.prepared = additional; Ok(()) } @@ -89,7 +113,7 @@ impl Pending { pub fn apply(&mut self, seq: u64, updates: impl IntoIterator) { for (btree, level, key, value) in updates { assert!(self.incoming.len() < self.prepared, "pending batch exceeded its reservation"); - self.incoming.push(Entry { btree, level, key, value, first: seq, order: self.next, live: true }); + self.incoming.push(Entry { btree, level, key, value, first: seq, last: seq, order: self.next, live: true }); self.next += 1; } self.prepared = 0; @@ -102,7 +126,7 @@ impl Pending { let mut chosen = None; let mut best = self.incoming.get(incoming_at).map(Entry::ordering); for (i, cursor) in at.iter_mut().enumerate().take(level) { - let run = self.runs[i].as_ref().expect("preceding run occupied"); + let run = &self.runs[i].as_ref().expect("preceding run occupied").entries; while run.get(*cursor).is_some_and(|e| !e.live) { *cursor += 1; } if let Some(e) = run.get(*cursor) { if best.is_none_or(|old| e.ordering() < old) { @@ -114,7 +138,7 @@ impl Pending { if best.is_none() { break; } let source = match chosen { Some(i) => { - let e = &mut self.runs[i].as_mut().unwrap()[at[i]]; + let e = &mut self.runs[i].as_mut().unwrap().entries[at[i]]; at[i] += 1; e } @@ -135,24 +159,73 @@ impl Pending { } self.incoming.clear(); for run in &mut self.runs[..level] { *run = None; } - self.runs[level] = Some(core::mem::take(&mut self.merged)); + self.pins.extend(0..self.merged.len()); + self.pins.sort_unstable_by_key(|&i| (self.merged[i].first, self.merged[i].order)); + self.runs[level] = Some(Run { entries: core::mem::take(&mut self.merged), pins: core::mem::take(&mut self.pins), oldest: 0 }); } pub fn oldest_pin(&self) -> Option { - self.runs.iter().flatten().flat_map(|r| r.iter()).filter(|e| e.live).map(|e| e.first).min() + self.oldest().map(|c| c.first) + } + + pub fn oldest(&self) -> Option { + self.runs.iter().flatten().filter_map(Run::oldest).min_by_key(|e| (e.first, e.order)) + .map(|e| Capture { position: e.position(), order: e.order, first: e.first }) + } + + #[cfg(test)] + pub fn allocated_bytes(&self) -> usize { + let entries = |entries: &Vec| entries.capacity() * core::mem::size_of::() + entries.iter().map(|e| e.value.capacity()).sum::(); + core::mem::size_of::() + entries(&self.incoming) + entries(&self.merged) + self.pins.capacity() * core::mem::size_of::() + + self.runs.iter().flatten().map(|r| entries(&r.entries) + r.pins.capacity() * core::mem::size_of::()).sum::() } + #[cfg(test)] pub fn iter(&self) -> Iter<'_> { self.between(None) } pub fn range(&self, btree: Btree, level: u8, min: Bpos, max: Bpos) -> Iter<'_> { self.between(Some(((btree, level, min), (btree, level, max)))) } + /// The logical range, preserving zero accounting values and their + /// versions so materialization cannot apply an old delta twice. + pub fn over(&self, btree: Btree, level: u8, min: Bpos, max: Bpos, keys: Vec<(Key, Vec)>) -> Result)>> { + let mut merged: BTreeMap<_, _> = keys.into_iter().map(|(k, v)| (k.pos, (k, v))).collect(); + let mut pending = self.range(btree, level, min, max).peekable(); + if btree == Btree::Accounting && level == 0 { + while let Some(first) = pending.peek() { + let pos = first.key().pos; + let (mut key, mut sums) = match merged.remove(&pos) { + Some((key, value)) => (key, journal::counters(&value)?), + None => (Key::new(kind::ACCOUNTING, pos), Vec::new()), + }; + let since = key.version; + while pending.peek().is_some_and(|entry| entry.key().pos == pos) { + let entry = pending.next().unwrap(); + let version = entry.key().version; + if version <= since { continue; } + let delta = journal::counters(entry.value())?; + sums.resize(sums.len().max(delta.len()), 0); + for (sum, d) in sums.iter_mut().zip(delta) { *sum = sum.wrapping_add(d); } + key.version = key.version.max(version); + } + merged.insert(pos, (key, super::value::counters(&sums))); + } + } else { + for entry in pending { + let key = *entry.key(); + if key.kind == kind::DELETED { merged.remove(&key.pos); } + else { merged.insert(key.pos, (key, entry.value().to_vec())); } + } + } + Ok(merged.into_values().collect()) + } + fn between(&self, bounds: Option<(Position, Position)>) -> Iter<'_> { let mut at = [0; LEVELS]; if let Some((min, _)) = bounds { for (i, run) in self.runs.iter().enumerate() { - if let Some(run) = run { at[i] = run.partition_point(|e| e.position() < min); } + if let Some(run) = run { at[i] = run.entries.partition_point(|e| e.position() < min); } } } Iter { pending: self, at, end: bounds.map(|(_, max)| max), saved: None } @@ -163,21 +236,22 @@ impl Pending { pub fn transfer(&mut self, capture: Capture, install: impl FnOnce(Transferred)) -> bool { let wanted = (capture.position, capture.order); let source = self.runs.iter_mut().flatten().find_map(|run| { - let at = run.partition_point(|e| e.ordering() < wanted); - run.get_mut(at).filter(|e| e.live && e.ordering() == wanted) + let at = run.entries.partition_point(|e| e.ordering() < wanted); + run.entries.get_mut(at).filter(|e| e.live && e.ordering() == wanted) }); let Some(source) = source else { return false }; let accounting = source.accounting(); let value = core::mem::take(&mut source.value); - install(Transferred { key: source.key, value, first: capture.first }); + install(Transferred { key: source.key, value, first: capture.first, last: source.last }); for run in self.runs.iter_mut().flatten() { - let at = run.partition_point(|e| e.position() < capture.position); - for entry in run[at..].iter_mut().take_while(|e| e.position() == capture.position && e.order <= capture.order) { + let at = run.entries.partition_point(|e| e.position() < capture.position); + for entry in run.entries[at..].iter_mut().take_while(|e| e.position() == capture.position && e.order <= capture.order) { if entry.live && (entry.order == capture.order || !accounting) { entry.live = false; entry.value = Vec::new(); } } + run.retire(); } true } @@ -196,7 +270,7 @@ impl<'a> Iter<'a> { let mut chosen = None; let mut best = None; for (i, run) in self.pending.runs.iter().enumerate() { - let Some(run) = run else { continue }; + let Some(Run { entries: run, .. }) = run else { continue }; while run.get(self.at[i]).is_some_and(|e| !e.live) { self.at[i] += 1; } if let Some(entry) = run.get(self.at[i]) { if self.end.is_none_or(|end| entry.position() <= end) && best.is_none_or(|old| entry.ordering() < old) { @@ -206,7 +280,7 @@ impl<'a> Iter<'a> { } } let i = chosen?; - let entry = &self.pending.runs[i].as_ref().unwrap()[self.at[i]]; + let entry = &self.pending.runs[i].as_ref().unwrap().entries[self.at[i]]; self.at[i] += 1; Some(entry) } @@ -231,6 +305,9 @@ impl<'a> Iterator for Iter<'a> { } } +#[cfg(test)] +pub(crate) use tests::allocation::after as fail_pending_allocation_after; + #[cfg(test)] mod tests { use super::*; @@ -238,17 +315,27 @@ mod tests { use alloc::vec; use std::collections::BTreeMap; - mod allocation { + pub(super) mod allocation { use core::cell::Cell; use std::alloc::{GlobalAlloc, Layout, System}; - std::thread_local! { static FAIL: Cell> = const { Cell::new(None) }; } + std::thread_local! { + static FAIL: Cell> = const { Cell::new(None) }; + static PENDING: Cell = const { Cell::new(false) }; + } + + pub struct Scope; + impl Scope { + pub fn enter() -> Self { PENDING.set(true); Self } + } + impl Drop for Scope { fn drop(&mut self) { PENDING.set(false); } } struct Allocator; #[global_allocator] static ALLOCATOR: Allocator = Allocator; fn fail() -> bool { + if !PENDING.try_with(Cell::get).unwrap_or(false) { return false; } FAIL.try_with(|f| match f.get() { None => false, Some(0) => { f.set(None); true } @@ -348,8 +435,8 @@ mod tests { } #[test] - fn either_workspace_allocation_can_fail_before_any_key_or_pin_changes() { - for successful in 0..2 { + fn any_workspace_allocation_can_fail_before_any_key_or_pin_changes() { + for successful in 0..3 { let mut pending = Pending::default(); add(&mut pending, 3, vec![update(1, 11)]); let extra = pending.incoming.capacity() + 1; diff --git a/lib/bcachefs/src/engine/tests/admission.rs b/lib/bcachefs/src/engine/tests/admission.rs index f9d0fe0e4a3..c23859d230f 100644 --- a/lib/bcachefs/src/engine/tests/admission.rs +++ b/lib/bcachefs/src/engine/tests/admission.rs @@ -95,6 +95,37 @@ fn writers(dev: &Image, cache_nodes: usize, dirs: &[u64], writers: usize, per: u /// The least cache a mount takes: one maximum transaction beside reclaim. const SMALL_CACHE: usize = super::super::mount::MIN_CACHE; +#[test] +fn committed_keys_survive_without_resident_dirty_leaves_and_replay_stays_pending() { + let dev = formatted(64 << 20, None); + let options = MountOptions { cache_nodes: SMALL_CACHE }; + let mount = block_on_ready(Mount::open(&dev, options, random())).unwrap(); + for i in 0..64 { block_on_ready(mount.create(mount.root(), format!("pending{i}").as_bytes(), New::File, OWNER, NOW)).unwrap(); } + let before = mount.pending_pressure(); + assert!(before.0 >= 128); + assert_eq!(before.2, 0, "logical updates need no resident dirty leaf"); + block_on_ready(mount.commit()).unwrap(); + assert_eq!(mount.pending_pressure(), before, "commit journals the updates without a global materialization pass"); + eprintln!("64-name pending workload: {} records, {} allocated bytes", before.0, before.1); + drop(mount); + + let mount = block_on_ready(Mount::open(&dev, options, random())).unwrap(); + assert!(mount.pending_pressure().0 >= 128, "replay keeps journal ownership"); + assert_eq!(mount.pending_pressure().2, 0); + for i in 0..64 { + let name = format!("pending{i}"); + assert!(block_on_ready(mount.lookup(mount.root(), name.as_bytes())).unwrap().is_some()); + if i % 2 == 0 { block_on_ready(mount.unlink(mount.root(), name.as_bytes(), false, NOW)).unwrap(); } + } + block_on_ready(mount.check_metadata_budget()); + block_on_ready(mount.unmount()).unwrap(); + let volume = block_on_ready(Volume::open(&dev)).unwrap(); + for i in 0..64 { + assert_eq!(block_on_ready(volume.lookup(volume.root(), format!("pending{i}").as_bytes())).unwrap().is_some(), i % 2 != 0); + } + assert_eq!(block_on_ready(fsck(&volume)).unwrap(), []); +} + /// Creates past what the journal holds, with no commit: entries fill and are /// written, the ring fills with entries the dirty nodes pin, and a writer /// waits for journal space until reclaim writes the nodes. Red when a diff --git a/lib/bcachefs/src/engine/tests/buckets.rs b/lib/bcachefs/src/engine/tests/buckets.rs index 3d2bc2ab0e3..92b147fd19b 100644 --- a/lib/bcachefs/src/engine/tests/buckets.rs +++ b/lib/bcachefs/src/engine/tests/buckets.rs @@ -197,7 +197,7 @@ fn a_full_volume_refuses_changes_by_name_and_still_removes_names_and_unmounts_cl m.commit().await.unwrap(); assert_eq!(m.create(r, b"again", New::Dir, OWNER, NOW).await.err(), Some(Error::NoSpace("no room left for the btree nodes a change may write"))); for i in 0..20 { - m.unlink(r, format!("f{i}").as_bytes(), false, NOW).await.unwrap(); + m.unlink(r, format!("f{i}").as_bytes(), false, NOW).await.unwrap_or_else(|error| panic!("remove {i}: {error:?}; {}", m.dump())); } let orphan = m.lookup(r, b"f20").await.unwrap().unwrap().ino; m.unlink(r, b"f20", false, NOW).await.unwrap(); @@ -248,14 +248,30 @@ pub(super) fn reclaim_all(m: &Mount) -> Vec { #[test] fn a_full_volume_crashed_mounts_read_write_and_still_removes_names() { let mut data = Vec::new(); - let dev = nearly_full(&mut data); + let dev = full_crash_removal_and_reuse(nearly_full(&mut data), 1); + let volume = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(read_all(&volume, path(&volume, "big")).unwrap(), data); +} + +#[test] +fn metadata_only_capacity_returns_after_reopening_and_removing_all_names() { + full_crash_removal_and_reuse(formatted(64 << 20, None), 0); +} + +fn full_crash_removal_and_reuse(dev: Image, existing: usize) -> Image { + let name = |i| format!("f{i:06}-{}", "x".repeat(240)); let made = { let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); let made = Executor::new().block_on( async { let mut n = 0; - while m.create(m.root(), format!("f{n}").as_bytes(), New::File, OWNER, NOW).await.is_ok() { - n += 1; + loop { + match m.create(m.root(), name(n).as_bytes(), New::File, OWNER, NOW).await { + Ok(_) => n += 1, + Err(Error::NoSpace(_)) => break, + Err(error) => panic!("fill: {error:?}; {}", m.dump()), + } + assert!(n < 100_000); } m.commit().await.unwrap(); n @@ -271,21 +287,34 @@ fn a_full_volume_crashed_mounts_read_write_and_still_removes_names() { async { let r = m.root(); assert_eq!(m.create(r, b"again", New::File, OWNER, NOW).await.err(), Some(Error::NoSpace("no room left for the btree nodes a change may write"))); - for i in 0..20 { - m.unlink(r, format!("f{i}").as_bytes(), false, NOW).await.unwrap(); + for i in 0..made { + let removed = m.unlink(r, name(i).as_bytes(), false, NOW).await.unwrap_or_else(|error| panic!("remove {i}: {error:?}; {}", m.dump())); + assert!(removed.last_link); + m.release(removed.node).await.unwrap(); + if i % 32 == 0 { m.commit().await.unwrap(); } } - let orphan = m.lookup(r, b"f20").await.unwrap().unwrap().ino; - m.unlink(r, b"f20", false, NOW).await.unwrap(); - m.release(orphan).await.unwrap(); m.commit().await.unwrap(); + for i in 0..64 { + m.create(r, name(i).as_bytes(), New::File, OWNER, NOW).await.unwrap(); + let removed = m.unlink(r, name(i).as_bytes(), false, NOW).await.unwrap(); + m.release(removed.node).await.unwrap(); + m.create(r, name(i).as_bytes(), New::Dir, OWNER, NOW).await.unwrap(); + m.unlink(r, name(i).as_bytes(), true, NOW).await.unwrap(); + } + let ino = m.create(r, b"reused", New::File, OWNER, NOW).await.unwrap().ino; + assert_eq!(m.write(ino, 0, &vec![0x52; 256 << 10], NOW).await.unwrap(), 256 << 10, "removal returns useful capacity"); + m.check_metadata_budget().await; }, stalled, ); block_on_ready(m.unmount()).unwrap(); let v = block_on_ready(Volume::open(&dev)).unwrap(); let (entries, more) = block_on_ready(v.readdir(v.root(), 0, usize::MAX)).unwrap(); - assert_eq!((entries.len(), more), (made + 1 - 21, false)); + assert_eq!((entries.len(), more), (existing + 1, false)); + assert_eq!(read_all(&v, path(&v, "reused")).unwrap(), vec![0x52; 256 << 10]); assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); + drop(v); + dev } /// A bucket emptied by an entry not yet durable is not written again: a diff --git a/lib/bcachefs/src/engine/tests/compact.rs b/lib/bcachefs/src/engine/tests/compact.rs index 0e5269bf162..d281e0a9f5c 100644 --- a/lib/bcachefs/src/engine/tests/compact.rs +++ b/lib/bcachefs/src/engine/tests/compact.rs @@ -28,14 +28,28 @@ fn deleting_all_files_leaves_only_the_inode_spine_and_reuses_its_room() { #[test] fn a_clean_node_relocation_keeps_its_keys_and_updates_its_pointer() { let dev = formatted(64 << 20, None); + let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); + let ino = block_on_ready(m.create(m.root(), b"dependency", New::File, OWNER, NOW)).unwrap().ino; + block_on_ready(m.commit()).unwrap(); + let first = block_on_ready(m.inode(ino)).unwrap().journal_seq; + block_on_ready(m.set_mtime(ino, NOW + 17)).unwrap(); + let latest = block_on_ready(m.inode(ino)).unwrap().journal_seq; + assert!(latest > first); + block_on_ready(m.unmount()).unwrap(); let v = block_on_ready(Volume::open(&dev)).unwrap(); let root = *v.roots().iter().find(|r| r.btree == Btree::Inodes).unwrap(); + let stored = block_on_ready(super::super::read::read_node(&dev, v.superblock(), None, Btree::Inodes, root.level, &root.ptr, Bpos::MAX)).unwrap(); + assert!(stored.bset_seqs.iter().copied().max().unwrap() >= latest, "a replacement represents its latest journal record, not its first pin"); drop(v); let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); assert!(block_on_ready(m.relocate_node(Btree::Inodes, root.level, Bpos::MAX, root.ptr.ptr.sector)).unwrap()); block_on_ready(m.unmount()).unwrap(); let v = block_on_ready(Volume::open(&dev)).unwrap(); - assert_ne!(v.roots().iter().find(|r| r.btree == Btree::Inodes).unwrap().ptr.ptr.sector, root.ptr.ptr.sector); + let moved = v.roots().iter().find(|r| r.btree == Btree::Inodes).unwrap(); + assert_ne!(moved.ptr.ptr.sector, root.ptr.ptr.sector); + let stored = block_on_ready(super::super::read::read_node(&dev, v.superblock(), None, Btree::Inodes, moved.level, &moved.ptr, Bpos::MAX)).unwrap(); + assert!(stored.bset_seqs.iter().copied().max().unwrap() >= latest, "a clean relocation preserves the existing dependency"); + assert_eq!(block_on_ready(v.inode(ino)).unwrap().mtime, NOW + 17); assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } diff --git a/lib/bcachefs/src/engine/tests/crash.rs b/lib/bcachefs/src/engine/tests/crash.rs index df14162ba86..7688daa3387 100644 --- a/lib/bcachefs/src/engine/tests/crash.rs +++ b/lib/bcachefs/src/engine/tests/crash.rs @@ -310,7 +310,7 @@ fn renames_reusing_buckets_on_a_nearly_full_volume_survive_a_crash_at_every_requ fn encrypted_renames_reusing_buckets_survive_a_crash_at_every_request() { let secret = hex_secret(); let image = formatted(64 << 20, Some(Encryption { passphrase: &secret, kdf: MIN_KDF })).into_inner(); - let reused = ops_survive_a_crash_on(image, Some(&secret), PRESSED, &renames(8, 6), 4, 6); + let reused = ops_survive_a_crash_on(image, Some(&secret), PRESSED, &renames(8, 32), 4, 6); assert!(reused > 4, "{reused} buckets taken again"); } diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index a5ed43839db..bce8d520f1e 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -415,7 +415,8 @@ fn retained_small_files_reclaim_their_truncated_data_buckets() { let mut inodes = Vec::new(); for i in 0..1600 { let ino = block_on_ready(m.create(m.root(), format!("f{i}").as_bytes(), New::File, OWNER, NOW)).unwrap().ino; - assert_eq!(block_on_ready(m.write(ino, 0, &[i as u8; 32_768], NOW)).unwrap(), 32_768); + let written = block_on_ready(m.write(ino, 0, &[i as u8; 32_768], NOW)).unwrap(); + assert_eq!(written, 32_768, "file {i}: {}", m.dump()); block_on_ready(m.truncate(ino, 4096, NOW)).unwrap(); inodes.push(ino); if i % 32 == 0 { block_on_ready(m.commit()).unwrap(); } @@ -427,6 +428,32 @@ fn retained_small_files_reclaim_their_truncated_data_buckets() { assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } +#[test] +fn pending_allocation_failure_precedes_data_reservation_and_reserved_write_preparation() { + let dev = formatted(64 << 20, None); + let mount = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); + let ino = block_on_ready(mount.create(mount.root(), b"retry", New::File, OWNER, NOW)).unwrap().ino; + assert_eq!(block_on_ready(mount.write(ino, 0, &[0x11; 4096], NOW)).unwrap(), 4096); + let before = mount.space(); + let out = super::super::pending::fail_pending_allocation_after(0, || block_on_ready(mount.reserve(ino, 0, 4096))); + assert!(matches!(out, Err(Error::NoMemory))); + assert_eq!(mount.space(), before, "a refused preflight takes no DATA slot"); + let mut reservation = block_on_ready(mount.reserve(ino, 0, 4096)).unwrap(); + block_on_ready(mount.set_mtime(ino, NOW + 1)).unwrap(); + let out = super::super::pending::fail_pending_allocation_after(0, || block_on_ready(mount.write_reserved(&mut reservation, 0, &[0x22; 4096], NOW))); + assert_eq!(out, Err(Error::NoMemory)); + let mut old = [0; 4096]; + assert_eq!(block_on_ready(mount.read(ino, 0, &mut old)).unwrap(), old.len()); + assert_eq!(old, [0x11; 4096]); + assert_eq!(block_on_ready(mount.write_reserved(&mut reservation, 0, &[0x22; 4096], NOW)).unwrap(), 4096); + drop(reservation); + block_on_ready(mount.check_metadata_budget()); + block_on_ready(mount.unmount()).unwrap(); + let volume = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(read_all(&volume, ino).unwrap(), [0x22; 4096]); + assert_eq!(block_on_ready(fsck(&volume)).unwrap(), []); +} + fn interrupted_user_evacuation(encrypted: bool) { use core::cell::Cell; use toyos_blockdev::model::{Model, Overlay}; diff --git a/lib/bcachefs/src/engine/tests/interleave.rs b/lib/bcachefs/src/engine/tests/interleave.rs index c44e50534e9..767e1cbe9e1 100644 --- a/lib/bcachefs/src/engine/tests/interleave.rs +++ b/lib/bcachefs/src/engine/tests/interleave.rs @@ -391,13 +391,11 @@ fn a_node_write_failing_past_its_retries_stops_the_mount_and_its_waiters() { assert_eq!(unmounted, Err(Error::ReadOnly("btree node write failed"))); } -/// A plan that fails on the second node of its batch: after a crash every -/// node's first write is fresh, and on this volume each node takes a bucket -/// of its own, the second of which freespace lists but whose alloc key holds -/// data. The mount stops read-only by the failure's name, and the next -/// transaction and the unmount answer that rather than wait. +/// After a crash every node's first write is fresh. Reclaim encounters a +/// second free bucket whose alloc key holds data and stops by that error; +/// later operations and unmount report it instead of waiting. #[test] -fn a_reclaim_plan_failing_midway_stops_the_mount() { +fn an_allocation_failure_during_reclaim_stops_the_mount() { use super::super::bucket::Bucket; use toyos_blockdev::executor::Executor; let dev = formatted(64 << 20, None); @@ -425,8 +423,19 @@ fn a_reclaim_plan_failing_midway_stops_the_mount() { let m = open_rw(&dev, None); let stalled = || panic!("stalled; {}", m.dump()); create(&m, b"b"); - let first = Executor::new().block_on(m.write_batch(Order::OldestPin), stalled); - assert_eq!(first.err(), Some(Error::Corrupt("a bucket an index lists as empty whose alloc key holds otherwise"))); + let failure = Executor::new().block_on(async { + for _ in 0..4 { + match m.write_batch(Order::OldestPin).await { + Err(error) => return error, + Ok(plans) => { + assert!(!plans.is_empty(), "the corrupt second bucket must be reached"); + if let Err(error) = m.point_batch(plans).await { return error; } + } + } + } + panic!("reclaim did not reach the corrupt second bucket"); + }, stalled); + assert_eq!(failure, Error::Corrupt("a bucket an index lists as empty whose alloc key holds otherwise")); let late = Executor::new().block_on(async { m.create(m.root(), b"late", New::File, OWNER, NOW).await.map(|i| i.ino) }, stalled); assert_eq!(late, Err(Error::ReadOnly("a bucket an index lists as empty whose alloc key holds otherwise"))); assert_eq!(Executor::new().block_on(m.unmount(), || panic!("unmount stalled")), Err(Error::ReadOnly("a bucket an index lists as empty whose alloc key holds otherwise"))); diff --git a/lib/bcachefs/src/engine/tests/journal.rs b/lib/bcachefs/src/engine/tests/journal.rs index 46da1392d53..074a56d9070 100644 --- a/lib/bcachefs/src/engine/tests/journal.rs +++ b/lib/bcachefs/src/engine/tests/journal.rs @@ -407,15 +407,10 @@ fn a_window_spanning_2_62_sequences_is_checked_entry_by_entry() { assert_eq!(window_of(&gap), Err(Error::Corrupt("a journal entry missing from the replay window"))); } -/// A window whose keys a replay installs into one leaf past seven nodes' -/// worth: 110 inodes per 16 KiB of the volume's buckets, which are its -/// nodes, journaled across 13 entries, none yet in a node, as an upstream -/// volume a crash left can hold them. The reader replays it; the read-write -/// mount stops by name at its first write, since reclaim cannot split the -/// leaf in one transaction -/// (`issues/a-replayed-bcachefs-node-past-seven-nodes-is-refused.md`). +/// More than seven nodes' worth of replay keys target one physical leaf. +/// They remain journal-owned until bounded materialization splits the leaf. #[test] -fn a_replayed_leaf_past_seven_nodes_is_read_but_not_mounted_read_write() { +fn a_replay_larger_than_seven_nodes_materializes_in_bounded_ranges() { let dev = formatted(64 << 20, None); let bucket_bytes = u64::from(block_on_ready(Volume::open(&dev)).unwrap().superblock().member.bucket_sectors) * SECTOR as u64; unclean(&dev); @@ -439,7 +434,9 @@ fn a_replayed_leaf_past_seven_nodes_is_read_but_not_mounted_read_write() { assert!(block_on_ready(v.inode(ino - 1)).is_ok(), "the reader replays the window"); drop(v); let m = block_on_ready(Mount::open(&dev, MountOptions { cache_nodes: 256 }, super::mount::random())).unwrap(); - let why = "a btree node whose keys fill more than seven nodes"; - assert_eq!(block_on_ready(m.create(m.root(), b"f", New::File, super::mount::OWNER, super::mount::NOW)).err(), Some(Error::Unsupported(why))); - assert_eq!(block_on_ready(m.unmount()).err(), Some(Error::ReadOnly(why))); + block_on_ready(m.create(m.root(), b"f", New::File, super::mount::OWNER, super::mount::NOW)).unwrap(); + block_on_ready(m.unmount()).unwrap(); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert!(block_on_ready(v.inode(ino - 1)).is_ok(), "the final replay key reached a physical leaf"); + assert!(block_on_ready(v.lookup(v.root(), b"f")).unwrap().is_some()); } From 4e829b24473f85d6070d40ea0229a615b8b2a7e4 Mon Sep 17 00:00:00 2001 From: japabu Date: Mon, 2 Nov 2026 22:37:10 +0100 Subject: [PATCH 49/93] Keep ordinary writers out of journal reclaim's reserve New ordinary grants could spend space in an entry opened by reclaim even below the ordinary reserve floor. Reclaim repeatedly admitted more work instead of retiring the oldest pins; full-volume reboot and removal stopped at the recovery share. Refuse those new grants until the floor is restored, while preserving existing grants and both reclaim shares. Release superseded ordinary pins only after replacement records are durable, rebuild run pin indices, and preserve captured dependencies and additive accounting. Materialize positive node growth within the existing bound, coalesce shrinking changes across the same leaf, and retire durable values already represented by the cached node. Publish pin-only progress through an empty existing-format journal checkpoint when a ring slot is available. All 168 library tests pass, including complete removal and capacity reuse from both DATA-full and metadata-full crashed volumes. New plaintext and encrypted low-journal-headroom tests exercise torn/reordered publication, read-write recovery, committed-prefix preservation, drain and fsck. The pre-floor candidate fails the unchanged removal test at865, retained as the negative control. No-default-features library clippy passes. This fixes the demonstrated native-geometry starvation. It does not claim termination for node==block geometry, where every node write is fresh and creates further backpointer work; that geometry decision remains open. --- lib/bcachefs/src/engine/journal.rs | 3 + lib/bcachefs/src/engine/mount/mod.rs | 22 +++--- lib/bcachefs/src/engine/mount/reclaim.rs | 42 ++++++----- lib/bcachefs/src/engine/pending.rs | 90 ++++++++++++++++++++++++ lib/bcachefs/src/engine/tests/crash.rs | 62 +++++++++++++++- 5 files changed, 193 insertions(+), 26 deletions(-) diff --git a/lib/bcachefs/src/engine/journal.rs b/lib/bcachefs/src/engine/journal.rs index 2ee5d743a73..e215404ce5d 100644 --- a/lib/bcachefs/src/engine/journal.rs +++ b/lib/bcachefs/src/engine/journal.rs @@ -594,6 +594,9 @@ impl Writer { Class::Reclaim if self.replaying => 0, Class::Reclaim => self.reclaim_entries, }; + // An entry opened by reclaim must not restart ordinary admission + // while the ring is still below its ordinary reserve floor. + if class == Class::Ordinary && self.ring_free() < keep { return false; } let leaves = total <= self.room() + self.ring_free().saturating_sub(keep) * per_entry; let fits = now && leaves; if fits { diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 9ea53f7796f..06f045851c3 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -72,10 +72,7 @@ const PER_NODE: u64 = fresh_cost(MAX_REWRITE); /// dirties, and the nodes a batch's rewrites replace its nodes with. const RECLAIM: usize = RECLAIM_ROOM + REWRITTEN; pub(crate) const MIN_CACHE: usize = RECLAIM + 3 * FOOTPRINT; -/// The smallest btree node this writer writes. Past its first bset a -/// smaller one has room for about two whole-block appends, and reclaim's own -/// changes to the allocator's leaves then rewrite more nodes than they -/// settle ([`MAX_GROUPS`] needs less). +/// The smallest btree node this writer accepts. const MIN_NODE: usize = 32 << 10; /// Retries of a failed node or journal write, each of the same bytes. const RETRIES: usize = 2; @@ -84,7 +81,7 @@ const RW_FEATURES: u64 = 1 << 8 | 1 << 9 | 1 << 12 | 1 << 13 | 1 << 17 | 1 << 18 /// Feature bit 5: the superblock holds a blacklist. const BLACKLIST_FEATURE: u64 = 1 << 5; -/// Ring geometry that leaves two worst-case reclaim shares and an ordinary +/// Ring geometry that leaves two cache-sized reclaim shares and an ordinary /// quarter, with room for at least one maximum-size ordinary transaction. pub(crate) fn journal_buckets(bucket_sectors: u64, block: usize) -> u64 { let journal = Writer::new(alloc::vec![0], bucket_sectors, block, K_TX); @@ -1045,15 +1042,15 @@ impl Mount { } /// Whether anything can be freed now: pending runs to materialize, - /// entries to write, a dirty node to - /// reclaim, or a journal whose last sequence can move on or that holds - /// an emptying. + /// entries to write, a dirty node to reclaim, or a journal whose last + /// sequence can move on or that holds an emptying. fn can_free(&self) -> bool { let st = self.st.borrow(); let flush = st.journal_frees() || st.alloc.returning(); st.pending.oldest_pin().is_some() || !st.journal.closed.is_empty() || (flush && st.journal.open.is_some() && st.journal.may_close()) + || (st.journal_frees() && st.journal.open.is_none() && st.journal.openable() > 0) || st.sparse_metadata.iter().any(|&b| !st.alloc.metadata_open(b)) || st.cache.ids().into_iter().any(|id| { let n = st.cache.node(id); @@ -1330,6 +1327,7 @@ impl Mount { let mut st = self.st.borrow_mut(); st.journal.closed.pop_front(); st.journal.written(seq, last_seq, (bytes.len() / SECTOR) as u64); + st.pending.durable(seq); st.alloc.durable_through(seq); drop(st); self.admit_reclaim.wake(); @@ -1394,6 +1392,9 @@ impl Mount { let upto = { let mut st = self.st.borrow_mut(); let st = &mut *st; + if st.journal.open.is_none() && st.journal_frees() && st.journal.openable() > 0 { + st.journal.open_new(); + } if st.journal.open.is_some() && st.journal.may_close() { self.close(st, false); } @@ -1539,6 +1540,11 @@ impl Mount { (st.pending.iter().count(), st.pending.allocated_bytes(), st.cache.dirty + st.cache.unpointed) } + pub(crate) fn journal_room(&self) -> (u64, u64) { + let st = self.st.borrow(); + (st.journal.ring_free(), 2 * st.journal.reclaim_entries) + } + /// What a stalled test reports: the counts admission and reclaim look /// at, and every node not clean. pub(crate) fn dump(&self) -> alloc::string::String { diff --git a/lib/bcachefs/src/engine/mount/reclaim.rs b/lib/bcachefs/src/engine/mount/reclaim.rs index 2561eab665f..5cd54cabfe8 100644 --- a/lib/bcachefs/src/engine/mount/reclaim.rs +++ b/lib/bcachefs/src/engine/mount/reclaim.rs @@ -167,11 +167,14 @@ impl Mount { /// One batch of dirty nodes chosen by `order`, planned under the slot /// and written outside it; then their pointer updates journaled. The /// journal is written once that frees a bucket of its ring, so a sweep's - /// updates share entries. Whether there was any node to write. + /// updates share entries. Whether any work was retired or written. pub(super) async fn reclaim_round(&self, order: Order) -> Result { + let had_work = self.busy(); let plans = self.write_batch(order).await?; if plans.is_empty() { - return Ok(false); + let retired = had_work && !self.busy(); + let published = if self.st.borrow().journal_frees() { self.flush_open().await? } else { false }; + return Ok(retired || published); } self.point_batch(plans).await?; if self.st.borrow().journal_frees() { @@ -270,7 +273,7 @@ impl Mount { /// rewritten to fresh nodes, split past two thirds full. async fn plan(&self, order: Order, only: Option) -> Result> { if only.is_none() { - self.materialize(order).await?; + while self.materialize(order).await? && self.st.borrow().cache.dirty == 0 {} } let chosen: Vec = if let Some(id) = only { alloc::vec![id] } else { let st = self.st.borrow(); @@ -388,9 +391,9 @@ impl Mount { Ok(plans) } - /// Transfers one bounded range from journal ownership to a cached - /// node. No transaction is replayed and no allocation triggers repeat. - async fn materialize(&self, order: Order) -> Result<()> { + /// Transfers a node's range with bounded positive growth. Deletions do + /// not consume growth room; no allocation triggers repeat. + async fn materialize(&self, order: Order) -> Result { let next = { let st = self.st.borrow(); match order { @@ -400,38 +403,43 @@ impl Mount { })), } }; - let Some(next) = next else { return Ok(()) }; + let Some(next) = next else { return Ok(false) }; let (btree, level, pos) = next.position; let id = self.find(btree, level, pos).await?; let mut st = self.st.borrow_mut(); let n = st.cache.node(id); - if n.writing || n.captured.is_some() || n.held != 0 { return Ok(()); } + if n.writing || n.captured.is_some() || n.held != 0 { return Ok(false); } let mut captures = Vec::new(); - let mut bytes = 0; - let mut pending = st.pending.range(btree, level, pos, n.max).peekable(); + let mut growth = 0; + let mut pending = st.pending.range(btree, level, pos, n.max) + .chain(st.pending.range(btree, level, n.min, pos).take_while(|e| e.key().pos < pos)).peekable(); while let Some(first) = pending.peek() { let at = first.key().pos; let start = captures.len(); - let mut size = if btree == Btree::Accounting && level == 0 { - n.keys.get(n.at(at)).filter(|(k, _)| k.pos == at).map_or(0, |(_, v)| key_bytes(v)) - } else { 0 }; + let before = n.keys.get(n.at(at)).filter(|(k, _)| k.pos == at).map_or(0, |(_, v)| key_bytes(v)); + let mut size = if btree == Btree::Accounting && level == 0 { before } else { 0 }; while pending.peek().is_some_and(|e| e.key().pos == at) { let entry = pending.next().unwrap(); - size = size.max(key_bytes(entry.value())); + if entry.key().kind != kind::DELETED { size = size.max(key_bytes(entry.value())); } captures.try_reserve(1).map_err(|_| Error::NoMemory)?; captures.push(entry.capture()); } - if bytes + size > K_TX as usize * 8 && start != 0 { captures.truncate(start); break; } - bytes += size; + let added = size.saturating_sub(before); + if growth + added > K_TX as usize * 8 && start != 0 { captures.truncate(start); continue; } + growth += added; } let st = &mut *st; + let durable = st.journal.durable; let mut accounting = None; + let transferred = !captures.is_empty(); for capture in captures { let pos = capture.position.2; assert!(st.pending.transfer(capture, |entry| { st.cache.change(id, |n| { if btree != Btree::Accounting || level != 0 { let kv = (entry.key.kind != kind::DELETED).then_some((entry.key, entry.value)); + let current = n.keys.get(n.at(pos)).filter(|(k, _)| k.pos == pos); + if entry.last <= durable && current == kv.as_ref() { return; } n.put(pos, kv, entry.first, entry.last); return; } @@ -449,7 +457,7 @@ impl Mount { }); }), "materialization cannot race a pending replacement under the slot"); } - Ok(()) + Ok(transferred) } pub(super) fn minimum_fill(&self) -> usize { diff --git a/lib/bcachefs/src/engine/pending.rs b/lib/bcachefs/src/engine/pending.rs index 53a86725ff7..c9c02e1b480 100644 --- a/lib/bcachefs/src/engine/pending.rs +++ b/lib/bcachefs/src/engine/pending.rs @@ -173,6 +173,57 @@ impl Pending { .map(|e| Capture { position: e.position(), order: e.order, first: e.first }) } + /// A durable ordinary replacement no longer needs the replaced record. + /// Captures retain their original dependency; accounting remains additive. + pub fn durable(&mut self, seq: u64) { + let mut at = [0; LEVELS]; + let mut changed = [false; LEVELS]; + loop { + let mut position = None; + for (i, run) in self.runs.iter().enumerate() { + let Some(run) = run else { continue }; + while run.entries.get(at[i]).is_some_and(|e| !e.live) { at[i] += 1; } + if let Some(entry) = run.entries.get(at[i]) { + position = Some(position.map_or(entry.position(), |p: Position| p.min(entry.position()))); + } + } + let Some(position) = position else { break }; + let mut end = at; + let mut replacement = None; + for (i, run) in self.runs.iter().enumerate() { + let Some(run) = run else { continue }; + while let Some(entry) = run.entries.get(end[i]).filter(|e| e.position() == position) { + if entry.live && !entry.accounting() && entry.last <= seq { + replacement = Some(replacement.map_or(entry.order, |old: u64| old.max(entry.order))); + } + end[i] += 1; + } + } + for (i, run) in self.runs.iter_mut().enumerate() { + let Some(run) = run else { continue }; + for entry in &mut run.entries[at[i]..end[i]] { + if !entry.live || entry.accounting() { continue; } + if replacement.is_some_and(|newer| newer > entry.order) { + entry.live = false; + entry.value = Vec::new(); + changed[i] = true; + } else if entry.last <= seq && entry.first != entry.last { + entry.first = entry.last; + changed[i] = true; + } + } + } + at = end; + } + for (i, run) in self.runs.iter_mut().enumerate() { + if !changed[i] { continue; } + let run = run.as_mut().unwrap(); + run.pins.sort_unstable_by_key(|&at| (run.entries[at].first, run.entries[at].order)); + run.oldest = 0; + run.retire(); + } + } + #[cfg(test)] pub fn allocated_bytes(&self) -> usize { let entries = |entries: &Vec| entries.capacity() * core::mem::size_of::() + entries.iter().map(|e| e.value.capacity()).sum::(); @@ -423,6 +474,42 @@ mod tests { assert_eq!((latest.value(), latest.capture().first), (&33u64.to_le_bytes()[..], 3)); } + #[test] + fn durable_replacements_release_old_pins_across_runs_and_keep_captures() { + let mut pending = Pending::default(); + add(&mut pending, 3, vec![update(1, 11)]); + add(&mut pending, 4, vec![update(2, 22)]); + add(&mut pending, 7, vec![update(1, 33)]); + pending.durable(6); + assert_eq!(pending.oldest_pin(), Some(3)); + pending.durable(7); + assert_eq!(pending.oldest_pin(), Some(4)); + assert_eq!(pending.iter().count(), 2); + + add(&mut pending, 8, vec![update(2, 44)]); + let capture = pending.range(Btree::Inodes, 0, Bpos::new(0, 2, 0), Bpos::new(0, 2, 0)).next().unwrap().capture(); + assert_eq!(capture.first, 4); + pending.durable(8); + assert_eq!(pending.oldest_pin(), Some(7), "promoted indices must be re-sorted"); + assert!(pending.transfer(capture, |entry| { + assert_eq!((entry.first, entry.last, entry.value), (4, 8, 44u64.to_le_bytes().to_vec())); + })); + assert_eq!(pending.oldest_pin(), Some(7)); + } + + #[test] + fn a_forgotten_intermediate_replacement_does_not_release_a_pin() { + let mut pending = Pending::default(); + add(&mut pending, 3, vec![update(1, 11)]); + add(&mut pending, 4, vec![update(2, 22)]); + add(&mut pending, 5, vec![update(1, 33)]); + add(&mut pending, 6, vec![update(1, 44)]); + pending.durable(5); + assert_eq!(pending.oldest_pin(), Some(3)); + pending.durable(6); + assert_eq!(pending.oldest_pin(), Some(4)); + } + #[test] fn failed_capacity_preflight_preserves_the_visible_batch_and_pin() { let mut pending = Pending::default(); @@ -459,6 +546,9 @@ mod tests { add(&mut replay.pending, version, vec![(Btree::Accounting, 0, key, value::counters(&[delta]))]); } assert_eq!(replay.pending.iter().count(), 4); + replay.pending.durable(99); + assert_eq!(replay.pending.iter().count(), 4); + assert_eq!(replay.pending.oldest_pin(), Some(2)); let stored = Key { version: Version { hi: 0, lo: 2 }, ..Key::new(super::super::kind::ACCOUNTING, pos) }; let once = replay.over(Btree::Accounting, 0, Bpos::MIN, Bpos::MAX, vec![(stored, value::counters(&[10]))]).unwrap(); assert_eq!(journal::counters(&once[0].1).unwrap(), [7]); diff --git a/lib/bcachefs/src/engine/tests/crash.rs b/lib/bcachefs/src/engine/tests/crash.rs index 7688daa3387..454aa77a36b 100644 --- a/lib/bcachefs/src/engine/tests/crash.rs +++ b/lib/bcachefs/src/engine/tests/crash.rs @@ -301,7 +301,7 @@ fn renames(files: usize, rounds: usize) -> Vec { fn renames_reusing_buckets_on_a_nearly_full_volume_survive_a_crash_at_every_request() { let mut data = Vec::new(); let image = super::buckets::nearly_full(&mut data).into_inner(); - let reused = ops_survive_a_crash_on(image, None, PRESSED, &renames(8, 8), 4, 8); + let reused = ops_survive_a_crash_on(image, None, PRESSED, &renames(8, 32), 4, 8); assert!(reused > 4, "{reused} buckets taken again"); } @@ -314,6 +314,66 @@ fn encrypted_renames_reusing_buckets_survive_a_crash_at_every_request() { assert!(reused > 4, "{reused} buckets taken again"); } +#[test] +fn journal_pressure_recovers_after_torn_reordered_publication() { + let name = |i| format!("p{i:06}-{}", "x".repeat(240)).into_bytes(); + for passphrase in [None, Some(hex_secret())] { + let encryption = passphrase.as_ref().map(|secret| Encryption { passphrase: secret, kdf: MIN_KDF }); + let model = Model::new(formatted(64 << 20, encryption).into_inner(), 512); + let applied = Cell::new(0); + let committed = Cell::new(0); + let armed = Cell::new(false); + let checked = Cell::new(0u64); + let dev = Watched { + model: &model, + after: Box::new(|model| { + if !armed.get() || checked.get() == 8 { return; } + checked.set(checked.get() + 1); + model.check(4, 0xadd1_0000 + checked.get(), |bytes| { + let disk = Overlay::new(bytes); + let m = match &passphrase { + Some(secret) => block_on_ready(Mount::unlock(&disk, secret, MIN_LIMITS, MEMORY, random())), + None => block_on_ready(Mount::open(&disk, MEMORY, random())), + }.map_err(|e| format!("pressure recovery: {e:?}"))?; + let (entries, more) = block_on_ready(m.readdir(m.root(), 0, usize::MAX)).map_err(|e| format!("names: {e:?}"))?; + if more || !(committed.get()..=applied.get()).contains(&entries.len()) { + return Err(format!("{} names outside committed/applied bounds", entries.len())); + } + let seen: BTreeSet<_> = entries.into_iter().map(|e| e.name).collect(); + if seen != (0..seen.len()).map(&name).collect() { return Err("names are not a creation prefix".into()); } + block_on_ready(m.unmount()).map_err(|e| format!("pressure drain: {e:?}"))?; + let v = match &passphrase { + Some(secret) => block_on_ready(Volume::unlock(&disk, secret, MIN_LIMITS)), + None => block_on_ready(Volume::open(&disk)), + }.map_err(|e| format!("reopen: {e:?}"))?; + let problems = block_on_ready(fsck(&v)).map_err(|e| format!("fsck: {e:?}"))?; + if !problems.is_empty() { return Err(format!("fsck: {problems:?}")); } + Ok(()) + }).unwrap_or_else(|e| panic!("{e}")); + }), + }; + let m = match &passphrase { + Some(secret) => block_on_ready(Mount::unlock(&dev, secret, MIN_LIMITS, MEMORY, random())), + None => block_on_ready(Mount::open(&dev, MEMORY, random())), + }.unwrap(); + Executor::new().block_on(async { + for i in 0..2000 { + let (free, floor) = m.journal_room(); + armed.set(free <= floor); + applied.set(i + 1); + m.create(m.root(), &name(i), New::File, OWNER, NOW).await.unwrap(); + if (i + 1) % 32 == 0 { + m.commit().await.unwrap(); + committed.set(i + 1); + } + } + }, || panic!("pressure operation stalled; {}", m.dump())); + armed.set(false); + block_on_ready(m.unmount()).unwrap(); + assert_eq!(checked.get(), 8, "cutpoints must reach the ordinary journal reserve floor"); + } +} + /// The window upstream's reader takes (A2-torn-newest A2.T1, A2.13): every /// flush entry's header last sequence, its checksum good or not, drops each /// entry below it as the journal is read; then, from the highest down, no-flush From 25f19a0761808c34f6112fcab4f5046d46aacd07 Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 3 Nov 2026 20:31:39 +0100 Subject: [PATCH 50/93] Run the process-depth chain from an endowed immutable image Each link of process_tree's depth chain spawned the next from a copy on /tmp. With /tmp served by the file server, every link kept its own /tmp directory connection, so the chain met the server's 32-client share bound at link 32, before the kernel's depth refusal at 65. The test now reads its own program once as the immutable region the SYSTEM server holds it in, and each link starts the next with CommandExt::boot_image and endows it a copy, so no link asks a file server for anything. The depth-65 refusal and whole-subtree teardown assertions are unchanged; B's spawn of a malformed /tmp executable still covers a served-/tmp spawn refused after admission. toyos::process::Image::as_immutable exposes that region only when the image is one, so a writable image's handle still leaves only for a spawn. Co-Authored-By: Claude Opus 5.5 --- sdk/toyos/src/process.rs | 9 ++++++ .../guest/rust-tests/src/bin/process_tree.rs | 30 +++++++++++++------ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/sdk/toyos/src/process.rs b/sdk/toyos/src/process.rs index 9a751db1a39..b2cd0238184 100644 --- a/sdk/toyos/src/process.rs +++ b/sdk/toyos/src/process.rs @@ -173,6 +173,15 @@ impl Image { Self { object: Object::Immutable(image), len } } + /// The region nobody writes this image is, when its server held the + /// program so: a copy starts another child with no file read. + pub fn as_immutable(&self) -> Option<&Immutable> { + match &self.object { + Object::Immutable(image) => Some(image), + Object::Read(_) => None, + } + } + /// The program's bytes, as they were read. pub fn bytes(&self) -> &[u8] { let bytes = match &self.object { diff --git a/tests/guest/rust-tests/src/bin/process_tree.rs b/tests/guest/rust-tests/src/bin/process_tree.rs index c05784201d9..0a59550e919 100644 --- a/tests/guest/rust-tests/src/bin/process_tree.rs +++ b/tests/guest/rust-tests/src/bin/process_tree.rs @@ -15,9 +15,11 @@ //! //! **The other arms.** A `MANAGE`-only handle is no place, and the supervisor refuses a //! launch whose place is a pipe and answers the launches after it. A chain — -//! each link spawns the next directly — stops where the kernel refuses a -//! process more than `MAX_DEPTH` below the supervisor, and dies whole with its -//! first link. And a `cat` a shell `detach`es outlives the shell. +//! each link spawns the next directly, from this program's immutable image it +//! is endowed, so no link asks a file server for anything — stops where the +//! kernel refuses a process more than `MAX_DEPTH` below the supervisor, and +//! dies whole with its first link. And a `cat` a shell `detach`es outlives the +//! shell. //! //! This process holds the launcher test-runner endows every job, and hands a //! duplicate to each child that launches: a direct spawn inherits none. @@ -44,8 +46,6 @@ use std::os::toyos::process::ExitStatusExt; use toyos_abi::end::{End, Fault}; const SELF_PATH: &str = "/system/bin/test_rs_process_tree"; -/// The chain loads its executable through the temporary file server. -const LINK_PATH: &str = "/tmp/test_rs_process_tree"; const INVALID_PATH: &str = "/tmp/process_tree_invalid_elf"; /// `/system/bin/toybox` under the name of an applet that reads its input until /// it ends: a declared program, so a launch, and one that runs until killed @@ -56,6 +56,9 @@ const SHELL: &str = "/system/bin/shell"; /// The name B's namespace carries the port back to A under. const BACK: &str = "back"; +/// The label a chain's link is endowed this program's image under. +const LINK_IMAGE: &str = "link-image"; + /// B to A: the handles to C, D, B's own `self` and E, in that order. const MSG_GROWN: u32 = 1; /// A to B: fault. @@ -301,9 +304,9 @@ fn a_pipe_is_no_place() { /// the link says on the chain's one output. Killing the first link then ends /// every one. fn a_chain_stops_at_max_depth_and_dies_whole() { - std::fs::copy(SELF_PATH, LINK_PATH).expect("copy this program to /tmp for the chain"); - let mut first = Command::new(LINK_PATH) - .args(["link", "1"]) + let read = toyos::files::read_image(SELF_PATH.as_bytes()).expect("read this program's image"); + let image = read.as_immutable().expect("the system share holds this program as an immutable region"); + let mut first = link_command(1, image.as_handle()) .stdout(Stdio::piped()) .spawn() .expect("spawn the chain's first link"); @@ -339,13 +342,22 @@ fn link(k: u32) -> ! { println!("unbounded at link {k}"); park(); } - let next = Command::new(LINK_PATH).args(["link", &(k + 1).to_string()]).spawn(); + let image: OwnedHandle = Endowments::get().take(LINK_IMAGE).expect("a link is endowed its image"); + let next = link_command(k + 1, image.as_handle()).spawn(); if let Err(e) = next { println!("link {k}: its next link was refused: {e}"); } park(); } +/// Link `k`, run from `image` and endowed a copy of it for the next. +fn link_command(k: u32, image: toyos::BorrowedHandle<'_>) -> Command { + let copy = image.try_clone_to_owned().expect("a copy of the image for the link"); + let mut command = Command::new(SELF_PATH); + command.args(["link", &k.to_string()]).boot_image(image.into()).endow(LINK_IMAGE, copy.into()); + command +} + /// A `cat` a shell `detach`es is its session owner's child, so it runs on once the shell's /// end is published: it reads the input this process holds open. fn a_detached_program_outlives_its_shell() { From 48316fc0e3aae83224729674d25e1cb59ada0ea5 Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 3 Nov 2026 20:46:22 +0100 Subject: [PATCH 51/93] Hold served stdio, holes and served /tmp in their tests A served file at stdout now reaches the child libc's system starts: spawn_served_stdout puts a /tmp file at descriptor 1, runs `echo child` through system and reads the parent's then the child's line back from it. libccase starts the MEMORY file server it needs. A write a gibibyte past a DATA file's end lands as a hole: a 64 MiB volume could not bridge the gap, so the host test passes only because nothing is allocated before the written block. 206_libc_refusals stops expecting symlink on /tmp to be refused: /tmp is a file server's, which makes the link. lseek_past_eof reads its ceiling from toyos::fs::MAX_FILE_BYTES, the kernel's file cache it named being gone. Co-Authored-By: Claude Opus 5.5 --- system/fileserver/src/data.rs | 16 +++++++++++++ tests/corpus/tinycc/206_libc_refusals.c | 5 ++-- tests/corpus/tinycc/206_libc_refusals.expect | 1 - tests/guest/libc/spawn_served_stdout.c | 24 +++++++++++++++++++ .../rust-tests/src/bin/lseek_past_eof.rs | 18 +++++++------- tests/images/libccase.toml | 10 ++++++-- 6 files changed, 58 insertions(+), 16 deletions(-) create mode 100644 tests/guest/libc/spawn_served_stdout.c diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index d4abb21c33e..93298b0d2c8 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -586,6 +586,22 @@ mod tests { assert_eq!(&bytes[2 * BLOCK + 17..2 * BLOCK + 19], b"va"); assert!(bytes[2 * BLOCK + 19..].iter().all(|&b| b == 0)); } + /// The volume is 64 MiB, so the write lands only as a hole: nothing + /// bridges the gibibyte before it. + #[test] + fn a_write_a_gibibyte_past_the_end_leaves_a_hole() { + let mut v = volume(); + let f = v.open("home/far", CREATE).unwrap(); + let far = 1 << 30; + v.write(f, far, b"x").unwrap(); + v.sync().unwrap(); + assert_eq!(v.node_meta(f).unwrap().size, far + 1); + let mut tail = vec![0xa5; 2 * BLOCK]; + let from = far + 1 - tail.len() as u64; + assert_eq!(v.read(f, from, &mut Buf(&mut tail)).unwrap(), tail.len()); + assert_eq!(tail.pop(), Some(b'x')); + assert!(tail.iter().all(|&b| b == 0)); + } #[test] fn unlink_and_replacement_keep_live_inode_identity() { let mut v = volume(); diff --git a/tests/corpus/tinycc/206_libc_refusals.c b/tests/corpus/tinycc/206_libc_refusals.c index 557a4a4bb50..c2582d490eb 100644 --- a/tests/corpus/tinycc/206_libc_refusals.c +++ b/tests/corpus/tinycc/206_libc_refusals.c @@ -1,6 +1,6 @@ /* What libc refuses: each call answers failure in its own POSIX form, errno says why, and nothing is done: ENOSYS where ToyOS lacks the function, - ENOTSUP where the kernel's /tmp keeps no second name or set time, ECHILD + ENOTSUP where /tmp's server keeps no second name or access time, ECHILD for a wait with no child to wait for, and the errno POSIX names for a lock, a limit or a mapping it cannot take, or memory it cannot give. */ #include @@ -74,7 +74,6 @@ int main(void) { said("getsid", getsid(0)); said("gethostname", gethostname(buf, sizeof buf)); said("link", link(FILE_PATH, FILE_PATH ".link")); - said("symlink", symlink(FILE_PATH, FILE_PATH ".symlink")); said("fchown", fchown(fd, 0, 0)); said("chmod", chmod(FILE_PATH, 0600)); said("fchmod", fchmod(fd, 0600)); @@ -199,7 +198,7 @@ int main(void) { int named = stat(FILE_PATH, &by_name) == 0; printf("file: %ld bytes; mode %s through fstat, %s through stat; second name: %s\n", size, st.st_mode == mode ? "as it was" : "changed", named && by_name.st_mode == mode ? "as it was" : "changed", - access(FILE_PATH ".link", F_OK) == 0 || access(FILE_PATH ".symlink", F_OK) == 0 ? "made" : "none"); + access(FILE_PATH ".link", F_OK) == 0 ? "made" : "none"); page[0] = 'y'; printf("page after mprotect: %c\n", page[0]); printf("limit after getrlimit: %lu %lu\n", (unsigned long)limit.rlim_cur, (unsigned long)limit.rlim_max); diff --git a/tests/corpus/tinycc/206_libc_refusals.expect b/tests/corpus/tinycc/206_libc_refusals.expect index 12c39fdecba..05ec019f581 100644 --- a/tests/corpus/tinycc/206_libc_refusals.expect +++ b/tests/corpus/tinycc/206_libc_refusals.expect @@ -4,7 +4,6 @@ setsid: -1, ENOSYS getsid: -1, ENOSYS gethostname: -1, ENOSYS link: -1, ENOTSUP -symlink: -1, ENOSYS fchown: -1, ENOSYS chmod: -1, ENOSYS fchmod: -1, ENOSYS diff --git a/tests/guest/libc/spawn_served_stdout.c b/tests/guest/libc/spawn_served_stdout.c new file mode 100644 index 00000000000..8fab8d16c2b --- /dev/null +++ b/tests/guest/libc/spawn_served_stdout.c @@ -0,0 +1,24 @@ +/* A served file at stdout is given to the child system starts: the shell's + output lands in the file after the parent's, at the description's offset. */ +#include "check.h" +#include + +#define PATH "/tmp/spawn_served_stdout" + +int main(void) { + int file = open(PATH, O_RDWR | O_CREAT | O_TRUNC, 0644); + CHECK(file >= 0, "open %s", PATH); + CHECK(write(file, "parent\n", 7) == 7, "write the parent's line"); + int saved = dup(1); + CHECK(saved >= 0, "dup stdout"); + CHECK(dup2(file, 1) == 1, "dup2 the file to stdout"); + int status = system("echo child"); + CHECK(dup2(saved, 1) == 1, "restore stdout"); + CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 0, "system(\"echo child\") answered %#x", status); + char said[32] = {0}; + ssize_t n = pread(file, said, sizeof said - 1, 0); + CHECK(n == 13 && strcmp(said, "parent\nchild\n") == 0, "the file holds %zd bytes: %s", n, said); + CHECK(unlink(PATH) == 0, "unlink %s", PATH); + printf("spawn_served_stdout: system's child wrote to the served file at stdout\n"); + return 0; +} diff --git a/tests/guest/rust-tests/src/bin/lseek_past_eof.rs b/tests/guest/rust-tests/src/bin/lseek_past_eof.rs index 40011896a4b..e426db7381e 100644 --- a/tests/guest/rust-tests/src/bin/lseek_past_eof.rs +++ b/tests/guest/rust-tests/src/bin/lseek_past_eof.rs @@ -1,14 +1,12 @@ //! A seek past EOF is a position, not an error, and nothing silently moves it. //! -//! POSIX judges the offsets, the hole and the seek pointer; the `MAX_FILE_SIZE` -//! ceiling is this kernel's own, judged against the page index it protects. +//! POSIX judges the offsets, the hole and the seek pointer; the +//! `MAX_FILE_BYTES` ceiling is ToyOS's own (`toyos::fs`). use std::fs::{self, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; -const PAGE: u64 = 4096; -/// `file_cache::MAX_FILE_SIZE`: the page index is a `u32`, so `(u32::MAX + 1) * 4096`. -const MAX_FILE_SIZE: u64 = (u32::MAX as u64 + 1) * PAGE; +use toyos::fs::MAX_FILE_BYTES; fn pattern(len: usize) -> Vec { (0..len).map(|i| (i * 37 + 11) as u8 | 1).collect() @@ -90,22 +88,22 @@ fn the_ceiling_refuses(dir: &str) { .unwrap_or_else(|e| panic!("create {path}: {e}")); f.write_all(&seed).expect("seed page 0"); - f.seek(SeekFrom::Start(MAX_FILE_SIZE + 1)) + f.seek(SeekFrom::Start(MAX_FILE_BYTES + 1)) .expect_err("a seek past the ceiling was accepted"); let here = f.seek(SeekFrom::Current(0)).expect("tell"); assert_eq!(here, 8, "{dir}: the refused seek moved the pointer to {here}"); - f.set_len(MAX_FILE_SIZE + 1) + f.set_len(MAX_FILE_BYTES + 1) .expect_err("a truncate past the ceiling was accepted"); - let got = f.seek(SeekFrom::Start(MAX_FILE_SIZE)).expect("seek exactly to the ceiling"); - assert_eq!(got, MAX_FILE_SIZE, "{dir}: the ceiling itself is a position"); + let got = f.seek(SeekFrom::Start(MAX_FILE_BYTES)).expect("seek exactly to the ceiling"); + assert_eq!(got, MAX_FILE_BYTES, "{dir}: the ceiling itself is a position"); f.write_all(&[0x22]) .expect_err("a write whose end crosses the ceiling was accepted"); // The straddle a position-only guard passes: two bytes land under the // ceiling, two past it where `(abs_pos / 4096) as u32` wraps onto page 0. - let at = MAX_FILE_SIZE - 2; + let at = MAX_FILE_BYTES - 2; let got = f.seek(SeekFrom::Start(at)).expect("seek two bytes under the ceiling"); assert_eq!(got, at, "{dir}: a seek two bytes under the ceiling answered {got}"); f.write_all(&[0x33, 0x44, 0x55, 0x66]) diff --git a/tests/images/libccase.toml b/tests/images/libccase.toml index 43ecc34a8f6..339a24d8f4b 100644 --- a/tests/images/libccase.toml +++ b/tests/images/libccase.toml @@ -1,8 +1,9 @@ # The boot `libc_children` runs libc's C programs on: each starts children of # its own, `echo` among them, and `system` and `popen` run their commands by -# `/system/bin/shell -c`. +# `/system/bin/shell -c`; and `/tmp`'s file server, whose file +# `spawn_served_stdout` puts at the stdout `system`'s child inherits. [boot] -start = ["logkeeper", "test-runner"] +start = ["logkeeper", "fileserver", "test-runner"] [programs.logkeeper] service = true @@ -22,6 +23,11 @@ syscap = ["dup", "roster"] "bin/echo" = "/system/bin/toybox" "bin/cat" = "/system/bin/toybox" +[programs.fileserver] +restart = true +pager = true +roles = ["memory"] + # The virtio-serial driver, for the VM's console alone: this image runs no agent. [programs.virtio-console] drives = ["pci:07:80:00/1af4:1043"] From f32a98839715571464ac81c18469172cbab7d917 Mon Sep 17 00:00:00 2001 From: japabu Date: Tue, 3 Nov 2026 20:46:29 +0100 Subject: [PATCH 52/93] Close the storage records the consolidation resolved Deleted, each by its cause being gone: - a-kernel-files-fstat-answers-its-handles-mtime: the kernel holds no file; fstat is the file server's. - a-link-on-a-kernel-mount-to-a-served-path-leads-nowhere and the-kernel-resolves-a-path-by-rules-of-its-own: /tmp and /system are served, so every link resolves in fileserver::resolve, whose tests hold absolute hand-back, climbing by directory and loop refusal. - a-symbolic-link-on-tmp-displaces-its-name-and-lists-nowhere: the MEMORY volume refuses an existing name and lists links (symlinks_keep_their_names_and_never_replace_existing_entries); libc sends every symlink to its server. - a-listing-still-scans-the-whole-mount: tmpfs and the private format are gone; DATA lists one directory's dirents, MEMORY its children, ROOT a prefix range. - a-bcachefs-hole-costs-a-block-write-per-page and one-write-far-past-a-files-end-holds-data-s-server: native DATA writes only the block a write touches, so a gap is a hole (a_write_a_gibibyte_past_the_end_leaves_a_hole). - a-file-servers-refused-connection-reads-as-a-missing-program: no is_served routing falls back to a kernel path. On the T14 at 0507cbe36 (proctreecase) the chain's link 32 read the server's own refusal, "the file server serves as many connections as it takes". - a-served-file-at-stdio-stops-every-spawn: toyos::fd::copy exports a served file; spawn_served_stdout runs it through libc's system. With the export replaced by a refusal, that case alone fails, at system, errno EOPNOTSUPP. bcachefs-crate-is-not-bcachefs is renamed the-bcachefs-crate-must-implement-bcachefs: the private format is gone, and what remains is upstream fsck of the integrated writer and every accepted geometry finishing its unmount. four-shares-take-a-file-server records that /tmp now spends a share's 32 connections, one per live process that touched it, which stopped process_tree's /tmp chain at link 32. The storage and small-kernel tracks, the /tmp performance record and records naming removed kernel files describe the served tree. Comments that still described the kernel's /tmp, its mounts or its path normalization (toyos::launch, the supervisor, toyos-abi's SpawnArgs and Stat, toyos::fs, toyos-swap, osbench and guest tests) describe what serves them now. Co-Authored-By: Claude Opus 5.5 --- ...chefs-hole-costs-a-block-write-per-page.md | 32 -------- ...d-connection-reads-as-a-missing-program.md | 29 -------- ...l-files-fstat-answers-its-handles-mtime.md | 20 ----- ...el-mount-to-a-served-path-leads-nowhere.md | 21 ------ .../a-listing-still-scans-the-whole-mount.md | 45 ------------ ...-served-file-at-stdio-stops-every-spawn.md | 21 ------ ...-lives-where-any-process-can-rewrite-it.md | 8 +- ...mp-displaces-its-name-and-lists-nowhere.md | 29 -------- ...call-costs-toyos-up-to-six-times-linuxs.md | 30 +++----- ...-memory-swaps-to-an-encrypted-partition.md | 2 +- issues/bcachefs-crate-is-not-bcachefs.md | 73 ------------------- .../deferred-release-outlives-its-syscall.md | 5 +- ...rogram-sees-only-the-files-it-was-given.md | 4 +- issues/four-shares-take-a-file-server.md | 12 +++ ...bc-refuses-what-toyos-cannot-yet-answer.md | 4 - ...nd-through-a-bin-sh-toyos-does-not-have.md | 4 +- ...ar-past-a-files-end-holds-data-s-server.md | 21 ------ issues/probe-mounts-on-a-checksum.md | 10 +-- ...ge-is-layers-and-a-role-is-a-filesystem.md | 52 ++++++------- ...-bcachefs-crate-must-implement-bcachefs.md | 43 +++++++++++ ...-capability-end-state-is-twelve-answers.md | 2 +- ...-small-interrupts-post-and-threads-wait.md | 14 ++-- ...nel-resolves-a-path-by-rules-of-its-own.md | 45 ------------ ...4-reboots-through-ubuntu-for-every-test.md | 3 +- issues/toyos-builds-itself.md | 3 - lib/swap/src/lib.rs | 2 +- sdk/abi/src/syscall.rs | 15 ++-- sdk/toyos/src/fs.rs | 4 +- sdk/toyos/src/launch.rs | 14 ++-- system/supervisor/src/main.rs | 4 +- tests/guest/osbench/src/files.rs | 4 +- tests/guest/osbench/src/sys/toyos.rs | 3 +- .../rust-tests/src/bin/abuse_elf_loader.rs | 4 +- .../rust-tests/src/bin/fs_truncate_persist.rs | 7 +- .../rust-tests/src/bin/spawn_from_tmp.rs | 2 +- tests/toyos.rs | 6 +- 36 files changed, 151 insertions(+), 446 deletions(-) delete mode 100644 issues/a-bcachefs-hole-costs-a-block-write-per-page.md delete mode 100644 issues/a-file-servers-refused-connection-reads-as-a-missing-program.md delete mode 100644 issues/a-kernel-files-fstat-answers-its-handles-mtime.md delete mode 100644 issues/a-link-on-a-kernel-mount-to-a-served-path-leads-nowhere.md delete mode 100644 issues/a-listing-still-scans-the-whole-mount.md delete mode 100644 issues/a-served-file-at-stdio-stops-every-spawn.md delete mode 100644 issues/a-symbolic-link-on-tmp-displaces-its-name-and-lists-nowhere.md delete mode 100644 issues/bcachefs-crate-is-not-bcachefs.md delete mode 100644 issues/one-write-far-past-a-files-end-holds-data-s-server.md create mode 100644 issues/the-bcachefs-crate-must-implement-bcachefs.md delete mode 100644 issues/the-kernel-resolves-a-path-by-rules-of-its-own.md diff --git a/issues/a-bcachefs-hole-costs-a-block-write-per-page.md b/issues/a-bcachefs-hole-costs-a-block-write-per-page.md deleted file mode 100644 index 53928f8f68d..00000000000 --- a/issues/a-bcachefs-hole-costs-a-block-write-per-page.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-02 ---- - -# A hole on bcachefs costs one block write per page, because the extent list cannot say "hole" - -`Mounted::allocate` (`lib/bcachefs/src/fs.rs`) now zeroes every block -it allocates only to bridge a gap, because the allocator hands back the blocks a -shrink most recently freed and a hole that reads the previous tenant's bytes is -a data leak. That is correct and it is what FAT32 already does (`Fat32::set_len` -calls `zero_range` on every grow). The cost is linear in the hole: reaching page -N over a gap of M pages writes M zero blocks, one device write each, inside the -file server's write that reaches the page (`system/fileserver/src/data.rs`). - -A sequential append reaches `target` in one block, so the zeroing loop -does not run. The paths that pay are a `lseek` past the end followed by a -write, and a shrink-then-write-above-the-mark, whose gap is bounded by the -file. - -The reason it is linear is the representation: `block_for` walks the extent list -accumulating `block_count`, so page K's block is decided by how many blocks -precede it. A list like that has no way to say "these pages are not allocated" — -every page below the size must have a block. FAT32 has the same property and the -same cost. - -**Exit condition.** An extent list that can express an unallocated run, so a -hole costs one list entry rather than one write per page, and a read of it -resolves to zeros without a block. That is an on-disk format change for -bcachefs, which is why it is not a small one — the same wall -`issues/a-listing-still-scans-the-whole-mount.md` ends at. diff --git a/issues/a-file-servers-refused-connection-reads-as-a-missing-program.md b/issues/a-file-servers-refused-connection-reads-as-a-missing-program.md deleted file mode 100644 index 3a2885bc96d..00000000000 --- a/issues/a-file-servers-refused-connection-reads-as-a-missing-program.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-23 ---- - -# A file server's refused connection reads as a missing program - -`toyos::files::is_served` (`sdk/toyos/src/files.rs`) answers `false` for any -path whose route fails, a refusal included: `toyos_fs_is_served` -(`sdk/toyos/src/files/owner.rs`) answers the route's error, and `is_served` -reads every answer but `Ok(1)` as "the kernel's". So when a process's first -connection to the file server serving a path is refused, std's spawn takes -the kernel's path for a program on that server, and the kernel, which serves -no `/system`, answers `NotFound`. - -Seen on the T14 at `db6be8df1` and on `main` `35e930136`, `proctreecase`: -`process_tree`'s depth chain, each link a helper of the one before in the -job's share, stopped at link 31 with "its next link was refused: entity not -found", and the kernel logged `spawn: /system/bin/test_rs_process_tree: not -found`. SYSTEM's server serves one share at most `SERVED_SHARE` (32) clients -(`system/fileserver/src/bounds.rs`), and the 32nd process of that share to -connect is refused. The chain now runs from a copy on `/tmp`; the defect -stands for any program a share's 33rd process spawns off a file server. - -Exit: a refused connection reaches the spawn's caller as the server's -refusal, never as the kernel's `NotFound`; a guest arm spawns a `/system` -program from a process whose share is at the server's bound and reads that -refusal. diff --git a/issues/a-kernel-files-fstat-answers-its-handles-mtime.md b/issues/a-kernel-files-fstat-answers-its-handles-mtime.md deleted file mode 100644 index 3bbc018691b..00000000000 --- a/issues/a-kernel-files-fstat-answers-its-handles-mtime.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-29 ---- - -# A kernel file's `fstat` answers its handle's mtime, not the file's - -`ops::fstat` answers `OpenFileState::mtime` (`kernel/src/object/ops.rs`), which -is the file's stamp at the open, moved only by a write or `ftruncate` through -that handle. A write through another handle moves the file's own stamp -(`file_cache::touch`) and not this one. So on `/tmp` a file opened at `t0` and -written through a second handle at `t1` answers `t0` through the first and `t1` -after a reopen, while the size it answers beside it is the file's. - -**Mechanism read off the code; not reproduced.** - -**Exit condition.** `fstat` of a kernel file answers the stamp the file holds, -with a guest test that writes through one handle and `fstat`s another opened -before the write. diff --git a/issues/a-link-on-a-kernel-mount-to-a-served-path-leads-nowhere.md b/issues/a-link-on-a-kernel-mount-to-a-served-path-leads-nowhere.md deleted file mode 100644 index 67bdc775b6c..00000000000 --- a/issues/a-link-on-a-kernel-mount-to-a-served-path-leads-nowhere.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-27 ---- - -# A link on a kernel mount to a served path leads nowhere - -The kernel resolves a symlink on its own mounts — ROOT and `/tmp` — in its own -tree (`kernel/src/vfs.rs`), and its tree holds nothing under `/apps`, -`/config`, `/home`, `/state`, `/log` or `/boot` but ROOT's empty directories: -those are file servers', reached through a directory capability -(`toyos::files`, which std and libc share). So a link in `/tmp` whose target is -`/home/toy/notes` opens nothing (`NotFound`), where the same link on a served -directory is followed — a file server hands an absolute target back to the -client, which resolves it in its own table. - -**Exit**: the kernel answers a path that crosses a link into a directory it -does not serve with the target and the rest of the path, as a file server's -`LINK` reply does, and std resolves it again in the caller's table; with a -guest test that opens a `/home` file through a `/tmp` link. diff --git a/issues/a-listing-still-scans-the-whole-mount.md b/issues/a-listing-still-scans-the-whole-mount.md deleted file mode 100644 index d039f76c644..00000000000 --- a/issues/a-listing-still-scans-the-whole-mount.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-01 ---- - -# A listing costs the whole mount on the two mounts with no directory structure - -The `FileSystem::list` bound is per directory now (`kernel/src/vfs.rs`, -`under_directory`), and FAT32 pays only the named subtree because it descends -to it (`Fat32::walk`'s `under`). The other two formats still read every name -in the volume and drop the ones that do not match: - -- `TmpFs::list` (`kernel/src/tmpfs.rs`) iterates the whole `entries` map. -- `Mounted::list` (`lib/bcachefs/src/fs.rs`) walks the whole B+tree through - `btree::for_each_live`, decoding every leaf value to learn its name: ROOT - in the kernel, and DATA in the file server. - -So a `readdir` of one directory is O(volume) on `/tmp`, ROOT and `/home`, and it -decodes a leaf value per entry on the two bcachefs volumes. Only the -*allocation* is bounded by what the directory holds; the work is not. Nothing -measures it, and no test would notice the difference. - -**This is a bound that was there and is not any more.** The old `list(16_384)` -stopped at the 16,384th entry it materialised and refused, so a mount with a -million names cost 16,384 entries of work and then an error. The new one visits -every name in the mount and filters, so that same mount costs a million — the -refusal now counts only what the listed directory holds, and nothing counts the -walk. On a kernel mount it runs under the VFS lock a `SYS_READDIR` holds, which -every other kernel filesystem syscall waits behind; on DATA it holds the file -server. - -The end of it is a real directory index — a store keyed by (directory, name) -rather than a flat namespace of paths that happen to contain `/`. On bcachefs -that is an on-disk format change, so it is not a small one. - -**`SYS_CHDIR` and `SYS_SPAWN` pay the same worst case on ROOT.** Both judge -a directory on a kernel mount with `FileSystem::is_dir` under the VFS lock -(`Vfs::is_mounted_dir`; a served directory is the file server's to answer), and -`Mounted::is_dir` (`lib/bcachefs/src/fs.rs`) is a `for_each_live` walk that stops -at the first name beneath the directory — so a directory that is there costs -the walk up to its first entry, and one that is not costs every node on the -mount. Every spawn states a cwd, so this is on the path of every process start -whose cwd is on ROOT. Nothing is kept, and a node the walk -cannot read refuses that one call. The same (directory, name) index ends it. diff --git a/issues/a-served-file-at-stdio-stops-every-spawn.md b/issues/a-served-file-at-stdio-stops-every-spawn.md deleted file mode 100644 index 04b0e81b4cd..00000000000 --- a/issues/a-served-file-at-stdio-stops-every-spawn.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-21 ---- - -# A served file at stdio stops every spawn - -A file a file server holds is a descriptor naming an object of -`toyos::files`', not a kernel handle, and `toyos::fd::copy` refuses one for a -child (`EOPNOTSUPP`, std's `Unsupported`). libc's `posix_spawn` asks for -descriptors 0 to 2 on every spawn (`spawnreq::plan`), so once a C program has -put a served file there, by `dup2` or `freopen`, every `posix_spawn`, -`system` and `popen` it makes is refused. So is an `adddup2` of a served -descriptor and an `addopen` of a served path, and std's `Command` with -inherited stdio in the same case. Before served files were descriptors, no -served file could sit at 0 to 2. Read from the code; no guest case has run it. - -**Exit**: a served file can be given to a child, as PR 4 of the descriptor -work gives served kinds (`EXPORT`), and a guest C case redirects stdout to a -file under `/home`, runs `system`, and reads the child's output back from it. diff --git a/issues/a-swapped-binary-lives-where-any-process-can-rewrite-it.md b/issues/a-swapped-binary-lives-where-any-process-can-rewrite-it.md index 4bd6004aa08..2465fb9d7e6 100644 --- a/issues/a-swapped-binary-lives-where-any-process-can-rewrite-it.md +++ b/issues/a-swapped-binary-lives-where-any-process-can-rewrite-it.md @@ -8,7 +8,7 @@ opened: 2026-09-23 A service swap (`toyos-swap`) writes the binary the supervisor verified to `/tmp/swap//` and starts the service from that path. `/tmp` -is a tmpfs every process may write, because the filesystem is ambient by the +is the MEMORY file server's, which every process may write, because the filesystem is ambient by the owner's ruling, so the file the supervisor hashed is not the file the kernel loads by any guarantee — only by nobody having rewritten it in between. Two windows: @@ -18,9 +18,9 @@ guarantee — only by nobody having rewritten it in between. Two windows: requester has hung up, milliseconds to two seconds later; a process that rewrites the file in that window runs its own bytes holding the service's manifest row — its device claims included. -- **For the service's life**, closed: a spawn from `/tmp` runs a snapshot of - the file taken at the spawn (`file_cache::Snapshot`), which no later write - reaches. A restore after a failed swap spawns the previous installed path +- **For the service's life**, closed: a spawn from `/tmp` runs a copy of + the file read into a memory object at the spawn (`Image::read_for`), which + no later write reaches. A restore after a failed swap spawns the previous installed path again, so it is the first window again. Nothing in the tree does either; the authority to ask for a swap is diff --git a/issues/a-symbolic-link-on-tmp-displaces-its-name-and-lists-nowhere.md b/issues/a-symbolic-link-on-tmp-displaces-its-name-and-lists-nowhere.md deleted file mode 100644 index a614807125b..00000000000 --- a/issues/a-symbolic-link-on-tmp-displaces-its-name-and-lists-nowhere.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-01 ---- - -# A symbolic link on /tmp displaces its name and lists nowhere - -Read from the code, not run. The kernel's `/tmp` (`kernel/src/tmpfs.rs`) -makes a link over whatever file its name held, where POSIX's `symlink` refuses -a name that exists `EEXIST`, so libc's `symlink` refuses `ENOSYS` for a link -the kernel would hold (`sdk/libc/src/posix_io.rs`). Its `list` answers files -alone, so no `readdir` of a directory shows a link it holds. - -libc's `symlink` already sends a link a file server is to hold to that -server, through `toyos::files` as std's does, which refuses a name that -exists, answered `EEXIST`, and `readdir` there names the link; corpus case -`211_home_files` asserts both under `/home`. What is left is `/tmp`. - -**Exit**: `/tmp` is a file server's, so every `symlink` is one, sent with -nothing asked first and answered `EEXIST` for a name that exists, and libc's -`readdir` of `/tmp` names the link, each asserted by a test. The kernel's `SYS_SYMLINK` -and its symlink code (`kernel/src/vfs.rs`, `kernel/src/tmpfs.rs`) go wherever -nothing calls them. That `EEXIST` is how LLVM's `LockFileManager` takes its -lock (`create_link`, `::symlink` in `llvm/lib/Support/Unix/Path.inc`). An LLVM -build on ToyOS reaches `symlink` too: -`LLVM_USE_SYMLINKS` is on for a UNIX host, so `add_llvm_tool_symlink` -(`llvm/cmake/modules/AddLLVM.cmake`) makes each tool's aliases with CMake's -`create_symlink`. diff --git a/issues/a-tmpfs-metadata-call-costs-toyos-up-to-six-times-linuxs.md b/issues/a-tmpfs-metadata-call-costs-toyos-up-to-six-times-linuxs.md index 30f31aa391b..69afb79af4b 100644 --- a/issues/a-tmpfs-metadata-call-costs-toyos-up-to-six-times-linuxs.md +++ b/issues/a-tmpfs-metadata-call-costs-toyos-up-to-six-times-linuxs.md @@ -7,8 +7,8 @@ opened: 2026-10-28 # A tmpfs metadata call costs ToyOS up to six times Linux's `osbench files` runs the same operations under each system's memory -filesystem: ToyOS's `/tmp`, which the kernel serves through syscalls today, -and Linux's `/dev/shm`, tmpfs through the VFS. +filesystem: ToyOS's `/tmp`, which the kernel served through syscalls when +this was measured, and Linux's `/dev/shm`, tmpfs through the VFS. From the 2026-10-28 scoreboard (evidence branch `scoreboard-2026-10-28`): ToyOS at 069cbbeb8, four boots off the internal NVMe, against five runs of @@ -31,24 +31,18 @@ On the T14 ToyOS loses all eight, 1.7x to 6.5x, each resolved. On the AMD it loses `open_close` (2.1x) and `rename` (1.4x), both resolved, wins `tree.100`, `create_write_unlink` and `write_4k_at`, and is unresolved on the other three. There the HPET explains Linux's spread: its TSC runs read -`open_close` at 2.9 µs and its HPET runs at 4.3 µs. The listing -(`fs.mem.readdir.1000`) is `issues/a-listing-still-scans-the-whole-mount.md`'s, -and the sequential copies are +`open_close` at 2.9 µs and its HPET runs at 4.3 µs. The sequential copies are `issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md`'s. -**The kernel's tmpfs is not where this is fixed.** Step 9 of -`issues/the-kernel-is-small-interrupts-post-and-threads-wait.md`, under the -owner's rulings of 2026-09-25 and 2026-10-03, deletes `tmpfs.rs`, -`fs_rename.rs`, `file_cache.rs` and `vfs.rs`, and moves `/tmp` to -userland per session. An `fs.mem` call then becomes a file server request, -which costs 3 to 12 times a kernel file call today +`/tmp` is now the MEMORY file server's (`system/fileserver/src/memory.rs`), so +each `fs.mem` call is a file server request, which cost 3 to 12 times a kernel +file call when both existed (`issues/a-file-server-request-costs-three-to-twelve-times-a-kernel-file-call.md`). -So the move makes this gap larger, unless the file server's round trip -shrinks first. +These rows have not been measured since the move; the gap is expected to have +grown. -**Exit**: once `/tmp` is served from userland, these eight rows within -Linux's on both benches in a labelled current-state comparison, or each -remaining difference measured to its parts and justified. +**Exit**: these eight rows within Linux's on both benches in a labelled +current-state comparison, or each remaining difference measured to its parts +and justified. -**Owner**: whatever serves `/tmp` through step 9's migration. Until then, -nothing in the kernel's tmpfs is owed this work. +**Owner**: the file server. diff --git a/issues/anonymous-memory-swaps-to-an-encrypted-partition.md b/issues/anonymous-memory-swaps-to-an-encrypted-partition.md index 4b47019cc92..c7103f92d2a 100644 --- a/issues/anonymous-memory-swaps-to-an-encrypted-partition.md +++ b/issues/anonymous-memory-swaps-to-an-encrypted-partition.md @@ -351,7 +351,7 @@ Each of these changes at the stage named: (the RAM/8 window cap) becomes the compositor's wire allowance over on-screen buffers. - **Deleted.** `system/fileserver/src/cache.rs` goes at its prerequisite. - `kernel/src/file_cache.rs` behind kernel tmpfs goes at P3's memory format. + `kernel/src/file_cache.rs` went with kernel tmpfs (#1152). - **When text maps its source without the copy** (`issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md` stage 5), the copied-segment rule goes with the copies. diff --git a/issues/bcachefs-crate-is-not-bcachefs.md b/issues/bcachefs-crate-is-not-bcachefs.md deleted file mode 100644 index 8b58dfe52c0..00000000000 --- a/issues/bcachefs-crate-is-not-bcachefs.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -status: open -kind: track -opened: 2026-08-01 ---- - -# The `lib/bcachefs/` crate does not implement bcachefs - -ToyOS's `lib/bcachefs/` crate implements a ToyOS-native on-disk format written from scratch. -It shares a name with Linux bcachefs and nothing else, and **the magics alone settle -it**: ours is a four-byte `MAGIC = b"BCFS"` plus -`DESIGNATION_MAGIC = b"TOYOS-FORMAT-ME\0"` (`superblock.rs:5,24`) and `NODE_MAGIC = b"BTND"` -(`btree.rs:7`), against upstream's 16-byte UUID `BCHFS_MAGIC`, its per-bset -`BSET_MAGIC ^ sb.uuid` and its `JSET_MAGIC`. Neither implementation could mount -what the other writes, and neither would get past the first block trying. - -That collision has already cost this project once: research into the *upstream* -format was filed under this crate's name and needed a warning at its top to stop -a reader taking it for documentation of what we ship. Warning the reader fixes -the document and not the collision — a crate that does not implement the format -it is named after is a hazard we keep paying for, in exactly this way. Renaming -it is the owner's call, not something to do in a docs pass. - -## Answered by the owner, 2026-08-15 - -> *"bcachefs is the default filesystem for toyos. the crate must be an -> implementation of the spec."* - -The resolution is neither a rename nor a deletion. **`lib/bcachefs/` must become a -real implementation of the bcachefs on-disk format, and that format is ToyOS's -default filesystem.** The name stops being wrong by the crate growing into it. - -This entry therefore stops being a question and becomes work: what is owed is the -implementation, and the gap between the two formats recorded above is the measure -of it. The format is implemented clean-room (root `CLAUDE.md`): an analyst reads -upstream and writes the specification, and a different agent implements from it. - -**Where it stands.** `lib/bcachefs/src/engine` reads single-device 1.39 -volumes, crc32c and ChaCha20/Poly1305, replaying the journal of one not cleanly -unmounted, and formats a clean volume holding a tree -of directories, files, symlinks and hard links (directories alone when -encrypted), with all its allocator and accounting state, that upstream's -`bcachefs fsck -n -K` passes. Its fsck derives -that state from a volume's keys and compares, and the derivation agrees with -what upstream stored in its own volumes, a replayed journal's included; it -checks the journal against the superblock. It mounts a volume read-write -(`Mount`): namespace operations as journaled transactions, nodes appended to or -split by reclaim, a crash replayed and a clean unmount, which upstream's fsck -also passes. It refuses to write a volume of blocks under 4 KiB or nodes under -32 KiB (`issues/the-bcachefs-writer-refuses-small-blocks-and-nodes.md`). It -reuses a btree bucket once the entry that emptied it is durable. File data is -not yet written (B7), so symlinks are not yet created online, and its reclaim reserves are -sized for common rewrites -(`issues/bcachefs-reclaim-reserves-are-sized-for-common-rewrites.md`). It -replaced a read side written from upstream's headers. Its tests read volumes upstream's tools wrote -(`lib/bcachefs/tests/fixtures/`). The build writes ROOT with it, at the -smallest size that leaves no bucket free, and the loader, `imgstat` and the -SYSTEM file server read ROOT through its reader. The rest of the crate is -still the interim format, which DATA uses until the engine replaces it. - -The track itself is not planned here. - -One fact about the tree that a real-bcachefs track inherits whichever way it is -sequenced: the kernel must parse this format to reach `/system/bin/init` at all, so a -machine whose ROOT it cannot mount has no userland. The observation the ruling -inverts is the other half — a home-grown format has no second implementation to -be judged against, and upstream bcachefs is exactly such a judge. - -**How that judge is used, since a judge is not a dependency:** upstream's `bcachefs-tools` is a *development* -instrument, run by a builder on a volume ToyOS wrote, with its `fsck` and its -readback pasted into the pull request. What the suite runs against is committed -fixtures — volumes upstream's tools wrote, recorded in `NOTICE`. Nothing is -fetched and no distribution is depended on. diff --git a/issues/deferred-release-outlives-its-syscall.md b/issues/deferred-release-outlives-its-syscall.md index 5ddbbb1a96d..1bf8cb53150 100644 --- a/issues/deferred-release-outlives-its-syscall.md +++ b/issues/deferred-release-outlives-its-syscall.md @@ -258,9 +258,8 @@ The ruling, in the words it was recorded in, where "wall 4" is this question: The queue landed with #257 (`7ab9367b0`) and left the kernel in `80a1f1ceb`, when nothing the kernel flushed reached a device any more; the conversion it -was sequenced before is planned by nothing. Today `OpenFileState::drop` calls -`file_cache::release` and takes "the file cache's lock and no other", so -`File` needs no release site that parks. +was sequenced before is planned by nothing. `OpenFileState` and the file cache +whose release it ran went in #1152. ## A process's own releases run before its end, ruled 2026-10-13 diff --git a/issues/every-program-sees-only-the-files-it-was-given.md b/issues/every-program-sees-only-the-files-it-was-given.md index 2951b52c1de..9604831c344 100644 --- a/issues/every-program-sees-only-the-files-it-was-given.md +++ b/issues/every-program-sees-only-the-files-it-was-given.md @@ -100,8 +100,8 @@ nameable by every program and make confused-deputy bugs structural. what it holds, and a package launched from `/apps` is minted its own directory read-only and its own folder of the session's home read-write, `/system` read-only, and nothing else a file server serves - (`toyos_manifest::Program::view`). It still reaches `/tmp`, which the - kernel serves to every program. Every row the image declares still sees + (`toyos_manifest::Program::view`). It still reaches `/tmp`, the one tree + the MEMORY file server serves every view. Every row the image declares still sees the whole tree, `/system` read-only and the rest read-write but `/state`, of which each row holds only its own directory where it keeps state (`service = true` or `state = true`: the daemons' own state is built). diff --git a/issues/four-shares-take-a-file-server.md b/issues/four-shares-take-a-file-server.md index 768955ce31d..c300e59ce27 100644 --- a/issues/four-shares-take-a-file-server.md +++ b/issues/four-shares-take-a-file-server.md @@ -38,6 +38,18 @@ so it waits for one of two exits: a measured parallel build on ToyOS shows which bound it reaches and by how much, or a share's streams are bounded by the bytes they cost its client's job rather than by count. +**`/tmp` now spends the same bound.** Since #1152 `/tmp` is the MEMORY file +server's, and the SDK keeps one connection per directory capability for the +life of the process. So every live process of a share that has touched `/tmp` +holds one of that share's 32 connections there, and the 33rd is refused. Seen +on the T14 at `0507cbe36` (`proctreecase`): a chain of processes each +spawning the next from a copy of its program on `/tmp` stopped at link 32, +refused "the file server serves as many connections as it takes". A parallel +build whose compilers and linkers each write a temporary there reaches it at +32 concurrent processes, before any stream bound. Nothing runs that load now: +`process_tree`'s chain starts each link from an endowed image and touches no +file server, and `bounds.rs`' unit tests hold the count alone. + Owner: the file server. ## Exit condition diff --git a/issues/libc-refuses-what-toyos-cannot-yet-answer.md b/issues/libc-refuses-what-toyos-cannot-yet-answer.md index a4000d786a8..dc2951c8495 100644 --- a/issues/libc-refuses-what-toyos-cannot-yet-answer.md +++ b/issues/libc-refuses-what-toyos-cannot-yet-answer.md @@ -10,10 +10,6 @@ These answer failure in their POSIX form and do nothing (`sdk/libc/src/refused.rs`, `posix_io.rs`), each asserted by `tests/corpus/tinycc/206_libc_refusals.c`. Each waits on what it names: -- `symlink` of a link the kernel would hold, `ENOSYS`: `SYS_SYMLINK` - displaces what its name held - (`issues/a-symbolic-link-on-tmp-displaces-its-name-and-lists-nowhere.md`). A - file server makes one and refuses a name that exists, as POSIX has it. - `chmod`, `fchmod` and `fchmodat`, `ENOSYS`: a file has no mode bits to set. - `statvfs` and `fstatvfs`, `ENOSYS`: no call answers a filesystem's size or free space. diff --git a/issues/ninja-runs-every-command-through-a-bin-sh-toyos-does-not-have.md b/issues/ninja-runs-every-command-through-a-bin-sh-toyos-does-not-have.md index b760c170622..da53b5083db 100644 --- a/issues/ninja-runs-every-command-through-a-bin-sh-toyos-does-not-have.md +++ b/issues/ninja-runs-every-command-through-a-bin-sh-toyos-does-not-have.md @@ -9,8 +9,8 @@ opened: 2026-09-30 Ninja starts each build edge as `/bin/sh -c ` (v1.13.1 `src/subprocess-posix.cc:132-134`), and the command is the build file's own text, written for a POSIX shell; tokio's `tests/process_smoke.rs` runs `sh -c` -too. ToyOS has no `/bin`: `/` holds exactly the names `kernel/src/vfs.rs`'s -`ROOT_ENTRIES` lists. Its shell is `/system/bin/shell`, which takes `-c`; +too. ToyOS has no `/bin`: `/` holds only the directories a process's view +names (`toyos_manifest::ROLES`). Its shell is `/system/bin/shell`, which takes `-c`; nothing establishes that it runs the POSIX shell language build files are written in. diff --git a/issues/one-write-far-past-a-files-end-holds-data-s-server.md b/issues/one-write-far-past-a-files-end-holds-data-s-server.md deleted file mode 100644 index 938a1d0eaed..00000000000 --- a/issues/one-write-far-past-a-files-end-holds-data-s-server.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-27 ---- - -# One write far past a file's end holds DATA's server for as long as the gap - -DATA's file server (`system/fileserver`) takes a `WRITE` at any offset up to `toyos::fs::MAX_FILE_BYTES` (16 TiB), -and `Mounted::allocate` bridges the gap from the file's end to it -by allocating and zeroing every block between -(`issues/a-bcachefs-hole-costs-a-block-write-per-page.md` is why). -The server is one thread, so the one request runs until the volume is full or the gap -is bridged, and every other client of `/apps`, `/config`, `/home` and `/state` -waits the whole of it: any program holding `/home` can stop every program's -files with one `seek` and one byte. - -**Exit**: a write whose gap is past what a request may cost is refused by name -before anything is allocated — or the format expresses a hole — with a guest -test that writes one byte a gigabyte past a file's end and has another client -answered meanwhile. diff --git a/issues/probe-mounts-on-a-checksum.md b/issues/probe-mounts-on-a-checksum.md index 78bd535ee05..0d4b53b7b27 100644 --- a/issues/probe-mounts-on-a-checksum.md +++ b/issues/probe-mounts-on-a-checksum.md @@ -14,12 +14,10 @@ against the crate: read-write mount: `sync()` rewrites both superblocks, and any file operation writes the bitmap, btree nodes and data. So mounting a stranger's disk modifies it, which is a weaker form of the wrong the designation stamp exists to prevent. -Accidental collision is not the risk — random block-0 bytes satisfy 4 bytes of -magic, 4 of version and a 32-bit CRC with probability about 2^-64 — and neither -is a *genuine* upstream bcachefs volume, which does not begin with ASCII `BCFS` -(this crate shares the name and nothing else; `issues/bcachefs-crate-is-not-bcachefs.md`). The risk is a **deliberately -crafted block 0** on a disk somebody hands you, which is the metal track's -situation exactly. +Accidental collision is not the risk: random bytes do not pass a superblock's +magic and checksum. The risk is a volume this machine did not write on a +partition typed as DATA — a **deliberately crafted** one on a disk somebody +hands you, which is the metal track's situation exactly. Recommendation, for the owner to decide: diff --git a/issues/storage-is-layers-and-a-role-is-a-filesystem.md b/issues/storage-is-layers-and-a-role-is-a-filesystem.md index 3252db436e7..1e090fa3ecc 100644 --- a/issues/storage-is-layers-and-a-role-is-a-filesystem.md +++ b/issues/storage-is-layers-and-a-role-is-a-filesystem.md @@ -19,12 +19,13 @@ mount protocol — is superseded, and nothing here builds it. ## Block -A block device is a service: one program per controller (`blockd` for NVMe; -the xHCI's own program for USB mass storage, once it has one), serving each -partition as a session over `toyos-blockring`. A partition has one holder, -every request is bounded to it before the device sees it, and a flush answers -for its own writer's writes. GPT is the one partition scheme, read by the -service that drives the disk. Caching is the file server's, never the block +A block device is a service: one driver per controller (`nvme`, and `usbd` +for USB mass storage) offers its disks to the block service (`diskserver`), +which opens a partition for the client its grant admits and hands the session, +over `toyos-blockring`, to the driver that serves it. A partition has one +holder, every request is bounded to it before the device sees it, and a flush +answers for its own writer's writes. GPT is the one partition scheme, read by +the driver of the disk. Caching is the file server's, never the block service's. ## Volume @@ -38,21 +39,21 @@ mounts degraded, or refuses — a member vanishing is never silent. A role on on disk is the one-member case. Two filesystems claiming one unique role are refused rather than guessed, which `boot_partition_identity` already asserts for the boot volume. A blank volume joins the system by the designation stamp -the bcachefs crate already carries (`DESIGNATION_MAGIC`, -`lib/bcachefs/src/superblock.rs`): the probe formats a designated volume and -refuses to reformat a used one (`issues/probe-mounts-on-a-checksum.md` +the bcachefs crate already carries (`lib/bcachefs/src/designation.rs`): the +probe formats a designated volume and refuses to reformat a used one +(`issues/probe-mounts-on-a-checksum.md` is what that refusal still owes). Microsoft's basic-data type is a foreign volume, handed to whichever server recognises its superblock. | partition | filesystem | mounts as | writable | |---|---|---|---| -| ESP | FAT32, firmware's rule | `/boot` | kernel only | +| ESP | FAT32, firmware's rule | `/boot` | no; BOOT's server serves it read-only | | ROOT | bcachefs image the build writes | `/system` | no; signed per release | | DATA | bcachefs, formatted on first boot | `/apps`, `/config`, `/home`, `/state` | yes | | LOG | FAT32 while a Mac has to read the dev stick | `/log` | yes | -`/tmp` has no backing. The dev loop keeps the ESP and ROOT on the stick and -DATA on the internal NVMe; the installed product carries all four on one disk. +`/tmp` is the MEMORY server's, in RAM, with no partition. The dev loop keeps +the ESP and ROOT on the stick and DATA on the internal NVMe; the installed product carries all four on one disk. Roles come from partitions, so both shapes are the same code. ## Filesystem @@ -67,16 +68,17 @@ and the updater write it offline, checkpoint it and mark the journal clean; a server that finds a dirty journal under `/system` refuses by name, because replay is a write-path job and a system image needing one is a broken install. -**Every filesystem is a file server per role** — SYSTEM, LOG, DATA, BOOT — -SYSTEM over ROOT's bytes in memory and every other over its block service's -sessions, linking the whole crate it serves: a server that crashes loses its -own role and nothing else, and no filesystem parser runs in the kernel. +**Every filesystem is a file server per role** — SYSTEM, LOG, DATA, BOOT, +MEMORY — SYSTEM over ROOT's bytes in memory, MEMORY over paged objects alone +and every other over its block service's sessions, linking the whole crate it +serves: a server that crashes loses its own role and nothing else, and no +filesystem parser runs in the kernel. **Multi-device is a filesystem property, not a block-layer subsystem.** Pooling across disks, replicas per subvolume, tiering with a fast foreground device, erasure coding, snapshots, checksums and scrub, compression, encryption, subvolume per user and per app: every one is a feature of upstream bcachefs's -format, which is why `issues/bcachefs-crate-is-not-bcachefs.md`'s +format, which is why `issues/the-bcachefs-crate-must-implement-bcachefs.md`'s ruling — the crate becomes a real implementation — is what makes them reachable. The block layer stays dumb. LVM- and md-shaped aggregation is refused: it cannot tell metadata from data, place a file's replicas on @@ -95,12 +97,10 @@ NTFS by the same shape if wanted. ## Paths The tree, its names and how a program finds them are -`issues/where-everything-lives.md`. What this track owes it is the -mount structure: a mount point is exactly one top-level name -(`kernel/src/vfs.rs`, `ROOT_ENTRIES` and the array indexed by it), so +`issues/where-everything-lives.md`. A client routes a path to the directory +capability with the longest matching prefix (`toyos::files`), so `/media/

{ } let flags: [u64; 7] = core::array::from_fn(|i| le64(b, 144 + 8 * i)); match field(flags[0], 2, 6) { - 0 => {} + 0 => return Err(Error::Unsupported("superblock without a checksum")), 1 if crc32c(&b[16..total]) == le32(b, 0) => {} 1 => return Err(Error::Checksum("superblock")), _ => return Err(Error::Unsupported("superblock checksum other than crc32c")), diff --git a/lib/bcachefs/src/engine/tests/mod.rs b/lib/bcachefs/src/engine/tests/mod.rs index fcef437c12f..cb057c29f89 100644 --- a/lib/bcachefs/src/engine/tests/mod.rs +++ b/lib/bcachefs/src/engine/tests/mod.rs @@ -1358,3 +1358,39 @@ fn an_upstream_volumes_file_in_one_run_is_in_place_and_inline_data_is_not() { let (at, len) = block_on_ready(v.in_place(path(&v, "Documents/deep/seq.txt"))).unwrap().expect("one run"); assert_eq!(dev.get()[at as usize..(at + len) as usize], seq()[..]); } + +/// What ToyOS writes always carries a checksum, so a superblock, a btree +/// node or an extent that declares none is refused rather than read +/// unverified. +#[test] +fn metadata_and_data_without_a_checksum_are_refused() { + let dev = formatted(8 << 20, None); + rewrite_superblock(&dev, |header, _| header.flags[0] &= !(0x3f << 2)); + assert_eq!(open_err(&dev), Error::Unsupported("superblock without a checksum")); + + let dev = formatted(8 << 20, None); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + let (ptr, _) = leaf_of(&v, Btree::Inodes, Bpos::MIN); + let flags = ptr.ptr.sector * SECTOR as u64 + 152; + let mut block = [[0u8; BLOCK]]; + block_on_ready(dev.read(flags / BLOCK as u64, &mut block)).unwrap(); + block[0][(flags % BLOCK as u64) as usize] &= !0xf; + block_on_ready(dev.write(flags / BLOCK as u64, &block)).unwrap(); + assert_eq!(block_on_ready(v.inode(4096)), Err(Error::Unsupported("btree node without a checksum"))); + + let data = [0x5au8; BLOCK]; + let mut tree = Tree::new(); + tree.file(Tree::ROOT, b"f", &data, Meta { perm: 0o644, uid: 0, gid: 0 }).unwrap(); + let dev = Bytes::new(vec![0u8; 8 << 20]); + let options = Options { label: b"toyos", time: 1_761_177_600_000_000_000, encryption: None }; + block_on_ready(mkfs(&dev, &options, &tree, &mut Seeded(0x5eed))).unwrap(); + let ino = path(&block_on_ready(Volume::open(&dev)).unwrap(), "f"); + assert_eq!(read_all(&block_on_ready(Volume::open(&dev)).unwrap(), ino).unwrap(), data, "the file reads before its checksum goes"); + rewrite_leaf(&dev, Btree::Extents, Bpos::MIN, 0, |keys| { + for (key, value) in keys.iter_mut().filter(|(k, _)| k.kind == crate::engine::kind::EXTENT) { + let extent = Extent::decode(value).unwrap(); + *value = Extent::encode(key.size, 0, 0, extent.ptr); + } + }); + assert_eq!(read_all(&block_on_ready(Volume::open(&dev)).unwrap(), ino), Err(Error::Unsupported("data without a checksum"))); +} From 39df624fa4ba8b6d84f26c623f44e21a89a226ce Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 08:38:49 +0100 Subject: [PATCH 67/93] bcachefs: mkfs leaves no other format's bytes where it reserves or past it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mkfs wrote each superblock copy's few kilobytes at the start of its 1 MiB reserve and nothing after its last copy. A partition it formatted kept whatever another format had there: on both benches, the private format's backup superblock in the last block, which a main image then mounted. Each copy is now written padded to its whole reserve, and the device past the last copy's reserve is zeroed, before block 0 — still the last write, which consumes the designation. A known foreign signature in those places (a backup superblock at the device's end, one inside the first megabyte) does not survive the format; the buckets in between are the volume's own and read only through its checksummed metadata. Test: mkfs over a device every byte of which is another writer's, its last block a private-format superblock, leaves zeros past each copy within its reserve, past the last copy, and in the last block. Co-Authored-By: Claude Opus 5.5 --- lib/bcachefs/src/engine/mkfs.rs | 16 +++++++++++++++- lib/bcachefs/src/engine/tests/mod.rs | 28 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/lib/bcachefs/src/engine/mkfs.rs b/lib/bcachefs/src/engine/mkfs.rs index e39ef1c7fe6..81417284c54 100644 --- a/lib/bcachefs/src/engine/mkfs.rs +++ b/lib/bcachefs/src/engine/mkfs.rs @@ -114,8 +114,22 @@ pub(crate) async fn mkfs_shaped(dev: &impl BlockDev, options: &Options<'_>, tree for (sector, bytes) in &volume.nodes { write(dev, *sector, bytes).await?; } + // Each copy fills its whole reserve and the device past the last one is + // zeroed: no other format's superblock, a backup at the device's end + // among them, outlives the format in a place this volume never writes. + let reserve = (1usize << volume.header.layout.max_bits) * SECTOR; for &offset in &volume.header.layout.offsets { - write(dev, offset, &Superblock::encode(&volume.header, &volume.fields, offset)).await?; + let mut copy = Superblock::encode(&volume.header, &volume.fields, offset); + copy.resize(reserve, 0); + write(dev, offset, ©).await?; + } + let tail = volume.header.layout.offsets.last().expect("a layout lists copies") * SECTOR as u64 + reserve as u64; + let zeros = alloc::vec![[0u8; BLOCK]; 256]; + let mut block = tail / BLOCK as u64; + while block < dev.blocks() { + let n = (dev.blocks() - block).min(zeros.len() as u64); + dev.write(block, &zeros[..n as usize]).await?; + block += n; } // Sectors 0–6 are zeros here; the layout copy is sector 7. let mut first = alloc::vec![0u8; BLOCK]; diff --git a/lib/bcachefs/src/engine/tests/mod.rs b/lib/bcachefs/src/engine/tests/mod.rs index cb057c29f89..c1bb9e998c4 100644 --- a/lib/bcachefs/src/engine/tests/mod.rs +++ b/lib/bcachefs/src/engine/tests/mod.rs @@ -1394,3 +1394,31 @@ fn metadata_and_data_without_a_checksum_are_refused() { }); assert_eq!(read_all(&block_on_ready(Volume::open(&dev)).unwrap(), ino), Err(Error::Unsupported("data without a checksum"))); } + +/// A device another format filled: past each copy within its reserve and +/// past the last copy to the device's end, mkfs leaves zeros, so no foreign +/// superblock there — a backup in the last block among them — survives it. +/// The device is a bucket's fraction past a whole number of buckets, so the +/// last copy's reserve ends before the device does, as on the benches. +#[test] +fn mkfs_leaves_nothing_of_another_format_in_the_places_it_reserves() { + let bytes = (8 << 20) + (8 << 10); + let mut foreign = vec![0xabu8; bytes]; + foreign[bytes - BLOCK..bytes - BLOCK + 4].copy_from_slice(b"BCFS"); + let dev = Bytes::new(foreign); + let options = Options { label: b"toyos", time: 1_761_177_600_000_000_000, encryption: None }; + block_on_ready(mkfs(&dev, &options, &Tree::new(), &mut Seeded(0x5eed))).unwrap(); + let sb = block_on_ready(Volume::open(&dev)).unwrap().superblock().clone(); + let layout = sb.header.layout.clone(); + let raw = dev.into_inner(); + let reserve = (1usize << layout.max_bits) * SECTOR; + for &offset in &layout.offsets { + let at = offset as usize * SECTOR; + let copy = super::superblock::Superblock::encode(&sb.header, &sb.fields, offset).len(); + assert!(raw[at + copy..at + reserve].iter().all(|&b| b == 0), "the reserve of the copy at sector {offset} keeps another writer's bytes"); + } + let tail = *layout.offsets.last().unwrap() as usize * SECTOR + reserve; + assert!(tail < bytes - BLOCK, "the last copy's reserve reaches the last block: no tail to clear"); + assert!(raw[tail..].iter().all(|&b| b == 0), "the tail keeps another writer's bytes"); + assert!(raw[bytes - BLOCK..].iter().all(|&b| b == 0), "the last block keeps the other format's backup"); +} From 164f391a5c9ef4fc4ab32b970cc57b50f86fccff Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 08:39:06 +0100 Subject: [PATCH 68/93] fileserver: osbench's DATA sequence over the benches' DATA states, refused or clean MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes issues/data-mounted-a-volume-another-format-wrote-over.md. What happened. The native preparation boots (disks format data --commit, 2026-11-02, T14 20:12 UTC, AMD 20:31 UTC) formatted DATA partitions holding the private format's live volume. mkfs zeroed block 0 and wrote superblock copies at sectors 8 and 2056 — the private format's bitmap blocks 1 and 257 — and left the last block, the private format's backup superblock. The next main image (T14, 2026-11-03 19:39 UTC) found no superblock at block 0, fell back to the backup silently, mounted the old volume ("22 names") and wrote it for a day with a bitmap whose blocks 1 and 257 were native superblocks: 1330 of the blocks below 1728, the superblock and 1057 bitmap blocks among them, read free. Its allocator gave out live blocks; osbench's data reached a node, and ab3/2-b4096's mount refused BadMagic { BTND, [212, 213, 214, 215] }. Evidence, read-only captures decoded on the host: both private superblocks intact (root 1334530); bitmap block 1 holding a native header (version 0x0427, "ToyOS DATA", seq 10, written 20:12:08), block 257 a whole native copy, the same on the AMD (20:31:08); the private tree's block 1758 reached twice, live node 15052 marked free. A replay of this sequence through main's private-format server on a model of the T14's partition, with a walker that shares no code with the format: intact, no write over the committed tree and a clean check; after the native mkfs, a silent mount, 41 writes over committed nodes and the next mount's BadMagic. The cache, the write-back and the block service placed nothing wrong. The native format now refuses such a partition (a superblock place another writer took, Error::Overwritten), accepts nothing without a checksum, and leaves no other format's superblock where mkfs reserves or past its last copy. This test runs every kind of call the sequence makes, a few times each, against the native DATA role on a 1 GiB partition laid out as the benches' front and end were: a fresh designated partition is formatted and survives; the private format's volume is refused as foreign until designated, then formatted with no private superblock left; a native volume the private format's server then wrote, in the T14's shape and the AMD's, is refused by name and untouched until designated, then formatted clean. A mounted volume is judged by fsck and a fresh mount. The rows' own repetitions are not repeated here: at the native writer's cost per call on the host model they take hours, where main's private format took seconds. Evidence: https://github.com/ToyOSOrg/evidence/tree/b30cf0b9c0e2adbb8882674b8a306f543d4247b8/data-mixed-volume Co-Authored-By: Claude Opus 5.5 --- ...nted-a-volume-another-format-wrote-over.md | 54 --- system/fileserver/tests/osbench_replay.rs | 368 ++++++++++++++++++ 2 files changed, 368 insertions(+), 54 deletions(-) delete mode 100644 issues/data-mounted-a-volume-another-format-wrote-over.md create mode 100644 system/fileserver/tests/osbench_replay.rs diff --git a/issues/data-mounted-a-volume-another-format-wrote-over.md b/issues/data-mounted-a-volume-another-format-wrote-over.md deleted file mode 100644 index c86a0795554..00000000000 --- a/issues/data-mounted-a-volume-another-format-wrote-over.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -status: assigned -kind: defect -opened: 2026-11-04 ---- - -# DATA mounted a volume another format had written over, and wrote file data over its nodes - -Held by the storage-consolidation branch's fixes to the native format's -superblock read, checksum types and mkfs, which close it. - -**What happened on both benches.** The native format's preparation boots -(`disks format data --commit`, 2026-11-02, T14 20:12 UTC, AMD 20:31 UTC) ran -mkfs over DATA partitions that held the private format's live volume. mkfs -zeroed block 0, wrote superblock copies at sectors 8 and 2056 — the private -format's bitmap blocks 1 and 257 — and never wrote the partition's last -block, which held the private format's backup superblock. The next boot of a -main image (T14, 2026-11-03 19:39 UTC) found no superblock at block 0, fell -back to the backup without saying so, mounted the old volume ("22 names") and -rewrote block 0 as its own. Its bitmap now read the native superblock's bytes: -1330 of the blocks below 1728 and most of 0..32767 free, among them the -superblock, 1057 bitmap blocks and the old tree's nodes. Its allocator handed -those out; a day of boots later osbench's file data lay on a node and the next -mount refused `BadMagic { expected: "BTND", got: [212, 213, 214, 215] }`. -The AMD's DATA is the same mixture, not yet failed. - -**What the evidence shows.** Captured read-only from the T14's p4 (the first -16 MiB, the last block, blocks 1325056..1361919) and the AMD's (the first -16 MiB): both private superblocks intact and identical (root 1334530, the -mount used block 0); bitmap block 1 beginning with a native superblock header -(version 0x0427, label "ToyOS DATA", seq 10, time base 2026-11-02 20:12:08 -UTC) and block 257 holding a whole native copy; a tree of 190 nodes in which -block 1758 is reached twice and live node 15052 is marked free. - -**What a replay shows.** osbench's persist and fs.disk sequence on the private -format over a model of the T14's partition: on an intact volume no write lands -on a node the committed tree reaches and the volume checks clean; after the -native mkfs, the mount succeeds silently, 41 writes land on committed nodes, -nodes are reached twice, and the next mount refuses with file bytes where a -node's magic was. The shared write path — the cache, the write-back, the -block service's ordering — placed nothing wrong; the private format's silent -backup fallback and unchecked bitmap did, on a partition the native mkfs left -carrying both formats. - -**What the native format owes, which the fix provides.** A mount that finds -another writer's bytes where its layout puts a superblock copy or the layout -copy refuses the volume by name, and one that finds a copy a crash tore says -so and rewrites it; no metadata or data is accepted without a checksum; mkfs -leaves no other format's superblock in the places it reserves or past its -last copy. - -**What the benches need.** Both DATA partitions hold neither volume whole. -The native format refuses them by name; `disks format data --commit` on each -formats them clean. diff --git a/system/fileserver/tests/osbench_replay.rs b/system/fileserver/tests/osbench_replay.rs new file mode 100644 index 00000000000..7a00fd86d5e --- /dev/null +++ b/system/fileserver/tests/osbench_replay.rs @@ -0,0 +1,368 @@ +//! osbench's `persist` and `fs.disk` file sequence and the shutdown sync, +//! against the DATA role on a model of the bench's partition, in the states +//! the benches' DATA went through: fresh, the native mkfs over the private +//! format's volume, and that volume after the private format's server, which +//! mounted it from its backup superblock, wrote it again. The role either refuses +//! the partition by name and writes nothing, or formats it whole; a volume it +//! mounts is judged by `fsck` and a fresh mount after the sequence. + +use std::cell::Cell; + +use bcachefs::engine::{fsck, Volume as Native}; +use fileserver::cache::Disk; +use fileserver::data::{DataVolume, Probed}; +use fileserver::ram::Ram; +use fileserver::volume::{Buf, OpenHow, Volume, Writes}; +use toyos_blockdev::{block_on_ready, Block, BlockDev, Error, BLOCK}; + +const MIB: usize = 1 << 20; +const CREATE: OpenHow = OpenHow { create: true, create_new: false, truncate: false }; +const CREATE_TRUNC: OpenHow = OpenHow { create: true, create_new: false, truncate: true }; +const PLAIN: OpenHow = OpenHow { create: false, create_new: false, truncate: false }; +const ROOTS: [&str; 4] = ["apps", "config", "home", "state"]; + +/// A DATA partition of 1 GiB, sparse in memory: past the places that +/// matter, the native copies' and the private format's superblocks', the +/// benches' 170 and 600 GiB add only buckets. +const BLOCKS: u64 = 1 << 18; +/// The private format's bitmap on the T14: blocks 1 to 1360, the native +/// copies' places among them. +const BITMAP: u64 = 1360; +/// The sequential rows' file: 64 MiB on the bench, a 4 KiB call 512 times here. +const SEQ: usize = 2 * MIB; + +fn clock() -> u64 { + 1_793_650_328_000_000_000 +} + +/// The partition, counting the writes it takes. +struct Model { + ram: Ram, + writes: Cell, +} + +impl Model { + fn new() -> Self { + Self { ram: Ram::new(BLOCKS), writes: Cell::new(0) } + } + + fn block(&self, lba: u64) -> Block { + let mut b = [[0u8; BLOCK]]; + block_on_ready(self.ram.read(lba, &mut b)).unwrap(); + b[0] + } + + fn put(&self, lba: u64, block: Block) { + block_on_ready(self.ram.write(lba, &[block])).unwrap(); + } +} + +impl BlockDev for Model { + fn blocks(&self) -> u64 { + self.ram.blocks() + } + async fn read(&self, lba: u64, out: &mut [Block]) -> Result<(), Error> { + self.ram.read(lba, out).await + } + async fn write(&self, lba: u64, data: &[Block]) -> Result<(), Error> { + self.writes.set(self.writes.get() + data.len() as u64); + self.ram.write(lba, data).await + } + async fn flush(&self) -> Result<(), Error> { + Ok(()) + } +} + +impl Disk for &Model {} + +type Vol<'a> = DataVolume<&'a Model>; + +fn probe(m: &Model) -> Probed<&Model> { + DataVolume::probe(m, 16 * 1024, &ROOTS, clock) +} + +/// The private format's superblock, as its server wrote both copies. +fn private_superblock() -> Block { + let mut b = [0u8; BLOCK]; + b[..4].copy_from_slice(b"BCFS"); + b[4] = 2; + b[12..20].copy_from_slice(&BLOCKS.to_le_bytes()); + b +} + +/// The partition as the private format left it before any native format: +/// its superblock at both ends, a full bitmap, nodes and file data. +fn private_volume(m: &Model) { + m.put(0, private_superblock()); + m.put(BLOCKS - 1, private_superblock()); + for b in 1..=BITMAP { + m.put(b, [0xff; BLOCK]); + } + for b in BITMAP + 1..BITMAP + 4000 { + let mut block = [(b % 251) as u8; BLOCK]; + if b % 3 == 0 { + block[..4].copy_from_slice(b"BTND"); + } + m.put(b, block); + } +} + +/// Block 0 designates the partition for a format, as `disks` writes it. +fn designate(m: &Model) { + let mut stamp = [0u8; BLOCK]; + stamp.copy_from_slice(&bcachefs::designation(BLOCKS)); + m.put(0, stamp); +} + +/// The private format's server mounting the volume the native mkfs left, +/// from its backup: both its superblocks written, its bitmap's first and +/// 257th blocks read, ored and written back over the native copies there +/// (`ored_from` bytes on, as far as its allocations reached), and its nodes +/// and data written into blocks the native volume lays out. +fn private_server_writes(m: &Model, ored_from: usize) { + m.put(0, private_superblock()); + m.put(BLOCKS - 1, private_superblock()); + for b in [1, 257] { + let mut block = m.block(b); + for byte in &mut block[ored_from..] { + *byte = 0xff; + } + m.put(b, block); + } + for b in (1361..40_000).step_by(7) { + m.put(b, [0x2d; BLOCK]); + } +} + +fn put(v: &mut Vol, path: &str, data: &[u8]) { + let n = v.open(path, CREATE_TRUNC).unwrap(); + v.write(n, 0, data).unwrap(); + v.close(n).unwrap(); +} + +fn remove_all(v: &mut Vol, dir: &str) { + for e in v.list(dir).unwrap() { + let p = format!("{dir}/{}", e.name); + if v.lstat(&p).unwrap().kind == fileserver::volume::Kind::Dir { + remove_all(v, &p); + } else { + v.unlink(&p).unwrap(); + } + } + v.rmdir(dir).unwrap(); +} + +fn page_at(i: usize, pages: usize) -> u64 { + let x = (i as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15) >> 20; + (x % pages as u64) * 4096 +} + +/// `tests/guest/osbench/src/storage.rs`'s durable rows, then `files.rs`'s +/// suite under the same directory, then the shutdown sync, each call as the +/// file server receives it. Every kind of call the rows make, a few times +/// each and on small files: what judges the volume is each call's effect, +/// which the rows' thousands of repetitions repeat. +fn osbench(v: &mut Vol) { + let d = "home/osbench-1"; + v.mkdir(d).unwrap(); + let block = [0x6bu8; 4096]; + put(v, &format!("{d}/over"), &vec![1u8; MIB]); + let f = v.open(&format!("{d}/over"), PLAIN).unwrap(); + for _ in 0..3 { + v.write(f, 0, &block).unwrap(); + v.sync().unwrap(); + } + for _ in 0..3 { + v.sync().unwrap(); + } + v.close(f).unwrap(); + let log = v.open(&format!("{d}/log"), CREATE).unwrap(); + for _ in 0..3 { + let at = v.node_meta(log).unwrap().size; + v.write(log, at, &block).unwrap(); + v.sync().unwrap(); + } + v.close(log).unwrap(); + for _ in 0..3 { + let n = v.open(&format!("{d}/fresh"), CREATE_TRUNC).unwrap(); + v.write(n, 0, &block).unwrap(); + v.sync().unwrap(); + v.close(n).unwrap(); + v.unlink(&format!("{d}/fresh")).unwrap(); + } + let chunk = vec![0x2du8; MIB]; + for _ in 0..2 { + let n = v.open(&format!("{d}/seq"), CREATE_TRUNC).unwrap(); + for i in 0..2 { + v.write(n, (i * MIB) as u64, &chunk).unwrap(); + } + v.sync().unwrap(); + v.close(n).unwrap(); + } + v.unlink(&format!("{d}/seq")).unwrap(); + remove_all(v, d); + + v.mkdir(d).unwrap(); + let fill: Vec = (0..MIB).map(|i| (i % 251) as u8).collect(); + put(v, &format!("{d}/small"), &fill[..4096]); + for _ in 0..3 { + let n = v.open(&format!("{d}/fresh"), CREATE_TRUNC).unwrap(); + v.write(n, 0, &[7u8; 100]).unwrap(); + v.close(n).unwrap(); + v.unlink(&format!("{d}/fresh")).unwrap(); + } + let big = format!("{d}/big"); + let n = v.open(&big, CREATE_TRUNC).unwrap(); + for i in 0..2 { + v.write(n, (i * MIB) as u64, &fill).unwrap(); + } + v.close(n).unwrap(); + let n = v.open(&big, PLAIN).unwrap(); + let mut buf = vec![0u8; 4096]; + for i in 0..8 { + v.read(n, page_at(i, 512), &mut Buf(&mut buf)).unwrap(); + let at = page_at(i, 512) as usize % MIB; + assert_eq!(buf, fill[at..at + 4096], "page {i} of the big file"); + } + for i in 0..8 { + v.write(n, page_at(i, 512), &buf).unwrap(); + } + v.close(n).unwrap(); + v.unlink(&big).unwrap(); + let listed = format!("{d}/listed"); + v.mkdir(&listed).unwrap(); + for i in 0..10 { + let n = v.open(&format!("{listed}/entry-{i:04}"), CREATE_TRUNC).unwrap(); + v.close(n).unwrap(); + } + assert_eq!(v.list(&listed).unwrap().len(), 10); + remove_all(v, &listed); + let n = v.open(&format!("{d}/a"), CREATE_TRUNC).unwrap(); + v.close(n).unwrap(); + for i in 0..8 { + let (from, to) = if i % 2 == 0 { ("a", "b") } else { ("b", "a") }; + v.rename(&format!("{d}/{from}"), &format!("{d}/{to}")).unwrap(); + } + for _ in 0..3 { + v.mkdir(&format!("{d}/sub")).unwrap(); + v.rmdir(&format!("{d}/sub")).unwrap(); + } + for _ in 0..2 { + let tree = format!("{d}/tree"); + v.mkdir(&tree).unwrap(); + for i in 0..10 { + put(v, &format!("{tree}/f{i:03}"), &[3u8; 64]); + } + for i in 0..10 { + v.unlink(&format!("{tree}/f{i:03}")).unwrap(); + } + v.rmdir(&tree).unwrap(); + } + let seq: Vec = (0..MIB).map(|i| (i % 253) as u8).collect(); + let path = format!("{d}/seq"); + for call in [4096usize, MIB] { + for _ in 0..2 { + let n = v.open(&path, CREATE_TRUNC).unwrap(); + for c in 0..SEQ / call { + let at = c * call % MIB; + v.write(n, (c * call) as u64, &seq[at..at + call]).unwrap(); + } + v.close(n).unwrap(); + } + } + let n = v.open(&path, PLAIN).unwrap(); + let mut back = vec![0u8; SEQ]; + assert_eq!(v.read(n, 0, &mut Buf(&mut back)).unwrap(), back.len()); + assert!(back.chunks(MIB).all(|c| c == seq), "the sequential file reads back what was written"); + v.close(n).unwrap(); + v.unlink(&path).unwrap(); + put(v, "home/kept", &seq); + remove_all(v, d); + v.sync().unwrap(); +} + +/// The sequence on a mounted volume, then the volume judged: `fsck` finds +/// nothing, and a fresh mount finds the one file left as it was written. +fn replay_and_judge(m: &Model) { + let Probed::Mounted(mut v) = probe(m) else { panic!("the volume did not mount") }; + osbench(&mut v); + drop(v); + let findings = block_on_ready(fsck(&block_on_ready(Native::open(m)).unwrap())).unwrap(); + assert_eq!(findings, [], "fsck after the sequence"); + let Probed::Mounted(mut again) = probe(m) else { panic!("the volume did not mount again") }; + let names: Vec = again.list("home").unwrap().into_iter().map(|e| e.name).collect(); + assert_eq!(names, ["kept"]); + let n = again.open("home/kept", PLAIN).unwrap(); + let mut back = vec![0u8; MIB]; + again.read(n, 0, &mut Buf(&mut back)).unwrap(); + assert!(back.iter().enumerate().all(|(i, &b)| b == (i % 253) as u8), "the file kept reads back"); +} + +/// The probe of a partition it must refuse: by name, writing nothing. +fn refused(m: &Model, why: &str) { + let before = m.writes.get(); + match probe(m) { + Probed::Unmountable(said) => assert!(said.contains(why), "refused as {said:?}, not for {why:?}"), + Probed::Foreign => assert_eq!(why, "foreign", "refused as foreign"), + Probed::Mounted(_) => panic!("mounted a partition it should refuse ({why})"), + } + assert_eq!(m.writes.get(), before, "a refusal wrote to the partition"); +} + +/// The native mkfs leaves none of the private format's superblocks: block 0 +/// is its own and the last block, where the private format kept its backup, +/// zeros. +fn no_private_superblock(m: &Model) { + assert_ne!(&m.block(0)[..4], b"BCFS"); + assert_eq!(m.block(BLOCKS - 1), [0; BLOCK], "the private format's backup survived the format"); +} + +#[test] +fn a_fresh_designated_partition_is_formatted_and_survives_osbench() { + let m = Model::new(); + refused(&m, "foreign"); + designate(&m); + replay_and_judge(&m); +} + +/// The benches' first step: the private format's volume, then the native +/// format. Undesignated, it is refused and untouched; designated, formatted +/// whole, with nothing of the private format left where a reader of that +/// format looks. +#[test] +fn the_private_format_s_volume_is_refused_until_designated_then_formatted_clean() { + let m = Model::new(); + private_volume(&m); + refused(&m, "foreign"); + designate(&m); + let Probed::Mounted(v) = probe(&m) else { panic!("a designated partition did not format") }; + drop(v); + no_private_superblock(&m); + replay_and_judge(&m); +} + +/// The benches' second step: a native volume the private format's server +/// then mounted from its backup and wrote — the T14's shape, its bitmap's +/// allocations ored over all but the first copy's header, and the AMD's, +/// where they reached nothing of the copy. Refused by name, untouched, until +/// designated again; then formatted clean. +#[test] +fn a_native_volume_the_private_format_wrote_over_is_refused_by_name() { + for (bench, ored_from) in [("T14", 216), ("AMD", 2960)] { + let m = Model::new(); + private_volume(&m); + designate(&m); + let Probed::Mounted(v) = probe(&m) else { panic!("{bench}: a designated partition did not format") }; + drop(v); + // What the native mkfs that formatted the benches left: the private + // format's backup at the last block. + m.put(BLOCKS - 1, private_superblock()); + private_server_writes(&m, ored_from); + refused(&m, "Overwritten { sector: 7 }"); + designate(&m); + let Probed::Mounted(v) = probe(&m) else { panic!("{bench}: a designated partition did not format") }; + drop(v); + no_private_superblock(&m); + replay_and_judge(&m); + } +} From 94fdd7faa4169074b634ea0efe8b7e96a19bdef9 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 15:20:02 +0100 Subject: [PATCH 69/93] bcachefs: an ordinary transaction's keys go into the nodes it holds Every runtime change was staged in pending, the journal-owned key runs, and stayed there until journal pressure made reclaim materialize it. On the T14's 170 GiB DATA geometry, pending reached 326k live entries during a tenth of osbench's call mix, and every transaction paid for the whole set: each lookup rebuilt a node's logical keys over every pending key in its range, accounting re-summed every delta at its position, the iterator walked dead entries past its range, a scan passed every deletion, and each journal flush scanned all of pending and re-sorted its pins. Measured on a host model of that partition with no device latency: about 500 s, against 0.47 s for main's private format. Now, as upstream bcachefs does, an ordinary transaction puts its keys into the leaves it already holds at apply. Each such node first takes what pending holds of its range, so nothing older lies over a key put in place. Pending keeps what no node can take yet: a replay's keys, reclaim's own pointer and allocation updates, and a change to a node whose written bset awaits its pointer. Reclaim's updates stay pending because putting them in place dirties the next node with each node written, and on nodes of one block reclaim then never settles (the geometry tests stall). Pending can still hold a whole journal after a replay, so its operations cost a seek per run and the entries they touch: - `State::keys`, which copied a node's logical keys, is replaced by `Pending::overlay`, merging the node's keys with pending in place; a point lookup is one position, `Source::get` and `Source::walk` are the mount's own (a leaf at a time), and `keys`/`rootless` leave the trait. - The iterator stops skipping dead entries at its range's end, and a run compacts in place once half its entries are dead. - `durable` releases only the positions updated since it last ran, and a pin it lifts joins a queue in sequence order instead of re-sorting runs. - Live entries are counted per btree, leaves apart from interior levels, so a node of a btree with nothing pending is read alone. - A node's oldest unwritten pin is kept as changes arrive, not recomputed over all of them on every cache change. Tests: pending's lookup, durable and compaction costs counted in entries visited; the admission test states the new ownership (64 creates leave pending empty, replay stays pending). Tests whose construction depended on keys being pending at unmount get a larger file count (nonces) or the kept inode that gives their publication shape (member left idle). The reserved-write preflight test remounts clean before its first refusal and drops pending's spare workspace before its second, since a change put in place leaves that workspace unused. Negative controls, each applied and restored by script: the unbounded dead skip, durable rescanning everything, no compaction, keys left pending, and reclaim's updates put in place (stalls past 420 s), the idle member written, the unwritten pin not kept, the live count ignored: all red. Harness (scratch, in the PR's evidence), back to back on this Mac: main 0.46 s; this change 7.3 s. Metadata rows are at or under main's (rename x2100 0.021 s vs 0.057 s); the remainder is data writes, each updating extents, inode, alloc, backpointers, LRU, freespace and accounting. Co-Authored-By: Claude Opus 5.5 --- lib/bcachefs/src/engine/cache.rs | 45 ++- lib/bcachefs/src/engine/fsck.rs | 2 +- lib/bcachefs/src/engine/mount/mod.rs | 155 ++++++-- lib/bcachefs/src/engine/mount/reclaim.rs | 18 +- lib/bcachefs/src/engine/pending.rs | 440 +++++++++++++++------ lib/bcachefs/src/engine/read.rs | 96 +++-- lib/bcachefs/src/engine/tests/admission.rs | 7 +- lib/bcachefs/src/engine/tests/data.rs | 10 +- lib/bcachefs/src/engine/tests/geometry.rs | 9 +- 9 files changed, 559 insertions(+), 223 deletions(-) diff --git a/lib/bcachefs/src/engine/cache.rs b/lib/bcachefs/src/engine/cache.rs index add03c9de2d..1edc9c05064 100644 --- a/lib/bcachefs/src/engine/cache.rs +++ b/lib/bcachefs/src/engine/cache.rs @@ -53,7 +53,7 @@ pub(crate) struct Cached { pub bytes: usize, /// Positions changed since capture, with their oldest lifetime pin and /// latest represented journal dependency. - pub unwritten: BTreeMap, + pub unwritten: Unwritten, /// The oldest sequence a written bset holds whose pointer update is not /// applied. pub captured: Option, @@ -72,7 +72,7 @@ pub(crate) struct Cached { impl Cached { pub(crate) fn pin(&self) -> Option { - let oldest = self.unwritten.values().map(|c| c.first).chain(self.compact).min(); + let oldest = self.unwritten.oldest.into_iter().chain(self.compact).min(); match (oldest, self.captured) { (Some(a), Some(b)) => Some(a.min(b)), (a, b) => a.or(b), @@ -113,10 +113,47 @@ impl Cached { } (None, false) => {} } - self.unwritten.entry(pos).and_modify(|c| { c.first = c.first.min(first); c.last = c.last.max(last); }).or_insert(Change { first, last }); + self.unwritten.put(pos, first, last); } } +/// A node's changes since capture, and the oldest pin among them, which +/// every change to the node reads. +#[derive(Default)] +pub(crate) struct Unwritten { + changes: BTreeMap, + oldest: Option, +} + +impl Unwritten { + fn put(&mut self, pos: Bpos, first: u64, last: u64) { + self.changes.entry(pos).and_modify(|c| { c.first = c.first.min(first); c.last = c.last.max(last); }).or_insert(Change { first, last }); + self.oldest = Some(self.oldest.map_or(first, |o| o.min(first))); + } + + pub(crate) fn clear(&mut self) { + *self = Self::default(); + } + + pub(crate) fn take(&mut self) -> BTreeMap { + self.oldest = None; + core::mem::take(&mut self.changes) + } +} + +impl FromIterator<(Bpos, Change)> for Unwritten { + fn from_iter>(changes: I) -> Self { + let changes: BTreeMap = changes.into_iter().collect(); + let oldest = changes.values().map(|c| c.first).min(); + Self { changes, oldest } + } +} + +impl core::ops::Deref for Unwritten { + type Target = BTreeMap; + fn deref(&self) -> &Self::Target { &self.changes } +} + /// The bytes a key with `value` takes in a bset. pub(crate) fn key_bytes(value: &[u8]) -> usize { 8 * (super::bkey::KEY_U64S + value.len().div_ceil(8)) @@ -297,7 +334,7 @@ mod tests { journal_seq: 1, keys: Rc::new(Vec::new()), bytes: 0, - unwritten: BTreeMap::new(), + unwritten: Unwritten::default(), captured: None, compact: None, writing: false, diff --git a/lib/bcachefs/src/engine/fsck.rs b/lib/bcachefs/src/engine/fsck.rs index f6478178594..b1f1bea68ba 100644 --- a/lib/bcachefs/src/engine/fsck.rs +++ b/lib/bcachefs/src/engine/fsck.rs @@ -14,7 +14,7 @@ use super::bkey::{Bpos, Key}; use super::derive::{Space, Tally}; use super::mkfs::{self, COMPAT_CLEAN, COMPAT_INITIALIZED, DOWNGRADE, EXTENT_TYPE_U64S, FEATURES}; use super::journal::{records, Found, Record}; -use super::read::{hash, read_bytes, Source, Volume}; +use super::read::{hash, read_bytes, Volume}; use super::superblock::{field_type, member_flag, Root, Superblock, HEADER}; use super::value::mode::{S_IFDIR, S_IFLNK, S_IFMT, S_IFREG}; use super::value::{self, csum_type, data_type, inode_field, inode_flag, AllocV4, Dirent, DirentType, Extent, InodeValue, NodePtr, Subvolume, SUBVOLUME_LIVE}; diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index a4a63b07e16..3e667993885 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -1,6 +1,8 @@ //! A volume mounted read-write: one transaction at a time under the writer -//! slot (§3.3), journal-owned logical changes, and bounded materialization -//! into btree nodes so the journal can move on. +//! slot (§3.3). An ordinary transaction's keys go into the nodes it holds; +//! replayed keys, reclaim's own updates and a change a node cannot take yet +//! wait in pending, and reclaim materializes them in bounded steps so the +//! journal can move on. //! //! **Order of waits.** A transaction takes admission first — journal space //! and cache room at its bound — and the slot second. Nothing waits under the @@ -14,6 +16,7 @@ //! pointer update that makes it reachable. Until that update is in an entry, //! the node's keys keep the journal pinned and the node stays in memory. +use alloc::borrow::Cow; use alloc::boxed::Box; use alloc::collections::{BTreeMap, BTreeSet}; use alloc::rc::Rc; @@ -25,12 +28,12 @@ use toyos_crypto::{scrypt, Random}; use super::bkey::{Bpos, Key, Version, KEY_U64S}; use super::bucket::Bucket; -use super::cache::{key_bytes, Cache, Cached, Change, Keys, NodeId}; +use super::cache::{key_bytes, Cache, Cached, NodeId, Unwritten}; use super::crypt::MasterKey; use super::journal::{self, Class, Closed, Jset, Replayed, Writer, SKIP}; use super::mkfs::{self, COMPAT_CLEAN, ROOT_SNAPSHOT}; use super::node::{self, Build, FIRST_KEYS}; -use super::pending::Pending; +use super::pending::{Overlay, Pending}; use super::read::{self, Inode, Parts, Source, Volume}; use super::superblock::{field_type, Field, Header, Member, Root, Superblock}; use super::sync::{Broadcast, Queue}; @@ -711,7 +714,7 @@ impl Mount { journal_seq: node.bset_seqs.iter().copied().max().unwrap_or(0), bytes: node.keys.iter().map(|(_, v)| key_bytes(v)).sum(), keys: Rc::new(node.keys), - unwritten: BTreeMap::new(), + unwritten: Unwritten::default(), captured: None, compact: None, writing: false, @@ -740,10 +743,17 @@ impl Mount { /// no root is given an empty one, in memory until it is first written. /// Nothing holds it: for a caller that changes it before its next await. async fn find(&self, btree: Btree, level: u8, pos: Bpos) -> Result { - let mut id = match self.root_node(btree).await? { + let root = match self.root_node(btree).await? { Some(id) => id, None => self.new_root(btree), }; + self.descend(root, level, pos).await + } + + /// The node at `level` spanning `pos` below `id`, cached; held as + /// [`Mount::find`]'s is. + async fn descend(&self, mut id: NodeId, level: u8, pos: Bpos) -> Result { + let btree = self.st.borrow().cache.node(id).btree; loop { let (child, ptr, max) = { let st = self.st.borrow(); @@ -754,9 +764,8 @@ impl Mount { if n.level < level { return Err(Error::Corrupt("a key above its btree's root")); } - let keys = st.keys(id)?; - let (key, value) = keys.get(keys.partition_point(|(k, _)| k.pos < pos)).ok_or(Error::Corrupt("interior node short of its range"))?; - (n.level - 1, NodePtr::decode(value)?, key.pos) + let (key, value) = st.logical(id, pos).next().ok_or(Error::Corrupt("interior node short of its range"))??; + (n.level - 1, NodePtr::decode(&value)?, key.pos) }; id = self.load(btree, child, &ptr, max).await?; } @@ -782,7 +791,7 @@ impl Mount { journal_seq: 0, bytes: 0, keys: Rc::new(Vec::new()), - unwritten: BTreeMap::new(), + unwritten: Unwritten::default(), captured: None, compact: None, writing: false, @@ -813,9 +822,8 @@ impl Mount { } else if n.level < level { return Err(Error::Corrupt("a key above its btree's root")); } else { - let keys = st.keys(id)?; - let (key, value) = keys.get(keys.partition_point(|(k, _)| k.pos < pos)).ok_or(Error::Corrupt("interior node short of its range"))?; - Some((n.level - 1, NodePtr::decode(value)?, key.pos)) + let (key, value) = st.logical(id, pos).next().ok_or(Error::Corrupt("interior node short of its range"))??; + Some((n.level - 1, NodePtr::decode(&value)?, key.pos)) } }; match next { @@ -888,10 +896,17 @@ impl Mount { } impl State { - fn keys(&self, id: NodeId) -> Result { + /// Node `id`'s keys from `from` on, as the pending keys over them make + /// them. + fn logical(&self, id: NodeId, from: Bpos) -> Overlay<'_> { let n = self.cache.node(id); - if self.pending.range(n.btree, n.level, n.min, n.max).next().is_none() { return Ok(Rc::clone(&n.keys)); } - Ok(Rc::new(self.pending.over(n.btree, n.level, n.min, n.max, n.keys.to_vec())?)) + self.pending.overlay(n.btree, n.level, from.max(n.min), n.max, &n.keys) + } + + /// The logical key at `pos` of node `id`. + fn get(&self, id: NodeId, pos: Bpos) -> Result)>> { + let n = self.cache.node(id); + self.pending.overlay(n.btree, n.level, pos, pos, &n.keys).next().transpose() } fn oldest_pin(&self) -> Option { @@ -908,6 +923,25 @@ impl State { } } +/// Puts `key` into `n`, journaled at `seq`: an accounting key adds its +/// delta to the node's. +fn put(n: &mut Cached, key: Key, value: Vec, seq: u64) { + let pos = key.pos; + if n.btree != Btree::Accounting || n.level != 0 { + n.put(pos, (key.kind != kind::DELETED).then_some((key, value)), seq, seq); + return; + } + let (mut sum, mut sums) = match n.keys.get(n.at(pos)) { + Some((stored, counters)) if stored.pos == pos => (*stored, journal::counters(counters).expect("cached accounting decodes")), + _ => (Key::new(kind::ACCOUNTING, pos), Vec::new()), + }; + let delta = journal::counters(&value).expect("a transaction's accounting decodes"); + sums.resize(sums.len().max(delta.len()), 0); + for (s, d) in sums.iter_mut().zip(delta) { *s = s.wrapping_add(d); } + sum.version = sum.version.max(key.version); + n.put(pos, Some((sum, value::counters(&sums))), seq, seq); +} + /// Ends the holds on `ids`. fn release(st: &mut State, ids: &[NodeId]) { for &id in ids { @@ -927,9 +961,8 @@ fn held(st: &State, btree: Btree, level: u8, pos: Bpos) -> NodeId { if n.level == level { return id; } - let keys = st.keys(id).expect("a held logical node decodes"); - let (_, value) = &keys[keys.partition_point(|(k, _)| k.pos < pos)]; - let ptr = NodePtr::decode(value).expect("a pointer the cache holds decodes"); + let (_, value) = st.logical(id, pos).next().expect("a held node spans its range").expect("a held logical node decodes"); + let ptr = NodePtr::decode(&value).expect("a pointer the cache holds decodes"); id = st.cache.id_at(ptr.ptr.sector).expect("a held node in the cache"); } } @@ -961,14 +994,32 @@ impl Source for Mount { st.roots.get(&btree).cloned() } - fn rootless(&self, btree: Btree) -> Result)>> { - let st = self.st.borrow(); - st.new_roots.get(&btree).map_or_else(|| Ok(Vec::new()), |&id| st.keys(id).map(|keys| keys.to_vec())) + async fn get(&self, btree: Btree, pos: Bpos) -> Result)>> { + let _reading = read::Reading(self, self.begin_read()); + let Some(root) = self.root_node(btree).await? else { return Ok(None) }; + let leaf = self.descend(root, 0, pos).await?; + Ok(self.st.borrow().get(leaf, pos)?.map(|(k, v)| (k, v.into_owned()))) } - async fn keys(&self, btree: Btree, level: u8, ptr: &NodePtr, max: Bpos) -> Result { - let id = self.load(btree, level, ptr, max).await?; - self.st.borrow().keys(id) + /// A leaf at a time, each found from the root: pending keys are read + /// where they lie, never copied over a whole node. + async fn walk(&self, btree: Btree, mut from: Bpos, mut visit: impl FnMut(&Key, &[u8]) -> Result) -> Result<()> { + let _reading = read::Reading(self, self.begin_read()); + loop { + let Some(root) = self.root_node(btree).await? else { return Ok(()) }; + let leaf = self.descend(root, 0, from).await?; + let st = self.st.borrow(); + for kv in st.logical(leaf, from) { + let (key, value) = kv?; + if !visit(&key, &value)? { + return Ok(()); + } + } + match st.cache.node(leaf).max.successor() { + Some(next) => from = next, + None => return Ok(()), + } + } } fn begin_read(&self) -> u64 { @@ -1205,25 +1256,25 @@ impl Mount { } for (&(btree, level, pos), update) in &txn.updates { if level != 0 { continue; } - let keys = st.keys(held(st, btree, level, pos)).expect("a prepared logical node decodes"); - let prior = keys.get(keys.partition_point(|(k, _)| k.pos < pos)).filter(|(k, _)| k.pos == pos); - let charge = |entry: Option<&(Key, Vec)>| entry.map_or(0, |(_, v)| allocator::charge(btree, v)); - st.metadata_bytes = st.metadata_bytes + charge(update.as_ref()) - charge(prior); + let prior = st.get(held(st, btree, level, pos), pos).expect("a prepared logical node decodes"); + let prior = prior.as_ref().map(|(_, v)| &v[..]); + let update = update.as_ref().map(|(_, v)| &v[..]); + let charge = |value: Option<&[u8]>| value.map_or(0, |v| allocator::charge(btree, v)); + let tree_node = |value: Option<&[u8]>| value.is_some_and(|v| v.get(2) == Some(&data_type::BTREE)); + let (added, removed, before, after) = (charge(update), charge(prior), tree_node(prior), tree_node(update)); + st.metadata_bytes = st.metadata_bytes + added - removed; if btree == Btree::Backpointers { - let before = prior.is_some_and(|(_, value)| value.get(2) == Some(&data_type::BTREE)); - let after = update.as_ref().is_some_and(|(_, value)| value.get(2) == Some(&data_type::BTREE)); st.btree_nodes = st.btree_nodes + u64::from(after) - u64::from(before); if before && !after { st.baseline_nodes.remove(&(pos.offset >> self.bp_shift())); } } } for (&pos, delta) in &txn.accounting { - let keys = st.keys(held(st, Btree::Accounting, 0, pos)).expect("a prepared accounting node decodes"); - let prior = keys.get(keys.partition_point(|(k, _)| k.pos < pos)).filter(|(k, _)| k.pos == pos); - let before = prior.map_or(0, |(_, v)| key_bytes(v) as u64); - let counters = prior.map_or(0, |(_, v)| v.len() / 8).max(delta.len()); + let prior = st.get(held(st, Btree::Accounting, 0, pos), pos).expect("a prepared accounting node decodes"); + let before = prior.as_ref().map_or(0, |(_, v)| key_bytes(v) as u64); + let counters = prior.as_ref().map_or(0, |(_, v)| v.len() / 8).max(delta.len()); st.metadata_bytes = st.metadata_bytes + (KEY_U64S * 8 + counters * 8) as u64 - before; } - self.apply(st, txn, bound); + self.apply(st, txn, bound, class); for sector in allocated { st.alloc.unpin(sector / u64::from(self.sb.member.bucket_sectors)); } } else { release(st, &txn.held); @@ -1245,7 +1296,7 @@ impl Mount { /// Appends the transaction's records, transfers its keys to pending /// ownership, and publishes roots and completed node writes. No await /// separates these logical changes. - fn apply(&self, st: &mut State, mut txn: Txn, bound: u64) { + fn apply(&self, st: &mut State, mut txn: Txn, bound: u64, class: Class) { let installs = core::mem::take(&mut txn.installs); let holds = core::mem::take(&mut txn.held); let filled = core::mem::take(&mut txn.filled); @@ -1280,7 +1331,21 @@ impl Mount { let key = Key { version: versions[&pos], ..Key::new(kind::ACCOUNTING, pos) }; (Btree::Accounting, 0, key, value::counters(&delta)) }); - st.pending.apply(seq, updates.chain(accounting)); + // An ordinary transaction's keys go into the nodes it holds, as + // reclaim's materialization would put them; each node first takes + // what pending holds of its range, so nothing older lies over them. + // A node that cannot take them now, and reclaim's own updates, + // leave them to pending and reclaim. + let mut left = Vec::new(); + for (btree, level, key, value) in updates.chain(accounting) { + let id = (class == Class::Ordinary).then(|| held(st, btree, level, key.pos)).filter(|&id| st.cache.node(id).captured.is_none()); + let Some(id) = id.filter(|&id| self.drained(st, id, key.pos)) else { + left.push((btree, level, key, value)); + continue; + }; + st.cache.change(id, |n| put(n, key, value, seq)); + } + st.pending.apply(seq, left); for root in core::mem::take(&mut txn.roots) { st.roots.insert(root.btree, root); } @@ -1291,6 +1356,15 @@ impl Mount { } } + /// Whether node `id` takes a key at `pos` in place: it first takes what + /// pending holds of its range, and pending holds nothing at `pos` after. + fn drained(&self, st: &mut State, id: NodeId, pos: Bpos) -> bool { + let n = st.cache.node(id); + let (btree, level, min, max) = (n.btree, n.level, n.min, n.max); + if st.pending.holds(btree, level, min, max) && self.take_into(st, id, min).is_err() { return false; } + !st.pending.holds(btree, level, pos, pos) + } + /// Closes the open entry: its last sequence the oldest any node pins, /// its tail the roots, datetime, key version and clocks as they stand /// now. A pin released by an update a later entry carries cannot move @@ -1576,6 +1650,11 @@ impl Mount { #[cfg(test)] impl Mount { + /// Drops pending's spare workspace, so its next batch allocates. + pub(crate) fn shrink_pending(&self) { + self.st.borrow_mut().pending.shrink(); + } + /// The dirty-node limit, reclaim's room, the most dirty and unpointed /// nodes held at once, and the ordinary admissions that waited for /// journal space and for cache room. diff --git a/lib/bcachefs/src/engine/mount/reclaim.rs b/lib/bcachefs/src/engine/mount/reclaim.rs index 70ba1d60b6f..2d59af335c0 100644 --- a/lib/bcachefs/src/engine/mount/reclaim.rs +++ b/lib/bcachefs/src/engine/mount/reclaim.rs @@ -418,6 +418,15 @@ impl Mount { let mut st = self.st.borrow_mut(); let n = st.cache.node(id); if n.writing || n.captured.is_some() || n.held != 0 { return Ok(false); } + self.take_into(&mut st, id, pos) + } + + /// Transfers pending keys of node `id`'s range into it, from `pos` on + /// and round to it, with bounded positive growth. The caller has found + /// the node free to change. + pub(super) fn take_into(&self, st: &mut State, id: NodeId, pos: Bpos) -> Result { + let n = st.cache.node(id); + let (btree, level) = (n.btree, n.level); let mut captures = Vec::new(); let mut growth = 0; let mut pending = st.pending.range(btree, level, pos, n.max) @@ -437,7 +446,6 @@ impl Mount { if growth + added > K_TX as usize * 8 && start != 0 { captures.truncate(start); continue; } growth += added; } - let st = &mut *st; let durable = st.journal.durable; let mut accounting = None; let transferred = !captures.is_empty(); @@ -488,7 +496,7 @@ impl Mount { let parent = self.find(btree, level + 1, max).await?; let adjacent = { let mut st = self.st.borrow_mut(); - let keys = st.keys(parent)?; + let keys = st.logical(parent, Bpos::MIN).collect::>>()?; let at = keys.partition_point(|(k, _)| k.pos < max); if keys.len() == 1 { let seq = st.journal.open.as_ref().map_or(st.journal.next_seq, |o| o.seq); @@ -903,7 +911,7 @@ impl Mount { let btree = st.cache.change(id, |n| { n.captured = None; keys.extend(n.keys.iter().cloned()); - unwritten.append(&mut n.unwritten); + unwritten.append(&mut n.unwritten.take()); n.btree }); st.cache.remove(id); @@ -911,7 +919,7 @@ impl Mount { } for p in by { let mine: Vec<(Key, Vec)> = keys.iter().filter(|(k, _)| (p.min..=p.max).contains(&k.pos)).cloned().collect(); - let changes: BTreeMap = unwritten.range(p.min..=p.max).map(|(k, v)| (*k, *v)).collect(); + let changes: Unwritten = unwritten.range(p.min..=p.max).map(|(k, v)| (*k, *v)).collect(); let id = st.cache.insert(fresh_cached(&p, mine)); let seq = st.journal.open.as_ref().map_or(st.journal.next_seq, |o| o.seq); st.cache.change(id, |n| { @@ -979,7 +987,7 @@ fn fresh_cached(p: &Planned, keys: Vec<(Key, Vec)>) -> Cached { journal_seq: p.journal_seq, bytes: keys.iter().map(|(_, v)| key_bytes(v)).sum(), keys: Rc::new(keys), - unwritten: BTreeMap::new(), + unwritten: Unwritten::default(), captured: None, compact: None, writing: false, diff --git a/lib/bcachefs/src/engine/pending.rs b/lib/bcachefs/src/engine/pending.rs index c9c02e1b480..fd3bd9de724 100644 --- a/lib/bcachefs/src/engine/pending.rs +++ b/lib/bcachefs/src/engine/pending.rs @@ -1,7 +1,16 @@ -//! Journal-owned keys, independent of the leaves that will materialize them. - +//! Journal-owned keys no node holds yet: a replay's, reclaim's own updates, +//! and an ordinary change to a node that could not take it then. After a +//! replay it can hold a whole journal, so an operation costs a seek per run +//! and, amortized, the entries it returns or changes, never a pass over the +//! whole set. + +use alloc::borrow::Cow; +use alloc::collections::VecDeque; use alloc::vec::Vec; -use alloc::collections::BTreeMap; +#[cfg(test)] +use core::cell::Cell; +use core::iter::Peekable; +use core::slice; use super::bkey::{Bpos, Key}; use super::{Btree, Error, Result}; @@ -20,28 +29,103 @@ struct Entry { last: u64, order: u64, live: bool, + /// Its index once its run is compacted. + slot: usize, +} + +impl Entry { + fn position(&self) -> Position { (self.btree, self.level, self.key.pos) } + fn ordering(&self) -> (Position, u64) { (self.position(), self.order) } + fn accounting(&self) -> bool { self.btree == Btree::Accounting && self.level == 0 } } struct Run { entries: Vec, - pins: Vec, + /// Each entry's pin as it stood when the run was made, oldest first. A + /// pin `durable` later moved is the lifted queue's. + pins: Vec<(u64, usize)>, oldest: usize, + dead: usize, +} + +// Entries the calling thread's pending operations have looked at: what the +// tests that pin each operation's cost count. +#[cfg(test)] +std::thread_local! { static VISITED: Cell = const { Cell::new(0) }; } + +#[cfg(test)] +fn visit() { VISITED.with(|v| v.set(v.get() + 1)); } +#[cfg(not(test))] +fn visit() {} + +#[cfg(test)] +fn visited() -> usize { VISITED.with(Cell::get) } + +/// What `f` returns, and the entries pending looked at meanwhile. +#[cfg(test)] +fn visits(f: impl FnOnce() -> T) -> (T, usize) { + let before = visited(); + let out = f(); + (out, visited() - before) } impl Run { + fn pinned(entries: &[Entry], &(first, i): &(u64, usize)) -> bool { + entries[i].live && entries[i].first == first + } + fn oldest(&self) -> Option<&Entry> { - self.pins.get(self.oldest).map(|&i| &self.entries[i]) + self.pins.get(self.oldest).map(|&(_, i)| &self.entries[i]) } fn retire(&mut self) { - while self.pins.get(self.oldest).is_some_and(|&i| !self.entries[i].live) { self.oldest += 1; } + while self.pins.get(self.oldest).is_some_and(|pin| !Self::pinned(&self.entries, pin)) { self.oldest += 1; } + } + + /// The entries at `position`. + fn at(&self, position: Position) -> core::ops::Range { + let from = self.entries.partition_point(|e| e.position() < position); + from..from + self.entries[from..].partition_point(|e| e.position() == position) + } + + fn find(&self, wanted: (Position, u64)) -> Option { + let at = self.entries.partition_point(|e| e.ordering() < wanted); + self.entries.get(at).is_some_and(|e| e.live && e.ordering() == wanted).then_some(at) + } + + fn kill(&mut self, i: usize, live: &mut Live) { + let entry = &mut self.entries[i]; + entry.live = false; + entry.value = Vec::new(); + live.remove(entry); + self.dead += 1; + } + + /// Drops the dead entries once they are half the run, in place, so a + /// range never walks more dead entries than live ones it passes. + fn compact(&mut self) { + if self.dead * 2 <= self.entries.len() { return; } + for (slot, entry) in self.entries.iter_mut().filter(|e| e.live).enumerate() { + entry.slot = slot; + } + let entries = &self.entries; + self.pins.retain(|pin| Self::pinned(entries, pin)); + for pin in &mut self.pins { pin.1 = entries[pin.1].slot; } + self.entries.retain(|e| e.live); + self.dead = 0; + self.oldest = 0; } } -impl Entry { - fn position(&self) -> Position { (self.btree, self.level, self.key.pos) } - fn ordering(&self) -> (Position, u64) { (self.position(), self.order) } - fn accounting(&self) -> bool { self.btree == Btree::Accounting && self.level == 0 } +/// Live entries of each btree, its leaves apart from its interior levels: +/// where none are, a lookup reads the node alone. +struct Live([[usize; 2]; Btree::ALL.len()]); + +impl Live { + fn of(&mut self, entry: &Entry) -> &mut usize { &mut self.0[entry.btree as usize][usize::from(entry.level > 0)] } + fn add(&mut self, entry: &Entry) { *self.of(entry) += 1; } + fn remove(&mut self, entry: &Entry) { *self.of(entry) -= 1; } + fn any(&self, btree: Btree, level: u8) -> bool { self.0[btree as usize][usize::from(level > 0)] > 0 } } /// An identity survives run merges; a newer update at its position does not @@ -58,10 +142,10 @@ pub(crate) struct View<'a> { first: u64, } -impl View<'_> { +impl<'a> View<'a> { pub fn position(&self) -> Position { self.entry.position() } - pub fn key(&self) -> &Key { &self.entry.key } - pub fn value(&self) -> &[u8] { &self.entry.value } + pub fn key(&self) -> &'a Key { &self.entry.key } + pub fn value(&self) -> &'a [u8] { &self.entry.value } pub fn capture(&self) -> Capture { Capture { position: self.position(), order: self.entry.order, first: self.first } } } @@ -76,14 +160,29 @@ pub(crate) struct Pending { runs: [Option; LEVELS], incoming: Vec, merged: Vec, - pins: Vec, + pins: Vec<(u64, usize)>, next: u64, prepared: usize, + /// Ordinary updates in sequence order, until `durable` reaches theirs. + undurable: VecDeque<(u64, Position)>, + /// Entries `durable` moved to their own sequence's pin, in that order. + lifted: VecDeque<(u64, Position, u64)>, + live: Live, } impl Default for Pending { fn default() -> Self { - Self { runs: core::array::from_fn(|_| None), incoming: Vec::new(), merged: Vec::new(), pins: Vec::new(), next: 0, prepared: 0 } + Self { + runs: core::array::from_fn(|_| None), + incoming: Vec::new(), + merged: Vec::new(), + pins: Vec::new(), + next: 0, + prepared: 0, + undurable: VecDeque::new(), + lifted: VecDeque::new(), + live: Live([[0; 2]; Btree::ALL.len()]), + } } } @@ -96,14 +195,18 @@ impl Pending { let mut total = additional; for run in &self.runs { let Some(run) = run else { break }; - total = total.checked_add(run.entries.iter().filter(|e| e.live).count()).ok_or(Error::NoMemory)?; + total = total.checked_add(run.entries.len() - run.dead).ok_or(Error::NoMemory)?; } if self.runs.iter().all(Option::is_some) || u64::try_from(additional).ok().and_then(|n| self.next.checked_add(n)).is_none() { return Err(Error::NoMemory); } + // Each update `durable` releases lifts at most one entry. + let lifts = self.undurable.len().checked_add(additional).ok_or(Error::NoMemory)?; self.incoming.try_reserve(additional).map_err(|_| Error::NoMemory)?; self.merged.try_reserve(total).map_err(|_| Error::NoMemory)?; self.pins.try_reserve(total).map_err(|_| Error::NoMemory)?; + self.undurable.try_reserve(additional).map_err(|_| Error::NoMemory)?; + self.lifted.try_reserve(lifts).map_err(|_| Error::NoMemory)?; self.prepared = additional; Ok(()) } @@ -113,7 +216,10 @@ impl Pending { pub fn apply(&mut self, seq: u64, updates: impl IntoIterator) { for (btree, level, key, value) in updates { assert!(self.incoming.len() < self.prepared, "pending batch exceeded its reservation"); - self.incoming.push(Entry { btree, level, key, value, first: seq, last: seq, order: self.next, live: true }); + let entry = Entry { btree, level, key, value, first: seq, last: seq, order: self.next, live: true, slot: 0 }; + if !entry.accounting() { self.undurable.push_back((seq, entry.position())); } + self.live.add(&entry); + self.incoming.push(entry); self.next += 1; } self.prepared = 0; @@ -150,18 +256,22 @@ impl Pending { }; let mut entry = Entry { value: core::mem::take(&mut source.value), ..*source }; source.live = false; - if !entry.accounting() && self.merged.last().is_some_and(|old| old.position() == entry.position()) { - entry.first = entry.first.min(self.merged.last().unwrap().first); - *self.merged.last_mut().unwrap() = entry; - } else { - self.merged.push(entry); + match self.merged.last_mut() { + Some(old) if old.position() == entry.position() && !entry.accounting() => { + entry.first = entry.first.min(old.first); + self.live.remove(old); + *old = entry; + } + _ => self.merged.push(entry), } } self.incoming.clear(); for run in &mut self.runs[..level] { *run = None; } - self.pins.extend(0..self.merged.len()); - self.pins.sort_unstable_by_key(|&i| (self.merged[i].first, self.merged[i].order)); - self.runs[level] = Some(Run { entries: core::mem::take(&mut self.merged), pins: core::mem::take(&mut self.pins), oldest: 0 }); + let merged = &self.merged; + self.pins.extend(merged.iter().enumerate().map(|(i, e)| (e.first, i))); + self.pins.sort_unstable_by_key(|&(first, i)| (first, merged[i].order)); + self.runs[level] = Some(Run { entries: core::mem::take(&mut self.merged), pins: core::mem::take(&mut self.pins), oldest: 0, dead: 0 }); + self.settle(); } pub fn oldest_pin(&self) -> Option { @@ -169,71 +279,91 @@ impl Pending { } pub fn oldest(&self) -> Option { - self.runs.iter().flatten().filter_map(Run::oldest).min_by_key(|e| (e.first, e.order)) + let lifted = self.lifted.front().and_then(|&(first, position, order)| { + self.entry((position, order)).filter(|e| e.first == first) + }); + self.runs.iter().flatten().filter_map(Run::oldest).chain(lifted).min_by_key(|e| (e.first, e.order)) .map(|e| Capture { position: e.position(), order: e.order, first: e.first }) } + fn entry(&self, wanted: (Position, u64)) -> Option<&Entry> { + self.runs.iter().flatten().find_map(|run| run.find(wanted).map(|i| &run.entries[i])) + } + + /// Drops lifted pins whose entries moved on, so `oldest` finds a live one + /// first. + fn settle(&mut self) { + while let Some(&(first, position, order)) = self.lifted.front() { + if self.entry((position, order)).is_some_and(|e| e.first == first) { break; } + self.lifted.pop_front(); + } + } + /// A durable ordinary replacement no longer needs the replaced record. /// Captures retain their original dependency; accounting remains additive. + /// Only positions updated since the last call can change. pub fn durable(&mut self, seq: u64) { - let mut at = [0; LEVELS]; - let mut changed = [false; LEVELS]; - loop { - let mut position = None; - for (i, run) in self.runs.iter().enumerate() { - let Some(run) = run else { continue }; - while run.entries.get(at[i]).is_some_and(|e| !e.live) { at[i] += 1; } - if let Some(entry) = run.entries.get(at[i]) { - position = Some(position.map_or(entry.position(), |p: Position| p.min(entry.position()))); - } - } - let Some(position) = position else { break }; - let mut end = at; - let mut replacement = None; - for (i, run) in self.runs.iter().enumerate() { - let Some(run) = run else { continue }; - while let Some(entry) = run.entries.get(end[i]).filter(|e| e.position() == position) { - if entry.live && !entry.accounting() && entry.last <= seq { - replacement = Some(replacement.map_or(entry.order, |old: u64| old.max(entry.order))); - } - end[i] += 1; - } - } - for (i, run) in self.runs.iter_mut().enumerate() { - let Some(run) = run else { continue }; - for entry in &mut run.entries[at[i]..end[i]] { - if !entry.live || entry.accounting() { continue; } - if replacement.is_some_and(|newer| newer > entry.order) { - entry.live = false; - entry.value = Vec::new(); - changed[i] = true; - } else if entry.last <= seq && entry.first != entry.last { - entry.first = entry.last; - changed[i] = true; - } + while let Some(&(at, position)) = self.undurable.front() { + if at > seq { break; } + self.undurable.pop_front(); + self.release(position, seq); + } + for run in self.runs.iter_mut().flatten() { run.retire(); } + self.settle(); + } + + fn release(&mut self, position: Position, seq: u64) { + let replacement = self.runs.iter().flatten() + .flat_map(|run| run.entries[run.at(position)].iter()) + .inspect(|_| visit()) + .filter(|e| e.live && !e.accounting() && e.last <= seq) + .map(|e| e.order) + .max(); + let Some(newer) = replacement else { return }; + for slot in &mut self.runs { + let Some(run) = slot else { continue }; + for i in run.at(position) { + let entry = &mut run.entries[i]; + if !entry.live || entry.accounting() { continue; } + if entry.order < newer { + run.kill(i, &mut self.live); + } else if entry.order == newer && entry.first != entry.last { + entry.first = entry.last; + assert!(self.lifted.len() < self.lifted.capacity(), "a lifted pin past its reservation"); + self.lifted.push_back((entry.last, position, entry.order)); } } - at = end; - } - for (i, run) in self.runs.iter_mut().enumerate() { - if !changed[i] { continue; } - let run = run.as_mut().unwrap(); - run.pins.sort_unstable_by_key(|&at| (run.entries[at].first, run.entries[at].order)); - run.oldest = 0; - run.retire(); + run.compact(); + if run.entries.is_empty() { *slot = None; } } } #[cfg(test)] pub fn allocated_bytes(&self) -> usize { let entries = |entries: &Vec| entries.capacity() * core::mem::size_of::() + entries.iter().map(|e| e.value.capacity()).sum::(); - core::mem::size_of::() + entries(&self.incoming) + entries(&self.merged) + self.pins.capacity() * core::mem::size_of::() - + self.runs.iter().flatten().map(|r| entries(&r.entries) + r.pins.capacity() * core::mem::size_of::()).sum::() + let pins = |pins: &Vec<(u64, usize)>| pins.capacity() * core::mem::size_of::<(u64, usize)>(); + core::mem::size_of::() + entries(&self.incoming) + entries(&self.merged) + pins(&self.pins) + + self.undurable.capacity() * core::mem::size_of::<(u64, Position)>() + self.lifted.capacity() * core::mem::size_of::<(u64, Position, u64)>() + + self.runs.iter().flatten().map(|r| entries(&r.entries) + pins(&r.pins)).sum::() } #[cfg(test)] pub fn iter(&self) -> Iter<'_> { self.between(None) } + #[cfg(test)] + pub fn shrink(&mut self) { + self.incoming.shrink_to_fit(); + self.merged.shrink_to_fit(); + self.pins.shrink_to_fit(); + self.undurable.shrink_to_fit(); + self.lifted.shrink_to_fit(); + } + + /// Whether a live entry lies in `[min, max]` of `btree` at `level`. + pub fn holds(&self, btree: Btree, level: u8, min: Bpos, max: Bpos) -> bool { + self.live.any(btree, level) && self.range(btree, level, min, max).next().is_some() + } + pub fn range(&self, btree: Btree, level: u8, min: Bpos, max: Bpos) -> Iter<'_> { self.between(Some(((btree, level, min), (btree, level, max)))) } @@ -241,35 +371,17 @@ impl Pending { /// The logical range, preserving zero accounting values and their /// versions so materialization cannot apply an old delta twice. pub fn over(&self, btree: Btree, level: u8, min: Bpos, max: Bpos, keys: Vec<(Key, Vec)>) -> Result)>> { - let mut merged: BTreeMap<_, _> = keys.into_iter().map(|(k, v)| (k.pos, (k, v))).collect(); - let mut pending = self.range(btree, level, min, max).peekable(); - if btree == Btree::Accounting && level == 0 { - while let Some(first) = pending.peek() { - let pos = first.key().pos; - let (mut key, mut sums) = match merged.remove(&pos) { - Some((key, value)) => (key, journal::counters(&value)?), - None => (Key::new(kind::ACCOUNTING, pos), Vec::new()), - }; - let since = key.version; - while pending.peek().is_some_and(|entry| entry.key().pos == pos) { - let entry = pending.next().unwrap(); - let version = entry.key().version; - if version <= since { continue; } - let delta = journal::counters(entry.value())?; - sums.resize(sums.len().max(delta.len()), 0); - for (sum, d) in sums.iter_mut().zip(delta) { *sum = sum.wrapping_add(d); } - key.version = key.version.max(version); - } - merged.insert(pos, (key, super::value::counters(&sums))); - } - } else { - for entry in pending { - let key = *entry.key(); - if key.kind == kind::DELETED { merged.remove(&key.pos); } - else { merged.insert(key.pos, (key, entry.value().to_vec())); } - } - } - Ok(merged.into_values().collect()) + self.overlay(btree, level, min, max, &keys).map(|kv| kv.map(|(k, v)| (k, v.into_owned()))).collect() + } + + /// `stored`, a node's keys in position order, with the pending keys of + /// `[min, max]` laid over them from `min` on: [`Pending::over`] a key at + /// a time. + pub fn overlay<'a>(&'a self, btree: Btree, level: u8, min: Bpos, max: Bpos, stored: &'a [(Key, Vec)]) -> Overlay<'a> { + let from = stored.partition_point(|(k, _)| k.pos < min); + let to = stored.partition_point(|(k, _)| k.pos <= max).max(from); + let entries = self.live.any(btree, level).then(|| self.range(btree, level, min, max).peekable()); + Overlay { pending: self, btree, level, stored: stored[from..to].iter().peekable(), entries } } fn between(&self, bounds: Option<(Position, Position)>) -> Iter<'_> { @@ -286,28 +398,76 @@ impl Pending { /// It is infallible and cannot yield. `false` means capture was superseded. pub fn transfer(&mut self, capture: Capture, install: impl FnOnce(Transferred)) -> bool { let wanted = (capture.position, capture.order); - let source = self.runs.iter_mut().flatten().find_map(|run| { - let at = run.entries.partition_point(|e| e.ordering() < wanted); - run.entries.get_mut(at).filter(|e| e.live && e.ordering() == wanted) - }); - let Some(source) = source else { return false }; + let Some(source) = self.runs.iter_mut().flatten().find_map(|run| run.find(wanted).map(|i| &mut run.entries[i])) else { return false }; let accounting = source.accounting(); let value = core::mem::take(&mut source.value); install(Transferred { key: source.key, value, first: capture.first, last: source.last }); - for run in self.runs.iter_mut().flatten() { - let at = run.entries.partition_point(|e| e.position() < capture.position); - for entry in run.entries[at..].iter_mut().take_while(|e| e.position() == capture.position && e.order <= capture.order) { - if entry.live && (entry.order == capture.order || !accounting) { - entry.live = false; - entry.value = Vec::new(); - } + for slot in &mut self.runs { + let Some(run) = slot else { continue }; + for i in run.at(capture.position) { + let entry = &run.entries[i]; + if entry.live && entry.order <= capture.order && (entry.order == capture.order || !accounting) { run.kill(i, &mut self.live); } } + run.compact(); run.retire(); + if run.entries.is_empty() { *slot = None; } } + self.settle(); true } } +/// A node's keys with the pending keys of its range laid over them. +pub(crate) struct Overlay<'a> { + pending: &'a Pending, + btree: Btree, + level: u8, + stored: Peekable)>>, + /// None where pending holds nothing of this btree and level. + entries: Option>>, +} + +impl<'a> Overlay<'a> { + /// An accounting key adds every pending delta newer than its version. + fn accounted(&self, pos: Bpos, stored: Option<&(Key, Vec)>) -> Result<(Key, Cow<'a, [u8]>)> { + let (mut key, mut sums) = match stored { + Some((key, value)) => (*key, journal::counters(value)?), + None => (Key::new(kind::ACCOUNTING, pos), Vec::new()), + }; + let since = key.version; + for entry in self.pending.range(self.btree, self.level, pos, pos) { + let version = entry.key().version; + if version <= since { continue; } + let delta = journal::counters(entry.value())?; + sums.resize(sums.len().max(delta.len()), 0); + for (sum, d) in sums.iter_mut().zip(delta) { *sum = sum.wrapping_add(d); } + key.version = key.version.max(version); + } + Ok((key, Cow::Owned(super::value::counters(&sums)))) + } +} + +impl<'a> Iterator for Overlay<'a> { + type Item = Result<(Key, Cow<'a, [u8]>)>; + fn next(&mut self) -> Option { + let stored_next = |stored: &mut Peekable)>>| stored.next().map(|(k, v)| Ok((*k, Cow::Borrowed(&v[..])))); + let Some(entries) = &mut self.entries else { return stored_next(&mut self.stored) }; + loop { + let stored = self.stored.peek().map(|(k, _)| k.pos); + let Some(pos) = entries.peek().map(|e| e.key().pos).filter(|p| stored.is_none_or(|s| *p <= s)) else { + return stored_next(&mut self.stored); + }; + let stored = self.stored.next_if(|(k, _)| k.pos == pos); + if self.btree == Btree::Accounting && self.level == 0 { + while entries.next_if(|e| e.key().pos == pos).is_some() {} + return Some(self.accounted(pos, stored)); + } + let latest = entries.next().unwrap(); + if latest.key().kind != kind::DELETED { return Some(Ok((*latest.key(), Cow::Borrowed(latest.value())))); } + } + } +} + pub(crate) struct Iter<'a> { pending: &'a Pending, at: [usize; LEVELS], @@ -322,9 +482,14 @@ impl<'a> Iter<'a> { let mut best = None; for (i, run) in self.pending.runs.iter().enumerate() { let Some(Run { entries: run, .. }) = run else { continue }; - while run.get(self.at[i]).is_some_and(|e| !e.live) { self.at[i] += 1; } - if let Some(entry) = run.get(self.at[i]) { - if self.end.is_none_or(|end| entry.position() <= end) && best.is_none_or(|old| entry.ordering() < old) { + let within = |e: &Entry| self.end.is_none_or(|end| e.position() <= end); + while run.get(self.at[i]).is_some_and(|e| !e.live && within(e)) { + visit(); + self.at[i] += 1; + } + visit(); + if let Some(entry) = run.get(self.at[i]).filter(|e| within(e)) { + if best.is_none_or(|old| entry.ordering() < old) { chosen = Some(i); best = Some(entry.ordering()); } @@ -427,6 +592,51 @@ mod tests { pending.apply(seq, updates); } + fn at(pos: u64) -> Bpos { Bpos::new(0, pos, 0) } + + + /// 2047 batches of eight positions each, in eleven runs. + fn crowded() -> Pending { + let mut pending = Pending::default(); + for batch in 0..2047u64 { add(&mut pending, batch + 1, (0..8).map(|i| update(batch * 8 + i, i)).collect()); } + assert_eq!(runs(&pending), 11); + pending + } + + fn runs(pending: &Pending) -> usize { pending.runs.iter().flatten().count() } + + #[test] + fn a_lookup_visits_what_its_position_holds_past_a_materialized_stretch() { + let mut pending = crowded(); + let captures: Vec<_> = pending.range(Btree::Inodes, 0, at(1001), at(4000)).map(|v| v.capture()).collect(); + for capture in captures { assert!(pending.transfer(capture, |_| ())); } + let (found, visited) = visits(|| pending.range(Btree::Inodes, 0, at(1000), at(1000)).count()); + assert_eq!(found, 1); + assert!(visited <= 4 * runs(&pending), "{visited} entries for one position"); + } + + #[test] + fn durable_visits_only_what_changed_since_it_last_ran() { + let mut pending = crowded(); + pending.durable(2047); + add(&mut pending, 3000, vec![update(5, 1)]); + let (_, visited) = visits(|| pending.durable(3000)); + assert!(visited <= 2 * runs(&pending), "{visited} entries for one update"); + assert_eq!(pending.oldest_pin(), Some(1)); + } + + #[test] + fn materialized_entries_leave_no_more_dead_than_live_in_a_run() { + let mut pending = Pending::default(); + for batch in 0..2048u64 { add(&mut pending, batch + 1, (0..8).map(|i| update(batch * 8 + i, i)).collect()); } + let captures: Vec<_> = pending.range(Btree::Inodes, 0, at(0), at(12_287)).map(|v| v.capture()).collect(); + for capture in captures { assert!(pending.transfer(capture, |_| ())); } + let live = pending.iter().count(); + assert_eq!(live, 4096); + let held: usize = pending.runs.iter().flatten().map(|r| r.entries.len()).sum(); + assert!(held <= 2 * live, "{held} entries held for {live} live"); + } + #[test] fn batches_keep_latest_values_and_original_pins_without_cloning_values() { let mut pending = Pending::default(); diff --git a/lib/bcachefs/src/engine/read.rs b/lib/bcachefs/src/engine/read.rs index 5d65a377594..403bfec554b 100644 --- a/lib/bcachefs/src/engine/read.rs +++ b/lib/bcachefs/src/engine/read.rs @@ -205,6 +205,14 @@ impl Volume { self.replay.as_ref() } + /// The keys of `btree` while it has no root. + pub(crate) fn rootless(&self, btree: Btree) -> Result)>> { + match &self.replay { + Some((_, replay)) => replay.over(btree, 0, Bpos::MIN, Bpos::MAX, Vec::new()), + None => Ok(Vec::new()), + } + } + /// The node `ptr` points at, `level` of `btree`, spanning `[min, max]`, /// with the journal's replay laid over its keys. pub(crate) async fn node(&self, btree: Btree, level: u8, ptr: &NodePtr, max: Bpos) -> Result { @@ -222,7 +230,7 @@ impl Volume { } /// A traversal in progress, ended when dropped. -struct Reading<'a, S: Source + ?Sized>(&'a S, u64); +pub(crate) struct Reading<'a, S: Source + ?Sized>(pub(crate) &'a S, pub(crate) u64); impl Drop for Reading<'_, S> { fn drop(&mut self) { @@ -242,12 +250,6 @@ pub(crate) trait Source { /// The root directory's inode number. fn root_dir(&self) -> u64; fn root_of(&self, btree: Btree) -> Option; - /// The keys of `btree` while it has no root. - fn rootless(&self, btree: Btree) -> Result)>>; - /// The keys of the node `ptr` points at, `level` of `btree`, spanning - /// `[ptr.min, max]`. - async fn keys(&self, btree: Btree, level: u8, ptr: &NodePtr, max: Bpos) -> Result; - /// A traversal that may follow pointers it read before an await begins, /// and its ticket: until [`Source::end_read`] with it, a mount writes no /// bucket that a pointer read now may lead into. @@ -259,42 +261,7 @@ pub(crate) trait Source { /// Visits `btree`'s keys from `from` in order until `visit` answers /// `false`. - async fn walk(&self, btree: Btree, from: Bpos, mut visit: impl FnMut(&Key, &[u8]) -> Result) -> Result<()> { - // Interior keys below are read once and followed after awaits. - let _reading = Reading(self, self.begin_read()); - let Some(root) = self.root_of(btree) else { - for (key, value) in self.rootless(btree)?.iter().filter(|(k, _)| k.pos >= from) { - if !visit(key, value)? { - break; - } - } - return Ok(()); - }; - let keys = self.keys(btree, root.level, &root.ptr, root.key.pos).await?; - let start = keys.partition_point(|(k, _)| k.pos < from); - let mut stack: Vec<(u8, Keys, usize)> = alloc::vec![(root.level, keys, start)]; - while let Some((level, keys, i)) = stack.last_mut() { - let Some((key, value)) = keys.get(*i) else { - stack.pop(); - continue; - }; - *i += 1; - if *level == 0 { - if !visit(key, value)? { - return Ok(()); - } - continue; - } - if key.kind != kind::BTREE_PTR_V2 { - return Err(Error::Unsupported("interior key other than btree_ptr_v2, such as a v1 btree_ptr")); - } - let (level, ptr, max) = (*level - 1, NodePtr::decode(value)?, key.pos); - let child = self.keys(btree, level, &ptr, max).await?; - let start = child.partition_point(|(k, _)| k.pos < from); - stack.push((level, child, start)); - } - Ok(()) - } + async fn walk(&self, btree: Btree, from: Bpos, visit: impl FnMut(&Key, &[u8]) -> Result) -> Result<()>; /// The key at `pos` in `btree`, if any. async fn get(&self, btree: Btree, pos: Bpos) -> Result)>> { @@ -666,16 +633,45 @@ impl Source for Volume { self.roots.iter().find(|r| r.btree == btree).cloned() } - fn rootless(&self, btree: Btree) -> Result)>> { - match &self.replay { - Some((_, replay)) => replay.over(btree, 0, Bpos::MIN, Bpos::MAX, Vec::new()), - None => Ok(Vec::new()), + /// Visits `btree`'s keys from `from` in order until `visit` answers + /// `false`. + async fn walk(&self, btree: Btree, from: Bpos, mut visit: impl FnMut(&Key, &[u8]) -> Result) -> Result<()> { + // Interior keys below are read once and followed after awaits. + let _reading = Reading(self, self.begin_read()); + let Some(root) = self.root_of(btree) else { + for (key, value) in self.rootless(btree)?.iter().filter(|(k, _)| k.pos >= from) { + if !visit(key, value)? { + break; + } + } + return Ok(()); + }; + let keys = Rc::new(self.node(btree, root.level, &root.ptr, root.key.pos).await?.keys); + let start = keys.partition_point(|(k, _)| k.pos < from); + let mut stack: Vec<(u8, Keys, usize)> = alloc::vec![(root.level, keys, start)]; + while let Some((level, keys, i)) = stack.last_mut() { + let Some((key, value)) = keys.get(*i) else { + stack.pop(); + continue; + }; + *i += 1; + if *level == 0 { + if !visit(key, value)? { + return Ok(()); + } + continue; + } + if key.kind != kind::BTREE_PTR_V2 { + return Err(Error::Unsupported("interior key other than btree_ptr_v2, such as a v1 btree_ptr")); + } + let (level, ptr, max) = (*level - 1, NodePtr::decode(value)?, key.pos); + let child = Rc::new(self.node(btree, level, &ptr, max).await?.keys); + let start = child.partition_point(|(k, _)| k.pos < from); + stack.push((level, child, start)); } + Ok(()) } - async fn keys(&self, btree: Btree, level: u8, ptr: &NodePtr, max: Bpos) -> Result { - Ok(Rc::new(self.node(btree, level, ptr, max).await?.keys)) - } } /// The namespace reads, as [`Source`] gives them. diff --git a/lib/bcachefs/src/engine/tests/admission.rs b/lib/bcachefs/src/engine/tests/admission.rs index c23859d230f..e642b6041e1 100644 --- a/lib/bcachefs/src/engine/tests/admission.rs +++ b/lib/bcachefs/src/engine/tests/admission.rs @@ -96,17 +96,16 @@ fn writers(dev: &Image, cache_nodes: usize, dirs: &[u64], writers: usize, per: u const SMALL_CACHE: usize = super::super::mount::MIN_CACHE; #[test] -fn committed_keys_survive_without_resident_dirty_leaves_and_replay_stays_pending() { +fn committed_keys_go_into_the_leaves_they_change_and_replay_stays_pending() { let dev = formatted(64 << 20, None); let options = MountOptions { cache_nodes: SMALL_CACHE }; let mount = block_on_ready(Mount::open(&dev, options, random())).unwrap(); for i in 0..64 { block_on_ready(mount.create(mount.root(), format!("pending{i}").as_bytes(), New::File, OWNER, NOW)).unwrap(); } let before = mount.pending_pressure(); - assert!(before.0 >= 128); - assert_eq!(before.2, 0, "logical updates need no resident dirty leaf"); + assert_eq!(before.0, 0, "an ordinary transaction's keys go into the leaves it holds"); + assert!(before.2 > 0); block_on_ready(mount.commit()).unwrap(); assert_eq!(mount.pending_pressure(), before, "commit journals the updates without a global materialization pass"); - eprintln!("64-name pending workload: {} records, {} allocated bytes", before.0, before.1); drop(mount); let mount = block_on_ready(Mount::open(&dev, options, random())).unwrap(); diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index bce8d520f1e..c84d0137cfa 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -431,15 +431,21 @@ fn retained_small_files_reclaim_their_truncated_data_buckets() { #[test] fn pending_allocation_failure_precedes_data_reservation_and_reserved_write_preparation() { let dev = formatted(64 << 20, None); + let ino = mounted(&dev, async |m: &Mount<&Image>| { + let ino = m.create(m.root(), b"retry", New::File, OWNER, NOW).await.unwrap().ino; + assert_eq!(m.write(ino, 0, &[0x11; 4096], NOW).await.unwrap(), 4096); + ino + }); + // Remounted clean, so no reclaim runs before the preflights refused here. let mount = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); - let ino = block_on_ready(mount.create(mount.root(), b"retry", New::File, OWNER, NOW)).unwrap().ino; - assert_eq!(block_on_ready(mount.write(ino, 0, &[0x11; 4096], NOW)).unwrap(), 4096); let before = mount.space(); let out = super::super::pending::fail_pending_allocation_after(0, || block_on_ready(mount.reserve(ino, 0, 4096))); assert!(matches!(out, Err(Error::NoMemory))); assert_eq!(mount.space(), before, "a refused preflight takes no DATA slot"); let mut reservation = block_on_ready(mount.reserve(ino, 0, 4096)).unwrap(); block_on_ready(mount.set_mtime(ino, NOW + 1)).unwrap(); + // The change left nothing in pending, which keeps its workspace. + mount.shrink_pending(); let out = super::super::pending::fail_pending_allocation_after(0, || block_on_ready(mount.write_reserved(&mut reservation, 0, &[0x22; 4096], NOW))); assert_eq!(out, Err(Error::NoMemory)); let mut old = [0; 4096]; diff --git a/lib/bcachefs/src/engine/tests/geometry.rs b/lib/bcachefs/src/engine/tests/geometry.rs index b1d065d82db..ef340650ca9 100644 --- a/lib/bcachefs/src/engine/tests/geometry.rs +++ b/lib/bcachefs/src/engine/tests/geometry.rs @@ -157,7 +157,7 @@ fn encrypted_publications_take_a_nonce_each() { let m = open(&dev, Some(&secret), MEMORY); oracle::reset(); Executor::new().block_on(async { - for i in 0..400 { + for i in 0..1000 { let name = format!("e{i}").into_bytes(); let ino = m.create(m.root(), &name, New::File, OWNER, NOW).await.unwrap().ino; expect.dirs.entry(m.root()).or_default().insert(name, ino); @@ -294,7 +294,8 @@ fn an_encrypted_node_of_one_block_survives_a_crash_at_every_request() { /// - Data buckets a quarter full, so the LRU spans several leaves split by /// fill. The first leaf holds the quarter-full entries. /// - One inode changed, its leaf and the accounting leaf to be rewritten (as -/// after an unclean mount); every other node appended to. +/// after an unclean mount); every other node appended to. Which inode +/// gives the shape below depends on the layout the churn leaves. /// /// The inode leaf's new node leaves its bucket a quarter full: that LRU key /// lands in the first leaf, which joins. The accounting leaf, joining in @@ -333,11 +334,11 @@ fn a_member_left_with_nothing_to_append_is_not_written() { block_on_ready(super::super::read::Source::walk(&m, btree, Bpos::MIN, |_, _| Ok(true))).unwrap(); m.rewrite_next(btree); } - block_on_ready(m.set_mtime(kept[0], NOW + 9)).unwrap(); + block_on_ready(m.set_mtime(kept[404], NOW + 9)).unwrap(); let published = block_on_ready(m.unmount_in_one_publication()).unwrap(); assert!(published.idle > 0, "no member was left with nothing to append: {published:?}"); let v = block_on_ready(Volume::open(&dev)).unwrap(); - assert_eq!(block_on_ready(v.inode(kept[0])).unwrap().mtime, NOW + 9); + assert_eq!(block_on_ready(v.inode(kept[404])).unwrap().mtime, NOW + 9); assert_eq!(block_on_ready(v.names()).unwrap().len(), kept.len()); assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } From f1b2f3e389624e4af00bf825539246d38169b9f0 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 15:45:52 +0100 Subject: [PATCH 70/93] tests: the DATA images start the UART driver and logkeeper holds the origins K4 (#1175) moved the UART to a userland driver fed by logkeeper's wire and made `log-origins` logkeeper's to hold; `every_boot_config_runs_logkeeper` refuses an image without it. datacase.toml and formatcase.toml, which this branch added, take designatecase.toml's rows. Co-Authored-By: Claude Opus 5.5 --- tests/images/datacase.toml | 13 ++++++++++--- tests/images/formatcase.toml | 13 ++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/images/datacase.toml b/tests/images/datacase.toml index b0e925456d7..6e10ae7642d 100644 --- a/tests/images/datacase.toml +++ b/tests/images/datacase.toml @@ -1,11 +1,18 @@ # Mounted DATA tests on both benches. No formatting or designation authority. [boot] -start = ["logkeeper", "diskserver", "fileserver", "acpiserver", "test-runner"] +start = ["logkeeper", "uart", "diskserver", "fileserver", "acpiserver", "test-runner"] [programs.logkeeper] service = true -syscap = ["logread"] -serves = ["log"] +syscap = ["logread", "console"] +serves = ["log", "log-wire"] +holds = ["log-origins"] + +[programs.uart] +devices = ["uart"] +holds = ["console-input"] +receives = ["log-wire"] +restart = true [programs.acpiserver] service = true diff --git a/tests/images/formatcase.toml b/tests/images/formatcase.toml index 6f752174fe4..56db7fcd360 100644 --- a/tests/images/formatcase.toml +++ b/tests/images/formatcase.toml @@ -1,12 +1,19 @@ # Explicit destructive DATA preparation only. No DATA file server holds the # partition; only this image's boot-started runner gets raw DATA authority. [boot] -start = ["logkeeper", "diskserver", "fileserver", "acpiserver", "test-runner"] +start = ["logkeeper", "uart", "diskserver", "fileserver", "acpiserver", "test-runner"] [programs.logkeeper] service = true -syscap = ["logread"] -serves = ["log"] +syscap = ["logread", "console"] +serves = ["log", "log-wire"] +holds = ["log-origins"] + +[programs.uart] +devices = ["uart"] +holds = ["console-input"] +receives = ["log-wire"] +restart = true [programs.acpiserver] service = true From f56cc636b930279afc3c486b973b5259a9f3ae8c Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 16:55:22 +0100 Subject: [PATCH 71/93] storage: a write is one extent for each run of contiguous blocks The file server reserved and wrote back DATA a 4 KiB page at a time, and the engine made each page a transaction and an extent of its own: a 64 MiB file was 16,384 transactions to reserve, 16,384 to write and as many to truncate. On the benches osbench's seq_write ran at 28-45 MB/s, 0.12-0.29x main. Now, as upstream does, a reservation is runs of contiguous sectors of at most 64 KiB (upstream's default extent size) and a write puts one extent in each run it crosses: - `Mount::reserve` covers the whole blocks of a range, a run per transaction; each block keeps a transaction's metadata credit until it is written, so a reserved write still cannot be refused for space, and the blocks never written return when the reservation is dropped. - `write_reserved` and `write` go through one `write_run`: a transaction that removes at most two old extents, as one block's write always could, so an overwrite of more goes in parts. A block written in part keeps the rest of its bytes. - An allocation for a run takes what the open data bucket has left, where the extent then ends, instead of closing it: with whole runs refused, a write that did not fit closed the bucket evacuation had just packed, and evacuation and the write chased each other forever. - The file server keeps one reservation per write and its pages by ticket, and writes each run of pages under one write's capacity back at once. `Writes::write`, for a file with no paged object, writes its pages back in one call and reads only a page it covers in part. The per-page reservation path is gone. Tests: one extent per 64 KiB run, reserved or not; an overwrite across 16 single-block extents goes in parts and reads back; 2000 reservations of 64 KiB written 16 KiB and dropped fit a 64 MiB volume. The partial write-back test refuses the read of the last page's kept bytes, as whole pages now read nothing. a_member_left_with_nothing_to_append_is_not_written builds its shape from 3600 files and the 485th kept inode. Co-Authored-By: Claude Opus 5.5 --- lib/bcachefs/src/engine/mount/allocator.rs | 15 +- lib/bcachefs/src/engine/mount/data.rs | 289 ++++++++++++++------- lib/bcachefs/src/engine/mount/mod.rs | 3 - lib/bcachefs/src/engine/tests/data.rs | 72 +++++ lib/bcachefs/src/engine/tests/geometry.rs | 8 +- system/fileserver/src/data.rs | 100 ++++--- system/fileserver/src/pages/tests.rs | 15 +- system/fileserver/src/volume.rs | 50 ++-- 8 files changed, 387 insertions(+), 165 deletions(-) diff --git a/lib/bcachefs/src/engine/mount/allocator.rs b/lib/bcachefs/src/engine/mount/allocator.rs index 507eab01de7..e2ab98e6e6c 100644 --- a/lib/bcachefs/src/engine/mount/allocator.rs +++ b/lib/bcachefs/src/engine/mount/allocator.rs @@ -234,17 +234,22 @@ impl Allocator { self.data = Some((b, 0, gen)); } - pub(super) fn next_data(&mut self, bucket_sectors: u64, sectors: u64, fixed: bool) -> Option<(u64, u8)> { + /// Contiguous data sectors: `sectors` of them, or with `partial` as many + /// as the open bucket has left, so it is filled before another opens. A + /// hole, one block a reservation gave back, serves `fixed` alone. + pub(super) fn next_data(&mut self, bucket_sectors: u64, sectors: u64, fixed: bool, partial: bool) -> Option<(u64, u8, u64)> { if fixed { - if let Some((_, sector)) = self.holes.pop_first() { return Some((sector, 0)); } + if let Some((_, sector)) = self.holes.pop_first() { return Some((sector, 0, sectors)); } } let (b, next, gen) = self.data?; - if sectors > bucket_sectors - next { + let room = bucket_sectors - next; + if sectors > room && !partial { self.data = None; return None; } - self.data = (next + sectors < bucket_sectors).then_some((b, next + sectors, gen)); - Some((b * bucket_sectors + next, gen)) + let got = sectors.min(room); + self.data = (next + got < bucket_sectors).then_some((b, next + got, gen)); + Some((b * bucket_sectors + next, gen, got)) } pub(super) fn data_fits(&self, nodes: usize, reclaim_room: usize, draw: Draw) -> bool { diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index 3fecd3d1689..7e04e0fc2b7 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -7,49 +7,74 @@ use super::*; const VERSIONS: u64 = 1024; -#[derive(Clone, Copy)] -pub(super) struct ReservedBlock { - ino: u64, +/// The largest extent this writer makes, upstream's default: what a read of +/// one block past the page cache verifies at most. +const EXTENT_MAX: u64 = 64 << 10; + +/// Contiguous sectors for consecutive blocks of a file, each written once. +struct Run { + first: u64, sector: u64, + blocks: u32, + written: u128, +} + +pub(super) struct Reserved { + ino: u64, + block_sectors: u64, + runs: Vec, +} + +impl Reserved { + fn unwritten(&self) -> u64 { + self.runs.iter().map(|r| u64::from(r.blocks - r.written.count_ones())).sum() + } } #[derive(Default)] pub(super) struct Reservations { - pub(super) active: BTreeMap, + pub(super) active: BTreeMap, canceled: Vec, next: u64, } impl Reservations { + /// Metadata credit the others hold: a transaction's worth for each + /// block not yet written, which a part's own transaction may then spend. pub(super) fn pending(&self, draw: Draw) -> u64 { let own = match draw { - Draw::Reserved(id) => usize::from(self.active.contains_key(&id)), - _ => 0, + Draw::Reserved(id) => Some(id), + _ => None, }; - (self.active.len() - own) as u64 * K_TX * 8 + self.active.iter().filter(|(&id, _)| Some(id) != own).map(|(_, r)| r.unwritten()).sum::() * K_TX * 8 } } -/// Capacity for one filesystem block. Dropping an unused reservation -/// returns its physical slot and metadata credit on the next operation. +/// Capacity for the whole blocks of a range: runs of contiguous sectors of at +/// most [`EXTENT_MAX`]. Dropping it returns the blocks never written on the +/// next operation. pub struct Reservation { owner: Rc>, id: u64, first: u64, - bytes: usize, + bytes: u64, } impl Reservation { pub fn range(&self) -> (u64, u64) { - (self.first, self.first + self.bytes as u64) + (self.first, self.first + self.bytes) } } impl Drop for Reservation { fn drop(&mut self) { let mut pool = self.owner.borrow_mut(); - if let Some(block) = pool.active.remove(&self.id) { - pool.canceled.push(block.sector); + if let Some(reserved) = pool.active.remove(&self.id) { + for run in &reserved.runs { + for block in (0..run.blocks).filter(|b| run.written & 1 << b == 0) { + pool.canceled.push(run.sector + u64::from(block) * reserved.block_sectors); + } + } } } } @@ -84,60 +109,108 @@ impl Mount { } } - /// Reserves the complete block containing `[offset, offset + len)`. - /// A range crossing a filesystem block needs separate reservations. + /// Reserves the whole blocks of `[offset, offset + len)`, a run of + /// contiguous sectors at a time: a later write into it cannot be refused + /// for space. pub async fn reserve(&self, ino: u64, offset: u64, len: usize) -> Result { - let bytes = self.sb.block_bytes(); - let first = offset / bytes as u64 * bytes as u64; - if len == 0 || offset.checked_add(len as u64).is_none_or(|end| end > i64::MAX as u64 || end > first.saturating_add(bytes as u64)) { - return Err(Error::Invalid("reservation outside one filesystem block")); - } + let block = self.sb.block_bytes() as u64; + let first = offset / block * block; + let end = offset.checked_add(len as u64).filter(|&end| len > 0 && i64::try_from(end).is_ok()).ok_or(Error::Invalid("reservation of nothing or past 2^63-1"))?.div_ceil(block) * block; + let bucket = u64::from(self.sb.member.bucket_sectors); + let longest = EXTENT_MAX.min(bucket * SECTOR as u64) / block * block; let _data = self.data_slot().await; - let id = loop { - let out = self - .transact(K_TX, FOOTPRINT, Draw::Ordinary, async |txn: &mut Txn| { - regular(&self.inode_in(txn, ino).await?)?; - self.block_extents(ino, first, first + bytes as u64).await?; - let (sector, _) = self.data_sector(self.sb.block_bytes() as u64 / SECTOR as u64, Draw::Ordinary, true).await?; - self.st.borrow_mut().alloc.pin(sector / u64::from(self.sb.member.bucket_sectors)); - let mut pool = self.reservations.borrow_mut(); - let id = pool.next; - pool.next = pool.next.checked_add(1).expect("reservation identifiers exhausted"); - pool.active.insert(id, ReservedBlock { ino, sector }); - Ok(id) - }) - .await; - match out { - Err(Error::NoSpace(_)) if self.compact_user().await? => continue, - other => break other?, - } + let id = { + let mut pool = self.reservations.borrow_mut(); + let id = pool.next; + pool.next = pool.next.checked_add(1).expect("reservation identifiers exhausted"); + pool.active.insert(id, Reserved { ino, block_sectors: block / SECTOR as u64, runs: Vec::new() }); + id }; - Ok(Reservation { owner: Rc::clone(&self.reservations), id, first, bytes }) + let reservation = Reservation { owner: Rc::clone(&self.reservations), id, first, bytes: end - first }; + let mut at = first; + while at < end { + let wanted = (end - at).min(longest) / SECTOR as u64; + let got = loop { + let out = self + .transact(K_TX, FOOTPRINT, Draw::Ordinary, async |txn: &mut Txn| { + regular(&self.inode_in(txn, ino).await?)?; + let (sector, got) = if wanted * SECTOR as u64 == block { + (self.data_sector(wanted, Draw::Ordinary, true).await?.0, wanted) + } else { + self.data_run(wanted, Draw::Ordinary).await? + }; + let blocks = (got * SECTOR as u64 / block) as u32; + let mut st = self.st.borrow_mut(); + for _ in 0..blocks { st.alloc.pin(sector / bucket); } + let run = Run { first: at, sector, blocks, written: 0 }; + self.reservations.borrow_mut().active.get_mut(&id).expect("a reservation being made").runs.push(run); + Ok(got) + }) + .await; + match out { + Err(Error::NoSpace(_)) if self.compact_user().await? => continue, + other => break other?, + } + }; + at += got * SECTOR as u64; + } + Ok(reservation) } - /// Writes inside the reserved block. Successful application consumes the - /// reservation; a failed write leaves it reserved for retry until dropped. + /// Writes into blocks of the reservation not yet written, as one extent + /// for each run the bytes cross; a block written in part keeps the rest + /// of its bytes, and counts as written. Bytes written; a + /// failure after some returns their count, and the rest stay reserved + /// for a retry until dropped. pub async fn write_reserved(&self, reservation: &mut Reservation, offset: u64, data: &[u8], now: i64) -> Result { if !Rc::ptr_eq(&reservation.owner, &self.reservations) { return Err(Error::Invalid("reservation belongs to another mount")); } + let block = self.sb.block_bytes() as u64; if offset < reservation.first || data.is_empty() || offset.checked_add(data.len() as u64).is_none_or(|end| end > reservation.range().1) { - return Err(Error::Invalid("write outside its reserved block")); + return Err(Error::Invalid("write outside its reserved blocks")); } let _slot = self.data_slot().await; - let block = self.reservations.borrow().active.get(&reservation.id).copied().ok_or(Error::Invalid("reservation already consumed"))?; self.observe(now); - let version = self.data_version().await?; - let out = self.write_chunk(block.ino, offset, data, now, version, Some((reservation.id, block.sector))).await; - if !self.reservations.borrow().active.contains_key(&reservation.id) { + let bucket = u64::from(self.sb.member.bucket_sectors); + let mut done = 0; + while done < data.len() { + let at = offset + done as u64; + let (ino, sector, end) = { + let pool = self.reservations.borrow(); + let reserved = pool.active.get(&reservation.id).ok_or(Error::Invalid("reservation already written"))?; + let run = reserved.runs.iter().find(|r| (r.first..r.first + u64::from(r.blocks) * block).contains(&at)).expect("a reservation covers its range"); + let end = (offset + data.len() as u64).min(run.first + u64::from(run.blocks) * block); + let (from, to) = (((at - run.first) / block) as u32, ((end - run.first).div_ceil(block)) as u32); + if (from..to).any(|b| run.written & 1 << b != 0) { + return Err(Error::Invalid("a reserved block written twice")); + } + (reserved.ino, run.sector + (at - run.first) / SECTOR as u64, end) + }; + let version = match self.data_version().await { + Ok(version) => version, + Err(_) if done > 0 => return Ok(done), + Err(e) => return Err(e), + }; + let n = match self.write_run(ino, at, &data[done..(end - offset) as usize], now, version, Some((reservation.id, sector))).await { + Ok(n) => n, + Err(_) if done > 0 => return Ok(done), + Err(e) => return Err(e), + }; + let mut pool = self.reservations.borrow_mut(); let mut st = self.st.borrow_mut(); - if out.is_ok() { - st.alloc.unpin(block.sector / u64::from(self.sb.member.bucket_sectors)); - } else { - st.alloc.cancel_data(block.sector, u64::from(self.sb.member.bucket_sectors)); + let reserved = pool.active.get_mut(&reservation.id).expect("a reservation being written"); + let run = reserved.runs.iter_mut().find(|r| (r.first..r.first + u64::from(r.blocks) * block).contains(&at)).expect("the run just written"); + for b in ((at - run.first) / block) as u32..((at + n as u64 - run.first).div_ceil(block)) as u32 { + run.written |= 1 << b; + st.alloc.unpin(run.sector / bucket); + } + if reserved.unwritten() == 0 { + pool.active.remove(&reservation.id); } + done += n; } - out + Ok(done) } pub(super) async fn create_symlink(&self, dir: u64, name: &[u8], target: &[u8], owner: Owner, now: i64) -> Result { @@ -180,17 +253,22 @@ impl Mount { Source::readlink(self, ino).await } - /// Writes as many bytes as fit. A later chunk's failure returns the - /// completed byte count; `commit` is the durability barrier. + /// Writes as many bytes as fit, an extent of at most [`EXTENT_MAX`] at a + /// time. A later extent's failure returns the completed byte count; + /// `commit` is the durability barrier. pub async fn write(&self, ino: u64, offset: u64, data: &[u8], now: i64) -> Result { offset.checked_add(data.len() as u64).filter(|n| i64::try_from(*n).is_ok()).ok_or(Error::Invalid("file size past 2^63-1"))?; let _slot = self.data_slot().await; self.observe(now); + let block = self.sb.block_bytes() as u64; + let longest = EXTENT_MAX.min(u64::from(self.sb.member.bucket_sectors) * SECTOR as u64) / block * block; let mut done = 0; while done < data.len() { - let version = self.data_version().await; - let out = match version { - Ok(version) => self.write_chunk(ino, offset + done as u64, &data[done..], now, version, None).await, + let at = offset + done as u64; + let room = (at / block * block + longest - at) as usize; + let part = &data[done..done + room.min(data.len() - done)]; + let out = match self.data_version().await { + Ok(version) => self.write_run(ino, at, part, now, version, None).await, Err(e) => Err(e), }; let out = match out { @@ -309,20 +387,55 @@ impl Mount { Ok(found) } - async fn write_chunk(&self, ino: u64, offset: u64, input: &[u8], now: i64, version: Version, reserved: Option<(u64, u64)>) -> Result { + /// One extent of `input` at `at`, onto `reserved`'s sectors or new ones: + /// the whole blocks it spans, the first and last read first where it + /// covers them in part. A transaction removes at most two old extents, + /// as one block's write always could, so a write over more stops before + /// the third. Bytes written. + async fn write_run(&self, ino: u64, at: u64, input: &[u8], now: i64, version: Version, reserved: Option<(u64, u64)>) -> Result { let draw = reserved.map_or(Draw::Ordinary, |(id, _)| Draw::Reserved(id)); self.transact(K_TX, FOOTPRINT, draw, async |txn: &mut Txn| { let mut inode = self.inode_in(txn, ino).await?; regular(&inode)?; let block = self.sb.block_bytes() as u64; - let first = offset / block * block; - let mut bytes = alloc::vec![0u8; block as usize]; - Source::read(self, ino, first, &mut bytes).await?; - let at = (offset - first) as usize; - let n = input.len().min(bytes.len() - at); - bytes[at..at + n].copy_from_slice(&input[..n]); - self.replace_block(txn, ino, &mut inode, first, &bytes, version, reserved.map(|(_, sector)| sector)).await?; - inode.size = inode.size.max(offset + n as u64); + let first = at / block * block; + let mut end = (at + input.len() as u64).div_ceil(block) * block; + let mut old = self.extents_in(ino, first, end, 3).await?; + if old.len() == 3 { + end = start(&old[2].0)? / block * block; + if end <= first { + return Err(Error::Unsupported("more than two data extents in one filesystem block")); + } + old.truncate(2); + old.retain(|(k, _)| start(k).is_ok_and(|s| s < end)); + } + let sector = match reserved { + Some((_, sector)) => sector, + None => { + let (sector, got) = if end - first == block { + (self.data_sector(block / SECTOR as u64, Draw::Ordinary, true).await?.0, block / SECTOR as u64) + } else { + self.data_run((end - first) / SECTOR as u64, Draw::Ordinary).await? + }; + end = first + got * SECTOR as u64; + old.retain(|(k, _)| start(k).is_ok_and(|s| s < end)); + self.st.borrow_mut().alloc.pin(sector / u64::from(self.sb.member.bucket_sectors)); + txn.allocated_data.push(sector); + sector + } + }; + let n = (input.len() as u64).min(end - at) as usize; + let mut bytes = alloc::vec![0u8; (end - first) as usize]; + if at > first { + Source::read(self, ino, first, &mut bytes[..block as usize]).await?; + } + if (at + n as u64) < end { + let last = (end - block - first) as usize; + Source::read(self, ino, end - block, &mut bytes[last..]).await?; + } + bytes[(at - first) as usize..(at - first) as usize + n].copy_from_slice(&input[..n]); + self.replace_range(txn, ino, &mut inode, first, &bytes, old, version, sector).await?; + inode.size = inode.size.max(at + n as u64); self.touch(&mut inode, now); self.set_inode(txn, ino, &inode).await?; Ok(n) @@ -331,11 +444,21 @@ impl Mount { } pub(super) async fn data_sector(&self, sectors: u64, draw: Draw, fixed: bool) -> Result<(u64, u8)> { + self.data_space(sectors, draw, fixed, false).await.map(|(sector, gen, _)| (sector, gen)) + } + + /// Up to `sectors` contiguous data sectors, as many as the open bucket + /// has left before another is taken: where an extent ends. + async fn data_run(&self, sectors: u64, draw: Draw) -> Result<(u64, u64)> { + self.data_space(sectors, draw, false, true).await.map(|(sector, _, got)| (sector, got)) + } + + async fn data_space(&self, sectors: u64, draw: Draw, fixed: bool, partial: bool) -> Result<(u64, u8, u64)> { let bucket = u64::from(self.sb.member.bucket_sectors); if sectors > bucket { return Err(Error::Unsupported("data extent larger than a bucket")); } - if let Some(at) = self.st.borrow_mut().alloc.next_data(bucket, sectors, fixed) { + if let Some(at) = self.st.borrow_mut().alloc.next_data(bucket, sectors, fixed, partial) { return Ok(at); } { @@ -347,10 +470,12 @@ impl Mount { } } self.take_bucket(true).await?; - Ok(self.st.borrow_mut().alloc.next_data(bucket, sectors, fixed).expect("a fresh bucket holds the extent")) + Ok(self.st.borrow_mut().alloc.next_data(bucket, sectors, fixed, partial).expect("a fresh bucket holds the extent")) } - async fn block_extents(&self, ino: u64, first: u64, end: u64) -> Result)>> { + /// Up to `most` live extents of `ino` overlapping `[first, end)`, in + /// order, each of a kind this writer can replace on this volume. + async fn extents_in(&self, ino: u64, first: u64, end: u64, most: usize) -> Result)>> { let mut old = Vec::new(); Source::walk(self, Btree::Extents, Bpos::new(ino, first / SECTOR as u64 + 1, 0), |k, v| { if k.pos.inode != ino || start(k)? >= end { @@ -359,12 +484,9 @@ impl Mount { if k.pos.snapshot == ROOT_SNAPSHOT { old.push((*k, v.to_vec())); } - Ok(true) + Ok(old.len() < most) }) .await?; - if old.len() > 2 || (old.len() == 2 && !old.iter().any(|(k, _)| k.kind == kind::INLINE_DATA)) { - return Err(Error::Unsupported("multiple data extents in one filesystem block")); - } for (k, v) in &old { match k.kind { kind::EXTENT => { @@ -380,10 +502,11 @@ impl Mount { Ok(old) } + /// `bytes`, whole blocks from `first`, as one new extent at `sector` in + /// place of `old`, whose parts outside the range stay. #[allow(clippy::too_many_arguments)] - async fn replace_block(&self, txn: &mut Txn, ino: u64, inode: &mut InodeValue, first: u64, bytes: &[u8], version: Version, reserved: Option) -> Result<()> { + async fn replace_range(&self, txn: &mut Txn, ino: u64, inode: &mut InodeValue, first: u64, bytes: &[u8], old: Vec<(Key, Vec)>, version: Version, sector: u64) -> Result<()> { let end = first + bytes.len() as u64; - let old = self.block_extents(ino, first, end).await?; let mut retained = Vec::new(); for (k, v) in &old { let from = start(k)?; @@ -395,23 +518,15 @@ impl Mount { retained.push(slice(k, v, end, to)?); } } - let blocks = bytes.len().div_ceil(self.sb.block_bytes()) * self.sb.block_bytes() / BLOCK; + let blocks = bytes.len() / BLOCK; let mut data = alloc::vec![[0u8; BLOCK]; blocks]; - data.as_flattened_mut()[..bytes.len()].copy_from_slice(bytes); - let physical = (blocks * BLOCK / SECTOR) as u32; - let sector = match reserved { - Some(sector) => sector, - None => self.data_sector(u64::from(physical), Draw::Ordinary, true).await?.0, - }; - if reserved.is_none() { - self.st.borrow_mut().alloc.pin(sector / u64::from(self.sb.member.bucket_sectors)); - txn.allocated_data.push(sector); - } + data.as_flattened_mut().copy_from_slice(bytes); + let physical = (bytes.len() / SECTOR) as u32; let gen = self.bucket(txn, sector / u64::from(self.sb.member.bucket_sectors)).await?.reused_gen(); let ptr = Ptr { sector, dev: 0, gen, cached: false, unwritten: false }; let value = self.encode_data(&mut data, version, ptr, self.sb.flag(1, 9, 1) != 0)?; - let pos = Bpos::new(ino, (first + bytes.len() as u64) / SECTOR as u64, ROOT_SNAPSHOT); - let key = Key { size: (bytes.len() / SECTOR) as u32, version, ..Key::new(kind::EXTENT, pos) }; + let pos = Bpos::new(ino, end / SECTOR as u64, ROOT_SNAPSHOT); + let key = Key { size: physical, version, ..Key::new(kind::EXTENT, pos) }; self.reissue("file data write failed", async || { self.dev.write_data(sector * SECTOR as u64 / BLOCK as u64, &data).await?; Ok(()) diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 3e667993885..40d8dd21ae9 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -1209,9 +1209,6 @@ impl Mount { let mut txn = Txn::default(); let out = prepare(&mut txn).await; self.finish(txn, out.is_ok(), Class::Ordinary, bound, nodes)?; - if out.is_ok() { - if let Draw::Reserved(id) = draw { self.reservations.borrow_mut().active.remove(&id).expect("a completed reserved write consumes its reservation"); } - } out } diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index c84d0137cfa..7c58e6a10cc 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -552,3 +552,75 @@ fn evacuation_keeps_small_reads_within_one_physical_block() { block_on_ready(m.unmount()).unwrap(); } } + +/// Every live extent of `ino`: its start and length in bytes. +fn extents(v: &Volume<&Image>, ino: u64) -> Vec<(u64, u64)> { + let mut out = Vec::new(); + block_on_ready(Source::walk(v, Btree::Extents, Bpos::new(ino, 0, 0), |k, _| { + if k.pos.inode != ino { return Ok(false); } + if k.kind == kind::EXTENT { + out.push(((k.pos.offset - u64::from(k.size)) * SECTOR as u64, u64::from(k.size) * SECTOR as u64)); + } + Ok(true) + })) + .unwrap(); + out +} + +/// A write is an extent for each 64 KiB run, reserved or not: 256 KiB is +/// four, not a block each. +#[test] +fn a_write_is_one_extent_for_each_run_of_contiguous_blocks() { + let dev = formatted(256 << 20, None); + let data: Vec = (0..256 << 10).map(|i| (i * 7) as u8).collect(); + let (plain, reserved) = mounted(&dev, async |m: &Mount<&Image>| { + let plain = m.create(m.root(), b"plain", New::File, OWNER, NOW).await.unwrap().ino; + assert_eq!(m.write(plain, 0, &data, NOW).await.unwrap(), data.len()); + let reserved = m.create(m.root(), b"reserved", New::File, OWNER, NOW).await.unwrap().ino; + let mut token = m.reserve(reserved, 0, data.len()).await.unwrap(); + assert_eq!(m.write_reserved(&mut token, 0, &data, NOW).await.unwrap(), data.len()); + (plain, reserved) + }); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + for ino in [plain, reserved] { + assert_eq!(read_all(&v, ino).unwrap(), data); + assert_eq!(extents(&v, ino), [(0, 65536), (65536, 65536), (131072, 65536), (196608, 65536)]); + } + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} + +/// A write over more old extents than one transaction may remove goes in +/// parts, each removing at most two, and the file reads as written. +#[test] +fn an_overwrite_across_many_old_extents_goes_in_parts() { + let dev = formatted(256 << 20, None); + let ino = mounted(&dev, async |m: &Mount<&Image>| { + let ino = m.create(m.root(), b"f", New::File, OWNER, NOW).await.unwrap().ino; + // Every block its own extent: each written alone, in reverse. + for b in (0..16u64).rev() { m.write(ino, b * 4096, &[b as u8; 4096], NOW).await.unwrap(); } + let mut token = m.reserve(ino, 0, 64 << 10).await.unwrap(); + assert_eq!(m.write_reserved(&mut token, 0, &[0xab; 64 << 10], NOW).await.unwrap(), 64 << 10); + ino + }); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(read_all(&v, ino).unwrap(), [0xab; 64 << 10]); + let parts = extents(&v, ino); + assert!(parts.len() >= 8 && parts.iter().all(|&(_, len)| len >= 2 * 4096), "{parts:?}"); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} + +/// Blocks of a reservation never written return when it is dropped: a +/// reservation of 64 KiB written 16 KiB and dropped, 2000 times over one +/// file, fits a 64 MiB volume that could not hold the blocks never written. +#[test] +fn a_dropped_reservation_returns_its_unwritten_blocks() { + let dev = formatted(64 << 20, None); + mounted(&dev, async |m: &Mount<&Image>| { + let ino = m.create(m.root(), b"f", New::File, OWNER, NOW).await.unwrap().ino; + for i in 0..2000u32 { + let mut token = m.reserve(ino, 0, 64 << 10).await.unwrap_or_else(|e| panic!("reservation {i}: {e:?}")); + assert_eq!(m.write_reserved(&mut token, 0, &[i as u8; 16 << 10], NOW).await.unwrap(), 16 << 10); + if i % 64 == 0 { m.commit().await.unwrap(); } + } + }); +} diff --git a/lib/bcachefs/src/engine/tests/geometry.rs b/lib/bcachefs/src/engine/tests/geometry.rs index ef340650ca9..6bada031496 100644 --- a/lib/bcachefs/src/engine/tests/geometry.rs +++ b/lib/bcachefs/src/engine/tests/geometry.rs @@ -309,13 +309,13 @@ fn a_member_left_with_nothing_to_append_is_not_written() { let m = open(&dev, None, MEMORY); let kept = Executor::new().block_on(async { let mut kept = Vec::new(); - for i in 0..2800usize { + for i in 0..3600usize { let ino = m.create(m.root(), format!("d{i}").as_bytes(), New::File, OWNER, NOW).await.unwrap().ino; m.write(ino, 0, &vec![i as u8; quarter], NOW).await.unwrap(); if i % 4 == 0 { kept.push(ino); } if i % 64 == 63 { m.commit().await.unwrap(); } } - for i in 0..2800usize { + for i in 0..3600usize { if i % 4 != 0 { let removed = m.unlink(m.root(), format!("d{i}").as_bytes(), false, NOW).await.unwrap(); m.release(removed.node).await.unwrap(); @@ -334,11 +334,11 @@ fn a_member_left_with_nothing_to_append_is_not_written() { block_on_ready(super::super::read::Source::walk(&m, btree, Bpos::MIN, |_, _| Ok(true))).unwrap(); m.rewrite_next(btree); } - block_on_ready(m.set_mtime(kept[404], NOW + 9)).unwrap(); + block_on_ready(m.set_mtime(kept[485], NOW + 9)).unwrap(); let published = block_on_ready(m.unmount_in_one_publication()).unwrap(); assert!(published.idle > 0, "no member was left with nothing to append: {published:?}"); let v = block_on_ready(Volume::open(&dev)).unwrap(); - assert_eq!(block_on_ready(v.inode(kept[404])).unwrap().mtime, NOW + 9); + assert_eq!(block_on_ready(v.inode(kept[485])).unwrap().mtime, NOW + 9); assert_eq!(block_on_ready(v.names()).unwrap().len(), kept.len()); assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index 09b75273a71..0a8c15ac428 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -18,17 +18,32 @@ impl toyos_crypto::Random for OsRandom { getrandom::fill(out).map_err(|_| toyos_crypto::Error::Entropy) } } -struct Pending { - ticket: u64, +/// A write's capacity, and how many of its pages still need it. +struct Capacity { reservation: Reservation, + pages: usize, } struct Open { holders: usize, size: u64, mtime: u64, changed: bool, - pages: BTreeMap, - incoming: BTreeMap, + /// Each write's capacity, by its ticket. + capacity: BTreeMap, + /// Pages copied and not yet written back, each with its write's ticket. + pages: BTreeMap, + /// Pages the current write reserved and has not copied yet. + incoming: BTreeMap, +} +impl Open { + /// A page no longer needs `ticket`'s capacity. + fn release(&mut self, ticket: u64) { + let capacity = self.capacity.get_mut(&ticket).expect("a page's write holds capacity"); + capacity.pages -= 1; + if capacity.pages == 0 { + self.capacity.remove(&ticket); + } + } } pub struct DataVolume { fs: Mount>>, @@ -317,6 +332,7 @@ impl Volume for DataVolume { size: inode.size, mtime: inode.mtime.max(0) as u64, changed: false, + capacity: BTreeMap::new(), pages: BTreeMap::new(), incoming: BTreeMap::new(), }) @@ -340,6 +356,7 @@ impl Volume for DataVolume { let open = self.open.get_mut(&node).unwrap(); open.pages.clear(); open.incoming.clear(); + open.capacity.clear(); mapped("orphan close", self.cache.block_on(self.fs.release(node)))?; } else { self.persist(node)?; @@ -381,18 +398,12 @@ impl Volume for DataVolume { if len == 0 { return Ok(reserved); } - for page in offset / BLOCK as u64..end.div_ceil(BLOCK as u64) { - match self.cache.block_on(self.fs.reserve(node, page * BLOCK as u64, BLOCK)) { - Ok(reservation) => { - self.open.get_mut(&node).unwrap().incoming.insert(page, Pending { ticket, reservation }); - } - Err(error) => { - self.unreserve(node, reserved, 0)?; - return Err(word(error)); - } - } - } + let pages = offset / BLOCK as u64..end.div_ceil(BLOCK as u64); + let bytes = ((pages.end - pages.start) as usize).checked_mul(BLOCK).ok_or(Error::FileTooLarge)?; + let reservation = self.cache.block_on(self.fs.reserve(node, pages.start * BLOCK as u64, bytes)).map_err(word)?; let open = self.open.get_mut(&node).unwrap(); + open.capacity.insert(ticket, Capacity { reservation, pages: (pages.end - pages.start) as usize }); + open.incoming.extend(pages.map(|page| (page, ticket))); open.size = open.size.max(end); open.mtime = (self.clock)(); open.changed = true; @@ -402,15 +413,24 @@ impl Volume for DataVolume { if len == 0 { return; } let open = self.open.get_mut(&node).expect("a reserved inode"); for page in offset / BLOCK as u64..(offset + len as u64).div_ceil(BLOCK as u64) { - let pending = open.incoming.remove(&page).expect("each copied page has incoming capacity"); - assert_eq!(pending.ticket, reserved.ticket, "the write's incoming capacity"); - open.pages.insert(page, pending); + let ticket = open.incoming.remove(&page).expect("each copied page has incoming capacity"); + assert_eq!(ticket, reserved.ticket, "the write's incoming capacity"); + if let Some(older) = open.pages.insert(page, ticket) { + open.release(older); + } } } fn unreserve(&mut self, node: Node, reserved: Reserved, landed: u64) -> Result<(), Error> { let open = self.open.get_mut(&node).ok_or(Error::Stale)?; - open.incoming.retain(|_, pending| pending.ticket != reserved.ticket); - open.pages.retain(|&page, pending| pending.ticket != reserved.ticket || page * (BLOCK as u64) < landed); + let ticket = reserved.ticket; + let cut = |page: u64, held: u64, from: u64| held == ticket && page * BLOCK as u64 >= from; + let gone: Vec = open.incoming.iter().filter(|&(&p, &t)| cut(p, t, 0)).map(|(&p, _)| p) + .chain(open.pages.iter().filter(|&(&p, &t)| cut(p, t, landed)).map(|(&p, _)| p)).collect(); + for page in gone { + if open.incoming.remove(&page).or_else(|| open.pages.remove(&page)).is_some() { + open.release(ticket); + } + } open.size = reserved.size.max(landed); Ok(()) } @@ -421,23 +441,31 @@ impl Volume for DataVolume { { return Err((0, Error::InvalidRequest)); } - for (i, bytes) in data.chunks(BLOCK).enumerate() { - let page = offset / BLOCK as u64 + i as u64; - let done = i * BLOCK; - let pending = open.pages.get_mut(&page).ok_or((done, Error::Io))?; + let first = offset / BLOCK as u64; + let pages = data.len().div_ceil(BLOCK) as u64; + let mut done = 0; + // Each run of pages under one write's capacity goes out at once. + while done < data.len() { + let page = first + (done / BLOCK) as u64; + let ticket = *open.pages.get(&page).ok_or((done, Error::Io))?; + let run = (page..first + pages).take_while(|p| open.pages.get(p) == Some(&ticket)).count(); + let len = (run * BLOCK).min(data.len() - done); + let capacity = &mut open.capacity.get_mut(&ticket).expect("a page's write holds capacity").reservation; let written = mapped( "writeback", - self.cache.block_on(self.fs.write_reserved( - &mut pending.reservation, - page * BLOCK as u64, - bytes, - time(open.mtime), - )), - ).map_err(|error| (done, error))?; - if written != bytes.len() { - return Err((done, Error::Io)); + self.cache.block_on(self.fs.write_reserved(capacity, offset + done as u64, &data[done..done + len], time(open.mtime))), + ) + .map_err(|error| (done, error))?; + for p in page..page + written.div_ceil(BLOCK).min(run) as u64 { + if written >= ((p - page + 1) as usize * BLOCK).min(len) { + open.pages.remove(&p); + open.release(ticket); + } + } + if written != len { + return Err((done + written / BLOCK * BLOCK, Error::Io)); } - open.pages.remove(&page); + done += len; } Ok(()) } @@ -449,7 +477,9 @@ impl Volume for DataVolume { let now = (self.clock)(); mapped("truncate", self.cache.block_on(self.fs.truncate(node, size, time(now))))?; let open = self.open.get_mut(&node).unwrap(); - open.pages.split_off(&size.div_ceil(BLOCK as u64)); + for (_, ticket) in open.pages.split_off(&size.div_ceil(BLOCK as u64)) { + open.release(ticket); + } open.size = size; open.mtime = now; open.changed = true; diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index aa0f13ce5e5..1dbb9b65383 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -10,6 +10,9 @@ use crate::ram::Ram; use crate::volume::OpenHow; use toyos_blockdev::{Block, BlockDev}; +/// The bytes past the first page of the partial write-back's file. +const TAIL: usize = 100; + struct RefusingRead { ram: Ram, refusing: Rc>, @@ -19,7 +22,7 @@ impl BlockDev for RefusingRead { fn blocks(&self) -> u64 { self.ram.blocks() } async fn read(&self, lba: u64, out: &mut [Block]) -> Result<(), toyos_blockdev::Error> { self.ram.read(lba, out).await?; - if self.refusing.get() && out.contains(&[0xb2; toyos_blockdev::BLOCK]) { + if self.refusing.get() && out.iter().any(|block| block[..TAIL] == [0xb2; TAIL]) { return Err(toyos_blockdev::Error::Device); } Ok(()) @@ -521,13 +524,16 @@ fn an_overwrite_keeps_incoming_capacity_while_its_older_dirty_page_is_flushed() } #[test] +/// Two pages each under its own write's capacity, written back in one run: +/// the second, the file's last page in part, reads the old block it keeps +/// the rest of, and that read is refused. fn a_partial_writeback_cleans_only_its_prefix_and_retries_the_same_reserved_suffix() { let refusing = Rc::new(Cell::new(false)); let disk = RefusingRead { ram: Ram::new(16384), refusing: Rc::clone(&refusing) }; let mut v = DataVolume::format(disk, 256, &["home"], clock).unwrap(); let node = v.open("home/f", CREATE).unwrap(); let mut old = vec![0xa1; PAGE as usize]; - old.extend(vec![0xb2; PAGE as usize]); + old.extend([0xb2; TAIL]); v.write(node, 0, &old).unwrap(); v.close(node).unwrap(); v.sync().unwrap(); @@ -535,8 +541,9 @@ fn a_partial_writeback_cleans_only_its_prefix_and_retries_the_same_reserved_suff let fake = Fake::new(4, 4); let mut files = Files::new(Some(fake.clone())); let (node, id) = open(&mut files, &mut v, "home/f"); - let new = pattern(2 * PAGE as usize, 8); - files.write(&mut v, id, node, 0, &new).unwrap(); + let new = pattern(PAGE as usize + TAIL, 8); + files.write(&mut v, id, node, 0, &new[..PAGE as usize]).unwrap(); + files.write(&mut v, id, node, PAGE, &new[PAGE as usize..]).unwrap(); refusing.set(true); assert_eq!(files.fsync(&mut v, Some((id, node))), Err(Error::Io)); let object = files.object(id).unwrap(); diff --git a/system/fileserver/src/volume.rs b/system/fileserver/src/volume.rs index 1804d4269e3..ea9a745b285 100644 --- a/system/fileserver/src/volume.rs +++ b/system/fileserver/src/volume.rs @@ -252,42 +252,38 @@ impl Reserved { /// A write straight to the volume, for a file with no paged object — on a /// server with no pager, or one whose object went — and for a volume's -/// tests: the reserve, then each page it touches written back, what the page -/// held below the old contents' end read first. +/// tests: the reserve, then the pages it touches written back at once, what +/// a page it covers in part held below the old contents' end read first. pub trait Writes: Volume { fn write(&mut self, node: Node, offset: u64, data: &[u8]) -> Result<(), Error> { const PAGE: u64 = toyos_blockdev::BLOCK as u64; if data.is_empty() { return Ok(()); } let reserved = self.reserve(node, offset, data.len() as u64)?; - let size = match self.node_meta(node) { - Ok(meta) => meta.size, - Err(error) => { - self.unreserve(node, reserved, 0)?; - return Err(error); - } - }; let end = offset + data.len() as u64; - let mut landed = 0; - for start in (offset / PAGE * PAGE..end).step_by(PAGE as usize) { - let mut page = vec![0u8; PAGE as usize]; - let below = reserved.contents().saturating_sub(start).min(PAGE) as usize; - let (from, to) = (offset.max(start), end.min(start + PAGE)); - let written = self.read(node, start, &mut Buf(&mut page[..below])).and_then(|_| { - page[(from - start) as usize..(to - start) as usize] - .copy_from_slice(&data[(from - offset) as usize..(to - offset) as usize]); - self.landed(node, reserved, from, (to - from) as usize); - self.write_back(node, start, &page[..(size - start).min(PAGE) as usize]) - .map_err(|(_, error)| error) - }); - if let Err(e) = written { - if let Err(again) = self.unreserve(node, reserved, landed) { - println!("fileserver: a refused write was not cut back ({again:?})"); + let (first, last) = (offset / PAGE * PAGE, end.div_ceil(PAGE) * PAGE); + let mut pages = vec![0u8; (last - first) as usize]; + let mut put = || -> Result<(), (u64, Error)> { + let size = self.node_meta(node).map_err(|error| (0, error))?.size; + // A page the bytes cover whole needs nothing read. + let partial = [first, last - PAGE].into_iter().filter(|&start| offset > start || end < start + PAGE); + for start in partial.collect::>() { + let below = reserved.contents().saturating_sub(start).min(PAGE) as usize; + let at = (start - first) as usize; + if below > 0 { + self.read(node, start, &mut Buf(&mut pages[at..at + below])).map_err(|error| (0, error))?; } - return Err(e); } - landed = to; + pages[(offset - first) as usize..(end - first) as usize].copy_from_slice(data); + self.landed(node, reserved, offset, data.len()); + self.write_back(node, first, &pages[..(size - first).min(last - first) as usize]) + .map_err(|(done, error)| ((first + done as u64).min(end).max(offset) - offset, error)) + }; + let Err((landed, error)) = put() else { return Ok(()) }; + let landed = if landed == 0 { 0 } else { offset + landed }; + if let Err(again) = self.unreserve(node, reserved, landed) { + println!("fileserver: a refused write was not cut back ({again:?})"); } - Ok(()) + Err(error) } } From 8f52abe993db536d1bc3fbd3f9ed7e9efea2ac1a Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 16:55:34 +0100 Subject: [PATCH 72/93] tests: nvme_disk_boots expects DATA's own line DATA's server says "bcachefs DATA: / bytes free" since the native engine replaced the private format; the row still expected the private format's "bcachefs, ", and failed at f1b2f3e38 on jan-pc having seen the server serve. Co-Authored-By: Claude Opus 5.5 --- tests/toyos.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/toyos.rs b/tests/toyos.rs index f4f658fdec7..61dac2ef092 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -5387,7 +5387,7 @@ fn a_host_disk_beside_the_stick_is_left_alone(test_config: &Path) -> Result<(), const NVME_ROLES: [&str; 3] = [ "Log serving /log — FAT32, ", "Boot serving /boot — FAT32 read-only, ", - "Data serving /apps, /config, /home, /state — bcachefs, ", + "Data serving /apps, /config, /home, /state — bcachefs DATA: ", ]; /// Run `command` as a job of the boot, to exit 0: everything said in its /// window, the kernel's records with the program lines. From 45283fbc5a29b0fb9c455d709adf07d2f736eac9 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 17:18:59 +0100 Subject: [PATCH 73/93] fileserver: a DATA path is resolved once a call, from the names it knows A stat of DATA cost 40-50 us on the benches against main's 11.6. The client sends one STAT either way; the difference was the server. Main's private format answered a path from a map in memory; the native DATA volume walked it through the engine from the root for every question, and a STAT asked four: `resolve` looked up each prefix of the path to see whether it was a link, then `stat_path` asked `lstat` and `identity` of the whole path again. On the host model that was 1.7 us of engine CPU a lookup and no device reads (the counted blocks read are zero), so the time is CPU, repeated. Now: - The DATA volume remembers what each path it looked up names, inode and type, so a known path costs one inode read. Only an unlink, rmdir or rename takes a name away; each forgets the path and what lies under it. The memory is bounded. - `Volume::link_at` answers resolution's question, and a remembered name that is no link answers without the engine. `Volume::stat` answers a path's metadata and identity from one lookup. - STAT and OPEN resolve the directories, look the name up once, and resolve again following it only where it is a link and the call follows. A test renames, removes and makes again remembered names and finds what is there afterwards. Co-Authored-By: Claude Opus 5.5 --- system/fileserver/src/data.rs | 87 +++++++++++++++++++++++++++++++-- system/fileserver/src/main.rs | 39 ++++++++++++--- system/fileserver/src/volume.rs | 14 ++++++ 3 files changed, 128 insertions(+), 12 deletions(-) diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index 0a8c15ac428..73ce53af6ea 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -4,12 +4,15 @@ use crate::cache::{Cache, Disk}; use crate::volume::{Kind, Listed, Meta, Node, OpenHow, Out, Reserved, Volume, WriteBack, parent}; use bcachefs::engine::{self, DirentType, Mount, MountOptions, New, Owner, Reservation}; +use std::cell::RefCell; use std::collections::BTreeMap; use std::rc::Rc; use toyos::fs::Error; use toyos_blockdev::{BLOCK, BlockDev}; const NODES: usize = 512; +/// Names a volume remembers, at most. +const NAMES: usize = 4096; const READ_CHUNK: usize = 256 << 10; const OWNER: Owner = Owner { perm: 0o644, uid: 0, gid: 0 }; struct OsRandom; @@ -50,6 +53,10 @@ pub struct DataVolume { cache: Rc>, roots: Vec, open: BTreeMap, + /// What each path looked up names, by path, so the next lookup of it + /// reads its inode alone. Only an unlink, rmdir or rename takes a name + /// away; each forgets the path and what lies under it. + names: RefCell>, next_ticket: u64, clock: fn() -> u64, } @@ -167,6 +174,7 @@ impl DataVolume { cache, roots: roots.iter().map(|root| root.to_string()).collect(), open: BTreeMap::new(), + names: RefCell::new(BTreeMap::new()), next_ticket: 1, clock, }; @@ -190,16 +198,39 @@ impl DataVolume { Ok(volume) } fn lookup(&self, path: &str) -> Result { + let names: Vec<&str> = path.split('/').filter(|name| !name.is_empty()).collect(); self.cache .block_on(async { - let mut ino = self.fs.root(); - for name in path.split('/').filter(|name| !name.is_empty()) { - ino = self.fs.lookup(ino, name.as_bytes()).await?.ok_or(engine::Error::NotFound)?.ino; - } + let (ino, _) = self.entry(&names).await?; self.fs.inode(ino).await }) .map_err(word) } + /// What `names` reach from the root, from the deepest path known. + async fn entry(&self, names: &[&str]) -> engine::Result<(u64, DirentType)> { + let known = (1..=names.len()).rev().find_map(|n| self.names.borrow().get(&names[..n].join("/")).map(|&e| (n, e))); + let (mut at, mut found) = known.unwrap_or((0, (self.fs.root(), DirentType::Directory))); + while at < names.len() { + if found.1 != DirentType::Directory { + return Err(engine::Error::NotDirectory); + } + let entry = self.fs.lookup(found.0, names[at].as_bytes()).await?.ok_or(engine::Error::NotFound)?; + found = (entry.ino, entry.kind); + at += 1; + let mut known = self.names.borrow_mut(); + if known.len() >= NAMES { + known.clear(); + } + known.insert(names[..at].join("/"), found); + } + Ok(found) + } + /// `path` and everything beneath it moved or gone. + fn forget(&self, path: &str) { + let path = path.split('/').filter(|name| !name.is_empty()).collect::>().join("/"); + let below = format!("{path}/"); + self.names.borrow_mut().retain(|known, _| *known != path && !known.starts_with(&below)); + } fn destination<'a>(&self, path: &'a str) -> Result<(u64, &'a [u8]), Error> { let name = path.rsplit('/').next().filter(|name| !name.is_empty()).ok_or(Error::Root)?; let dir = self.lookup(parent(path))?; @@ -270,6 +301,22 @@ impl Volume for DataVolume { fn lstat(&mut self, path: &str) -> Result { self.meta(&self.lookup(path)?) } + fn stat(&mut self, path: &str) -> Result<(Meta, u64), Error> { + let inode = self.lookup(path)?; + Ok((self.meta(&inode)?, inode.ino)) + } + fn link_at(&mut self, path: &str) -> Result, Error> { + let known = path.split('/').filter(|name| !name.is_empty()).collect::>().join("/"); + if self.names.borrow().get(&known).is_some_and(|&(_, kind)| kind != DirentType::Symlink) { + return Ok(None); + } + let inode = self.lookup(path)?; + if kind(&inode)? != Kind::Symlink { + return Ok(None); + } + let bytes = mapped("readlink", self.cache.block_on(self.fs.readlink(inode.ino)))?; + String::from_utf8(bytes).map(Some).map_err(|_| Error::Io) + } fn read_link(&mut self, path: &str) -> Result { let inode = self.lookup(path)?; if kind(&inode)? != Kind::Symlink { @@ -513,6 +560,7 @@ impl Volume for DataVolume { return Err(Error::NotADirectory); } let (dir, name) = self.destination(path)?; + self.forget(path); mapped("rmdir", self.cache.block_on(self.fs.unlink(dir, name, true, time((self.clock)())))).map(drop) } fn unlink(&mut self, path: &str) -> Result<(), Error> { @@ -520,6 +568,7 @@ impl Volume for DataVolume { return Err(Error::Root); } let (dir, name) = self.destination(path)?; + self.forget(path); let removed = mapped("unlink", self.cache.block_on(self.fs.unlink(dir, name, false, time((self.clock)()))))?; self.released(Some(removed)) } @@ -529,6 +578,8 @@ impl Volume for DataVolume { } let source = self.destination(from)?; let target = self.destination(to)?; + self.forget(from); + self.forget(to); let removed = mapped( "rename", self.cache.block_on(self.fs.rename(source, target, engine::Rename::Replace, time((self.clock)()))), @@ -665,6 +716,34 @@ mod tests { let mut v = reopen(v); assert!(v.list("home").unwrap().is_empty()); } + /// A path, once looked up, is remembered; an unlink, rmdir or rename + /// forgets it and everything under it. + #[test] + fn a_remembered_name_follows_its_renames_and_removal() { + let mut v = volume(); + v.mkdir("home/a").unwrap(); + v.mkdir("home/a/b").unwrap(); + let f = v.open("home/a/b/f", CREATE).unwrap(); + v.close(f).unwrap(); + assert_eq!(v.lstat("home/a/b/f").unwrap().kind, Kind::File); + v.rename("home/a", "home/c").unwrap(); + assert_eq!(v.lstat("home/a/b/f"), Err(Error::NotFound)); + assert_eq!(v.lstat("home/c/b/f").unwrap().kind, Kind::File); + v.unlink("home/c/b/f").unwrap(); + v.rmdir("home/c/b").unwrap(); + v.mkdir("home/c/b").unwrap(); + let g = v.open("home/c/b/g", CREATE).unwrap(); + v.close(g).unwrap(); + assert_eq!(v.stat("home/c/b/g").unwrap().1, g, "the new directory's entry, not the removed one's"); + assert_eq!(v.lstat("home/c/b/f"), Err(Error::NotFound)); + v.unlink("home/c/b/g").unwrap(); + assert_eq!(v.lstat("home/c/b/g"), Err(Error::NotFound)); + let again = v.open("home/c/b/g", CREATE).unwrap(); + v.close(again).unwrap(); + assert_eq!(v.stat("home/c/b/g").unwrap().1, again, "a name made again names its new file"); + assert_ne!(again, g); + } + #[test] fn directory_moves_preserve_descendants_and_symlinks() { let mut v = volume(); diff --git a/system/fileserver/src/main.rs b/system/fileserver/src/main.rs index 67dc895e9da..1efc3963e44 100644 --- a/system/fileserver/src/main.rs +++ b/system/fileserver/src/main.rs @@ -706,9 +706,9 @@ impl Server { bounds::relative(rel)?; let (root, start) = self.start(id, fid)?; let volume = &mut self.volume; - let mut lookup = |p: &str| match volume.lstat(p) { - Ok(meta) if meta.kind == Kind::Symlink => volume.read_link(p).map(Found::Link).map_err(drop), - Ok(_) | Err(Error::NotFound) => Ok(Found::Other), + let mut lookup = |p: &str| match volume.link_at(p) { + Ok(Some(target)) => Ok(Found::Link(target)), + Ok(None) | Err(Error::NotFound) => Ok(Found::Other), Err(_) => Err(()), }; resolve::resolve(&root, &start, rel, follow_last, &mut lookup).map_err(|e| match e { @@ -801,8 +801,7 @@ impl Server { /// What the volume path `path` is. fn stat_path(&mut self, path: &str) -> Result { - let meta = self.volume.lstat(path)?; - let identity = self.volume.identity(path)?; + let (meta, identity) = self.volume.stat(path)?; Ok(Answer::Reply(stat_reply(meta, identity, self.volume_id, 1))) } @@ -907,7 +906,18 @@ impl Server { let file = self.settled(id, r.fid)?; return self.stat_held(&file); } - match self.resolve(id, r.fid, &rel, r.flags & FOLLOW_LAST != 0)? { + // The name is stated as it is, and followed only where it is + // a link to follow: one lookup of it in the common case. + let follow = r.flags & FOLLOW_LAST != 0; + let path = match self.resolve(id, r.fid, &rel, false)? { + Resolved::Absolute(p) => return self.follow(id, &r, &p), + Resolved::Path(p) => p, + }; + let (meta, identity) = self.volume.stat(&path)?; + if !(follow && meta.kind == Kind::Symlink) { + return Ok(Answer::Reply(stat_reply(meta, identity, self.volume_id, 1))); + } + match self.resolve(id, r.fid, &rel, true)? { Resolved::Absolute(p) => self.follow(id, &r, &p), Resolved::Path(p) => self.stat_path(&p), } @@ -1096,7 +1106,9 @@ impl Server { return Err(Error::InvalidRequest); } let rel = self.path(id, r)?; - let path = match self.resolve(id, r.fid, &rel, r.flags & O_CREATE_NEW == 0)? { + // The name is looked up as it is, and followed only where it is a + // link to follow: one lookup of it in the common case. + let mut path = match self.resolve(id, r.fid, &rel, false)? { Resolved::Absolute(p) => return self.follow(id, r, &p), Resolved::Path(p) => p, }; @@ -1104,11 +1116,22 @@ impl Server { return Err(Error::TooManyOpen); } let writes = r.flags & (O_WRITE | O_APPEND | O_TRUNCATE) != 0; - let found = match self.volume.lstat(&path) { + let mut found = match self.volume.lstat(&path) { Ok(meta) => Some(meta.kind), Err(Error::NotFound) => None, Err(e) => return Err(e), }; + if found == Some(Kind::Symlink) && r.flags & O_CREATE_NEW == 0 { + path = match self.resolve(id, r.fid, &rel, true)? { + Resolved::Absolute(p) => return self.follow(id, r, &p), + Resolved::Path(p) => p, + }; + found = match self.volume.lstat(&path) { + Ok(meta) => Some(meta.kind), + Err(Error::NotFound) => None, + Err(e) => return Err(e), + }; + } let (held, meta) = match found { Some(_) if r.flags & O_CREATE_NEW != 0 => return Err(Error::AlreadyExists), Some(Kind::Dir) if writes => return Err(Error::IsADirectory), diff --git a/system/fileserver/src/volume.rs b/system/fileserver/src/volume.rs index ea9a745b285..2b29716cbd3 100644 --- a/system/fileserver/src/volume.rs +++ b/system/fileserver/src/volume.rs @@ -125,6 +125,20 @@ pub trait Volume { /// What the symlink at `path` holds. fn read_link(&mut self, path: &str) -> Result; + /// What `path` is and its [`Self::identity`], without following it. + fn stat(&mut self, path: &str) -> Result<(Meta, u64), Error> { + Ok((self.lstat(path)?, self.identity(path)?)) + } + + /// What the symlink at `path` holds, or `None` for anything else: a + /// component of a path being resolved. + fn link_at(&mut self, path: &str) -> Result, Error> { + match self.lstat(path)?.kind { + Kind::Symlink => self.read_link(path).map(Some), + _ => Ok(None), + } + } + /// The entries directly under `dir`. fn list(&mut self, dir: &str) -> Result, Error>; From c836688ceb94f971601437147aa10e68945e86b5 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 22:59:17 +0100 Subject: [PATCH 74/93] bcachefs: a node takes a position's pending deltas in one pass, once a transaction osbench_persist hung on both benches in its first DATA mkdir after a reboot. Reproduced on a host model of the T14's DATA: a volume written with osbench's call sequence and dropped unclean replays 1.5 million journal keys into pending, most of them accounting deltas at a handful of positions. The first transaction's leaf drain captured every delta and transferred them one at a time, and each transfer walked every entry at its position: about 0.45 ms a delta, hours for the leaf. The mount had finished; the mkdir never returned. Now take_into captures a position once, and Pending::transfer installs every accounting delta at it up to the capture in one pass over the position; an ordinary key still installs its newest value. On the same image the first mkdir takes 0.33 s. The drain in apply also changes (r3 BLOCKER 2): a node takes from pending at most once a transaction, within the room the transaction's own records leave of K_TX, so its keys and what its nodes take grow a node by at most one transaction's bound. That is what MAX_GROUPS' "two transactions more" assumes; before, a node drained again for each of the transaction's keys. take_into's first-position exception goes: a key is at most MAX_KEY, under K_TX's bytes, so materialization still always takes its first position. Tests: the first transaction after a replay of 2 000 creates looks at at most 16 pending entries per replayed one (the per-delta transfer looked at 262 657 for 1 551, red); renames over files and creates after such a replay grow no node by more than K_TX*8 bytes (draining for every key: 6 872 bytes, red; a room of the whole bound: 4 224 bytes, red). Mutations applied, run and restored in scratch. Co-Authored-By: Claude Opus 5.5 --- ...achefs-node-past-seven-nodes-is-refused.md | 30 ---------- lib/bcachefs/src/engine/mount/mod.rs | 44 +++++++++++--- lib/bcachefs/src/engine/mount/reclaim.rs | 22 +++---- lib/bcachefs/src/engine/pending.rs | 26 ++++---- lib/bcachefs/src/engine/tests/admission.rs | 60 +++++++++++++++++++ 5 files changed, 122 insertions(+), 60 deletions(-) delete mode 100644 issues/a-replayed-bcachefs-node-past-seven-nodes-is-refused.md diff --git a/issues/a-replayed-bcachefs-node-past-seven-nodes-is-refused.md b/issues/a-replayed-bcachefs-node-past-seven-nodes-is-refused.md deleted file mode 100644 index 6a8466dc7a9..00000000000 --- a/issues/a-replayed-bcachefs-node-past-seven-nodes-is-refused.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-23 ---- - -# A replayed bcachefs node whose keys fill more than seven nodes stops the read-write mount - -The writer splits a node in one reclaim transaction, into at most seven -nodes and a root (`MAX_GROUPS` in `lib/bcachefs/src/engine/mount/reclaim.rs`). -That bound holds for a node it wrote, and for one read from the medium, packed, -plus two transactions. It does not hold for a node a replay fills: -`Mount::install_replay` puts every key of the journal window into the leaf -its on-disk tree leads to, with no bound. A window that puts more than seven -nodes' keys into one leaf is reachable on an unclean upstream volume. Examples -are many inodes journaled since their leaf was last written, or keys journaled -into split nodes whose pointer update was not yet journaled. - -The reader replays such a volume in memory and reads it whole. The -read-write mount opens it, but its first write that needs the leaf split -stops the mount, read-only by name: "a btree node whose keys fill more than -seven nodes". `tests/journal.rs`, -`a_replayed_leaf_past_seven_nodes_is_read_but_not_mounted_read_write`, builds -the case: about 2 860 inodes in 13 entries of a 64 MiB volume's window. - -**Exit.** A split that spans several reclaim transactions for a node no -single transaction can split. For instance, write the window's keys out as a -run of new leaves under the old leaf's parent, a transaction at a time, -before the old leaf's pointer is replaced. Alternatively, a replay that -installs keys into nodes bounded as it goes. diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 40d8dd21ae9..f9d2c2901e0 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -156,6 +156,9 @@ struct State { /// The most dirty and unpointed nodes ever held together. #[cfg(test)] peak: usize, + /// The most bytes one ordinary apply grew a node by. + #[cfg(test)] + grown: usize, /// Ordinary admissions that waited for journal space, and for cache /// room, as tests construct them. #[cfg(test)] @@ -485,6 +488,8 @@ impl Mount { #[cfg(test)] peak: 0, #[cfg(test)] + grown: 0, + #[cfg(test)] waits: [0; 2], #[cfg(test)] journal_bytes: [0; 3], @@ -1329,19 +1334,27 @@ impl Mount { (Btree::Accounting, 0, key, value::counters(&delta)) }); // An ordinary transaction's keys go into the nodes it holds, as - // reclaim's materialization would put them; each node first takes - // what pending holds of its range, so nothing older lies over them. - // A node that cannot take them now, and reclaim's own updates, - // leave them to pending and reclaim. + // reclaim's materialization would put them, where pending holds + // nothing older at their position. A node that cannot take them now, + // and reclaim's own updates, leave them to pending and reclaim. + let room = match class { + Class::Ordinary => K_TX.checked_sub(size).expect("an ordinary transaction within K_TX") as usize * 8, + Class::Reclaim => 0, + }; + let mut drained = Vec::new(); let mut left = Vec::new(); for (btree, level, key, value) in updates.chain(accounting) { let id = (class == Class::Ordinary).then(|| held(st, btree, level, key.pos)).filter(|&id| st.cache.node(id).captured.is_none()); - let Some(id) = id.filter(|&id| self.drained(st, id, key.pos)) else { + let Some(id) = id.filter(|&id| self.drained(st, id, key.pos, room, &mut drained)) else { left.push((btree, level, key, value)); continue; }; st.cache.change(id, |n| put(n, key, value, seq)); } + #[cfg(test)] + for &(id, before) in &drained { + st.grown = st.grown.max(st.cache.node(id).bytes.saturating_sub(before)); + } st.pending.apply(seq, left); for root in core::mem::take(&mut txn.roots) { st.roots.insert(root.btree, root); @@ -1353,12 +1366,18 @@ impl Mount { } } - /// Whether node `id` takes a key at `pos` in place: it first takes what - /// pending holds of its range, and pending holds nothing at `pos` after. - fn drained(&self, st: &mut State, id: NodeId, pos: Bpos) -> bool { + /// Whether node `id` takes a key at `pos` in place: pending holds + /// nothing at `pos`. A node the transaction has not yet `drained` first + /// takes what pending holds of its range within `room`, the transaction + /// bound its own keys leave, so together they grow it by at most one + /// transaction's. + fn drained(&self, st: &mut State, id: NodeId, pos: Bpos, room: usize, drained: &mut Vec<(NodeId, usize)>) -> bool { let n = st.cache.node(id); let (btree, level, min, max) = (n.btree, n.level, n.min, n.max); - if st.pending.holds(btree, level, min, max) && self.take_into(st, id, min).is_err() { return false; } + if !drained.iter().any(|&(d, _)| d == id) { + drained.push((id, n.bytes)); + if st.pending.holds(btree, level, min, max) && self.take_into(st, id, min, room).is_err() { return false; } + } !st.pending.holds(btree, level, pos, pos) } @@ -1652,6 +1671,13 @@ impl Mount { self.st.borrow_mut().pending.shrink(); } + /// The most bytes one ordinary transaction grew a node by, its keys and + /// what the node took from pending together. + #[cfg(test)] + pub(crate) fn grown(&self) -> usize { + self.st.borrow().grown + } + /// The dirty-node limit, reclaim's room, the most dirty and unpointed /// nodes held at once, and the ordinary admissions that waited for /// journal space and for cache room. diff --git a/lib/bcachefs/src/engine/mount/reclaim.rs b/lib/bcachefs/src/engine/mount/reclaim.rs index 2d59af335c0..920e84ebf90 100644 --- a/lib/bcachefs/src/engine/mount/reclaim.rs +++ b/lib/bcachefs/src/engine/mount/reclaim.rs @@ -132,8 +132,9 @@ pub(super) const fn fresh_nodes(nodes: usize) -> usize { /// five (A3.15-A3.17). Either takes at most two transactions more before /// reclaim, which takes an oversized node first, plans it: the one holding /// the slot, as no transaction enters it while a changed node is oversized -/// ([`Mount::transact`]), and the pointer update of the one reclaim round in -/// flight, each at most `K_TX` u64s of keys. Spread evenly, all that fills +/// ([`Mount::transact`]), its keys and what its nodes take from pending +/// together, and the pointer update of the one reclaim round in flight, +/// each at most `K_TX` u64s of keys. Spread evenly, all that fills /// this many nodes of any size the writer accepts, each a key short of full. /// Materialization takes at most one transaction of distinct key positions, /// including during replay. @@ -418,13 +419,13 @@ impl Mount { let mut st = self.st.borrow_mut(); let n = st.cache.node(id); if n.writing || n.captured.is_some() || n.held != 0 { return Ok(false); } - self.take_into(&mut st, id, pos) + self.take_into(&mut st, id, pos, K_TX as usize * 8) } /// Transfers pending keys of node `id`'s range into it, from `pos` on - /// and round to it, with bounded positive growth. The caller has found - /// the node free to change. - pub(super) fn take_into(&self, st: &mut State, id: NodeId, pos: Bpos) -> Result { + /// and round to it, each position whose growth still fits `room` bytes; + /// deletions take none. The caller has found the node free to change. + pub(super) fn take_into(&self, st: &mut State, id: NodeId, pos: Bpos, room: usize) -> Result { let n = st.cache.node(id); let (btree, level) = (n.btree, n.level); let mut captures = Vec::new(); @@ -433,18 +434,19 @@ impl Mount { .chain(st.pending.range(btree, level, n.min, pos).take_while(|e| e.key().pos < pos)).peekable(); while let Some(first) = pending.peek() { let at = first.key().pos; - let start = captures.len(); let before = n.keys.get(n.at(at)).filter(|(k, _)| k.pos == at).map_or(0, |(_, v)| key_bytes(v)); let mut size = if btree == Btree::Accounting && level == 0 { before } else { 0 }; + let mut newest = None; while pending.peek().is_some_and(|e| e.key().pos == at) { let entry = pending.next().unwrap(); if entry.key().kind != kind::DELETED { size = size.max(key_bytes(entry.value())); } - captures.try_reserve(1).map_err(|_| Error::NoMemory)?; - captures.push(entry.capture()); + newest = Some(entry.capture()); } let added = size.saturating_sub(before); - if growth + added > K_TX as usize * 8 && start != 0 { captures.truncate(start); continue; } + if growth + added > room { continue; } growth += added; + captures.try_reserve(1).map_err(|_| Error::NoMemory)?; + captures.push(newest.expect("a position pending holds has an entry")); } let durable = st.journal.durable; let mut accounting = None; diff --git a/lib/bcachefs/src/engine/pending.rs b/lib/bcachefs/src/engine/pending.rs index fd3bd9de724..8fe5ef908f8 100644 --- a/lib/bcachefs/src/engine/pending.rs +++ b/lib/bcachefs/src/engine/pending.rs @@ -63,7 +63,7 @@ fn visited() -> usize { VISITED.with(Cell::get) } /// What `f` returns, and the entries pending looked at meanwhile. #[cfg(test)] -fn visits(f: impl FnOnce() -> T) -> (T, usize) { +pub(crate) fn visits(f: impl FnOnce() -> T) -> (T, usize) { let before = visited(); let out = f(); (out, visited() - before) @@ -394,19 +394,23 @@ impl Pending { Iter { pending: self, at, end: bounds.map(|(_, max)| max), saved: None } } - /// The callback installs the value and its pin before the source retires. - /// It is infallible and cannot yield. `false` means capture was superseded. - pub fn transfer(&mut self, capture: Capture, install: impl FnOnce(Transferred)) -> bool { - let wanted = (capture.position, capture.order); - let Some(source) = self.runs.iter_mut().flatten().find_map(|run| run.find(wanted).map(|i| &mut run.entries[i])) else { return false }; - let accounting = source.accounting(); - let value = core::mem::take(&mut source.value); - install(Transferred { key: source.key, value, first: capture.first, last: source.last }); + /// The callback installs the captured value, or every accounting delta + /// at its position up to it, each with its pin, before the sources + /// retire: one pass over the position. It is infallible and cannot + /// yield. `false` means capture was superseded. + pub fn transfer(&mut self, capture: Capture, mut install: impl FnMut(Transferred)) -> bool { + if self.entry((capture.position, capture.order)).is_none() { return false; } for slot in &mut self.runs { let Some(run) = slot else { continue }; for i in run.at(capture.position) { - let entry = &run.entries[i]; - if entry.live && entry.order <= capture.order && (entry.order == capture.order || !accounting) { run.kill(i, &mut self.live); } + visit(); + let entry = &mut run.entries[i]; + if !entry.live || entry.order > capture.order { continue; } + if entry.accounting() || entry.order == capture.order { + let first = if entry.accounting() { entry.first } else { capture.first }; + install(Transferred { key: entry.key, value: core::mem::take(&mut entry.value), first, last: entry.last }); + } + run.kill(i, &mut self.live); } run.compact(); run.retire(); diff --git a/lib/bcachefs/src/engine/tests/admission.rs b/lib/bcachefs/src/engine/tests/admission.rs index e642b6041e1..75e32d787f9 100644 --- a/lib/bcachefs/src/engine/tests/admission.rs +++ b/lib/bcachefs/src/engine/tests/admission.rs @@ -125,6 +125,66 @@ fn committed_keys_go_into_the_leaves_they_change_and_replay_stays_pending() { assert_eq!(block_on_ready(fsck(&volume)).unwrap(), []); } +/// An unclean mount replays every transaction's accounting deltas, many at +/// each accounting position. The first transaction after takes its nodes' +/// pending keys, each entry looked at a bounded number of times. Red when +/// each delta's transfer passes over all its position's: quadratic in the +/// journal, which stalled the first write after a crash on a large volume. +#[test] +fn the_first_transaction_after_a_replay_takes_each_delta_once() { + let dev = formatted(64 << 20, None); + let options = MountOptions { cache_nodes: SMALL_CACHE }; + let mount = block_on_ready(Mount::open(&dev, options, random())).unwrap(); + for i in 0..2000 { block_on_ready(mount.create(mount.root(), format!("delta{i}").as_bytes(), New::File, OWNER, NOW)).unwrap(); } + drop(mount); + + let mount = block_on_ready(Mount::open(&dev, options, random())).unwrap(); + let (replayed, _, _) = mount.pending_pressure(); + assert!(replayed >= 1000, "replay holds a thousand entries and more: {replayed}"); + let (created, visited) = super::super::pending::visits(|| block_on_ready(mount.create(mount.root(), b"after", New::File, OWNER, NOW))); + created.unwrap(); + // A few passes over each entry, where a delta's transfer passing over + // its position's would look at about a thousand per delta. + assert!(visited <= 16 * replayed, "{visited} pending entries looked at for {replayed} replayed"); + block_on_ready(mount.unmount()).unwrap(); + let volume = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(block_on_ready(fsck(&volume)).unwrap(), []); +} + +/// After a replay left many pending keys in the ranges of the root's leaves: +/// renames over other files, each changing two neighbouring inodes, and, on +/// another such replay, creates. What a leaf takes from pending and a +/// transaction's keys grow it by at most one transaction's bound, as +/// reclaim's split bound assumes (`MAX_GROUPS`). Red when a node takes from +/// pending for each key (the renames), or past the room the keys leave (the +/// creates). +#[test] +fn a_transaction_after_a_replay_grows_a_node_by_one_transaction_at_most() { + const OPTIONS: MountOptions = MountOptions { cache_nodes: SMALL_CACHE }; + for what in ["renames", "creates"] { + let dev = formatted(64 << 20, None); + let mount = block_on_ready(Mount::open(&dev, OPTIONS, random())).unwrap(); + for i in 0..2000 { block_on_ready(mount.create(mount.root(), format!("file{i}").as_bytes(), New::File, OWNER, NOW)).unwrap(); } + block_on_ready(mount.commit()).unwrap(); + drop(mount); + let mount = block_on_ready(Mount::open(&dev, OPTIONS, random())).unwrap(); + if what == "renames" { + for i in (1960..2000).step_by(2) { + let (from, to) = (format!("file{i}"), format!("file{}", i + 1)); + block_on_ready(mount.rename((mount.root(), from.as_bytes()), (mount.root(), to.as_bytes()), Rename::Replace, NOW)).unwrap(); + } + } else { + for i in 0..16 { block_on_ready(mount.create(mount.root(), format!("after{i}").as_bytes(), New::File, OWNER, NOW)).unwrap(); } + } + let grown = mount.grown(); + assert!(grown > 0, "{what} put keys in place"); + assert!(grown <= super::super::mount::K_TX as usize * 8, "{what} grew a node by {grown} bytes in one transaction"); + block_on_ready(mount.unmount()).unwrap(); + let volume = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(block_on_ready(fsck(&volume)).unwrap(), [], "{what}"); + } +} + /// Creates past what the journal holds, with no commit: entries fill and are /// written, the ring fills with entries the dirty nodes pin, and a writer /// waits for journal space until reclaim writes the nodes. Red when a From af6ae82938804d6817cb1d81f5640f5583b9c980 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 22:59:18 +0100 Subject: [PATCH 75/93] bcachefs: a reserved write's extent points at the block holding its offset write_reserved computed the extent's sector from `at` rather than from the block holding it, while write_run writes from that block's start: on a volume of 16 KiB blocks, a write at 4096 pointed 8 sectors past its block, and its last 4 KiB landed past the reservation (r3 BLOCKER 1). Test: on 16 KiB blocks, a reserved write at 4096 reads back, the file written beside it is unchanged, and fsck is clean. With the old sector it reads as Checksum("extent"), red. Co-Authored-By: Claude Opus 5.5 --- lib/bcachefs/src/engine/mount/data.rs | 3 ++- lib/bcachefs/src/engine/tests/data.rs | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index 7e04e0fc2b7..6dbf9a7c750 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -176,6 +176,7 @@ impl Mount { let mut done = 0; while done < data.len() { let at = offset + done as u64; + // The extent starts at the block holding `at`, and so do its sectors. let (ino, sector, end) = { let pool = self.reservations.borrow(); let reserved = pool.active.get(&reservation.id).ok_or(Error::Invalid("reservation already written"))?; @@ -185,7 +186,7 @@ impl Mount { if (from..to).any(|b| run.written & 1 << b != 0) { return Err(Error::Invalid("a reserved block written twice")); } - (reserved.ino, run.sector + (at - run.first) / SECTOR as u64, end) + (reserved.ino, run.sector + (at / block * block - run.first) / SECTOR as u64, end) }; let version = match self.data_version().await { Ok(version) => version, diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index 7c58e6a10cc..4234dcb602e 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -624,3 +624,25 @@ fn a_dropped_reservation_returns_its_unwritten_blocks() { } }); } + +/// A reserved write into a block past its first 4 KiB, on 16 KiB blocks: +/// the extent is the block's, at the block's own sectors. Its neighbour, +/// written just after the reservation took its run, keeps its bytes. +#[test] +fn a_reserved_write_inside_a_large_block_lands_on_that_block() { + let dev = super::geometry::shaped(64 << 20, 32, 32, None); + let (reserved, neighbour) = mounted(&dev, async |m: &Mount<&Image>| { + let reserved = m.create(m.root(), b"reserved", New::File, OWNER, NOW).await.unwrap().ino; + let mut token = m.reserve(reserved, 4096, 4096).await.unwrap(); + let neighbour = m.create(m.root(), b"neighbour", New::File, OWNER, NOW).await.unwrap().ino; + assert_eq!(m.write(neighbour, 0, &[0x5a; 16 << 10], NOW).await.unwrap(), 16 << 10); + assert_eq!(m.write_reserved(&mut token, 4096, &[0xc3; 4096], NOW).await.unwrap(), 4096); + (reserved, neighbour) + }); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + let mut want = vec![0u8; 8192]; + want[4096..].fill(0xc3); + assert_eq!(read_all(&v, reserved).unwrap(), want); + assert_eq!(read_all(&v, neighbour).unwrap(), [0x5a; 16 << 10], "the neighbour's bytes"); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} From dfa6e2166d2bf613141dd2095796ac83f10935f7 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 22:59:18 +0100 Subject: [PATCH 76/93] bcachefs: a whole superblock copy of another volume refuses the volume read_superblock checked identity only for copies that failed to read, so a valid copy naming another volume was accepted, and chosen if its sequence was higher (r3 BLOCKER 3). A readable copy whose internal UUID is not the primary's (or, without a readable primary, the chosen copy's) is now Overwritten at its offset. Test: another volume's whole copy in the second copy's place, from a format with another random source, is refused at 2056; before the fix it opened. The flipped-byte case is relabelled as the damaged copy it is. Co-Authored-By: Claude Opus 5.5 --- lib/bcachefs/src/engine/read.rs | 13 +++++++++---- lib/bcachefs/src/engine/tests/mod.rs | 11 ++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/bcachefs/src/engine/read.rs b/lib/bcachefs/src/engine/read.rs index 403bfec554b..8344ec989cf 100644 --- a/lib/bcachefs/src/engine/read.rs +++ b/lib/bcachefs/src/engine/read.rs @@ -820,10 +820,11 @@ pub(crate) async fn read_bytes(dev: &impl BlockDev, at: u64, len: usize) -> Resu /// did not read whole: what a crash in a copy's write leaves. /// /// Every place the layout gives the volume must hold the volume's own bytes: -/// a copy whose first sector does not name this volume and its own offset, -/// or a layout copy other than the chosen copy's layout, is another writer's, -/// and the volume is refused as [`Error::Overwritten`] rather than mounted -/// from what is left. +/// a whole copy of another volume than the primary's, a copy whose first +/// sector does not name this volume and its own offset, or a layout copy +/// other than the chosen copy's layout, is another writer's, and the volume +/// is refused as [`Error::Overwritten`] rather than mounted from what is +/// left. pub(crate) async fn read_superblock(dev: &impl BlockDev) -> Result<(Superblock, Vec)> { let primary = read_copy(dev, PRIMARY).await; let layout_copy = read_bytes(dev, LAYOUT * SECTOR as u64, Layout::BYTES).await?; @@ -856,6 +857,10 @@ pub(crate) async fn read_superblock(dev: &impl BlockDev) -> Result<(Superblock, let (_, failure) = copies.into_iter().next().expect("the primary is read"); return Err(failure.err().unwrap_or(Error::Corrupt("no superblock copy"))); }; + let own = copies[0].1.as_ref().map_or(best.header.internal_uuid, |sb| sb.header.internal_uuid); + if let Some((offset, _)) = copies.iter().find(|(_, read)| read.as_ref().is_ok_and(|sb| sb.header.internal_uuid != own)) { + return Err(Error::Overwritten { sector: *offset }); + } if Layout::decode(&layout_copy).ok().as_ref() != Some(&best.header.layout) { return Err(Error::Overwritten { sector: LAYOUT }); } diff --git a/lib/bcachefs/src/engine/tests/mod.rs b/lib/bcachefs/src/engine/tests/mod.rs index c1bb9e998c4..a88f6d6783d 100644 --- a/lib/bcachefs/src/engine/tests/mod.rs +++ b/lib/bcachefs/src/engine/tests/mod.rs @@ -247,7 +247,7 @@ fn a_torn_primary_superblock_is_read_from_another_copy_and_named() { } /// A copy's place holding what is not this volume's copy there — nothing, -/// another volume's copy, a copy of this one at another offset, or another +/// another volume's copy, whole or damaged, a copy of this one at another offset, or another /// layout — is another writer's, and the volume is refused, not opened from /// the copies left. #[test] @@ -260,10 +260,15 @@ fn a_superblock_place_another_writer_took_refuses_the_volume() { }; let primary = 8 * SECTOR; let second = 2056 * SECTOR; + let other = Bytes::new(vec![0u8; 32 << 20]); + let options = Options { label: b"toyos", time: 1_761_177_600_000_000_000, encryption: None }; + block_on_ready(mkfs(&other, &options, &Tree::new(), &mut Seeded(0x07e4))).unwrap(); + let other = other.into_inner(); type Edit<'a> = &'a dyn Fn(&mut Vec); - let cases: [(&str, Edit, u64); 6] = [ + let cases: [(&str, Edit, u64); 7] = [ ("a zeroed primary", &|raw| raw[primary..primary + BLOCK].fill(0), 8), - ("another volume's UUID", &|raw| raw[second + 40] ^= 1, 2056), + ("another volume's damaged copy", &|raw| raw[second + 40] ^= 1, 2056), + ("another volume's whole copy", &|raw| raw[second..second + BLOCK].copy_from_slice(&other[second..second + BLOCK]), 2056), ("the primary's bytes in the second copy's place", &|raw| raw.copy_within(primary..primary + BLOCK, second), 2056), ("a zeroed layout copy", &|raw| raw[7 * SECTOR..8 * SECTOR].fill(0), 7), ("another format's superblock over the front", &|raw| { From 537af08e7324a7f324d17a0620d5549f54760baa Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 22:59:18 +0100 Subject: [PATCH 77/93] issues: the two limits #1152 brings, and the replayed-leaf refusal it ended r3 BLOCKER 4: a reserved DATA block holds a transaction's metadata credit until it is written, and a small journal makes reclaim write nodes after nearly every transaction. Each record has its evidence, owner and exit. a-replayed-bcachefs-node-past-seven-nodes-is-refused is closed: since 11632e8b6 a replay's keys stay in pending and materialize in bounded ranges, and the test it cites became a_replay_larger_than_seven_nodes_materializes_in_bounded_ranges, which mounts such a volume read-write. No citation of it remains. The bcachefs track records that checksum-none volumes are refused (adb3b5655), a format option the crate otherwise claims. Co-Authored-By: Claude Opus 5.5 --- ...nsactions-metadata-credit-until-written.md | 27 ++++++++++++++++++ ...te-nodes-after-nearly-every-transaction.md | 28 +++++++++++++++++++ ...-bcachefs-crate-must-implement-bcachefs.md | 4 ++- 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 issues/a-reserved-data-block-holds-a-transactions-metadata-credit-until-written.md create mode 100644 issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md diff --git a/issues/a-reserved-data-block-holds-a-transactions-metadata-credit-until-written.md b/issues/a-reserved-data-block-holds-a-transactions-metadata-credit-until-written.md new file mode 100644 index 00000000000..50ec41d5879 --- /dev/null +++ b/issues/a-reserved-data-block-holds-a-transactions-metadata-credit-until-written.md @@ -0,0 +1,27 @@ +--- +status: open +kind: defect +opened: 2026-11-04 +--- + +# A reserved DATA block holds a transaction's metadata credit until it is written + +**Owner.** The storage track, which owns `lib/bcachefs` and the file server's +DATA volume. + +The file server reserves the blocks of a write before it copies the bytes, +and `Reservations::pending` (`lib/bcachefs/src/engine/mount/data.rs`) counts +one ordinary transaction's metadata credit, `K_TX * 8` bytes, for every +reserved block not yet written. A block of 4 KiB therefore holds as much +metadata room as data until write-back, and a large write needs about twice +its size free. Main's private format had no such limit. + +**Evidence.** On QEMU's 128 MiB DATA, osbench's 16 MiB setup file is refused +for space (#1152's QEMU run of the `fs.disk` rows). The benches' DATA +partitions are large enough that no row reaches it. + +**Exit.** Credit by what write-back will journal: one transaction's credit +for each run of contiguous reserved blocks, which `write_run` writes as one +extent, or for each transaction write-back is expected to take, instead of +for each block. A test that reserves a write of most of a small volume's free +space and writes it back. diff --git a/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md b/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md new file mode 100644 index 00000000000..82b1fab6085 --- /dev/null +++ b/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md @@ -0,0 +1,28 @@ +--- +status: open +kind: defect +opened: 2026-11-04 +--- + +# A small bcachefs journal makes reclaim write nodes after nearly every transaction + +**Owner.** The storage track, which owns `lib/bcachefs`. + +Each dirty node reserves its worst-case pointer update in the journal, and an +ordinary transaction is admitted only with a transaction's journal room left +(`lib/bcachefs/src/engine/mount/mod.rs`). Where the journal holds little, as +on the 64 MiB host test images and QEMU-sized DATA, a few dirty nodes fill +it, so reclaim writes nodes after nearly every transaction and the eager +put-in-place of #1152 gains nothing there. mkfs sizes the journal at 1/128 +of the device, at least eight buckets (`mkfs.rs`), so the benches' DATA +partitions are far from this; only DATA mounts this engine read-write. + +**Evidence.** The 64 MiB host images' tests: a crash after 2 000 creates +replays about a thousand journal keys of them +(`tests/admission.rs`, `the_first_transaction_after_a_replay_takes_each_delta_once`), +the rest already written by reclaim. + +**Exit.** A journal reservation for dirty nodes that does not grow with +each one, for instance one pointer update for each reclaim batch rather than +for each node, with the reclaim interval on a small volume measured before +and after. diff --git a/issues/the-bcachefs-crate-must-implement-bcachefs.md b/issues/the-bcachefs-crate-must-implement-bcachefs.md index 71e3e1bcd9e..e408620de20 100644 --- a/issues/the-bcachefs-crate-must-implement-bcachefs.md +++ b/issues/the-bcachefs-crate-must-implement-bcachefs.md @@ -27,7 +27,9 @@ allocator and accounting state from a volume's keys and compares. It refuses to write a volume of blocks under 4 KiB or nodes of 8 KiB and under (`issues/the-bcachefs-writer-refuses-small-blocks-and-nodes.md`), and its reclaim reserves are sized for common rewrites -(`issues/bcachefs-reclaim-reserves-are-sized-for-common-rewrites.md`). +(`issues/bcachefs-reclaim-reserves-are-sized-for-common-rewrites.md`). It +refuses a superblock, node or extent that declares no checksum, which the +format allows: a volume made with checksums off does not mount. **Exit**: the integrated writer's volumes, ROOT and a used DATA on each bench included, pass upstream's `bcachefs fsck -n -K`, with that evidence published. From c74862d03342dfd0ee3de7b6617ba903320812e3 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 22:59:18 +0100 Subject: [PATCH 78/93] fileserver: remembered DATA paths are bounded by bytes and forgotten by range r3 suggestions: forget walked the whole cache on every unlink, rmdir and rename; it now removes the path and the sorted range from "path/" to before "path0". The bound was 4096 entries of up to a whole path each; it is now 256 KiB of path bytes. Tests: forgetting "a" keeps "a-b", "a0", "ab" and "b", with the byte count following; 1 000 paths of 4 KiB stay within the bound. Co-Authored-By: Claude Opus 5.5 --- system/fileserver/src/data.rs | 81 +++++++++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index 73ce53af6ea..2a58644c5a3 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -11,8 +11,8 @@ use toyos::fs::Error; use toyos_blockdev::{BLOCK, BlockDev}; const NODES: usize = 512; -/// Names a volume remembers, at most. -const NAMES: usize = 4096; +/// Bytes of paths a volume remembers, at most. +const NAME_BYTES: usize = 256 << 10; const READ_CHUNK: usize = 256 << 10; const OWNER: Owner = Owner { perm: 0o644, uid: 0, gid: 0 }; struct OsRandom; @@ -48,15 +48,49 @@ impl Open { } } } +/// Paths and what they name, bounded by their bytes. +#[derive(Default)] +struct Names { + by_path: BTreeMap, + bytes: usize, +} + +impl Names { + fn get(&self, path: &str) -> Option<&(u64, DirentType)> { + self.by_path.get(path) + } + fn insert(&mut self, path: String, entry: (u64, DirentType)) { + if let Some(known) = self.by_path.get_mut(&path) { + *known = entry; + return; + } + if self.bytes + path.len() > NAME_BYTES { + self.by_path.clear(); + self.bytes = 0; + } + self.bytes += path.len(); + self.by_path.insert(path, entry); + } + /// `path` and the paths beneath it, which sort from `path/` to before `path0`. + fn forget(&mut self, path: &str) { + let below: Vec = self.by_path.range(format!("{path}/")..format!("{path}0")).map(|(known, _)| known.clone()).collect(); + for known in below.iter().map(String::as_str).chain([path]) { + if self.by_path.remove(known).is_some() { + self.bytes -= known.len(); + } + } + } +} + pub struct DataVolume { fs: Mount>>, cache: Rc>, roots: Vec, open: BTreeMap, - /// What each path looked up names, by path, so the next lookup of it - /// reads its inode alone. Only an unlink, rmdir or rename takes a name - /// away; each forgets the path and what lies under it. - names: RefCell>, + /// What each path looked up names, so the next lookup of it reads its + /// inode alone. Only an unlink, rmdir or rename takes a name away; each + /// forgets the path and what lies under it. + names: RefCell, next_ticket: u64, clock: fn() -> u64, } @@ -174,7 +208,7 @@ impl DataVolume { cache, roots: roots.iter().map(|root| root.to_string()).collect(), open: BTreeMap::new(), - names: RefCell::new(BTreeMap::new()), + names: RefCell::new(Names::default()), next_ticket: 1, clock, }; @@ -217,19 +251,14 @@ impl DataVolume { let entry = self.fs.lookup(found.0, names[at].as_bytes()).await?.ok_or(engine::Error::NotFound)?; found = (entry.ino, entry.kind); at += 1; - let mut known = self.names.borrow_mut(); - if known.len() >= NAMES { - known.clear(); - } - known.insert(names[..at].join("/"), found); + self.names.borrow_mut().insert(names[..at].join("/"), found); } Ok(found) } /// `path` and everything beneath it moved or gone. fn forget(&self, path: &str) { let path = path.split('/').filter(|name| !name.is_empty()).collect::>().join("/"); - let below = format!("{path}/"); - self.names.borrow_mut().retain(|known, _| *known != path && !known.starts_with(&below)); + self.names.borrow_mut().forget(&path); } fn destination<'a>(&self, path: &'a str) -> Result<(u64, &'a [u8]), Error> { let name = path.rsplit('/').next().filter(|name| !name.is_empty()).ok_or(Error::Root)?; @@ -718,6 +747,30 @@ mod tests { } /// A path, once looked up, is remembered; an unlink, rmdir or rename /// forgets it and everything under it. + #[test] + fn forgetting_a_path_forgets_what_lies_under_it_and_no_neighbour() { + let mut names = Names::default(); + for path in ["a", "a/b", "a/b/c", "a-b", "a0", "ab", "b"] { + names.insert(path.to_string(), (1, DirentType::Regular)); + } + names.forget("a"); + assert_eq!(names.by_path.keys().map(String::as_str).collect::>(), ["a-b", "a0", "ab", "b"]); + assert_eq!(names.bytes, names.by_path.keys().map(String::len).sum::()); + names.insert("ab".to_string(), (2, DirentType::Regular)); + assert_eq!(names.bytes, 8, "a path named again is counted once"); + } + + #[test] + fn remembered_paths_stay_within_their_bytes() { + let mut names = Names::default(); + let long = "n".repeat(4000); + for i in 0..1000 { + names.insert(format!("{long}{i}"), (i, DirentType::Regular)); + assert!(names.bytes <= NAME_BYTES); + } + assert!(!names.by_path.is_empty()); + } + #[test] fn a_remembered_name_follows_its_renames_and_removal() { let mut v = volume(); From a0098f4d4d4b0d2fdfa7fcd72eac15444686f112 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 22:59:18 +0100 Subject: [PATCH 79/93] bootlog: a boot whose job list ran past its bound is no pass osbench round 2's boots on both benches hung in osbench_persist; the test runner killed it at its bound and rebooted, and the boot's log, with its boot record and the supervisor's stop, read as passed in verdict.txt. The runner's deadline line reaches the log (the readbacks carry it), so the "console only" note was wrong. bootlog::verdict now refuses a log carrying the test runner's own deadline line, naming it; another program quoting the words is not it. Co-Authored-By: Claude Opus 5.5 --- build/bootlog.rs | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/build/bootlog.rs b/build/bootlog.rs index 625e255cc59..847d89ef92f 100644 --- a/build/bootlog.rs +++ b/build/bootlog.rs @@ -22,9 +22,8 @@ pub const REBOOTING: &str = "Rebooting."; pub const STOPPING: &str = toyos_logstream::STOPPING; /// What `tests/guest/test-runner` says when its job list runs past -/// `toyos_tco::JOB_BOUND_MS`, with the job it was inside as the next word. -/// **Console only**: a userland write reaches the serial backend and never a -/// log record, so no stick carries it. +/// `toyos_tco::JOB_BOUND_MS`, with the job it was inside as the next word, +/// before it kills that job and reboots: a boot that says it is no pass. pub const JOB_DEADLINE_SAID: &str = "test-runner: the job list ran past its bound, and the job it was inside is"; @@ -337,6 +336,9 @@ pub enum Unfit { /// The loader pass after the foreign-identity arm's reset read no /// [`FOREIGN_DONE`]: the stop it staged never finished. NoForeignDone, + /// The test runner's word that its job list ran past its bound: the + /// line, naming the job it killed. + JobsPastBound(String), } impl fmt::Display for Unfit { @@ -354,6 +356,10 @@ impl fmt::Display for Unfit { "the loader's pass after the reset carries no {HANDED_BACK:?} with {REBOOTING:?} \ under {LOG_TAIL:?}: the stop this boot asked for never reached the reset" ), + Self::JobsPastBound(said) => write!( + f, + "the test runner killed a job and rebooted at its bound: {said:?}" + ), Self::NoForeignDone => write!( f, "the loader's pass after the reset carries no {FOREIGN_DONE:?}: the stop this \ @@ -541,6 +547,12 @@ pub fn boot_millis(log: &str) -> Option { /// pass's ([`HANDED_BACK`], and [`REBOOTING`] under [`LOG_TAIL`]). pub fn verdict(log: &str) -> Result { let boot_ms = boot_millis(log).ok_or(Unfit::NoBootRecord)?; + let past_bound = |line: &&str| { + toyos_logstream::program_line(line).is_some_and(|said| said.tag == "test-runner" && said.text.starts_with(JOB_DEADLINE_SAID)) + }; + if let Some(line) = log.lines().find(past_bound) { + return Err(Unfit::JobsPastBound(line.trim().to_string())); + } if stopping_line(log).is_none() { let last = log.lines().rev().find(|line| !line.trim().is_empty()).unwrap_or_default(); return Err(Unfit::Unfinished(last.trim().to_string())); @@ -578,6 +590,19 @@ mod tests { assert_eq!(verdict(""), Err(Unfit::NoBootRecord)); } + /// A boot the test runner ended at its job bound stopped as a passing + /// one does, and is none. + #[test] + fn a_boot_whose_job_list_ran_past_its_bound_is_not_a_pass() { + let past = format!("[2026-11-04 17:07:50 535.506 test-runner tid=1] {JOB_DEADLINE_SAID} osbench_persist (525000 ms)\n"); + let log = format!( + "[ 1.151 cpu0 kernel] Boot: complete (1151ms)\n{past}[ 535.507 supervisor] {STOPPING} (Reboot)\n" + ); + assert_eq!(verdict(&log), Err(Unfit::JobsPastBound(past.trim().to_string()))); + let quoted = log.replace("test-runner tid=1] ", "osbench tid=4] said "); + assert_eq!(verdict("ed), Ok(1151)); + } + /// Only the supervisor's own stop line, naming a shutdown, is a power-off. #[test] fn a_power_off_is_the_supervisors_stop_naming_a_shutdown() { From a043c5e4ba0c5ca579c4c3090267ad1d305a4a66 Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 23:08:55 +0100 Subject: [PATCH 80/93] issues: a bcachefs replay holds its whole journal window in memory Found with the osbench_persist hang: the T14-sized host model's unclean replay holds about 1.5 million keys in pending and peaks at 1.0 GB resident. Recorded with its evidence, owner and exit. Co-Authored-By: Claude Opus 5.5 --- ...olds-its-whole-journal-window-in-memory.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 issues/a-bcachefs-replay-holds-its-whole-journal-window-in-memory.md diff --git a/issues/a-bcachefs-replay-holds-its-whole-journal-window-in-memory.md b/issues/a-bcachefs-replay-holds-its-whole-journal-window-in-memory.md new file mode 100644 index 00000000000..828a353d5ff --- /dev/null +++ b/issues/a-bcachefs-replay-holds-its-whole-journal-window-in-memory.md @@ -0,0 +1,29 @@ +--- +status: open +kind: defect +opened: 2026-11-04 +--- + +# A bcachefs replay holds its whole journal window in memory + +**Owner.** The storage track, which owns `lib/bcachefs`. + +An unclean mount puts every key of the journal window into pending +(`journal::replay`, `lib/bcachefs/src/engine/pending.rs`), each accounting +delta its own entry. mkfs sizes the journal at 1/128 of the device, and +reclaim trims it only under journal pressure, so on a large DATA the window +holds whatever ran since the last pressure. + +**Evidence.** A host model of the T14's DATA, written with osbench's +persist and `fs.disk` call sequence and dropped unclean, replays about +1.5 million keys, most of them accounting deltas at a handful of positions. +The file server's process peaks at 1.0 GB resident (`/usr/bin/time -l`) +and the mount takes 3 to 3.6 s, before the first transaction takes the +deltas into their leaf in 0.3 to 0.4 s (#1152, the osbench_persist hang +investigation; its harness and logs are in the evidence). + +**Exit.** Replay memory bounded by something smaller than the journal, for +instance accounting deltas summed per position as they replay, against the +version their leaf holds, and a journal window trimmed by its size as well +as by pressure. A host test that replays a window of many transactions and +bounds pending's bytes. From f433570e35de463f397d40c6428aec57cc62c6eb Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 23:40:00 +0100 Subject: [PATCH 81/93] review r4 of #1152: the name cache counts each entry's cost, and the doc goes back to its test BLOCKER 2: the doc comment of a_remembered_name_follows_its_renames_and_removal had been left above the new forgetting test; it is back on its own test. Suggestions: a remembered path costs its bytes and ENTRY_BYTES, its key and value and their share of a B-tree node, so many short paths cannot make several MB under the 256 KiB bound (the test adds 100 000 of them); materialization's first position fitting its room is a compile-time assert, MAX_KEY <= K_TX * 8; the superblock test's doc is reflowed. Co-Authored-By: Claude Opus 5.5 --- lib/bcachefs/src/engine/mount/mod.rs | 1 + lib/bcachefs/src/engine/tests/mod.rs | 6 +++--- system/fileserver/src/data.rs | 32 ++++++++++++++++++++-------- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index f9d2c2901e0..2f2d46fa16d 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -222,6 +222,7 @@ const MAX_KEY: usize = 8 * (KEY_U64S + super::bkey::MAX_VALUE_U64S); /// evenly over [`MAX_GROUPS`] nodes, each taking its share and at most a key /// past it. const fn splits_fit(node: usize) -> bool { +const _: () = assert!(MAX_KEY <= K_TX as usize * 8, "materialization's room takes any one key, so it always takes its first position"); (5 * (node - FIRST_KEYS) + 2 * K_TX as usize * 8).div_ceil(MAX_GROUPS) + MAX_KEY <= node - FIRST_KEYS } const _: () = assert!(splits_fit(super::mkfs::MIN_BUCKET as usize * SECTOR), "mkfs's smallest nodes are writable"); diff --git a/lib/bcachefs/src/engine/tests/mod.rs b/lib/bcachefs/src/engine/tests/mod.rs index a88f6d6783d..67bf838e842 100644 --- a/lib/bcachefs/src/engine/tests/mod.rs +++ b/lib/bcachefs/src/engine/tests/mod.rs @@ -247,9 +247,9 @@ fn a_torn_primary_superblock_is_read_from_another_copy_and_named() { } /// A copy's place holding what is not this volume's copy there — nothing, -/// another volume's copy, whole or damaged, a copy of this one at another offset, or another -/// layout — is another writer's, and the volume is refused, not opened from -/// the copies left. +/// another volume's copy, whole or damaged, a copy of this one at another +/// offset, or another layout — is another writer's, and the volume is +/// refused, not opened from the copies left. #[test] fn a_superblock_place_another_writer_took_refuses_the_volume() { let pristine = formatted(32 << 20, None).into_inner(); diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index 2a58644c5a3..88f74b5aeb9 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -11,8 +11,11 @@ use toyos::fs::Error; use toyos_blockdev::{BLOCK, BlockDev}; const NODES: usize = 512; -/// Bytes of paths a volume remembers, at most. +/// Bytes the paths a volume remembers take, at most. const NAME_BYTES: usize = 256 << 10; +/// What a remembered path costs past its bytes: its key and value, and +/// about their share of a B-tree node. +const ENTRY_BYTES: usize = core::mem::size_of::<(String, (u64, DirentType))>() + 16; const READ_CHUNK: usize = 256 << 10; const OWNER: Owner = Owner { perm: 0o644, uid: 0, gid: 0 }; struct OsRandom; @@ -48,7 +51,7 @@ impl Open { } } } -/// Paths and what they name, bounded by their bytes. +/// Paths and what they name, bounded by the memory they take. #[derive(Default)] struct Names { by_path: BTreeMap, @@ -64,11 +67,11 @@ impl Names { *known = entry; return; } - if self.bytes + path.len() > NAME_BYTES { + if self.bytes + cost(&path) > NAME_BYTES { self.by_path.clear(); self.bytes = 0; } - self.bytes += path.len(); + self.bytes += cost(&path); self.by_path.insert(path, entry); } /// `path` and the paths beneath it, which sort from `path/` to before `path0`. @@ -76,12 +79,16 @@ impl Names { let below: Vec = self.by_path.range(format!("{path}/")..format!("{path}0")).map(|(known, _)| known.clone()).collect(); for known in below.iter().map(String::as_str).chain([path]) { if self.by_path.remove(known).is_some() { - self.bytes -= known.len(); + self.bytes -= cost(known); } } } } +fn cost(path: &str) -> usize { + path.len() + ENTRY_BYTES +} + pub struct DataVolume { fs: Mount>>, cache: Rc>, @@ -745,8 +752,6 @@ mod tests { let mut v = reopen(v); assert!(v.list("home").unwrap().is_empty()); } - /// A path, once looked up, is remembered; an unlink, rmdir or rename - /// forgets it and everything under it. #[test] fn forgetting_a_path_forgets_what_lies_under_it_and_no_neighbour() { let mut names = Names::default(); @@ -755,11 +760,13 @@ mod tests { } names.forget("a"); assert_eq!(names.by_path.keys().map(String::as_str).collect::>(), ["a-b", "a0", "ab", "b"]); - assert_eq!(names.bytes, names.by_path.keys().map(String::len).sum::()); + let counted = |names: &Names| names.by_path.keys().map(|path| cost(path)).sum::(); + assert_eq!(names.bytes, counted(&names)); names.insert("ab".to_string(), (2, DirentType::Regular)); - assert_eq!(names.bytes, 8, "a path named again is counted once"); + assert_eq!(names.bytes, counted(&names), "a path named again is counted once"); } + /// Long paths and many short ones alike. #[test] fn remembered_paths_stay_within_their_bytes() { let mut names = Names::default(); @@ -768,9 +775,16 @@ mod tests { names.insert(format!("{long}{i}"), (i, DirentType::Regular)); assert!(names.bytes <= NAME_BYTES); } + for i in 0..100_000u64 { + names.insert(format!("{i:x}"), (i, DirentType::Regular)); + assert!(names.bytes <= NAME_BYTES); + } + assert!(names.by_path.len() <= NAME_BYTES / ENTRY_BYTES); assert!(!names.by_path.is_empty()); } + /// A path, once looked up, is remembered; an unlink, rmdir or rename + /// forgets it and everything under it. #[test] fn a_remembered_name_follows_its_renames_and_removal() { let mut v = volume(); From bdf062cd3ce89970403c2daf65c30b3e06954abb Mon Sep 17 00:00:00 2001 From: japabu Date: Wed, 4 Nov 2026 23:40:01 +0100 Subject: [PATCH 82/93] bcachefs: the journal's live entries are held to 32 MiB, and a scan keeps only the window's records r4 BLOCKER 1. A replay held its whole live window in memory, bounded only by the journal mkfs sizes at 1/128 of the device: about 1.3 GiB on the T14's 170 GiB DATA. Measured on the host model of that DATA, osbench's persist and fs.disk sequence at its full repetitions, dropped unclean, then mounted by the file server (`/usr/bin/time -l`): before: peak 3.7 GB resident, mount 12.5 s, first mkdir 0.78 s live cap: peak 1.7 GB, mount 1.19 s, first mkdir 0.017 s both: peak 0.20 GB, mount 1.10 s, first mkdir 0.018 s (The earlier record's 1.0 GB was this sequence at a tenth of its repetitions.) Both benches have 16 GB; 3.7 GB at a tenth of what the journal can hold is not comfortable, so the exit goes in here. - The writer keeps live entries to JOURNAL_LIVE (32 MiB) of the ring, or the least ring reclaim needs, wherever on the ring they lie: the ring counts as that many buckets for its free room, slots and shares. - The scan read every bucket's entries into memory, a whole ring of stale ones. An entry older than a good, unblacklisted flush entry's last sequence now keeps no records; the window it can never be in starts at the newest flush entry's last sequence, as last sequences do not go back. A window reaching such an entry is refused as Corrupt rather than replayed short. issues/a-bcachefs-replay-holds-its-whole-journal-window-in-memory.md, filed in a043c5e4b, closes by its exit; the small-journal record names the cap. Tests: a writer held to three of eight buckets offers what a three-bucket ring does as they fill (red without the cap: 32 against 12 entries); a scan of entries whose last sequences move on keeps the records of the window only, and one whose last sequence goes back is refused (red with every record kept). Co-Authored-By: Claude Opus 5.5 --- ...olds-its-whole-journal-window-in-memory.md | 29 ----------- ...te-nodes-after-nearly-every-transaction.md | 3 +- lib/bcachefs/src/engine/journal.rs | 48 +++++++++++++++---- lib/bcachefs/src/engine/mount/mod.rs | 9 +++- lib/bcachefs/src/engine/tests/journal.rs | 47 ++++++++++++++++++ 5 files changed, 97 insertions(+), 39 deletions(-) delete mode 100644 issues/a-bcachefs-replay-holds-its-whole-journal-window-in-memory.md diff --git a/issues/a-bcachefs-replay-holds-its-whole-journal-window-in-memory.md b/issues/a-bcachefs-replay-holds-its-whole-journal-window-in-memory.md deleted file mode 100644 index 828a353d5ff..00000000000 --- a/issues/a-bcachefs-replay-holds-its-whole-journal-window-in-memory.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-11-04 ---- - -# A bcachefs replay holds its whole journal window in memory - -**Owner.** The storage track, which owns `lib/bcachefs`. - -An unclean mount puts every key of the journal window into pending -(`journal::replay`, `lib/bcachefs/src/engine/pending.rs`), each accounting -delta its own entry. mkfs sizes the journal at 1/128 of the device, and -reclaim trims it only under journal pressure, so on a large DATA the window -holds whatever ran since the last pressure. - -**Evidence.** A host model of the T14's DATA, written with osbench's -persist and `fs.disk` call sequence and dropped unclean, replays about -1.5 million keys, most of them accounting deltas at a handful of positions. -The file server's process peaks at 1.0 GB resident (`/usr/bin/time -l`) -and the mount takes 3 to 3.6 s, before the first transaction takes the -deltas into their leaf in 0.3 to 0.4 s (#1152, the osbench_persist hang -investigation; its harness and logs are in the evidence). - -**Exit.** Replay memory bounded by something smaller than the journal, for -instance accounting deltas summed per position as they replay, against the -version their leaf holds, and a journal window trimmed by its size as well -as by pressure. A host test that replays a window of many transactions and -bounds pending's bytes. diff --git a/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md b/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md index 82b1fab6085..9bfa89911f0 100644 --- a/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md +++ b/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md @@ -14,7 +14,8 @@ ordinary transaction is admitted only with a transaction's journal room left on the 64 MiB host test images and QEMU-sized DATA, a few dirty nodes fill it, so reclaim writes nodes after nearly every transaction and the eager put-in-place of #1152 gains nothing there. mkfs sizes the journal at 1/128 -of the device, at least eight buckets (`mkfs.rs`), so the benches' DATA +of the device, at least eight buckets (`mkfs.rs`), and the writer holds its +live entries to 32 MiB of it (`JOURNAL_LIVE`), so the benches' DATA partitions are far from this; only DATA mounts this engine read-write. **Evidence.** The 64 MiB host images' tests: a crash after 2 000 creates diff --git a/lib/bcachefs/src/engine/journal.rs b/lib/bcachefs/src/engine/journal.rs index e215404ce5d..9f6f7d24ede 100644 --- a/lib/bcachefs/src/engine/journal.rs +++ b/lib/bcachefs/src/engine/journal.rs @@ -157,8 +157,12 @@ pub(crate) struct Found { pub offset: u64, /// Sectors it occupies, its last block's padding included. pub sectors: u64, - /// Its records, decrypted; empty when it is not `good`. + /// Its records, decrypted; empty when it is not `good`, or when it is + /// `past`. pub area: Vec, + /// Older than a good flush entry's last sequence: before the window, + /// so its records were not kept. + pub past: bool, } /// Every entry the ring holds, by sequence. @@ -167,18 +171,33 @@ pub(crate) struct Scan { } /// Reads every journal bucket in ring order (A2.9), each as -/// [`scan_bucket`] does. +/// [`scan_bucket`] does. An entry older than the last sequence of some good +/// flush entry the blacklist leaves keeps no records: the ring can hold far more than any window, +/// and [`Scan::window`] refuses a window that reaches one. pub(crate) async fn scan(dev: &impl BlockDev, sb: &Superblock, key: Option<&MasterKey>) -> Result { let mut entries: BTreeMap = BTreeMap::new(); + let mut floor = 0; for index in 0..sb.journal.iter().map(|&(_, count)| count as usize).sum() { - for found in scan_bucket(dev, sb, key, index).await? { + for mut found in scan_bucket(dev, sb, key, index).await? { + if found.seq < floor { + found.area = Vec::new(); + found.past = true; + } match entries.get(&found.seq) { - Some(had) if had.good && found.good && had.area != found.area => { + Some(had) if had.good && found.good && !had.past && !found.past && had.area != found.area => { return Err(Error::Corrupt("two journal entries of one sequence")); } Some(had) if had.good || !found.good => {} _ => { + let raised = (found.good && found.flush && found.last_seq <= found.seq && !sb.blacklisted(found.seq)).then_some(found.last_seq).filter(|&l| l > floor); entries.insert(found.seq, found); + if let Some(last_seq) = raised { + for (_, e) in entries.range_mut(floor..last_seq) { + e.area = Vec::new(); + e.past = true; + } + floor = last_seq; + } } } } @@ -235,7 +254,7 @@ fn entry(sb: &Superblock, key: Option<&MasterKey>, b: &[u8], bucket: usize, at: // An entry that runs past its bucket is cut there and fails its checksum. let Some(end) = end.filter(|&end| end <= b.len()) else { let sectors = (b.len() / SECTOR) as u64; - return Ok(Found { seq, last_seq: 0, flush, good: false, bucket, offset, sectors, area: Vec::new() }); + return Ok(Found { seq, last_seq: 0, flush, good: false, bucket, offset, sectors, area: Vec::new(), past: false }); }; let sectors = (end.next_multiple_of(block) / SECTOR) as u64; let mut bytes = b[..end].to_vec(); @@ -254,7 +273,7 @@ fn entry(sb: &Superblock, key: Option<&MasterKey>, b: &[u8], bucket: usize, at: }; let last_seq = if flush { le64(&bytes, 48) } else { 0 }; let area = if good { bytes.split_off(HEADER) } else { Vec::new() }; - Ok(Found { seq, last_seq, flush, good, bucket, offset, sectors, area }) + Ok(Found { seq, last_seq, flush, good, bucket, offset, sectors, area, past: false }) } /// The entries replay takes: from the newest flush entry `F`'s last @@ -299,6 +318,9 @@ impl Scan { if !e.good && !sb.blacklisted(e.seq) { return Err(Error::Checksum("journal entry")); } + if e.past { + return Err(Error::Corrupt("a journal window reaching behind another flush entry's last sequence")); + } next = e.seq.saturating_add(1); } let next = newest.checked_add(1).ok_or(Error::Corrupt("a journal sequence at its last value"))?; @@ -489,6 +511,8 @@ pub(crate) struct Closed { pub(crate) struct Writer { /// Each ring bucket's device bucket. pub ring: Vec, + /// Ring buckets live entries may hold at once: what a replay reads. + live: usize, pub bucket_sectors: u64, /// Entries are at most this many bytes, `per_bucket` of them to a bucket. entry: usize, @@ -525,6 +549,7 @@ impl Writer { assert!(entry > HEADER + TAIL + 2 * max_tx as usize * 8, "journal buckets of {bucket} bytes hold no transaction"); let newest = alloc::vec![None; ring.len()]; Writer { + live: ring.len(), ring, bucket_sectors, entry, @@ -545,9 +570,15 @@ impl Writer { } } - /// Entries the whole ring holds. + /// Keeps live entries to at most `buckets` of the ring, so a replay + /// reads no more, wherever on the ring they lie. + pub(crate) fn hold_live_to(&mut self, buckets: usize) { + self.live = buckets.clamp(1, self.ring.len()); + } + + /// Entries the live part of the ring holds. pub(crate) fn slots(&self) -> u64 { - self.ring.len() as u64 * self.per_bucket + self.live as u64 * self.per_bucket } /// Body u64s an entry holds for transactions. @@ -559,6 +590,7 @@ impl Writer { pub(crate) fn ring_free(&self) -> u64 { let outstanding = (self.closed.len() + usize::from(self.open.is_some())) as u64; let free = (0..self.ring.len()).filter(|&i| i != self.head && self.newest[i].is_none()).count() as u64; + let free = free.saturating_sub((self.ring.len() - self.live) as u64); let entry_sectors = (self.entry / SECTOR) as u64; (free * self.per_bucket + (self.bucket_sectors - self.head_sectors) / entry_sectors).saturating_sub(outstanding) } diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 2f2d46fa16d..43231ce76f6 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -83,6 +83,11 @@ const RW_FEATURES: u64 = 1 << 8 | 1 << 9 | 1 << 12 | 1 << 13 | 1 << 17 | 1 << 18 /// Feature bit 5: the superblock holds a blacklist. const BLACKLIST_FEATURE: u64 = 1 << 5; +/// Journal bytes live entries hold at most. A replay holds every key of +/// them in memory until its nodes take them, so this, not the journal +/// mkfs sized by the device, bounds an unclean mount's memory and time. +const JOURNAL_LIVE: u64 = 32 << 20; + /// Ring geometry that leaves two cache-sized reclaim shares and an ordinary /// quarter, with room for at least one maximum-size ordinary transaction. pub(crate) fn journal_buckets(bucket_sectors: u64, block: usize) -> u64 { @@ -217,12 +222,12 @@ const _: () = assert!(PER_NODE <= K_TX, "a reclaim batch of one node fits one re /// The largest key a node holds, as its reader refuses a larger value /// (A3.16). const MAX_KEY: usize = 8 * (KEY_U64S + super::bkey::MAX_VALUE_U64S); +const _: () = assert!(MAX_KEY <= K_TX as usize * 8, "materialization's room takes any one key, so it always takes its first position"); /// Whether nodes of `node` bytes hold reclaim's largest split: a node read /// at five times its bytes unpacked, and two transactions more, spread /// evenly over [`MAX_GROUPS`] nodes, each taking its share and at most a key /// past it. const fn splits_fit(node: usize) -> bool { -const _: () = assert!(MAX_KEY <= K_TX as usize * 8, "materialization's room takes any one key, so it always takes its first position"); (5 * (node - FIRST_KEYS) + 2 * K_TX as usize * 8).div_ceil(MAX_GROUPS) + MAX_KEY <= node - FIRST_KEYS } const _: () = assert!(splits_fit(super::mkfs::MIN_BUCKET as usize * SECTOR), "mkfs's smallest nodes are writable"); @@ -388,6 +393,8 @@ impl Mount { let ring: Vec = sb.journal.iter().flat_map(|&(first, count)| first..first + count).collect(); let bucket_sectors = u64::from(sb.member.bucket_sectors); let mut journal = Writer::new(ring, bucket_sectors, sb.block_bytes(), K_TX); + let live = (JOURNAL_LIVE / (bucket_sectors * SECTOR as u64)).max(journal_buckets(bucket_sectors, sb.block_bytes())); + journal.hold_live_to(usize::try_from(live).unwrap_or(usize::MAX)); let mut blacklist = sb.blacklist.clone(); let (clocks, restate) = match &replay { None => { diff --git a/lib/bcachefs/src/engine/tests/journal.rs b/lib/bcachefs/src/engine/tests/journal.rs index 6118e547889..0a070860ac2 100644 --- a/lib/bcachefs/src/engine/tests/journal.rs +++ b/lib/bcachefs/src/engine/tests/journal.rs @@ -297,6 +297,53 @@ fn a_window_with_an_entry_missing_or_failing_its_checksum_is_refused() { assert_eq!(window_of(&unclean_with(&[], |_, _| {})).unwrap().map(|w| (w.first, w.last, w.next)), None); } +/// A writer holding its live entries to three buckets of an eight-bucket +/// ring offers what a writer of a three-bucket ring does, as its buckets +/// fill, wherever on the ring they lie. +#[test] +fn a_writer_held_to_some_buckets_offers_what_a_ring_of_that_many_does() { + let (bucket_sectors, block) = (512, BLOCK); + let mut held = journal::Writer::new((0..8).collect(), bucket_sectors, block, super::super::mount::K_TX); + held.hold_live_to(3); + let mut small = journal::Writer::new((0..3).collect(), bucket_sectors, block, super::super::mount::K_TX); + assert_eq!(held.slots(), small.slots()); + assert_eq!(held.ring_free(), small.ring_free()); + (held.head, small.head) = (5, 1); + held.newest[4] = Some(9); + small.newest[0] = Some(9); + assert_eq!(held.ring_free(), small.ring_free()); + held.newest[6] = Some(10); + small.newest[2] = Some(10); + // Only what the head bucket has left. + assert_eq!(held.ring_free(), small.ring_free()); + held.head_sectors = bucket_sectors; + small.head_sectors = bucket_sectors; + assert_eq!((held.ring_free(), small.ring_free()), (0, 0)); +} + +/// The ring can hold far more than a window: an entry older than a good +/// flush entry's last sequence keeps no records, wherever in the ring it +/// lies. A window that reaches one has last sequences that went back, and +/// is refused. +#[test] +fn a_scan_keeps_no_records_of_entries_behind_a_flush_entrys_last_sequence() { + let roots = Ring::of(&formatted(8 << 20, None), None).roots; + let mut area = Vec::new(); + tail(&mut area, &roots); + let scanned = |dev: &Image| -> Vec<(u64, bool, bool)> { + block_on_ready(async { + let sb = super::super::read::read_superblock(dev).await.unwrap().0; + journal::scan(dev, &sb, None).await.unwrap().entries.values().map(|e| (e.seq, e.past, e.area.is_empty())).collect() + }) + }; + let moving = unclean_with(&[(1, 1, area.clone(), 0), (2, 1, area.clone(), 0), (3, 2, area.clone(), 0), (4, 3, area.clone(), 0)], |_, _| {}); + assert_eq!(scanned(&moving), [(1, true, true), (2, true, true), (3, false, false), (4, false, false)]); + assert_eq!(window_of(&moving).unwrap().map(|w| (w.first, w.last)), Some((3, 4))); + // Entry 3 said its window starts at 3, the newest that it starts at 2. + let back = unclean_with(&[(1, 1, area.clone(), 0), (2, 1, area.clone(), 0), (3, 3, area.clone(), 0), (4, 2, area.clone(), 0)], |_, _| {}); + assert_eq!(window_of(&back), Err(Error::Corrupt("a journal window reaching behind another flush entry's last sequence"))); +} + /// What the ring holds past a bucket's newest entry is left over from an /// earlier pass and is not read; past a damaged entry, the reader steps a /// block at a time to the next. From 4410a56642e8ec6b780af69c0af02863b850bd65 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 5 Nov 2026 00:22:32 +0100 Subject: [PATCH 83/93] fileserver: a block's place in the eviction order is the version it was cleaned at Found measuring r5's ring change: with DATA's journal ring at 32 MiB instead of 1.3 GiB, osbench's sequence at full repetitions on the host model took 179 s of data writes instead of 40 s. The sample put the time in Inner::evict's VecDeque::retain. A block cleaned again was queued again while its older entry stayed, and retain kept every entry of a clean block, so blocks written over and over (the journal's, now that the ring is small) left the queue longer than its compaction bound after each compaction, and every eviction compacted it again. The stale entry also evicted a block that had been cleaned since. An entry now carries the version its block was clean at; only the entry matching the block's current version is live, so each clean block has one, compaction leaves the queue the size of what it orders, and the oldest place evicts. The same run now takes 34.7 s. Test: block 1 cleaned after block 2 was read survives the eviction block 3 causes, and 100 000 rewrites of two blocks leave the queue within its bound; red when an entry is live by its block's cleanness alone. Co-Authored-By: Claude Opus 5.5 --- system/fileserver/src/cache.rs | 53 +++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/system/fileserver/src/cache.rs b/system/fileserver/src/cache.rs index 94a47a85ad2..13ace870efe 100644 --- a/system/fileserver/src/cache.rs +++ b/system/fileserver/src/cache.rs @@ -108,9 +108,10 @@ struct Inner { /// The most clean blocks kept. clean_limit: usize, slots: BTreeMap, - /// Clean blocks in the order they were read or cleaned; stale entries are - /// skipped when popped. - order: VecDeque, + /// Clean blocks in the order they were read or cleaned, each with the + /// version it was clean at: an entry whose block was written since is + /// stale and skipped, so each clean block has one live entry. + order: VecDeque<(u64, u64)>, dirty: usize, versions: u64, fetches: BTreeMap, @@ -211,7 +212,7 @@ impl Cache { None => { if inner.order.try_reserve(1).is_ok() && let Some(data) = boxed(data) { inner.slots.insert(at, Slot { data: Rc::new(data), dirty: false, version: 0 }); - inner.order.push_back(at); + inner.order.push_back((at, 0)); } } } @@ -450,7 +451,7 @@ impl Cache { // The disk holds it now, so a block whose place in the // eviction order is refused is dropped rather than kept. if inner.order.try_reserve(1).is_ok() { - inner.order.push_back(block); + inner.order.push_back((block, version)); } else { inner.slots.remove(&block); } @@ -464,18 +465,24 @@ impl Inner { /// Drop the oldest clean blocks past the limit. fn evict(&mut self) { while self.slots.len() - self.dirty > self.clean_limit { - let Some(block) = self.order.pop_front() else { break }; - if self.slots.get(&block).is_some_and(|s| !s.dirty) { - self.slots.remove(&block); + let Some(entry) = self.order.pop_front() else { break }; + if self.live(entry) { + self.slots.remove(&entry.0); } } // A queue of mostly stale entries is compacted, so it stays the size // of what it orders. if self.order.len() > 4 * (self.clean_limit + DIRTY_LIMIT) { - let slots = &self.slots; - self.order.retain(|b| slots.get(b).is_some_and(|s| !s.dirty)); + let mut order = core::mem::take(&mut self.order); + order.retain(|&entry| self.live(entry)); + self.order = order; } } + /// Whether `entry` is its block's place in the order: the block is + /// clean at the version the entry was queued at. + fn live(&self, (block, version): (u64, u64)) -> bool { + self.slots.get(&block).is_some_and(|s| !s.dirty && s.version == version) + } } #[cfg(test)] @@ -546,6 +553,32 @@ mod tests { Ok(out[0]) } + /// A block written and flushed again takes its place in the eviction + /// order anew; its earlier place no longer evicts it, and a queue of a + /// few blocks written over and over stays their size. + #[test] + fn a_block_cleaned_again_is_evicted_by_its_newest_place() { + let c = Cache::new(Counting::new(1024), 2); + block_on_ready(c.write(1, &[[1; BLOCK]])).unwrap(); + block_on_ready(c.flush()).unwrap(); + read_one(&c, 2).unwrap(); + block_on_ready(c.write(1, &[[2; BLOCK]])).unwrap(); + block_on_ready(c.flush()).unwrap(); + // A third clean block evicts the oldest place: block 2's. + read_one(&c, 3).unwrap(); + let reads = c.disk.reads.get(); + assert_eq!(read_one(&c, 1).unwrap(), [2; BLOCK]); + assert_eq!(c.disk.reads.get(), reads, "block 1 was cleaned after block 2 was read"); + read_one(&c, 2).unwrap(); + assert_eq!(c.disk.reads.get(), reads + 1, "block 2 was evicted"); + + for round in 0..100_000u64 { + block_on_ready(c.write(round % 2, &[[round as u8; BLOCK]])).unwrap(); + block_on_ready(c.flush()).unwrap(); + } + assert!(c.inner.borrow().order.len() <= 4 * (2 + DIRTY_LIMIT) + 2); + } + #[test] fn a_write_reaches_the_disk_at_the_flush_and_not_before() { let c = cache(); From 6ff184af823072ee414afcfda19e0cdbd00f8dc7 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 5 Nov 2026 00:22:33 +0100 Subject: [PATCH 84/93] bcachefs: mkfs's journal ring is the replay bound, and the writer's cap goes r5 BLOCKER 1. Holding live entries to 32 MiB bounded the replay window, but a mount still read and checksummed the whole ring mkfs laid out at a 128th of the device: about 1.3 GiB on the T14's DATA, 32 GiB on a 4 TB disk. mkfs now lays the ring out at a 128th of the device, at least eight buckets and at most JOURNAL_BYTES (32 MiB), never under what reclaim's shares need. The ring itself is then the bound on what a mount reads and what a replay holds, and the writer's own cap (Writer::hold_live_to, its `live` count and the mount's call) is redundant on every volume this engine formats; it is deleted. What stays for volumes another writer laid out: the scan's floor. An upstream ring can be far larger than any window, and a read-only mount of such a volume would otherwise hold every stale entry's records. Its doc now says so. A ring another writer laid out larger is used whole by this writer when mounted read-write, as that writer sized it. r5 suggestions: close asserts each entry's last sequence against the newest closed entry's, not only the newest written one, so the invariant the scan's refusal relies on fails at its source; the scan test gains a stale entry in a later bucket than the window, which takes the at-insert path; the scan's doc is reflowed. Tests: on an 8 GiB device, whose 128th is 64 MiB, mkfs's ring is JOURNAL_BYTES, and after the writer has gone round it an unclean mount's window is within it, fsck clean; red with the ring sized by the device. The wrapped scan case is red without the at-insert drop. The 171 GiB geometry test asserts the bound where it used to assert a 128th. Co-Authored-By: Claude Opus 5.5 --- ...te-nodes-after-nearly-every-transaction.md | 6 +- lib/bcachefs/src/engine/journal.rs | 25 +++----- lib/bcachefs/src/engine/mkfs.rs | 14 ++++- lib/bcachefs/src/engine/mount/mod.rs | 16 ++--- lib/bcachefs/src/engine/tests/buckets.rs | 2 +- lib/bcachefs/src/engine/tests/journal.rs | 63 ++++++++++++------- 6 files changed, 75 insertions(+), 51 deletions(-) diff --git a/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md b/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md index 9bfa89911f0..63680450be8 100644 --- a/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md +++ b/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md @@ -14,9 +14,9 @@ ordinary transaction is admitted only with a transaction's journal room left on the 64 MiB host test images and QEMU-sized DATA, a few dirty nodes fill it, so reclaim writes nodes after nearly every transaction and the eager put-in-place of #1152 gains nothing there. mkfs sizes the journal at 1/128 -of the device, at least eight buckets (`mkfs.rs`), and the writer holds its -live entries to 32 MiB of it (`JOURNAL_LIVE`), so the benches' DATA -partitions are far from this; only DATA mounts this engine read-write. +of the device, at least eight buckets and at most 32 MiB (`JOURNAL_BYTES` +in `mkfs.rs`), so the benches' DATA partitions are far from this; only +DATA mounts this engine read-write. **Evidence.** The 64 MiB host images' tests: a crash after 2 000 creates replays about a thousand journal keys of them diff --git a/lib/bcachefs/src/engine/journal.rs b/lib/bcachefs/src/engine/journal.rs index 9f6f7d24ede..87f6ecd221d 100644 --- a/lib/bcachefs/src/engine/journal.rs +++ b/lib/bcachefs/src/engine/journal.rs @@ -171,9 +171,10 @@ pub(crate) struct Scan { } /// Reads every journal bucket in ring order (A2.9), each as -/// [`scan_bucket`] does. An entry older than the last sequence of some good -/// flush entry the blacklist leaves keeps no records: the ring can hold far more than any window, -/// and [`Scan::window`] refuses a window that reaches one. +/// [`scan_bucket`] does. A ring another writer laid out can hold far more +/// than any window, so an entry older than the last sequence of a good flush +/// entry the blacklist leaves keeps no records, and [`Scan::window`] refuses +/// a window that reaches one. pub(crate) async fn scan(dev: &impl BlockDev, sb: &Superblock, key: Option<&MasterKey>) -> Result { let mut entries: BTreeMap = BTreeMap::new(); let mut floor = 0; @@ -511,8 +512,6 @@ pub(crate) struct Closed { pub(crate) struct Writer { /// Each ring bucket's device bucket. pub ring: Vec, - /// Ring buckets live entries may hold at once: what a replay reads. - live: usize, pub bucket_sectors: u64, /// Entries are at most this many bytes, `per_bucket` of them to a bucket. entry: usize, @@ -525,6 +524,9 @@ pub(crate) struct Writer { pub newest: Vec>, /// The last sequence of the newest durable flush entry. pub durable_last: u64, + /// The last sequence of the newest entry closed. No later one names an + /// older: the scan refuses a window that does. + pub closed_last: u64, /// The sequence of the newest flush entry on the medium. pub durable: u64, pub open: Option, @@ -549,7 +551,6 @@ impl Writer { assert!(entry > HEADER + TAIL + 2 * max_tx as usize * 8, "journal buckets of {bucket} bytes hold no transaction"); let newest = alloc::vec![None; ring.len()]; Writer { - live: ring.len(), ring, bucket_sectors, entry, @@ -558,6 +559,7 @@ impl Writer { head_sectors: 0, newest, durable_last: 0, + closed_last: 0, durable: 0, open: None, closed: VecDeque::new(), @@ -570,15 +572,9 @@ impl Writer { } } - /// Keeps live entries to at most `buckets` of the ring, so a replay - /// reads no more, wherever on the ring they lie. - pub(crate) fn hold_live_to(&mut self, buckets: usize) { - self.live = buckets.clamp(1, self.ring.len()); - } - - /// Entries the live part of the ring holds. + /// Entries the whole ring holds. pub(crate) fn slots(&self) -> u64 { - self.live as u64 * self.per_bucket + self.ring.len() as u64 * self.per_bucket } /// Body u64s an entry holds for transactions. @@ -590,7 +586,6 @@ impl Writer { pub(crate) fn ring_free(&self) -> u64 { let outstanding = (self.closed.len() + usize::from(self.open.is_some())) as u64; let free = (0..self.ring.len()).filter(|&i| i != self.head && self.newest[i].is_none()).count() as u64; - let free = free.saturating_sub((self.ring.len() - self.live) as u64); let entry_sectors = (self.entry / SECTOR) as u64; (free * self.per_bucket + (self.bucket_sectors - self.head_sectors) / entry_sectors).saturating_sub(outstanding) } diff --git a/lib/bcachefs/src/engine/mkfs.rs b/lib/bcachefs/src/engine/mkfs.rs index 81417284c54..c4048238e56 100644 --- a/lib/bcachefs/src/engine/mkfs.rs +++ b/lib/bcachefs/src/engine/mkfs.rs @@ -65,6 +65,11 @@ pub(crate) const MIN_BUCKET: u64 = 32; const MAX_BUCKET: u64 = 4096; const WANT_BUCKETS: u64 = 2048; const MIN_BUCKETS: u64 = 512; +/// The journal's bytes at most. Its live entries are what an unclean mount +/// replays, every key of them held in memory until its nodes take them, and +/// a mount reads the whole ring: so the ring, not the device, bounds a +/// replay's memory, reading and work. +pub(crate) const JOURNAL_BYTES: u64 = 32 << 20; /// Every btree a v1 volume writes, each given a root here: a node count /// that only grows with its keys keeps the layout's fixed point monotone. @@ -422,7 +427,10 @@ impl Geometry { if !block.is_power_of_two() || block < BLOCK_SECTORS || !node.is_power_of_two() || node < block || node > bucket.min(MAX_NODE_SECTORS) { return Err(Error::Invalid("a block or node size mkfs does not lay out")); } - let journal = (front, (nbuckets / 128).max(8).max(super::mount::journal_buckets(bucket, (block * SECTOR as u64) as usize))); + // A 128th of the device, at least eight buckets and at most + // JOURNAL_BYTES, and never less than reclaim's shares need. + let journal = (nbuckets / 128).max(8).min(JOURNAL_BYTES / (bucket * SECTOR as u64)); + let journal = (front, journal.max(super::mount::journal_buckets(bucket, (block * SECTOR as u64) as usize))); Ok(Geometry { block, bucket, nbuckets, node, layout, journal, data_buckets: 0 }) } @@ -777,7 +785,9 @@ mod tests { // 171 GiB, as `format` lays out a partition that size. let g = Geometry::new(171 << 21).unwrap(); assert_eq!((g.bucket, g.node), (4096, 512)); - assert_eq!(g.journal.1, g.nbuckets / 128); + // JOURNAL_BYTES, where a 128th would be 1.3 GiB. + assert_eq!(g.journal.1 * g.bucket * SECTOR as u64, JOURNAL_BYTES); + assert!(g.nbuckets / 128 > g.journal.1); } #[test] diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 43231ce76f6..1c44bc3fb9d 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -83,11 +83,6 @@ const RW_FEATURES: u64 = 1 << 8 | 1 << 9 | 1 << 12 | 1 << 13 | 1 << 17 | 1 << 18 /// Feature bit 5: the superblock holds a blacklist. const BLACKLIST_FEATURE: u64 = 1 << 5; -/// Journal bytes live entries hold at most. A replay holds every key of -/// them in memory until its nodes take them, so this, not the journal -/// mkfs sized by the device, bounds an unclean mount's memory and time. -const JOURNAL_LIVE: u64 = 32 << 20; - /// Ring geometry that leaves two cache-sized reclaim shares and an ordinary /// quarter, with room for at least one maximum-size ordinary transaction. pub(crate) fn journal_buckets(bucket_sectors: u64, block: usize) -> u64 { @@ -393,8 +388,6 @@ impl Mount { let ring: Vec = sb.journal.iter().flat_map(|&(first, count)| first..first + count).collect(); let bucket_sectors = u64::from(sb.member.bucket_sectors); let mut journal = Writer::new(ring, bucket_sectors, sb.block_bytes(), K_TX); - let live = (JOURNAL_LIVE / (bucket_sectors * SECTOR as u64)).max(journal_buckets(bucket_sectors, sb.block_bytes())); - journal.hold_live_to(usize::try_from(live).unwrap_or(usize::MAX)); let mut blacklist = sb.blacklist.clone(); let (clocks, restate) = match &replay { None => { @@ -1405,7 +1398,8 @@ impl Mount { let mut open = st.journal.open.take().expect("an entry open"); let pinned = st.oldest_pin().map_or(open.seq, |p| p.min(open.seq)); let last_seq = if own { pinned } else { pinned.min(st.journal.durable) }; - assert!(last_seq >= st.journal.durable_last, "a journal entry's last sequence moved back"); + assert!(last_seq >= st.journal.durable_last.max(st.journal.closed_last), "a journal entry's last sequence moved back"); + st.journal.closed_last = last_seq; let roots = encoded_roots(&st.roots); journal::tail(&mut open.area, roots.iter().map(|(b, l, k)| (*b, *l, &k[..])), Some(st.now), st.key_version, st.clocks); let entry = Jset { seq: open.seq, last_seq, area: &open.area }; @@ -1732,6 +1726,12 @@ impl Mount { Ok(self.st.borrow().published) } + /// The ring bucket the journal writes into. + #[cfg(test)] + pub(crate) fn journal_head(&self) -> usize { + self.st.borrow().journal.head + } + pub(crate) fn journal_room(&self) -> (u64, u64) { let st = self.st.borrow(); (st.journal.ring_free(), 2 * st.journal.reclaim_entries) diff --git a/lib/bcachefs/src/engine/tests/buckets.rs b/lib/bcachefs/src/engine/tests/buckets.rs index caf1decfdd4..d74aa325ab7 100644 --- a/lib/bcachefs/src/engine/tests/buckets.rs +++ b/lib/bcachefs/src/engine/tests/buckets.rs @@ -58,7 +58,7 @@ impl BlockDev for Sparse { } } -fn formatted_on(dev: &D) { +pub(super) fn formatted_on(dev: &D) { let options = Options { label: b"toyos", time: 1_761_177_600_000_000_000, encryption: None }; block_on_ready(mkfs(dev, &options, &Tree::new(), &mut Seeded(0x5eed))).unwrap(); } diff --git a/lib/bcachefs/src/engine/tests/journal.rs b/lib/bcachefs/src/engine/tests/journal.rs index 0a070860ac2..bc64f194833 100644 --- a/lib/bcachefs/src/engine/tests/journal.rs +++ b/lib/bcachefs/src/engine/tests/journal.rs @@ -297,31 +297,41 @@ fn a_window_with_an_entry_missing_or_failing_its_checksum_is_refused() { assert_eq!(window_of(&unclean_with(&[], |_, _| {})).unwrap().map(|w| (w.first, w.last, w.next)), None); } -/// A writer holding its live entries to three buckets of an eight-bucket -/// ring offers what a writer of a three-bucket ring does, as its buckets -/// fill, wherever on the ring they lie. +/// On a device a 128th of which is past JOURNAL_BYTES, mkfs's ring is +/// JOURNAL_BYTES, and an unclean mount after the writer has gone round it +/// replays a window within it. Red when mkfs sizes the ring by the device. #[test] -fn a_writer_held_to_some_buckets_offers_what_a_ring_of_that_many_does() { - let (bucket_sectors, block) = (512, BLOCK); - let mut held = journal::Writer::new((0..8).collect(), bucket_sectors, block, super::super::mount::K_TX); - held.hold_live_to(3); - let mut small = journal::Writer::new((0..3).collect(), bucket_sectors, block, super::super::mount::K_TX); - assert_eq!(held.slots(), small.slots()); - assert_eq!(held.ring_free(), small.ring_free()); - (held.head, small.head) = (5, 1); - held.newest[4] = Some(9); - small.newest[0] = Some(9); - assert_eq!(held.ring_free(), small.ring_free()); - held.newest[6] = Some(10); - small.newest[2] = Some(10); - // Only what the head bucket has left. - assert_eq!(held.ring_free(), small.ring_free()); - held.head_sectors = bucket_sectors; - small.head_sectors = bucket_sectors; - assert_eq!((held.ring_free(), small.ring_free()), (0, 0)); +fn a_large_volumes_ring_and_its_replay_window_stay_within_journal_bytes() { + let dev = super::buckets::Sparse::new(8 << 30); + super::buckets::formatted_on(&dev); + let sb = block_on_ready(super::super::read::read_superblock(&dev)).unwrap().0; + let bucket = u64::from(sb.member.bucket_sectors) * SECTOR as u64; + let ring: u64 = sb.journal.iter().map(|&(_, count)| count).sum(); + assert!(sb.member.nbuckets / 128 > ring, "a 128th of the device is more than the ring"); + assert_eq!(ring * bucket, mkfs::JOURNAL_BYTES); + + let mount = block_on_ready(Mount::open(&dev, MountOptions { cache_nodes: 256 }, super::mount::random())).unwrap(); + let mut last = mount.journal_head(); + let wrapped = (0..100_000).any(|i| { + block_on_ready(mount.create(mount.root(), format!("f{i}").as_bytes(), New::File, super::mount::OWNER, super::mount::NOW)).unwrap(); + block_on_ready(mount.commit()).unwrap(); + let head = mount.journal_head(); + let back = head < last; + last = head; + back + }); + assert!(wrapped, "the writer went round the ring"); + drop(mount); + + let v = block_on_ready(Volume::open(&dev)).unwrap(); + let (window, _) = v.replayed().expect("an unclean volume replays"); + let scan = block_on_ready(v.scan_journal()).unwrap(); + let bytes: u64 = scan.in_window(v.superblock(), window).map(|e| e.sectors * SECTOR as u64).sum(); + assert!(bytes <= mkfs::JOURNAL_BYTES, "a window of {bytes} bytes"); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } -/// The ring can hold far more than a window: an entry older than a good +/// A ring can hold far more than a window: an entry older than a good /// flush entry's last sequence keeps no records, wherever in the ring it /// lies. A window that reaches one has last sequences that went back, and /// is refused. @@ -339,6 +349,15 @@ fn a_scan_keeps_no_records_of_entries_behind_a_flush_entrys_last_sequence() { let moving = unclean_with(&[(1, 1, area.clone(), 0), (2, 1, area.clone(), 0), (3, 2, area.clone(), 0), (4, 3, area.clone(), 0)], |_, _| {}); assert_eq!(scanned(&moving), [(1, true, true), (2, true, true), (3, false, false), (4, false, false)]); assert_eq!(window_of(&moving).unwrap().map(|w| (w.first, w.last)), Some((3, 4))); + // A stale entry in a later bucket than the window, as a wrapped ring + // leaves it: read after the floor is known. + let dev = formatted(8 << 20, None); + let bucket_blocks = u64::from(block_on_ready(super::super::read::read_superblock(&dev)).unwrap().0.member.bucket_sectors) / 8; + let ring = Ring::of(&dev, None); + let used = (ring.entry(3, 2, &area).len() + ring.entry(4, 3, &area).len()) as u64; + let wrapped = unclean_with(&[(3, 2, area.clone(), 0), (4, 3, area.clone(), bucket_blocks - used), (1, 1, area.clone(), 0)], |_, _| {}); + assert_eq!(scanned(&wrapped), [(1, true, true), (3, false, false), (4, false, false)]); + assert_eq!(window_of(&wrapped).unwrap().map(|w| (w.first, w.last)), Some((3, 4))); // Entry 3 said its window starts at 3, the newest that it starts at 2. let back = unclean_with(&[(1, 1, area.clone(), 0), (2, 1, area.clone(), 0), (3, 3, area.clone(), 0), (4, 2, area.clone(), 0)], |_, _| {}); assert_eq!(window_of(&back), Err(Error::Corrupt("a journal window reaching behind another flush entry's last sequence"))); From cd639c3e6fffb0aeae14db324e39d0ed2e4fcc37 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 5 Nov 2026 00:50:21 +0100 Subject: [PATCH 85/93] review r6 of #1152: a journal larger than mkfs lays out is not written BLOCKER 1, by the coordinator's ruling: a ring another writer laid out larger than mkfs would for the same device was used whole by this writer, so an unclean mount of, say, a DATA formatted by upstream's tools would replay whatever that ring held. writable() now refuses it read-write by name, "writing a volume whose journal is larger than mkfs lays out", with mkfs's sizing factored into mkfs::ring_buckets so both read one rule. Read-only mounts keep the scan's floor. Test: a 64 MiB volume whose superblock names one ring bucket more is refused read-write by that name and still read; red with the check removed (the mount opened). Suggestions: the 8 GiB test is now what it can show, the writer going round a ring of 2 MiB buckets and an unclean mount replaying it clean (the window bound held by construction); each fetched block takes a fresh version, so only its newest place in the eviction order is live. Co-Authored-By: Claude Opus 5.5 --- lib/bcachefs/src/engine/mkfs.rs | 13 +++++++++---- lib/bcachefs/src/engine/mount/mod.rs | 6 ++++++ lib/bcachefs/src/engine/tests/journal.rs | 18 ++++++------------ lib/bcachefs/src/engine/tests/mount.rs | 16 ++++++++++++++++ system/fileserver/src/cache.rs | 11 +++++++---- 5 files changed, 44 insertions(+), 20 deletions(-) diff --git a/lib/bcachefs/src/engine/mkfs.rs b/lib/bcachefs/src/engine/mkfs.rs index c4048238e56..18bcd542382 100644 --- a/lib/bcachefs/src/engine/mkfs.rs +++ b/lib/bcachefs/src/engine/mkfs.rs @@ -71,6 +71,14 @@ const MIN_BUCKETS: u64 = 512; /// replay's memory, reading and work. pub(crate) const JOURNAL_BYTES: u64 = 32 << 20; +/// The journal buckets mkfs lays out for `nbuckets` buckets of `bucket` +/// sectors and blocks of `block` bytes: a 128th of the device, at least +/// eight buckets and at most JOURNAL_BYTES, and never less than reclaim's +/// shares need. +pub(crate) fn ring_buckets(nbuckets: u64, bucket: u64, block: usize) -> u64 { + (nbuckets / 128).max(8).min(JOURNAL_BYTES / (bucket * SECTOR as u64)).max(super::mount::journal_buckets(bucket, block)) +} + /// Every btree a v1 volume writes, each given a root here: a node count /// that only grows with its keys keeps the layout's fixed point monotone. const BTREES: [Btree; 15] = [ @@ -427,10 +435,7 @@ impl Geometry { if !block.is_power_of_two() || block < BLOCK_SECTORS || !node.is_power_of_two() || node < block || node > bucket.min(MAX_NODE_SECTORS) { return Err(Error::Invalid("a block or node size mkfs does not lay out")); } - // A 128th of the device, at least eight buckets and at most - // JOURNAL_BYTES, and never less than reclaim's shares need. - let journal = (nbuckets / 128).max(8).min(JOURNAL_BYTES / (bucket * SECTOR as u64)); - let journal = (front, journal.max(super::mount::journal_buckets(bucket, (block * SECTOR as u64) as usize))); + let journal = (front, ring_buckets(nbuckets, bucket, (block * SECTOR as u64) as usize)); Ok(Geometry { block, bucket, nbuckets, node, layout, journal, data_buckets: 0 }) } diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 1c44bc3fb9d..bdd9820840f 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -320,6 +320,12 @@ fn writable(sb: &Superblock) -> Result<()> { if sb.journal.is_empty() { return Err(Error::Unsupported("writing a volume without journal buckets")); } + // An unclean mount replays what the ring holds; mkfs's ring is the + // bound on that, so a larger one, another writer's, is not written. + let ring: u64 = sb.journal.iter().map(|&(_, count)| count).sum(); + if ring > super::mkfs::ring_buckets(sb.member.nbuckets, u64::from(sb.member.bucket_sectors), sb.block_bytes()) { + return Err(Error::Unsupported("writing a volume whose journal is larger than mkfs lays out")); + } if !splits_fit(sb.node_sectors() as usize * SECTOR) { return Err(Error::Unsupported("writing a volume of btree nodes too small for reclaim's largest split")); } diff --git a/lib/bcachefs/src/engine/tests/journal.rs b/lib/bcachefs/src/engine/tests/journal.rs index bc64f194833..9c525286b25 100644 --- a/lib/bcachefs/src/engine/tests/journal.rs +++ b/lib/bcachefs/src/engine/tests/journal.rs @@ -297,18 +297,15 @@ fn a_window_with_an_entry_missing_or_failing_its_checksum_is_refused() { assert_eq!(window_of(&unclean_with(&[], |_, _| {})).unwrap().map(|w| (w.first, w.last, w.next)), None); } -/// On a device a 128th of which is past JOURNAL_BYTES, mkfs's ring is -/// JOURNAL_BYTES, and an unclean mount after the writer has gone round it -/// replays a window within it. Red when mkfs sizes the ring by the device. +/// The writer goes round the ring of a production geometry, 2 MiB buckets +/// on an 8 GiB device, and an unclean mount after replays it to a volume +/// fsck finds clean. #[test] -fn a_large_volumes_ring_and_its_replay_window_stay_within_journal_bytes() { +fn a_writer_that_went_round_a_ring_of_2_mib_buckets_replays_clean() { let dev = super::buckets::Sparse::new(8 << 30); super::buckets::formatted_on(&dev); let sb = block_on_ready(super::super::read::read_superblock(&dev)).unwrap().0; - let bucket = u64::from(sb.member.bucket_sectors) * SECTOR as u64; - let ring: u64 = sb.journal.iter().map(|&(_, count)| count).sum(); - assert!(sb.member.nbuckets / 128 > ring, "a 128th of the device is more than the ring"); - assert_eq!(ring * bucket, mkfs::JOURNAL_BYTES); + assert_eq!(sb.member.bucket_sectors, 4096, "2 MiB buckets"); let mount = block_on_ready(Mount::open(&dev, MountOptions { cache_nodes: 256 }, super::mount::random())).unwrap(); let mut last = mount.journal_head(); @@ -324,10 +321,7 @@ fn a_large_volumes_ring_and_its_replay_window_stay_within_journal_bytes() { drop(mount); let v = block_on_ready(Volume::open(&dev)).unwrap(); - let (window, _) = v.replayed().expect("an unclean volume replays"); - let scan = block_on_ready(v.scan_journal()).unwrap(); - let bytes: u64 = scan.in_window(v.superblock(), window).map(|e| e.sectors * SECTOR as u64).sum(); - assert!(bytes <= mkfs::JOURNAL_BYTES, "a window of {bytes} bytes"); + assert!(v.replayed().is_some(), "an unclean volume replays"); assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } diff --git a/lib/bcachefs/src/engine/tests/mount.rs b/lib/bcachefs/src/engine/tests/mount.rs index 80c98aeeb4e..59bd7ff4bb5 100644 --- a/lib/bcachefs/src/engine/tests/mount.rs +++ b/lib/bcachefs/src/engine/tests/mount.rs @@ -112,6 +112,22 @@ fn the_unclean_fixture_of_16_kib_nodes_is_read_and_refused_read_write() { assert_eq!(read_all(&v, path(&v, "f200")).unwrap(), b"200\n"); } +/// A journal ring larger than mkfs lays out, as another writer may make +/// one, is refused read-write by name, since an unclean mount would replay +/// what it holds; it is still read. +#[test] +fn a_volume_whose_journal_is_larger_than_mkfs_lays_out_is_read_but_not_written() { + let dev = formatted(64 << 20, None); + rewrite_superblock(&dev, |_, fields| { + let ring = fields.iter_mut().find(|f| f.kind == super::super::superblock::field_type::JOURNAL_V2).unwrap(); + let count = u64::from_le_bytes(ring.body[8..16].try_into().unwrap()); + ring.body[8..16].copy_from_slice(&(count + 1).to_le_bytes()); + }); + assert_eq!(block_on_ready(Mount::open(&dev, MEMORY, random())).err(), Some(Error::Unsupported("writing a volume whose journal is larger than mkfs lays out"))); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert!(block_on_ready(v.readdir(v.root(), 0, 10)).is_ok()); +} + /// What this writer does not handle in full is refused read-write by name /// (§3.1): a superblock field of a type v1.39.4 does not define, and a /// malformed logged truncate it cannot safely resume. diff --git a/system/fileserver/src/cache.rs b/system/fileserver/src/cache.rs index 13ace870efe..85ab0098f18 100644 --- a/system/fileserver/src/cache.rs +++ b/system/fileserver/src/cache.rs @@ -92,8 +92,9 @@ struct Slot { /// takes a block of its own. data: SharedBlock, dirty: bool, - /// Which write put `data` here, so a flush that carried an older copy - /// leaves the block dirty. + /// Which write or fetch put `data` here, so a flush that carried an + /// older copy leaves the block dirty, and only the newest place in the + /// eviction order is live. version: u64, } @@ -211,8 +212,10 @@ impl Cache { Some(slot) => *data = **slot.data, None => { if inner.order.try_reserve(1).is_ok() && let Some(data) = boxed(data) { - inner.slots.insert(at, Slot { data: Rc::new(data), dirty: false, version: 0 }); - inner.order.push_back((at, 0)); + inner.versions += 1; + let version = inner.versions; + inner.slots.insert(at, Slot { data: Rc::new(data), dirty: false, version }); + inner.order.push_back((at, version)); } } } From 49fe4799316b1a31ec56678ec793f61b83866ecb Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 5 Nov 2026 01:13:26 +0100 Subject: [PATCH 86/93] storage: a write that continues the last extends its reservation, and reserving takes no transaction in an open bucket osbench at 6ff184af8: seq_write.4096 18 MB/s on both benches (main 133-211, Linux 624-742), write_4k_at 25-30 us (main 10-12). Measured on the host model of the T14's DATA before fixing, 4 KiB calls through the file server's reserve and its dirty-page write-back: reserve 8.7 us a call, write-back 23.6 us a page (1 MiB calls: 2.9 + 2.6) Every call was its own reservation: a transaction to take its sectors, and at write-back one transaction and one extent per page, since pages under different reservations are written apart. Now: - Mount::extend reserves the blocks past a reservation's end into it; where the new sectors follow the last run's in the same bucket, they join that run (up to the 64 KiB extent), so a write-back of both is one extent. Mount::reserve is an empty reservation extended. - Sectors of the open data bucket are taken in memory when the grant's own room judgement (Mount::room_for, what grant refuses NoSpace without) holds; a new bucket, or a judgement that may wait or refuse, is still a transaction's. - The file server's DATA volume extends the last write's ticket when a write starts where its reservation ends; a refused write cuts the extension back (Mount::cut), and an unreserve that landed nothing leaves the ticket's earlier pages alone. Same measurement now: reserve 0.9 us a call, write-back 3.6-3.9 us a page. Scattered 4 KiB overwrites (write_4k_at's shape): reserve 0.8 us, the overwrite's transaction itself 30.6 us; the latter is the copy-on-write replace of the old extent, unchanged here. Tests: extending a reservation block by block writes back as at most one extent per bucket of four blocks (64 extents without merging, red) and a cut gives back the last extension; on 2 MiB buckets a run never joins across into the next bucket (red without that condition: the harness had hit "data placed in a bucket its alloc key holds otherwise"); cancelling a write that extended the last one gives back its room (red without the cut). The partial write-back test writes its tail first so its two pages stay under two writes' capacity, as it states. Co-Authored-By: Claude Opus 5.5 --- lib/bcachefs/src/engine/mount/data.rs | 113 +++++++++++++++++++------- lib/bcachefs/src/engine/mount/mod.rs | 11 +++ lib/bcachefs/src/engine/tests/data.rs | 48 +++++++++++ system/fileserver/src/data.rs | 39 ++++++++- system/fileserver/src/pages/tests.rs | 9 +- 5 files changed, 183 insertions(+), 37 deletions(-) diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index 6dbf9a7c750..80470616e9c 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -93,7 +93,11 @@ fn start(k: &Key) -> Result { } fn regular(inode: &InodeValue) -> Result<()> { - match inode.mode() & mode::S_IFMT { + regular_mode(inode.mode()) +} + +fn regular_mode(bits: u16) -> Result<()> { + match bits & mode::S_IFMT { mode::S_IFREG | mode::S_IFLNK => Ok(()), mode::S_IFDIR => Err(Error::IsDirectory), _ => Err(Error::Invalid("file data on a special inode")), @@ -115,10 +119,7 @@ impl Mount { pub async fn reserve(&self, ino: u64, offset: u64, len: usize) -> Result { let block = self.sb.block_bytes() as u64; let first = offset / block * block; - let end = offset.checked_add(len as u64).filter(|&end| len > 0 && i64::try_from(end).is_ok()).ok_or(Error::Invalid("reservation of nothing or past 2^63-1"))?.div_ceil(block) * block; - let bucket = u64::from(self.sb.member.bucket_sectors); - let longest = EXTENT_MAX.min(bucket * SECTOR as u64) / block * block; - let _data = self.data_slot().await; + let end = offset.checked_add(len as u64).filter(|&end| len > 0 && i64::try_from(end).is_ok()).ok_or(Error::Invalid("reservation of nothing or past 2^63-1"))?; let id = { let mut pool = self.reservations.borrow_mut(); let id = pool.next; @@ -126,35 +127,89 @@ impl Mount { pool.active.insert(id, Reserved { ino, block_sectors: block / SECTOR as u64, runs: Vec::new() }); id }; - let reservation = Reservation { owner: Rc::clone(&self.reservations), id, first, bytes: end - first }; - let mut at = first; + let mut reservation = Reservation { owner: Rc::clone(&self.reservations), id, first, bytes: 0 }; + self.extend(&mut reservation, (end - first) as usize).await?; + Ok(reservation) + } + + /// Reserves the whole blocks of the `len` bytes past `reservation`'s end + /// into it: a write that continues the last one, whose blocks then join + /// its last run where the sectors follow on in the same bucket, so a + /// write-back of both is one extent. + pub async fn extend(&self, reservation: &mut Reservation, len: usize) -> Result<()> { + if !Rc::ptr_eq(&reservation.owner, &self.reservations) { + return Err(Error::Invalid("reservation belongs to another mount")); + } + let block = self.sb.block_bytes() as u64; + let (from, at_end) = reservation.range(); + let end = at_end.checked_add(len as u64).filter(|&end| len > 0 && i64::try_from(end).is_ok()).ok_or(Error::Invalid("reservation of nothing or past 2^63-1"))?.div_ceil(block) * block; + let bucket = u64::from(self.sb.member.bucket_sectors); + let longest = EXTENT_MAX.min(bucket * SECTOR as u64) / block * block; + let _data = self.data_slot().await; + let ino = self.reservations.borrow().active.get(&reservation.id).ok_or(Error::Invalid("reservation already written"))?.ino; + regular_mode(self.inode(ino).await?.mode)?; + let mut at = at_end; while at < end { let wanted = (end - at).min(longest) / SECTOR as u64; - let got = loop { - let out = self - .transact(K_TX, FOOTPRINT, Draw::Ordinary, async |txn: &mut Txn| { - regular(&self.inode_in(txn, ino).await?)?; - let (sector, got) = if wanted * SECTOR as u64 == block { - (self.data_sector(wanted, Draw::Ordinary, true).await?.0, wanted) - } else { - self.data_run(wanted, Draw::Ordinary).await? - }; - let blocks = (got * SECTOR as u64 / block) as u32; - let mut st = self.st.borrow_mut(); - for _ in 0..blocks { st.alloc.pin(sector / bucket); } - let run = Run { first: at, sector, blocks, written: 0 }; - self.reservations.borrow_mut().active.get_mut(&id).expect("a reservation being made").runs.push(run); - Ok(got) - }) - .await; - match out { - Err(Error::NoSpace(_)) if self.compact_user().await? => continue, - other => break other?, - } + let single = wanted * SECTOR as u64 == block; + // Sectors of the open bucket are taken in memory where room for + // their write is plain; a new bucket, or a judgement that may + // wait or refuse, is a transaction's. + let taken = if self.room_for(K_TX, FOOTPRINT) { self.st.borrow_mut().alloc.next_data(bucket, wanted, single, !single).map(|(sector, _, got)| (sector, got)) } else { None }; + let (sector, got) = match taken { + Some(taken) => taken, + None => loop { + let out = self + .transact(K_TX, FOOTPRINT, Draw::Ordinary, async |_: &mut Txn| { + if single { self.data_sector(wanted, Draw::Ordinary, true).await.map(|(sector, _)| (sector, wanted)) } else { self.data_run(wanted, Draw::Ordinary).await } + }) + .await; + match out { + Err(Error::NoSpace(_)) if self.compact_user().await? => continue, + other => break other?, + } + }, }; + let blocks = (got * SECTOR as u64 / block) as u32; + { + let mut st = self.st.borrow_mut(); + for _ in 0..blocks { st.alloc.pin(sector / bucket); } + } + let mut pool = self.reservations.borrow_mut(); + let runs = &mut pool.active.get_mut(&reservation.id).expect("a reservation being made").runs; + match runs.last_mut() { + Some(last) if last.first + u64::from(last.blocks) * block == at + && last.sector + u64::from(last.blocks) * block / SECTOR as u64 == sector + && last.sector / bucket == sector / bucket + && u64::from(last.blocks + blocks) * block <= longest => last.blocks += blocks, + _ => runs.push(Run { first: at, sector, blocks, written: 0 }), + } at += got * SECTOR as u64; + reservation.bytes = at - from; } - Ok(reservation) + Ok(()) + } + + /// Gives back the blocks of `reservation` from the one holding `to` on, + /// none of them written: what a write that extended it and was refused + /// had taken. + pub fn cut(&self, reservation: &mut Reservation, to: u64) { + let block = self.sb.block_bytes() as u64; + let to = to.max(reservation.first).div_ceil(block) * block; + let mut pool = self.reservations.borrow_mut(); + let Some(reserved) = pool.active.get_mut(&reservation.id) else { return }; + let mut canceled = Vec::new(); + reserved.runs.retain_mut(|run| { + let kept = (to.saturating_sub(run.first) / block).min(u64::from(run.blocks)) as u32; + for b in kept..run.blocks { + assert!(run.written & 1 << b == 0, "a reserved block cut after its write"); + canceled.push(run.sector + u64::from(b) * reserved.block_sectors); + } + run.blocks = kept; + kept > 0 + }); + pool.canceled.extend(canceled); + reservation.bytes = reservation.bytes.min(to - reservation.first); } /// Writes into blocks of the reservation not yet written, as one extent diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index bdd9820840f..4ca1c656398 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -1098,6 +1098,17 @@ impl Mount { /// left unwritten. Writing later changes leaves those ledgers as they /// are, and on a node of one block reclaim's writes never settle, as /// each one's keys dirty another node. + /// Whether an ordinary transaction of `journal` u64s and `nodes` nodes + /// would find room for what it may write: what [`Mount::grant`] refuses + /// `NoSpace` without, judged now. + pub(super) fn room_for(&self, journal: u64, nodes: usize) -> bool { + self.cancel_dropped(); + let pending = self.reservations.borrow().pending(Draw::Ordinary); + let st = self.st.borrow(); + let stable = self.metadata_nodes(&st, pending + st.granted_bytes + journal * 8); + st.read_only.is_none() && st.alloc.keeps(st.metadata_buckets, stable, nodes, st.reclaim_room, Draw::Ordinary, false) && st.alloc.fits(nodes, st.reclaim_room, Draw::Ordinary) + } + fn grant(&self, class: Class, draw: Draw, journal: u64, nodes: usize, arrived: u64) -> Result { self.cancel_dropped(); let pending = self.reservations.borrow().pending(draw); diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index 4234dcb602e..41b34740032 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -589,6 +589,54 @@ fn a_write_is_one_extent_for_each_run_of_contiguous_blocks() { assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } +/// Writes of a block each, every one extending the reservation the last +/// made, are written back as one extent for each run of contiguous sectors +/// in one bucket, as one write of them all is; a cut gives back what the +/// last extension took. On 16 KiB buckets the runs cross buckets often. +#[test] +fn an_extended_reservation_writes_back_as_one_write_would() { + let dev = formatted(64 << 20, None); + let data: Vec = (0..64 * 4096).map(|i| (i * 7) as u8).collect(); + let ino = mounted(&dev, async |m: &Mount<&Image>| { + let ino = m.create(m.root(), b"f", New::File, OWNER, NOW).await.unwrap().ino; + let mut token = m.reserve(ino, 0, 4096).await.unwrap(); + for _ in 1..64 { m.extend(&mut token, 4096).await.unwrap(); } + m.extend(&mut token, 4096).await.unwrap(); + m.cut(&mut token, 64 * 4096); + assert_eq!(token.range(), (0, 64 * 4096)); + assert_eq!(m.write_reserved(&mut token, 0, &data, NOW).await.unwrap(), data.len()); + ino + }); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(read_all(&v, ino).unwrap(), data); + let runs = extents(&v, ino); + assert!(runs.len() <= 64 / 4 + 1, "{} extents for 64 blocks on buckets of four", runs.len()); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} + +/// The same across the end of a bucket of 2 MiB, where the next bucket +/// taken can lie right after it: a run does not cross into it. +#[test] +fn an_extended_reservation_keeps_each_run_in_one_bucket() { + use super::buckets::Sparse; + let dev = Sparse::new(4 << 30); + block_on_ready(mkfs(&dev, &Options { label: b"data", time: NOW as u64, encryption: None }, &Tree::new(), &mut Seeded(7))).unwrap(); + let blocks = 2 * 512 + 64; + let data: Vec = (0..blocks * 4096).map(|i| (i * 7) as u8).collect(); + let m = block_on_ready(Mount::open(&dev, MEMORY, random())).unwrap(); + // Three blocks first, so runs of sixteen do not end where buckets do. + let pad = block_on_ready(m.create(m.root(), b"pad", New::File, OWNER, NOW)).unwrap().ino; + block_on_ready(m.write(pad, 0, &[1; 3 * 4096], NOW)).unwrap(); + let ino = block_on_ready(m.create(m.root(), b"f", New::File, OWNER, NOW)).unwrap().ino; + let mut token = block_on_ready(m.reserve(ino, 0, 4096)).unwrap(); + for _ in 1..blocks { block_on_ready(m.extend(&mut token, 4096)).unwrap(); } + assert_eq!(block_on_ready(m.write_reserved(&mut token, 0, &data, NOW)).unwrap(), data.len()); + block_on_ready(m.unmount()).unwrap(); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + assert_eq!(read_all(&v, ino).unwrap(), data); + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); +} + /// A write over more old extents than one transaction may remove goes in /// parts, each removing at most two, and the file reads as written. #[test] diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index 88f74b5aeb9..68a413876af 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -40,6 +40,11 @@ struct Open { pages: BTreeMap, /// Pages the current write reserved and has not copied yet. incoming: BTreeMap, + /// The last write's ticket, which a write starting where its + /// reservation ends extends, so a write-back of both is one extent; and + /// where that reservation ended before the current write extended it. + tail: Option, + extended_from: Option, } impl Open { /// A page no longer needs `ticket`'s capacity. @@ -418,6 +423,8 @@ impl Volume for DataVolume { capacity: BTreeMap::new(), pages: BTreeMap::new(), incoming: BTreeMap::new(), + tail: None, + extended_from: None, }) .holders += 1; if how.truncate @@ -483,14 +490,30 @@ impl Volume for DataVolume { } let pages = offset / BLOCK as u64..end.div_ceil(BLOCK as u64); let bytes = ((pages.end - pages.start) as usize).checked_mul(BLOCK).ok_or(Error::FileTooLarge)?; - let reservation = self.cache.block_on(self.fs.reserve(node, pages.start * BLOCK as u64, bytes)).map_err(word)?; let open = self.open.get_mut(&node).unwrap(); - open.capacity.insert(ticket, Capacity { reservation, pages: (pages.end - pages.start) as usize }); + let continued = open.tail.filter(|tail| open.capacity.get(tail).is_some_and(|c| c.reservation.range().1 == offset)); + open.extended_from = continued.map(|_| offset); + let ticket = match continued { + Some(tail) => { + let capacity = open.capacity.get_mut(&tail).unwrap(); + self.cache.block_on(self.fs.extend(&mut capacity.reservation, bytes)).map_err(word)?; + capacity.pages += (pages.end - pages.start) as usize; + tail + } + None => { + let reservation = self.cache.block_on(self.fs.reserve(node, pages.start * BLOCK as u64, bytes)).map_err(word)?; + let open = self.open.get_mut(&node).unwrap(); + open.capacity.insert(ticket, Capacity { reservation, pages: (pages.end - pages.start) as usize }); + ticket + } + }; + let open = self.open.get_mut(&node).unwrap(); + open.tail = Some(ticket); open.incoming.extend(pages.map(|page| (page, ticket))); open.size = open.size.max(end); open.mtime = (self.clock)(); open.changed = true; - Ok(reserved) + Ok(Reserved { ticket, ..reserved }) } fn landed(&mut self, node: Node, reserved: Reserved, offset: u64, len: usize) { if len == 0 { return; } @@ -507,8 +530,16 @@ impl Volume for DataVolume { let open = self.open.get_mut(&node).ok_or(Error::Stale)?; let ticket = reserved.ticket; let cut = |page: u64, held: u64, from: u64| held == ticket && page * BLOCK as u64 >= from; + // Copied pages of this ticket below `landed` are this write's or an + // earlier one's it continued; none landed leaves them all. + let from = if landed == 0 { u64::MAX } else { landed }; let gone: Vec = open.incoming.iter().filter(|&(&p, &t)| cut(p, t, 0)).map(|(&p, _)| p) - .chain(open.pages.iter().filter(|&(&p, &t)| cut(p, t, landed)).map(|(&p, _)| p)).collect(); + .chain(open.pages.iter().filter(|&(&p, &t)| cut(p, t, from)).map(|(&p, _)| p)).collect(); + // An extension gives its blocks back now; a reservation of its own + // goes with its last page. + if let (Some(from), Some(capacity)) = (open.extended_from.take(), open.capacity.get_mut(&ticket)) { + self.fs.cut(&mut capacity.reservation, from.max(landed)); + } for page in gone { if open.incoming.remove(&page).or_else(|| open.pages.remove(&page)).is_some() { open.release(ticket); diff --git a/system/fileserver/src/pages/tests.rs b/system/fileserver/src/pages/tests.rs index 1dbb9b65383..720cb309585 100644 --- a/system/fileserver/src/pages/tests.rs +++ b/system/fileserver/src/pages/tests.rs @@ -524,9 +524,10 @@ fn an_overwrite_keeps_incoming_capacity_while_its_older_dirty_page_is_flushed() } #[test] -/// Two pages each under its own write's capacity, written back in one run: -/// the second, the file's last page in part, reads the old block it keeps -/// the rest of, and that read is refused. +/// Two pages each under its own write's capacity, the tail written first so +/// the second write does not continue it, written back in one run: the +/// second page, the file's last in part, reads the old block it keeps the +/// rest of, and that read is refused. fn a_partial_writeback_cleans_only_its_prefix_and_retries_the_same_reserved_suffix() { let refusing = Rc::new(Cell::new(false)); let disk = RefusingRead { ram: Ram::new(16384), refusing: Rc::clone(&refusing) }; @@ -542,8 +543,8 @@ fn a_partial_writeback_cleans_only_its_prefix_and_retries_the_same_reserved_suff let mut files = Files::new(Some(fake.clone())); let (node, id) = open(&mut files, &mut v, "home/f"); let new = pattern(PAGE as usize + TAIL, 8); - files.write(&mut v, id, node, 0, &new[..PAGE as usize]).unwrap(); files.write(&mut v, id, node, PAGE, &new[PAGE as usize..]).unwrap(); + files.write(&mut v, id, node, 0, &new[..PAGE as usize]).unwrap(); refusing.set(true); assert_eq!(files.fsync(&mut v, Some((id, node))), Err(Error::Io)); let object = files.object(id).unwrap(); From 92dcd868595bf6f8980cbca4b3e9ed6b6226b90e Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 5 Nov 2026 01:35:44 +0100 Subject: [PATCH 87/93] issues: the two rows #1152 lands behind main By the coordinator's decision #1152 lands with these recorded, not fixed. osbench at a1f07627e, both benches, against nightly 2026-10-31-2316: - write_4k_at about 2.4x main: the copy-on-write overwrite transaction, 30.6 us of host CPU an overwrite, spread with no single cost. - readdir.1000 about 3x main: each listing reads every entry's inode with a lookup of its own (416 of 534 us on the host model), and every READDIR request lists the whole directory again. Each has its evidence, owner and exit. Co-Authored-By: Claude Opus 5.5 --- ...ite-on-data-costs-about-2-4-times-mains.md | 30 +++++++++++++++++++ ...-listing-reads-every-entrys-inode-alone.md | 29 ++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 issues/a-4-kib-overwrite-on-data-costs-about-2-4-times-mains.md create mode 100644 issues/a-data-directory-listing-reads-every-entrys-inode-alone.md diff --git a/issues/a-4-kib-overwrite-on-data-costs-about-2-4-times-mains.md b/issues/a-4-kib-overwrite-on-data-costs-about-2-4-times-mains.md new file mode 100644 index 00000000000..a659884424b --- /dev/null +++ b/issues/a-4-kib-overwrite-on-data-costs-about-2-4-times-mains.md @@ -0,0 +1,30 @@ +--- +status: open +kind: defect +opened: 2026-11-05 +--- + +# A 4 KiB overwrite on DATA costs about 2.4 times main's + +**Owner.** The storage track, which owns `lib/bcachefs` and the file +server's DATA volume. + +osbench's `fs.disk.write_4k_at`, 4 KiB writes at scattered pages of a +16 MiB file, at #1152's a1f07627e: 23.9 us on the T14 and 30.8 us on the +AMD, p50, one boot each. Main dc1a8642e, on its private format, took +10.2 and 12.2 us (nightly 2026-10-31-2316); Linux ext4 0.7 and 2.3 us. + +Each overwrite is a copy-on-write replace: the write lands in new sectors +and one transaction removes or trims the old extent, inserts the new one, +and updates its backpointer, bucket accounting and the inode. On the host +model of the T14's DATA (CPU only, no device latency), taking the sectors +costs 0.8 us and that transaction 30.6 us an overwrite. A macOS `sample` of +it puts about 11% of the busy samples in `Cached::put`'s moves of a node's +keys (`memmove` under `mount::put` in `apply`); the rest is spread over the +pending overlay, allocation and the block writes, with no single cost +(#1152's evidence, `storage-integration/r9-small-writes`, `perf.rs`'s +`scattered_overwrites` and `scatter-sample.txt`). + +**Exit.** `write_4k_at` within 1.5 times main's on both benches, measured +by osbench against the same nightly's main, with the host model's +transaction cost and a sample of it published. diff --git a/issues/a-data-directory-listing-reads-every-entrys-inode-alone.md b/issues/a-data-directory-listing-reads-every-entrys-inode-alone.md new file mode 100644 index 00000000000..867cc65a1dc --- /dev/null +++ b/issues/a-data-directory-listing-reads-every-entrys-inode-alone.md @@ -0,0 +1,29 @@ +--- +status: open +kind: defect +opened: 2026-11-05 +--- + +# A DATA directory listing reads every entry's inode alone + +**Owner.** The storage track, which owns the file server's DATA volume and +`lib/bcachefs`. + +osbench's `fs.disk.readdir.1000`, listing a directory of 1 000 files, at +#1152's a1f07627e: 2.86 ms on the T14 and 1.99 ms on the AMD, p50, one boot +each; main dc1a8642e 0.91 and 0.69 ms (nightly 2026-10-31-2316), about 3x. + +**Cause.** A listing entry carries its file's size, so `DataVolume::list` +(`system/fileserver/src/data.rs`) reads each entry's inode with a lookup of +its own, from the root of the inodes btree, after one readdir of the +dirents; and every READDIR request lists the whole directory again before +taking its batch from the cursor. On the host model of the T14's DATA (CPU +only), one whole listing of 1 000 entries costs 534 us: the dirents 39 us, +the 1 000 inode lookups 416 us, sorting and encoding the rest (#1152's +evidence, `storage-integration/r10-known-gaps`, `perf.rs`'s +`listing_a_thousand`). A client asks in 64 KiB batches, so 1 000 short +names are one or two requests. + +**Exit.** The inodes of a listing read in one ordered pass over the inodes +btree, or the listing kept across a directory's READDIR batches, with +`readdir.1000` within 1.5 times main's on both benches. From 8cee28ab7c77f3f65ecc6296efb804ee4ad188e4 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 5 Nov 2026 01:36:52 +0100 Subject: [PATCH 88/93] issues: the smaller DATA rows that trail main By the coordinator's ruling under no silent debt: tree.100 (1.41-1.53x), stat, fstat and their direct variants (1.17-1.26x), and seq_write.4096 on the AMD (0.78x) at a1f07627e, with what is known of their causes, an owner, and the exit "within 1.1x of main on both benches, or each row's cause shown to be inherent". Co-Authored-By: Claude Opus 5.5 --- ...ller-data-rows-trail-main-by-up-to-half.md | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 issues/smaller-data-rows-trail-main-by-up-to-half.md diff --git a/issues/smaller-data-rows-trail-main-by-up-to-half.md b/issues/smaller-data-rows-trail-main-by-up-to-half.md new file mode 100644 index 00000000000..563d0e18acd --- /dev/null +++ b/issues/smaller-data-rows-trail-main-by-up-to-half.md @@ -0,0 +1,45 @@ +--- +status: open +kind: defect +opened: 2026-11-05 +--- + +# Smaller DATA rows trail main by up to half + +**Owner.** The storage track, which owns `lib/bcachefs` and the file +server's DATA volume. + +osbench at #1152's a1f07627e, one boot a bench, p50, against main +dc1a8642e on its private format (nightly 2026-10-31-2316); T14 / AMD: + +| row | #1152 | main | ratio | +|---|---|---|---| +| `fs.disk.tree.100` | 21.39 / 17.94 ms | 13.96 / 12.73 ms | 1.53 / 1.41 | +| `fs.disk.fstat` | 10.86 / 12.08 us | 8.72 / 9.92 us | 1.25 / 1.22 | +| `fs.disk.direct.fstat` | 10.64 / 11.86 us | 8.45 / 9.75 us | 1.26 / 1.22 | +| `fs.disk.stat` | 13.58 / 14.93 us | 11.63 / 12.54 us | 1.17 / 1.19 | +| `fs.disk.direct.stat` | 12.49 / 13.88 us | 10.66 / 11.54 us | 1.17 / 1.20 | +| `fs.disk.seq_write.4096` | 162 / 164 MB/s | 133 / 211 MB/s | 1.22 / 0.78 | + +The larger gaps have records of their own +(`issues/a-4-kib-overwrite-on-data-costs-about-2-4-times-mains.md`, +`issues/a-data-directory-listing-reads-every-entrys-inode-alone.md`). + +**What is known of the causes.** +- `tree.100` creates, writes and removes 100 small files: each create, + write and unlink is its own journaled transaction of several btrees + (inode, dirent, extent, backpointer, accounting). Which of them makes + the difference is not measured. +- `stat` resolves its path from the server's name cache and reads one + inode (round 5 of #1152: 5.2 us of server time on QEMU/KVM). `fstat` + reads no path. Where their remaining 1.7 to 2.3 us goes is not + established; it was not measured that it is the engine. +- `seq_write.4096` is above main on the T14 and below it on the AMD, after + #1152's write that continues the last extends its reservation. The + AMD's remaining cost is not established. + +Evidence: #1152's evidence, `storage-integration/r10-known-gaps` +(`osbench-a1f07627e/` and `osbench-table-a1f07627e.md`). + +**Exit.** Each row within 1.1 times main's on both benches, or its cause +shown to be inherent. From 79cd415d471e2d124248d54d194510c6be414954 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 5 Nov 2026 02:08:18 +0100 Subject: [PATCH 89/93] review r7 of #1152: the ring check after the shapes, extensions whole or refused, each take judged for its credit BLOCKER 1: the journal-size refusal sized mkfs's ring, which builds a journal writer for the volume's buckets, before the shape refusals; a volume of 8 KiB buckets, which a reader accepts, reached Writer::new's assert in the file server. The check now comes last in writable(), after the refusals that rule such buckets out. The shape test gains an 8 KiB- bucket case, refused by name (it panicked before the move). BLOCKER 2: an extension refused partway kept the runs it had taken, pinned with their credit, and the file server's tail range grew though the write was refused. Mount::extend now cuts the reservation back to where it started on any error: whole or refused. Test: near full, an extension of 256 KiB refused NoSpace leaves the reservation's range and the volume's room as they were; red without the cut (the range grew by 96 KiB and the room fell to 0). BLOCKER 3: a take was judged for one transaction's credit and could promise sixteen blocks, each owed a transaction's credit as each may be written back alone. Each take is now judged for its blocks' credit: in memory by room_for, and on the transaction path, whose grant judged one, by room_for for the rest inside it, refused NoSpace where it does not fit. An assert holds every take to the credit judged for it; with the old judgement it fails in every multi-block reservation test. A near-full test that would show an over-promise directly could not be built: near full, the space judgement is node-granular and the binding limit is data room, so the at most fifteen credits a single take over-promised stayed under one node there. BLOCKER 4: room_for and its doc sit above grant's doc again. Suggestion: extended_from has its own doc line. Unchanged cost (host model): 4 KiB calls reserve 0.9 us and write back 4.1 us a page; scattered overwrites 30.8 us. Co-Authored-By: Claude Opus 5.5 --- lib/bcachefs/src/engine/mount/data.rs | 29 +++++++++++++++++---- lib/bcachefs/src/engine/mount/mod.rs | 36 ++++++++++++++------------ lib/bcachefs/src/engine/tests/data.rs | 18 +++++++++++++ lib/bcachefs/src/engine/tests/mount.rs | 11 ++++++++ system/fileserver/src/data.rs | 5 ++-- 5 files changed, 75 insertions(+), 24 deletions(-) diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index 80470616e9c..58ab3176413 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -141,26 +141,44 @@ impl Mount { return Err(Error::Invalid("reservation belongs to another mount")); } let block = self.sb.block_bytes() as u64; - let (from, at_end) = reservation.range(); + let at_end = reservation.range().1; let end = at_end.checked_add(len as u64).filter(|&end| len > 0 && i64::try_from(end).is_ok()).ok_or(Error::Invalid("reservation of nothing or past 2^63-1"))?.div_ceil(block) * block; + let _data = self.data_slot().await; + // Whole or refused: what a refused extension took goes back. + let taken = self.take_reserved(reservation, end).await; + if taken.is_err() { + self.cut(reservation, at_end); + } + taken + } + + /// Takes runs into `reservation` up to `end`. Each take is judged for + /// the metadata credit it adds, a transaction's for each of its blocks, + /// as each may be written back alone. + async fn take_reserved(&self, reservation: &mut Reservation, end: u64) -> Result<()> { + let block = self.sb.block_bytes() as u64; let bucket = u64::from(self.sb.member.bucket_sectors); let longest = EXTENT_MAX.min(bucket * SECTOR as u64) / block * block; - let _data = self.data_slot().await; let ino = self.reservations.borrow().active.get(&reservation.id).ok_or(Error::Invalid("reservation already written"))?.ino; regular_mode(self.inode(ino).await?.mode)?; - let mut at = at_end; + let (from, mut at) = reservation.range(); while at < end { let wanted = (end - at).min(longest) / SECTOR as u64; let single = wanted * SECTOR as u64 == block; + let credit = K_TX * 8 * (wanted * SECTOR as u64 / block); // Sectors of the open bucket are taken in memory where room for - // their write is plain; a new bucket, or a judgement that may + // their writes is plain; a new bucket, or a judgement that may // wait or refuse, is a transaction's. - let taken = if self.room_for(K_TX, FOOTPRINT) { self.st.borrow_mut().alloc.next_data(bucket, wanted, single, !single).map(|(sector, _, got)| (sector, got)) } else { None }; + let taken = if self.room_for(credit, FOOTPRINT) { self.st.borrow_mut().alloc.next_data(bucket, wanted, single, !single).map(|(sector, _, got)| (sector, got)) } else { None }; let (sector, got) = match taken { Some(taken) => taken, None => loop { let out = self .transact(K_TX, FOOTPRINT, Draw::Ordinary, async |_: &mut Txn| { + // The grant judged one transaction's credit. + if !self.room_for(credit - K_TX * 8, FOOTPRINT) { + return Err(Error::NoSpace("no room left for the btree nodes reserved blocks' writes may write")); + } if single { self.data_sector(wanted, Draw::Ordinary, true).await.map(|(sector, _)| (sector, wanted)) } else { self.data_run(wanted, Draw::Ordinary).await } }) .await; @@ -171,6 +189,7 @@ impl Mount { }, }; let blocks = (got * SECTOR as u64 / block) as u32; + assert!(u64::from(blocks) * K_TX * 8 <= credit, "a take promises more credit than was judged"); { let mut st = self.st.borrow_mut(); for _ in 0..blocks { st.alloc.pin(sector / bucket); } diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 4ca1c656398..33fb81c799d 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -320,12 +320,6 @@ fn writable(sb: &Superblock) -> Result<()> { if sb.journal.is_empty() { return Err(Error::Unsupported("writing a volume without journal buckets")); } - // An unclean mount replays what the ring holds; mkfs's ring is the - // bound on that, so a larger one, another writer's, is not written. - let ring: u64 = sb.journal.iter().map(|&(_, count)| count).sum(); - if ring > super::mkfs::ring_buckets(sb.member.nbuckets, u64::from(sb.member.bucket_sectors), sb.block_bytes()) { - return Err(Error::Unsupported("writing a volume whose journal is larger than mkfs lays out")); - } if !splits_fit(sb.node_sectors() as usize * SECTOR) { return Err(Error::Unsupported("writing a volume of btree nodes too small for reclaim's largest split")); } @@ -352,6 +346,14 @@ fn writable(sb: &Superblock) -> Result<()> { if members.len() < 2 || usize::from(super::le16(members, 0)) < Member::BYTES { return Err(Error::Unsupported("writing a volume whose member entry is shorter than 1.39's")); } + // An unclean mount replays what the ring holds; mkfs's ring is the + // bound on that, so a larger one, another writer's, is not written. + // Last: sizing mkfs's ring takes a journal writer for these buckets, + // which the shapes refused above cannot have. + let ring: u64 = sb.journal.iter().map(|&(_, count)| count).sum(); + if ring > super::mkfs::ring_buckets(sb.member.nbuckets, u64::from(sb.member.bucket_sectors), sb.block_bytes()) { + return Err(Error::Unsupported("writing a volume whose journal is larger than mkfs lays out")); + } Ok(()) } @@ -1090,6 +1092,17 @@ impl Mount { } } + /// Whether `credit` bytes more of metadata, past what is promised and + /// granted, and `nodes` nodes would find room: what [`Mount::grant`] + /// refuses `NoSpace` without, judged now. + pub(super) fn room_for(&self, credit: u64, nodes: usize) -> bool { + self.cancel_dropped(); + let pending = self.reservations.borrow().pending(Draw::Ordinary); + let st = self.st.borrow(); + let stable = self.metadata_nodes(&st, pending + st.granted_bytes + credit); + st.read_only.is_none() && st.alloc.keeps(st.metadata_buckets, stable, nodes, st.reclaim_room, Draw::Ordinary, false) && st.alloc.fits(nodes, st.reclaim_room, Draw::Ordinary) + } + /// Grants `class` its request now if it fits. An ordinary one that the /// room left could not hold is refused `NoSpace` once nothing written or /// emptied can change that: no other grant outstanding, as the ledgers @@ -1098,17 +1111,6 @@ impl Mount { /// left unwritten. Writing later changes leaves those ledgers as they /// are, and on a node of one block reclaim's writes never settle, as /// each one's keys dirty another node. - /// Whether an ordinary transaction of `journal` u64s and `nodes` nodes - /// would find room for what it may write: what [`Mount::grant`] refuses - /// `NoSpace` without, judged now. - pub(super) fn room_for(&self, journal: u64, nodes: usize) -> bool { - self.cancel_dropped(); - let pending = self.reservations.borrow().pending(Draw::Ordinary); - let st = self.st.borrow(); - let stable = self.metadata_nodes(&st, pending + st.granted_bytes + journal * 8); - st.read_only.is_none() && st.alloc.keeps(st.metadata_buckets, stable, nodes, st.reclaim_room, Draw::Ordinary, false) && st.alloc.fits(nodes, st.reclaim_room, Draw::Ordinary) - } - fn grant(&self, class: Class, draw: Draw, journal: u64, nodes: usize, arrived: u64) -> Result { self.cancel_dropped(); let pending = self.reservations.borrow().pending(draw); diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index 41b34740032..8599eef673a 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -637,6 +637,24 @@ fn an_extended_reservation_keeps_each_run_in_one_bucket() { assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); } +/// Near full, an extension refused for space takes nothing: the +/// reservation keeps its range and the volume its room. +#[test] +fn a_refused_extension_gives_back_what_it_took() { + let dev = formatted(64 << 20, None); + mounted(&dev, async |m: &Mount<&Image>| { + let ino = m.create(m.root(), b"f", New::File, OWNER, NOW).await.unwrap().ino; + let mut tokens = Vec::new(); + while let Ok(token) = m.reserve(ino, tokens.len() as u64 * 4096, 4096).await { tokens.push(token); } + // Room for some blocks again, fewer than the extension asks. + tokens.truncate(tokens.len() - 24); + let mut token = tokens.pop().unwrap(); + let (range, room) = (token.range(), m.space().1); + assert!(matches!(m.extend(&mut token, 256 << 10).await, Err(Error::NoSpace(_)))); + assert_eq!((token.range(), m.space().1), (range, room)); + }); +} + /// A write over more old extents than one transaction may remove goes in /// parts, each removing at most two, and the file reads as written. #[test] diff --git a/lib/bcachefs/src/engine/tests/mount.rs b/lib/bcachefs/src/engine/tests/mount.rs index 59bd7ff4bb5..074b19f9e7c 100644 --- a/lib/bcachefs/src/engine/tests/mount.rs +++ b/lib/bcachefs/src/engine/tests/mount.rs @@ -298,6 +298,17 @@ fn small_blocks_and_a_journal_position_off_a_block_are_refused() { let err = block_on_ready(Mount::open(&dev, MEMORY, random())).err(); assert_eq!(err, Some(Error::Unsupported("writing a volume of btree nodes too small for reclaim's largest split")), "{sectors} sectors"); } + // 8 KiB buckets of 8 KiB nodes, as a reader accepts them: refused by + // shape before anything sizes a journal writer for them. + let dev = formatted(64 << 20, None); + rewrite_superblock(&dev, |h, fields| { + h.flags[0] = h.flags[0] & !(0xffff << 12) | 16 << 12; + let members = fields.iter_mut().find(|f| f.kind == field_type::MEMBERS_V2).unwrap(); + members.body[8 + 26..8 + 28].copy_from_slice(&16u16.to_le_bytes()); + }); + assert_eq!(block_on_ready(Volume::open(&dev)).unwrap().superblock().member.bucket_sectors, 16, "a reader takes 8 KiB buckets"); + let err = block_on_ready(Mount::open(&dev, MEMORY, random())).err(); + assert_eq!(err, Some(Error::Unsupported("writing a volume of btree nodes too small for reclaim's largest split")), "8 KiB buckets"); let dev = formatted(32 << 20, None); assert_eq!(block_on_ready(Volume::open(&dev)).unwrap().superblock().node_sectors(), 32, "mkfs's 16 KiB nodes"); mounted(&dev, async |m: &Mount<&Image>| { m.create(m.root(), b"f", New::File, OWNER, NOW).await.unwrap(); }); diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index 68a413876af..b755adee018 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -41,9 +41,10 @@ struct Open { /// Pages the current write reserved and has not copied yet. incoming: BTreeMap, /// The last write's ticket, which a write starting where its - /// reservation ends extends, so a write-back of both is one extent; and - /// where that reservation ended before the current write extended it. + /// reservation ends extends, so a write-back of both is one extent. tail: Option, + /// Where the tail's reservation ended before this write extended it, + /// until the write lands or is cut. extended_from: Option, } impl Open { From 9fe8f779e30042eae85491a2cf9b56cdb0c8e789 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 5 Nov 2026 03:00:31 +0100 Subject: [PATCH 90/93] review r8 of #1152: a reservation's take is granted its whole credit r7 B3 continued. A take that opened a bucket went through a transaction whose grant judged and held one transaction's credit; the take's extra credit was then judged inside it against today's room, while data_space's own check for the new bucket saw only the grant's one credit, so neither judged both the take's credit and the bucket it lost; and the inside check refused at once where grant waits until settled. Now admission holds a metadata credit apart from the journal bound: admit, grant, ungrant, prepare_pending and finish carry it, granted_bytes counts it, and Mount::transact_holding grants a transaction its bound's records and a credit of at least them. transact is transact_holding at its bound's bytes. A take on the transaction path is granted its whole credit, so data_space's bucket check sees it and a refusal follows grant's settled rule; the inside check is gone. The in-memory take from the open bucket, which costs no capacity, stays judged by room_for for its whole credit. Evidence: the circular assert is replaced by the invariant itself, in tests, after every take: everything promised still fits (room_for(0, FOOTPRINT)). A near-full test fills the 64 MiB image (32 KiB buckets, eight-block runs) with 32 KiB and with 16 KiB reservations and writes every block back; red when the grant holds one transaction's credit for a take (the invariant fails). Reverting the in-memory judgement to one credit stays green on this image: there a bucket is eight blocks and so is a metadata bucket's worth of credit, and the fills tried end credit-bound with the open bucket full, or, with blocks written first to offset it, bound by data buckets with credit to spare; no buildable state put an in-memory take across the boundary. Suggestion: extended_from's doc says only where the tail's reservation ended before the current write extended it. Co-Authored-By: Claude Opus 5.5 --- lib/bcachefs/src/engine/mount/data.rs | 12 +++---- lib/bcachefs/src/engine/mount/mod.rs | 40 ++++++++++++++---------- lib/bcachefs/src/engine/mount/reclaim.rs | 6 ++-- lib/bcachefs/src/engine/tests/data.rs | 38 ++++++++++++++++++++++ system/fileserver/src/data.rs | 4 +-- 5 files changed, 73 insertions(+), 27 deletions(-) diff --git a/lib/bcachefs/src/engine/mount/data.rs b/lib/bcachefs/src/engine/mount/data.rs index 58ab3176413..3a6bf47d0ae 100644 --- a/lib/bcachefs/src/engine/mount/data.rs +++ b/lib/bcachefs/src/engine/mount/data.rs @@ -172,13 +172,11 @@ impl Mount { let taken = if self.room_for(credit, FOOTPRINT) { self.st.borrow_mut().alloc.next_data(bucket, wanted, single, !single).map(|(sector, _, got)| (sector, got)) } else { None }; let (sector, got) = match taken { Some(taken) => taken, + // The grant judges and holds the take's whole credit, so a + // bucket it opens is judged with it too. None => loop { let out = self - .transact(K_TX, FOOTPRINT, Draw::Ordinary, async |_: &mut Txn| { - // The grant judged one transaction's credit. - if !self.room_for(credit - K_TX * 8, FOOTPRINT) { - return Err(Error::NoSpace("no room left for the btree nodes reserved blocks' writes may write")); - } + .transact_holding(K_TX, credit, FOOTPRINT, Draw::Ordinary, async |_: &mut Txn| { if single { self.data_sector(wanted, Draw::Ordinary, true).await.map(|(sector, _)| (sector, wanted)) } else { self.data_run(wanted, Draw::Ordinary).await } }) .await; @@ -189,7 +187,6 @@ impl Mount { }, }; let blocks = (got * SECTOR as u64 / block) as u32; - assert!(u64::from(blocks) * K_TX * 8 <= credit, "a take promises more credit than was judged"); { let mut st = self.st.borrow_mut(); for _ in 0..blocks { st.alloc.pin(sector / bucket); } @@ -205,6 +202,9 @@ impl Mount { } at += got * SECTOR as u64; reservation.bytes = at - from; + drop(pool); + #[cfg(test)] + assert!(self.room_for(0, FOOTPRINT), "a take promised more than the room keeps"); } Ok(()) } diff --git a/lib/bcachefs/src/engine/mount/mod.rs b/lib/bcachefs/src/engine/mount/mod.rs index 33fb81c799d..ad288566aad 100644 --- a/lib/bcachefs/src/engine/mount/mod.rs +++ b/lib/bcachefs/src/engine/mount/mod.rs @@ -1058,7 +1058,7 @@ impl Mount { /// was committed before its arrival ([`Mount::grant`]). Reclaim's own share never waits /// on reclaim: past its closed entries, a short share is a bug, and the /// mount stops. - async fn admit(&self, class: Class, draw: Draw, journal: u64, nodes: usize) -> Result<()> { + async fn admit(&self, class: Class, draw: Draw, journal: u64, credit: u64, nodes: usize) -> Result<()> { let queue = match class { Class::Ordinary => &self.admit, Class::Reclaim => &self.admit_reclaim, @@ -1067,7 +1067,7 @@ impl Mount { let arrived = self.st.borrow().journal.next_seq; loop { let granted = place - .until(|| match self.grant(class, draw, journal, nodes, arrived) { + .until(|| match self.grant(class, draw, journal, credit, nodes, arrived) { Ok(true) => Some(Ok(true)), Err(e) => Some(Err(e)), Ok(false) if self.can_free() => Some(Ok(false)), @@ -1082,7 +1082,7 @@ impl Mount { Class::Ordinary => Box::pin(self.free()).await?, Class::Reclaim => { self.flush_closed().await?; - if !self.grant(class, draw, journal, nodes, arrived)? { + if !self.grant(class, draw, journal, credit, nodes, arrived)? { self.stop("journal reclaim's share exhausted"); return Err(Error::ReadOnly("journal reclaim's share exhausted")); } @@ -1111,7 +1111,7 @@ impl Mount { /// left unwritten. Writing later changes leaves those ledgers as they /// are, and on a node of one block reclaim's writes never settle, as /// each one's keys dirty another node. - fn grant(&self, class: Class, draw: Draw, journal: u64, nodes: usize, arrived: u64) -> Result { + fn grant(&self, class: Class, draw: Draw, journal: u64, credit: u64, nodes: usize, arrived: u64) -> Result { self.cancel_dropped(); let pending = self.reservations.borrow().pending(draw); let mut st = self.st.borrow_mut(); @@ -1121,7 +1121,7 @@ impl Mount { let room = match class { Class::Ordinary => { let held = st.cache.dirty + st.cache.unpointed + st.granted; - let stable = self.metadata_nodes(&st, pending + st.granted_bytes + journal * 8); + let stable = self.metadata_nodes(&st, pending + st.granted_bytes + credit); let keeps = st.alloc.keeps(st.metadata_buckets, stable, nodes, st.reclaim_room, draw, false); let physical = st.alloc.fits(nodes, st.reclaim_room, draw); let can_compact = st.sparse_metadata.iter().any(|&b| !st.alloc.metadata_open(b)); @@ -1140,7 +1140,7 @@ impl Mount { }; if room && st.journal.reserve(journal, class) { match class { - Class::Ordinary => { st.granted += nodes; st.granted_bytes += journal * 8; } + Class::Ordinary => { st.granted += nodes; st.granted_bytes += credit; } Class::Reclaim => st.reclaim_granted += nodes, } return Ok(true); @@ -1184,10 +1184,10 @@ impl Mount { } /// Returns what [`Mount::admit`] took. - fn ungrant(&self, st: &mut State, class: Class, journal: u64, nodes: usize) { + fn ungrant(&self, st: &mut State, class: Class, journal: u64, credit: u64, nodes: usize) { st.journal.unreserve(journal); match class { - Class::Ordinary => { st.granted -= nodes; st.granted_bytes -= journal * 8; } + Class::Ordinary => { st.granted -= nodes; st.granted_bytes -= credit; } Class::Reclaim => st.reclaim_granted -= nodes, } self.admit_reclaim.wake(); @@ -1219,28 +1219,36 @@ impl Mount { /// a transaction before it left so sends it back to wait for reclaim to /// split that node ([`MAX_GROUPS`]). async fn transact(&self, bound: u64, nodes: usize, draw: Draw, prepare: impl AsyncFnOnce(&mut Txn) -> Result) -> Result { + self.transact_holding(bound, bound * 8, nodes, draw, prepare).await + } + + /// [`Mount::transact`], its grant judging and holding `credit` bytes of + /// metadata room rather than its journal bound's: what it promises past + /// its own records, as a reservation's blocks each owe a transaction. + async fn transact_holding(&self, bound: u64, credit: u64, nodes: usize, draw: Draw, prepare: impl AsyncFnOnce(&mut Txn) -> Result) -> Result { assert!(nodes <= FOOTPRINT, "a grant of {nodes} nodes, past the largest any operation takes"); + assert!(credit >= bound * 8, "a grant's credit covers its own records"); self.check_writable()?; let _slot = loop { - self.admit(Class::Ordinary, draw, bound, nodes).await?; + self.admit(Class::Ordinary, draw, bound, credit, nodes).await?; let slot = self.slot(Class::Ordinary).await; let mut st = self.st.borrow_mut(); if st.cache.oversized == 0 { break slot; } - self.ungrant(&mut st, Class::Ordinary, bound, nodes); + self.ungrant(&mut st, Class::Ordinary, bound, credit, nodes); }; - self.prepare_pending(Class::Ordinary, bound, nodes)?; + self.prepare_pending(Class::Ordinary, bound, credit, nodes)?; let mut txn = Txn::default(); let out = prepare(&mut txn).await; - self.finish(txn, out.is_ok(), Class::Ordinary, bound, nodes)?; + self.finish(txn, out.is_ok(), Class::Ordinary, bound, credit, nodes)?; out } - fn prepare_pending(&self, class: Class, bound: u64, nodes: usize) -> Result<()> { + fn prepare_pending(&self, class: Class, bound: u64, credit: u64, nodes: usize) -> Result<()> { let mut st = self.st.borrow_mut(); if let Err(error) = st.pending.reserve(bound as usize / KEY_U64S) { - self.ungrant(&mut st, class, bound, nodes); + self.ungrant(&mut st, class, bound, credit, nodes); self.written.wake(); return Err(error); } @@ -1250,7 +1258,7 @@ impl Mount { /// Applies `txn` when `apply`, else drops it; either way releases its /// holds and its grant. `ReadOnly` when the mount stopped writing while /// it was prepared. - fn finish(&self, mut txn: Txn, apply: bool, class: Class, bound: u64, nodes: usize) -> Result<()> { + fn finish(&self, mut txn: Txn, apply: bool, class: Class, bound: u64, credit: u64, nodes: usize) -> Result<()> { let mut st = self.st.borrow_mut(); let st = &mut *st; // An ordinary grant is each operation's bound; reclaim's is capped @@ -1307,7 +1315,7 @@ impl Mount { st.peak = st.peak.max(st.cache.dirty + st.cache.unpointed); } assert!(st.cache.dirty + st.cache.unpointed <= st.limit + st.reclaim_room, "dirty and unpointed nodes past the cache's limit and reclaim's room"); - self.ungrant(st, class, bound, nodes); + self.ungrant(st, class, bound, credit, nodes); self.written.wake(); match stopped { Some(why) if apply => Err(Error::ReadOnly(why)), diff --git a/lib/bcachefs/src/engine/mount/reclaim.rs b/lib/bcachefs/src/engine/mount/reclaim.rs index 920e84ebf90..cb9e2c75799 100644 --- a/lib/bcachefs/src/engine/mount/reclaim.rs +++ b/lib/bcachefs/src/engine/mount/reclaim.rs @@ -268,12 +268,12 @@ impl Mount { // reclaim never waits on what it would itself have to do. self.flush_closed().await?; // Reclaim's room for nodes is its own share; no draw limits it. - self.admit(Class::Reclaim, Draw::Freeing, bound, nodes).await?; + self.admit(Class::Reclaim, Draw::Freeing, bound, bound * 8, nodes).await?; let _slot = self.slot(Class::Reclaim).await; - self.prepare_pending(Class::Reclaim, bound, nodes)?; + self.prepare_pending(Class::Reclaim, bound, bound * 8, nodes)?; let mut txn = Txn::default(); let out = self.point(&mut txn, plans).await; - self.finish(txn, out.is_ok(), Class::Reclaim, bound, nodes)?; + self.finish(txn, out.is_ok(), Class::Reclaim, bound, bound * 8, nodes)?; out } diff --git a/lib/bcachefs/src/engine/tests/data.rs b/lib/bcachefs/src/engine/tests/data.rs index 8599eef673a..33b72ff695d 100644 --- a/lib/bcachefs/src/engine/tests/data.rs +++ b/lib/bcachefs/src/engine/tests/data.rs @@ -655,6 +655,44 @@ fn a_refused_extension_gives_back_what_it_took() { }); } +/// Near full, reservations of several blocks fill the 64 MiB image's +/// 32 KiB buckets: of 32 KiB, a whole bucket each, every take opens a +/// bucket through a transaction; of 16 KiB, every second take is from the +/// open bucket, in memory. Each take owes a transaction's credit for each +/// of its blocks and is judged for all of it, so after each the promises +/// still fit the room (asserted in the take; red when the grant holds one +/// transaction's credit for a take), and every block reserved is then +/// written. +#[test] +fn reservations_of_several_blocks_fill_a_volume_and_are_all_written() { + for run in [32usize << 10, 16 << 10] { + let dev = formatted(64 << 20, None); + let ino = mounted(&dev, async |m: &Mount<&Image>| { + assert_eq!(m.superblock().member.bucket_sectors, 64, "32 KiB buckets"); + let ino = m.create(m.root(), b"f", New::File, OWNER, NOW).await.unwrap().ino; + let mut tokens = Vec::new(); + loop { + match m.reserve(ino, (tokens.len() * run) as u64, run).await { + Ok(token) => tokens.push(token), + Err(Error::NoSpace(_)) => break, + Err(e) => panic!("reserve: {e:?}"), + } + } + assert!(tokens.len() >= 16, "{} reservations of {run} bytes", tokens.len()); + for (i, token) in tokens.iter_mut().enumerate() { + assert_eq!(m.write_reserved(token, (i * run) as u64, &vec![i as u8; run], NOW).await.unwrap(), run); + } + m.commit().await.unwrap(); + ino + }); + let v = block_on_ready(Volume::open(&dev)).unwrap(); + for (i, chunk) in read_all(&v, ino).unwrap().chunks(run).enumerate() { + assert!(chunk.iter().all(|&b| b == i as u8), "chunk {i} of {run} bytes"); + } + assert_eq!(block_on_ready(fsck(&v)).unwrap(), []); + } +} + /// A write over more old extents than one transaction may remove goes in /// parts, each removing at most two, and the file reads as written. #[test] diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index b755adee018..9f45317e595 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -43,8 +43,8 @@ struct Open { /// The last write's ticket, which a write starting where its /// reservation ends extends, so a write-back of both is one extent. tail: Option, - /// Where the tail's reservation ended before this write extended it, - /// until the write lands or is cut. + /// Where the tail's reservation ended before the current write extended + /// it. extended_from: Option, } impl Open { From 6fd7151b65cec8d1e1b6be2b9c96f580146d00c4 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 5 Nov 2026 03:28:40 +0100 Subject: [PATCH 91/93] tests: two testcases members state what the served filesystem does Both were red on both benches at 9fe8f779e and not on the K4 baseline. - pipe_other_end_gone: this branch replaced its kernel file handle, an object with no other end that answers being readable, with a memory object when the kernel's File object went. A memory handle has no WAIT right, so the watch is refused PermissionDenied before the kernel asks whether the object has an other end, and the test's NotSupported never comes. The case now uses an event's wait side: watchable, readable when raised, and no pipe end. The kernel's order (rights first, then whether anything could answer) is right: a holder without WAIT learns nothing about the object. - 207_libc_names: readdir on /tmp gave DT_UNKNOWN for regular files under the kernel's tmpfs; /tmp is now the file server's memory volume, whose listing carries each entry's kind, so libc reports DT_REG. DT_REG is right: POSIX leaves d_type to the implementation, and Linux's tmpfs, ext4 and the rest report DT_REG for a regular file. The case prints DT_REG by name rather than "another type", and its expectation says it. Co-Authored-By: Claude Opus 5.5 --- tests/corpus/tinycc/207_libc_names.c | 3 ++- tests/corpus/tinycc/207_libc_names.expect | 4 ++-- .../rust-tests/src/bin/pipe_other_end_gone.rs | 18 +++++++++++------- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/corpus/tinycc/207_libc_names.c b/tests/corpus/tinycc/207_libc_names.c index 4edc7013dcd..b2a04fcb21b 100644 --- a/tests/corpus/tinycc/207_libc_names.c +++ b/tests/corpus/tinycc/207_libc_names.c @@ -176,7 +176,8 @@ static void listing(void) { while (count < 16 && (entry = readdir(dir)) != NULL) { char line[300]; snprintf(line, sizeof line, "%s %s", entry->d_name, - entry->d_type == DT_DIR ? "DT_DIR" : entry->d_type == DT_UNKNOWN ? "DT_UNKNOWN" : "another type"); + entry->d_type == DT_DIR ? "DT_DIR" : entry->d_type == DT_REG ? "DT_REG" + : entry->d_type == DT_UNKNOWN ? "DT_UNKNOWN" : "another type"); names[count++] = strdup(line); } closedir(dir); diff --git a/tests/corpus/tinycc/207_libc_names.expect b/tests/corpus/tinycc/207_libc_names.expect index 88c2be693c9..8f77a75e67a 100644 --- a/tests/corpus/tinycc/207_libc_names.expect +++ b/tests/corpus/tinycc/207_libc_names.expect @@ -17,8 +17,8 @@ setbuf NULL: file 3 bytes signal SIGALRM: was SIG_DFL, then SIG_IGN sigaction SIGALRM: 0 0, SIG_IGN, flags 0x10000000, mask 0x200 alarm 5: 0; alarm 0: 1 to 5; again: 0 -entry: a DT_UNKNOWN -entry: b DT_UNKNOWN +entry: a DT_REG +entry: b DT_REG entry: sub DT_DIR opendir of nothing: -1, ENOENT dladdr(main): 1; names this program: yes; starts where dl_iterate_phdr says: yes; symbol: none diff --git a/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs b/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs index 6fc279a8c88..bb616c40121 100644 --- a/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs +++ b/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs @@ -9,8 +9,8 @@ //! waiter left parked is a hang the runner's ceiling reds. A watch made after //! the leaving is answered by the submit that makes it. //! -//! **The refusal.** A connection and a memory object have no other end, and a watch that -//! asks for one is refused whatever else it asks. +//! **The refusal.** A connection and an event's wait side have no other end +//! to leave, and a watch that asks for one is refused whatever else it asks. //! //! **Its own close.** An end whose holder closes it while the other end lives //! answers its watch with `NotFound`: the poll is over, and nothing else would @@ -32,7 +32,7 @@ use toyos::ipc::Connection; use toyos::poller::{Poller, OTHER_END_GONE, READABLE}; use toyos::syscap::SysCap; use toyos::{namespace, pipe_pair, port, AsHandle, BorrowedHandle, OwnedHandle, Pipe}; -use toyos_abi::syscall::SyscallError; +use toyos_abi::syscall::{self, SyscallError}; #[path = "../roster.rs"] mod roster; @@ -73,10 +73,14 @@ fn test() { let (client, server) = connection(); refused(client.as_handle(), "a connection"); - let memory = toyos::shm::SharedMemory::create(4096).expect("a memory object"); - refused(memory.as_handle(), "a memory object"); - drop(memory); - println!(" a connection and a memory object were refused the question"); + // A handle the poller may watch, which answers being readable: a memory + // object has no wait right, and would be refused for that instead. + let event = syscall::event_create().expect("an event"); + // SAFETY: the kernel installed both handles for this call and no other. + let (signal, wait) = unsafe { (OwnedHandle::from_raw(event.signal), OwnedHandle::from_raw(event.wait)) }; + refused(wait.as_handle(), "an event's wait side"); + drop((signal, wait)); + println!(" a connection and an event's wait side were refused the question"); for own in [End::Read, End::Write] { let (reader, writer) = pipe(""); From 275a9e6b8e3f2d0a1bc10736fac6676902604ae0 Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 5 Nov 2026 03:33:33 +0100 Subject: [PATCH 92/93] tests: readdir_bound rides the shared boot as a member again The judge failed readdir_bound on both benches at 9fe8f779e: "no exit record, never ran". f916f41ed moved it off its own boot (testcases- readdir, needed while it filled the kernel's /tmp to its listing limit) onto TESTCASES, but kept it in RUST_SKIP and never added its job to that arm, so nothing ran it. On the K4 baseline 8ac4f2187 it had its own boot, which that run did not stage either. The test now cleans up after itself on the served /tmp, so it needs no boot of its own: its RUST_SKIP entry and its explicit row go, and it is a discovered member of the shared boot, judged on its exit like the others. The Arm doc's example of a test that must not share named it; it now says the rule without it. Co-Authored-By: Claude Opus 5.5 --- tests/common/metal.rs | 7 +++---- tests/toyos.rs | 9 --------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/tests/common/metal.rs b/tests/common/metal.rs index fb6a1d5f46f..5a79c769e44 100644 --- a/tests/common/metal.rs +++ b/tests/common/metal.rs @@ -35,10 +35,9 @@ pub struct Arm { /// config and parameters, and [`batches`] refuses a pair that does not. /// /// Named rather than derived from (config, parameters), because sharing is - /// not always safe and only the author knows: `readdir_bound` fills `/tmp` - /// to the VFS listing limit, so every later listing of `/tmp` on its boot - /// is refused. A test that must not share names its own; it costs a minute - /// and it says so. + /// not always safe and only the author knows: a test that leaves its boot + /// changed for every job after it must not share. A test that must not + /// share names its own; it costs a minute and it says so. pub boot: &'static str, /// The image config, relative to the repository root. pub config: &'static str, diff --git a/tests/toyos.rs b/tests/toyos.rs index 61dac2ef092..41e6e9c5553 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -74,8 +74,6 @@ const ACTUATOR_KERNEL: &[&str] = toyos_build::build::TEST_KERNEL; const RUST_SKIP: &[&str] = &[ // The DATA round trip has separate write and read boots, not a no-argument run. "data_roundtrip", - // Its explicit metal registration owns the directory listing workload. - "readdir_bound", // It writes the log disk for ever, so that its boot's deadline resets the // machine under it: `the_disk_survives_a_reset_under_load` runs it alone. "stick_load", @@ -903,13 +901,6 @@ const METAL: &[(&str, metal::Metal)] = &[ "memory_floor", metal::Metal { arms: FLOORCASE, judge: |b| b[0].job_passed("test_rs_memory_floor") }, ), - ( - "readdir_bound", - metal::Metal { - arms: TESTCASES, - judge: |b| b[0].job_passed("test_rs_readdir_bound"), - }, - ), ( "wake_storm_cost", metal::Metal { From 57dedf1f33269480dd637c15799536f31005955d Mon Sep 17 00:00:00 2001 From: japabu Date: Thu, 5 Nov 2026 03:42:55 +0100 Subject: [PATCH 93/93] review r10 of #1152: the refusal case takes the SDK's wake pair The event's wait side came from the raw event_create, adopted under unsafe; toyos::wake::pair() gives the same handles typed and safe, and the Bell is the wait side the poller watches. The import change and the manual drop go; the comment states only the contract. Co-Authored-By: Claude Opus 5.5 --- .../guest/rust-tests/src/bin/pipe_other_end_gone.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs b/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs index bb616c40121..95c8e5987fd 100644 --- a/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs +++ b/tests/guest/rust-tests/src/bin/pipe_other_end_gone.rs @@ -32,7 +32,7 @@ use toyos::ipc::Connection; use toyos::poller::{Poller, OTHER_END_GONE, READABLE}; use toyos::syscap::SysCap; use toyos::{namespace, pipe_pair, port, AsHandle, BorrowedHandle, OwnedHandle, Pipe}; -use toyos_abi::syscall::{self, SyscallError}; +use toyos_abi::syscall::SyscallError; #[path = "../roster.rs"] mod roster; @@ -73,13 +73,10 @@ fn test() { let (client, server) = connection(); refused(client.as_handle(), "a connection"); - // A handle the poller may watch, which answers being readable: a memory - // object has no wait right, and would be refused for that instead. - let event = syscall::event_create().expect("an event"); - // SAFETY: the kernel installed both handles for this call and no other. - let (signal, wait) = unsafe { (OwnedHandle::from_raw(event.signal), OwnedHandle::from_raw(event.wait)) }; - refused(wait.as_handle(), "an event's wait side"); - drop((signal, wait)); + // A watchable handle with no other end; a memory object lacks WAIT and is + // refused for that first. + let (_waker, bell) = toyos::wake::pair().expect("an event"); + refused(bell.as_handle(), "an event's wait side"); println!(" a connection and an event's wait side were refused the question"); for own in [End::Read, End::Write] {