diff --git a/Cargo.lock b/Cargo.lock index fd81e11be9e..e9c0b2b76f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -767,9 +767,14 @@ dependencies = [ name = "disks" version = "0.1.0" dependencies = [ + "bcachefs", "diskserver", + "getrandom 0.3.4", "toyos", + "toyos-blockdev", "toyos-blockring", + "toyos-blockserve", + "toyos-crypto", "toyos-gpt", "toyos-manifest", ] @@ -1010,6 +1015,7 @@ version = "0.0.0" dependencies = [ "bcachefs", "diskserver", + "getrandom 0.3.4", "toyos", "toyos-abi", "toyos-blockdev", diff --git a/build/bootlog.rs b/build/bootlog.rs index 625e255cc59..847d89ef92f 100644 --- a/build/bootlog.rs +++ b/build/bootlog.rs @@ -22,9 +22,8 @@ pub const REBOOTING: &str = "Rebooting."; pub const STOPPING: &str = toyos_logstream::STOPPING; /// What `tests/guest/test-runner` says when its job list runs past -/// `toyos_tco::JOB_BOUND_MS`, with the job it was inside as the next word. -/// **Console only**: a userland write reaches the serial backend and never a -/// log record, so no stick carries it. +/// `toyos_tco::JOB_BOUND_MS`, with the job it was inside as the next word, +/// before it kills that job and reboots: a boot that says it is no pass. pub const JOB_DEADLINE_SAID: &str = "test-runner: the job list ran past its bound, and the job it was inside is"; @@ -337,6 +336,9 @@ pub enum Unfit { /// The loader pass after the foreign-identity arm's reset read no /// [`FOREIGN_DONE`]: the stop it staged never finished. NoForeignDone, + /// The test runner's word that its job list ran past its bound: the + /// line, naming the job it killed. + JobsPastBound(String), } impl fmt::Display for Unfit { @@ -354,6 +356,10 @@ impl fmt::Display for Unfit { "the loader's pass after the reset carries no {HANDED_BACK:?} with {REBOOTING:?} \ under {LOG_TAIL:?}: the stop this boot asked for never reached the reset" ), + Self::JobsPastBound(said) => write!( + f, + "the test runner killed a job and rebooted at its bound: {said:?}" + ), Self::NoForeignDone => write!( f, "the loader's pass after the reset carries no {FOREIGN_DONE:?}: the stop this \ @@ -541,6 +547,12 @@ pub fn boot_millis(log: &str) -> Option { /// pass's ([`HANDED_BACK`], and [`REBOOTING`] under [`LOG_TAIL`]). pub fn verdict(log: &str) -> Result { let boot_ms = boot_millis(log).ok_or(Unfit::NoBootRecord)?; + let past_bound = |line: &&str| { + toyos_logstream::program_line(line).is_some_and(|said| said.tag == "test-runner" && said.text.starts_with(JOB_DEADLINE_SAID)) + }; + if let Some(line) = log.lines().find(past_bound) { + return Err(Unfit::JobsPastBound(line.trim().to_string())); + } if stopping_line(log).is_none() { let last = log.lines().rev().find(|line| !line.trim().is_empty()).unwrap_or_default(); return Err(Unfit::Unfinished(last.trim().to_string())); @@ -578,6 +590,19 @@ mod tests { assert_eq!(verdict(""), Err(Unfit::NoBootRecord)); } + /// A boot the test runner ended at its job bound stopped as a passing + /// one does, and is none. + #[test] + fn a_boot_whose_job_list_ran_past_its_bound_is_not_a_pass() { + let past = format!("[2026-11-04 17:07:50 535.506 test-runner tid=1] {JOB_DEADLINE_SAID} osbench_persist (525000 ms)\n"); + let log = format!( + "[ 1.151 cpu0 kernel] Boot: complete (1151ms)\n{past}[ 535.507 supervisor] {STOPPING} (Reboot)\n" + ); + assert_eq!(verdict(&log), Err(Unfit::JobsPastBound(past.trim().to_string()))); + let quoted = log.replace("test-runner tid=1] ", "osbench tid=4] said "); + assert_eq!(verdict("ed), Ok(1151)); + } + /// Only the supervisor's own stop line, naming a shutdown, is a power-off. #[test] fn a_power_off_is_the_supervisors_stop_naming_a_shutdown() { diff --git a/build/build.rs b/build/build.rs index 9d15793788a..b1b172009ad 100644 --- a/build/build.rs +++ b/build/build.rs @@ -2091,7 +2091,7 @@ fn shipped_parts(root: &Path, boot: &Boot, plan: &Plan) -> (Vec, Vec, Us /// a real one. /// /// Designates the result, because every caller here is making a scratch disk -/// for a guest that expects a working `/apps` and `/home`, and the kernel will +/// for a guest that expects a working `/apps` and `/home`, and the file server will /// not format an undesignated one. Leaving it to the call sites would mean two /// places to forget; forgetting is not silent (the boot says so and both paths /// are volatile) but it is not worth the chance. @@ -2104,12 +2104,12 @@ pub fn create_sparse(path: &Path, len: u64) { } /// Partition this image as a DATA disk and stamp the designation on that -/// partition, so the kernel is allowed to format it. +/// partition, so the file server is allowed to format it. /// -/// The kernel never formats a volume that does not carry the stamp, which is +/// The file server never formats a volume that does not carry the stamp, which is /// what stops it taking the disk of any machine it is booted on. So a throwaway /// image has to say so, and this is the whole of the test harness's opt-in: -/// **data on a scratch file, not a build flag.** The kernel binary and the +/// **data on a scratch file, not a build flag.** The file-server binary and the /// code path are identical either way — `probe` runs the same match /// on metal as it does here — so the configuration under test is the /// configuration that ships, which a `#[cfg]` could not have given us. @@ -3087,12 +3087,14 @@ mod tests { #[test] fn a_disk_changing_authorization_goes_to_no_row_a_login_can_start() { let gate = |text: &str| held_by_their_holders_alone(&toml::from_str(text).unwrap()); - assert!(gate("[programs.runner]\nauthorize = [\"designate:data\"]\n").is_ok()); + for word in ["designate:data", "format:data"] { + assert!(gate(&format!("[programs.runner]\nauthorize = [\"{word}\"]\n")).is_ok()); + let launched = gate(&format!("[programs.disks]\nauthorize = [\"{word}\"]\n[programs.shell]\nstarts = [\"disks\"]\n")); + assert!(launched.unwrap_err().contains("may start it")); + let login = gate(&format!("[boot]\nstart = [\"sshserver\"]\n[programs.sshserver]\nlogin = true\nauthorize = [\"{word}\"]\n")); + assert!(login.unwrap_err().contains("is a login authority")); + } assert!(gate("[programs.disks]\nauthorize = [\"tables\"]\n[programs.shell]\nstarts = [\"disks\"]\n").is_ok()); - let launched = gate("[programs.disks]\nauthorize = [\"designate:data\"]\n[programs.shell]\nstarts = [\"disks\"]\n"); - assert!(launched.unwrap_err().contains("may start it")); - let login = gate("[boot]\nstart = [\"sshserver\"]\n[programs.sshserver]\nlogin = true\nauthorize = [\"designate:data\"]\n"); - assert!(login.unwrap_err().contains("is a login authority")); assert!(gate("[programs.sshserver]\nlogin = true\nauthorize = [\"tables\"]\n").is_ok(), "reading the tables is no change"); assert!(gate("[programs.disks]\nauthorize = [\"format:everything\"]\n").unwrap_err().contains("no disk authorization")); } @@ -3403,6 +3405,8 @@ mod tests { "tests/images/benchcase.toml", "tests/images/clipboardcase.toml", "tests/images/designatecase.toml", + "tests/images/formatcase.toml", + "tests/images/datacase.toml", "tests/images/drivescase.toml", "tests/images/hostedclangcase.toml", "tests/images/hotplugcase.toml", diff --git a/build/image.rs b/build/image.rs index 19e6a2fdac3..58b4727a609 100644 --- a/build/image.rs +++ b/build/image.rs @@ -999,7 +999,9 @@ mod tests { #[test] fn file_sets_across_a_bucket_size_edge_fit_and_read_back() { let mut sides = std::collections::BTreeSet::new(); - for kib in (58880..59904).step_by(256) { + // Leave room for format bookkeeping on either side; the journal's + // reclaim reserve is part of the image, not of these file bytes. + for kib in [48 << 10, 56 << 10, 60 << 10, 64 << 10] { let big: Vec = (0..kib << 10).map(|i| (i / 4096 + kib) as u8).collect(); let files: Vec<(String, Vec)> = std::iter::once(("bin/big".to_string(), big)) .chain((0..200u32).map(|i| (format!("share/s{i:03}"), vec![i as u8; 3000 + i as usize]))) diff --git a/images/console.toml b/images/console.toml index aeef875eba8..e50e71340de 100644 --- a/images/console.toml +++ b/images/console.toml @@ -132,5 +132,5 @@ receives = ["disk", "input"] [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] diff --git a/images/system.toml b/images/system.toml index cd41c4e6076..71ad6532f05 100644 --- a/images/system.toml +++ b/images/system.toml @@ -386,12 +386,12 @@ receives = ["disk"] drives = ["pci:0c:03:30"] receives = ["disk", "input"] -# The file servers: DATA, the log and the boot volume, one process +# The file servers: DATA, temporary files, the log and boot volumes, one process # each, serving every program the directories of its role. Started again when # one ends, on the same ports. A pager: its readers wait on the pages it # supplies, which only a row of this file may make a program do. [programs.fileserver] restart = true pager = true -roles = ["data", "log", "boot"] +roles = ["data", "log", "boot", "memory"] receives = ["block"] diff --git a/issues/a-4-kib-overwrite-on-data-costs-about-2-4-times-mains.md b/issues/a-4-kib-overwrite-on-data-costs-about-2-4-times-mains.md new file mode 100644 index 00000000000..a659884424b --- /dev/null +++ b/issues/a-4-kib-overwrite-on-data-costs-about-2-4-times-mains.md @@ -0,0 +1,30 @@ +--- +status: open +kind: defect +opened: 2026-11-05 +--- + +# A 4 KiB overwrite on DATA costs about 2.4 times main's + +**Owner.** The storage track, which owns `lib/bcachefs` and the file +server's DATA volume. + +osbench's `fs.disk.write_4k_at`, 4 KiB writes at scattered pages of a +16 MiB file, at #1152's a1f07627e: 23.9 us on the T14 and 30.8 us on the +AMD, p50, one boot each. Main dc1a8642e, on its private format, took +10.2 and 12.2 us (nightly 2026-10-31-2316); Linux ext4 0.7 and 2.3 us. + +Each overwrite is a copy-on-write replace: the write lands in new sectors +and one transaction removes or trims the old extent, inserts the new one, +and updates its backpointer, bucket accounting and the inode. On the host +model of the T14's DATA (CPU only, no device latency), taking the sectors +costs 0.8 us and that transaction 30.6 us an overwrite. A macOS `sample` of +it puts about 11% of the busy samples in `Cached::put`'s moves of a node's +keys (`memmove` under `mount::put` in `apply`); the rest is spread over the +pending overlay, allocation and the block writes, with no single cost +(#1152's evidence, `storage-integration/r9-small-writes`, `perf.rs`'s +`scattered_overwrites` and `scatter-sample.txt`). + +**Exit.** `write_4k_at` within 1.5 times main's on both benches, measured +by osbench against the same nightly's main, with the host model's +transaction cost and a sample of it published. diff --git a/issues/a-bcachefs-hole-costs-a-block-write-per-page.md b/issues/a-bcachefs-hole-costs-a-block-write-per-page.md deleted file mode 100644 index 53928f8f68d..00000000000 --- a/issues/a-bcachefs-hole-costs-a-block-write-per-page.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-02 ---- - -# A hole on bcachefs costs one block write per page, because the extent list cannot say "hole" - -`Mounted::allocate` (`lib/bcachefs/src/fs.rs`) now zeroes every block -it allocates only to bridge a gap, because the allocator hands back the blocks a -shrink most recently freed and a hole that reads the previous tenant's bytes is -a data leak. That is correct and it is what FAT32 already does (`Fat32::set_len` -calls `zero_range` on every grow). The cost is linear in the hole: reaching page -N over a gap of M pages writes M zero blocks, one device write each, inside the -file server's write that reaches the page (`system/fileserver/src/data.rs`). - -A sequential append reaches `target` in one block, so the zeroing loop -does not run. The paths that pay are a `lseek` past the end followed by a -write, and a shrink-then-write-above-the-mark, whose gap is bounded by the -file. - -The reason it is linear is the representation: `block_for` walks the extent list -accumulating `block_count`, so page K's block is decided by how many blocks -precede it. A list like that has no way to say "these pages are not allocated" — -every page below the size must have a block. FAT32 has the same property and the -same cost. - -**Exit condition.** An extent list that can express an unallocated run, so a -hole costs one list entry rather than one write per page, and a read of it -resolves to zeros without a block. That is an on-disk format change for -bcachefs, which is why it is not a small one — the same wall -`issues/a-listing-still-scans-the-whole-mount.md` ends at. diff --git a/issues/a-data-directory-listing-reads-every-entrys-inode-alone.md b/issues/a-data-directory-listing-reads-every-entrys-inode-alone.md new file mode 100644 index 00000000000..867cc65a1dc --- /dev/null +++ b/issues/a-data-directory-listing-reads-every-entrys-inode-alone.md @@ -0,0 +1,29 @@ +--- +status: open +kind: defect +opened: 2026-11-05 +--- + +# A DATA directory listing reads every entry's inode alone + +**Owner.** The storage track, which owns the file server's DATA volume and +`lib/bcachefs`. + +osbench's `fs.disk.readdir.1000`, listing a directory of 1 000 files, at +#1152's a1f07627e: 2.86 ms on the T14 and 1.99 ms on the AMD, p50, one boot +each; main dc1a8642e 0.91 and 0.69 ms (nightly 2026-10-31-2316), about 3x. + +**Cause.** A listing entry carries its file's size, so `DataVolume::list` +(`system/fileserver/src/data.rs`) reads each entry's inode with a lookup of +its own, from the root of the inodes btree, after one readdir of the +dirents; and every READDIR request lists the whole directory again before +taking its batch from the cursor. On the host model of the T14's DATA (CPU +only), one whole listing of 1 000 entries costs 534 us: the dirents 39 us, +the 1 000 inode lookups 416 us, sorting and encoding the rest (#1152's +evidence, `storage-integration/r10-known-gaps`, `perf.rs`'s +`listing_a_thousand`). A client asks in 64 KiB batches, so 1 000 short +names are one or two requests. + +**Exit.** The inodes of a listing read in one ordered pass over the inodes +btree, or the listing kept across a directory's READDIR batches, with +`readdir.1000` within 1.5 times main's on both benches. diff --git a/issues/a-fat-files-mtime-through-its-handle-is-its-last-level.md b/issues/a-fat-files-mtime-through-its-handle-is-its-last-level.md deleted file mode 100644 index de11f4bb7e1..00000000000 --- a/issues/a-fat-files-mtime-through-its-handle-is-its-last-level.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-29 ---- - -# A FAT file's mtime through an open handle is its entry's last level, not its last write - -The file server's FAT volume answers a file's mtime off its directory entry -(`system/fileserver/src/fat.rs`, `node_meta` and `lstat`), and an entry is stamped -only when it is brought level: at the last close, an fsync or a sync. A write -moves nothing an `fstat` reads, so a writer that writes and `fstat`s reads the -stamp of the entry's last level, and the same file closed and opened again reads -the level that close made: one file, two mtimes. - -**Mechanism read off the code; not reproduced.** - -**Exit condition.** For a FAT file, the mtime `fstat` answers through a handle -after a write is the one it answers after the file is closed and opened again, -checked by a test that writes, `fstat`s, reopens and `fstat`s again. diff --git a/issues/a-file-servers-refused-connection-reads-as-a-missing-program.md b/issues/a-file-servers-refused-connection-reads-as-a-missing-program.md deleted file mode 100644 index 3a2885bc96d..00000000000 --- a/issues/a-file-servers-refused-connection-reads-as-a-missing-program.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-23 ---- - -# A file server's refused connection reads as a missing program - -`toyos::files::is_served` (`sdk/toyos/src/files.rs`) answers `false` for any -path whose route fails, a refusal included: `toyos_fs_is_served` -(`sdk/toyos/src/files/owner.rs`) answers the route's error, and `is_served` -reads every answer but `Ok(1)` as "the kernel's". So when a process's first -connection to the file server serving a path is refused, std's spawn takes -the kernel's path for a program on that server, and the kernel, which serves -no `/system`, answers `NotFound`. - -Seen on the T14 at `db6be8df1` and on `main` `35e930136`, `proctreecase`: -`process_tree`'s depth chain, each link a helper of the one before in the -job's share, stopped at link 31 with "its next link was refused: entity not -found", and the kernel logged `spawn: /system/bin/test_rs_process_tree: not -found`. SYSTEM's server serves one share at most `SERVED_SHARE` (32) clients -(`system/fileserver/src/bounds.rs`), and the 32nd process of that share to -connect is refused. The chain now runs from a copy on `/tmp`; the defect -stands for any program a share's 33rd process spawns off a file server. - -Exit: a refused connection reaches the spawn's caller as the server's -refusal, never as the kernel's `NotFound`; a guest arm spawns a `/system` -program from a process whose share is at the server's bound and reads that -refusal. diff --git a/issues/a-kernel-files-fstat-answers-its-handles-mtime.md b/issues/a-kernel-files-fstat-answers-its-handles-mtime.md deleted file mode 100644 index 3bbc018691b..00000000000 --- a/issues/a-kernel-files-fstat-answers-its-handles-mtime.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-29 ---- - -# A kernel file's `fstat` answers its handle's mtime, not the file's - -`ops::fstat` answers `OpenFileState::mtime` (`kernel/src/object/ops.rs`), which -is the file's stamp at the open, moved only by a write or `ftruncate` through -that handle. A write through another handle moves the file's own stamp -(`file_cache::touch`) and not this one. So on `/tmp` a file opened at `t0` and -written through a second handle at `t1` answers `t0` through the first and `t1` -after a reopen, while the size it answers beside it is the file's. - -**Mechanism read off the code; not reproduced.** - -**Exit condition.** `fstat` of a kernel file answers the stamp the file holds, -with a guest test that writes through one handle and `fstat`s another opened -before the write. diff --git a/issues/a-link-on-a-kernel-mount-to-a-served-path-leads-nowhere.md b/issues/a-link-on-a-kernel-mount-to-a-served-path-leads-nowhere.md deleted file mode 100644 index 67bdc775b6c..00000000000 --- a/issues/a-link-on-a-kernel-mount-to-a-served-path-leads-nowhere.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-27 ---- - -# A link on a kernel mount to a served path leads nowhere - -The kernel resolves a symlink on its own mounts — ROOT and `/tmp` — in its own -tree (`kernel/src/vfs.rs`), and its tree holds nothing under `/apps`, -`/config`, `/home`, `/state`, `/log` or `/boot` but ROOT's empty directories: -those are file servers', reached through a directory capability -(`toyos::files`, which std and libc share). So a link in `/tmp` whose target is -`/home/toy/notes` opens nothing (`NotFound`), where the same link on a served -directory is followed — a file server hands an absolute target back to the -client, which resolves it in its own table. - -**Exit**: the kernel answers a path that crosses a link into a directory it -does not serve with the target and the rest of the path, as a file server's -`LINK` reply does, and std resolves it again in the caller's table; with a -guest test that opens a `/home` file through a `/tmp` link. diff --git a/issues/a-listing-still-scans-the-whole-mount.md b/issues/a-listing-still-scans-the-whole-mount.md deleted file mode 100644 index d039f76c644..00000000000 --- a/issues/a-listing-still-scans-the-whole-mount.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-01 ---- - -# A listing costs the whole mount on the two mounts with no directory structure - -The `FileSystem::list` bound is per directory now (`kernel/src/vfs.rs`, -`under_directory`), and FAT32 pays only the named subtree because it descends -to it (`Fat32::walk`'s `under`). The other two formats still read every name -in the volume and drop the ones that do not match: - -- `TmpFs::list` (`kernel/src/tmpfs.rs`) iterates the whole `entries` map. -- `Mounted::list` (`lib/bcachefs/src/fs.rs`) walks the whole B+tree through - `btree::for_each_live`, decoding every leaf value to learn its name: ROOT - in the kernel, and DATA in the file server. - -So a `readdir` of one directory is O(volume) on `/tmp`, ROOT and `/home`, and it -decodes a leaf value per entry on the two bcachefs volumes. Only the -*allocation* is bounded by what the directory holds; the work is not. Nothing -measures it, and no test would notice the difference. - -**This is a bound that was there and is not any more.** The old `list(16_384)` -stopped at the 16,384th entry it materialised and refused, so a mount with a -million names cost 16,384 entries of work and then an error. The new one visits -every name in the mount and filters, so that same mount costs a million — the -refusal now counts only what the listed directory holds, and nothing counts the -walk. On a kernel mount it runs under the VFS lock a `SYS_READDIR` holds, which -every other kernel filesystem syscall waits behind; on DATA it holds the file -server. - -The end of it is a real directory index — a store keyed by (directory, name) -rather than a flat namespace of paths that happen to contain `/`. On bcachefs -that is an on-disk format change, so it is not a small one. - -**`SYS_CHDIR` and `SYS_SPAWN` pay the same worst case on ROOT.** Both judge -a directory on a kernel mount with `FileSystem::is_dir` under the VFS lock -(`Vfs::is_mounted_dir`; a served directory is the file server's to answer), and -`Mounted::is_dir` (`lib/bcachefs/src/fs.rs`) is a `for_each_live` walk that stops -at the first name beneath the directory — so a directory that is there costs -the walk up to its first entry, and one that is not costs every node on the -mount. Every spawn states a cwd, so this is on the path of every process start -whose cwd is on ROOT. Nothing is kept, and a node the walk -cannot read refuses that one call. The same (directory, name) index ends it. diff --git a/issues/a-read-and-a-write-of-a-paged-file-can-tear-across-pages.md b/issues/a-read-and-a-write-of-a-paged-file-can-tear-across-pages.md deleted file mode 100644 index a8b6236d619..00000000000 --- a/issues/a-read-and-a-write-of-a-paged-file-can-tear-across-pages.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-31 ---- - -# A read and a write of a paged file can tear across pages - -POSIX XSH 2.9.7 makes a `read` and a `write` of a regular file atomic with -respect to each other. A holder's `SYS_PAGED_READ` copies a pinned frame at a -time (`kernel/src/object/paged.rs`), and the file server's write lands page -by page under write pins, outside the object's lock. A read concurrent with a -write of several pages can return some pages before the write and some after -it, and a page can be torn at the bytes being copied. - -This is the contract memory stage 7a states (writes are visible at once, with -no atomicity beyond the hardware's), and it is not what POSIX asks. - -**Exit**: a read and a write of one file are atomic with respect to each -other, or each is atomic per call by a rule the file server and the kernel -state together, with a guest test that reds on a torn read. - -**Owner**: the storage track (`issues/storage-is-layers-and-a-role-is-a-filesystem.md`). diff --git a/issues/a-refused-carried-file-costs-every-other-files-fsync-or-leaks-its-blocks.md b/issues/a-refused-carried-file-costs-every-other-files-fsync-or-leaks-its-blocks.md new file mode 100644 index 00000000000..d1779ab1bb8 --- /dev/null +++ b/issues/a-refused-carried-file-costs-every-other-files-fsync-or-leaks-its-blocks.md @@ -0,0 +1,37 @@ +--- +status: open +kind: defect +opened: 2026-11-01 +--- + +# A carried file that is refused costs every other file's fsync, or leaks its blocks + +An fsync on DATA commits its own file and every file the commit carries +(`Volume::carried`): one whose entry was written, or whose reserve took +blocks, since the last commit (`system/fileserver/src/pages.rs`, +`Files::fsync`; `system/fileserver/src/data.rs`, `DataVolume::commit`). A +refusal on a carried file is paid by the files beside it: + +- **Its pages refused.** `Files::fsync` refuses the commit when a carried + file's write-back is refused, since its entry would name blocks without + its bytes. While that write-back stays refused, every fsync of every + file on the volume is refused with it. +- **Its entry refused.** `DataVolume::commit` says a carried entry whose + write was refused, and commits anyway. Where the file took blocks since + the last commit, the commit makes the allocator's take durable with no + entry naming them, and after a stop those blocks are leaked: not readable + by any file, and not free. This is not a disclosure. + +A file carried only by blocks taken (its entry unwritten since the last +commit) could be left out of the commit at the cost of that leak, rather +than refusing everyone's fsync. + +Found by review of #1141; no test drives either refusal into another file's +fsync or a stop. + +**Exit**: a refusal on one carried file costs only that file: an fsync of +another lands whole, and after a stop the volume holds as free every block +no entry names. A host test refuses one carried file's write-back, and +another's entry, fsyncs a third, stops, and counts the free blocks. + +**Owner**: the storage track (`issues/storage-is-layers-and-a-role-is-a-filesystem.md`). diff --git a/issues/a-replayed-bcachefs-node-past-seven-nodes-is-refused.md b/issues/a-replayed-bcachefs-node-past-seven-nodes-is-refused.md deleted file mode 100644 index 6a8466dc7a9..00000000000 --- a/issues/a-replayed-bcachefs-node-past-seven-nodes-is-refused.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-23 ---- - -# A replayed bcachefs node whose keys fill more than seven nodes stops the read-write mount - -The writer splits a node in one reclaim transaction, into at most seven -nodes and a root (`MAX_GROUPS` in `lib/bcachefs/src/engine/mount/reclaim.rs`). -That bound holds for a node it wrote, and for one read from the medium, packed, -plus two transactions. It does not hold for a node a replay fills: -`Mount::install_replay` puts every key of the journal window into the leaf -its on-disk tree leads to, with no bound. A window that puts more than seven -nodes' keys into one leaf is reachable on an unclean upstream volume. Examples -are many inodes journaled since their leaf was last written, or keys journaled -into split nodes whose pointer update was not yet journaled. - -The reader replays such a volume in memory and reads it whole. The -read-write mount opens it, but its first write that needs the leaf split -stops the mount, read-only by name: "a btree node whose keys fill more than -seven nodes". `tests/journal.rs`, -`a_replayed_leaf_past_seven_nodes_is_read_but_not_mounted_read_write`, builds -the case: about 2 860 inodes in 13 entries of a 64 MiB volume's window. - -**Exit.** A split that spans several reclaim transactions for a node no -single transaction can split. For instance, write the window's keys out as a -run of new leaves under the old leaf's parent, a transaction at a time, -before the old leaf's pointer is replaced. Alternatively, a replay that -installs keys into nodes bounded as it goes. diff --git a/issues/a-reserved-data-block-holds-a-transactions-metadata-credit-until-written.md b/issues/a-reserved-data-block-holds-a-transactions-metadata-credit-until-written.md new file mode 100644 index 00000000000..50ec41d5879 --- /dev/null +++ b/issues/a-reserved-data-block-holds-a-transactions-metadata-credit-until-written.md @@ -0,0 +1,27 @@ +--- +status: open +kind: defect +opened: 2026-11-04 +--- + +# A reserved DATA block holds a transaction's metadata credit until it is written + +**Owner.** The storage track, which owns `lib/bcachefs` and the file server's +DATA volume. + +The file server reserves the blocks of a write before it copies the bytes, +and `Reservations::pending` (`lib/bcachefs/src/engine/mount/data.rs`) counts +one ordinary transaction's metadata credit, `K_TX * 8` bytes, for every +reserved block not yet written. A block of 4 KiB therefore holds as much +metadata room as data until write-back, and a large write needs about twice +its size free. Main's private format had no such limit. + +**Evidence.** On QEMU's 128 MiB DATA, osbench's 16 MiB setup file is refused +for space (#1152's QEMU run of the `fs.disk` rows). The benches' DATA +partitions are large enough that no row reaches it. + +**Exit.** Credit by what write-back will journal: one transaction's credit +for each run of contiguous reserved blocks, which `write_run` writes as one +extent, or for each transaction write-back is expected to take, instead of +for each block. A test that reserves a write of most of a small volume's free +space and writes it back. diff --git a/issues/a-served-file-at-stdio-stops-every-spawn.md b/issues/a-served-file-at-stdio-stops-every-spawn.md deleted file mode 100644 index 04b0e81b4cd..00000000000 --- a/issues/a-served-file-at-stdio-stops-every-spawn.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-21 ---- - -# A served file at stdio stops every spawn - -A file a file server holds is a descriptor naming an object of -`toyos::files`', not a kernel handle, and `toyos::fd::copy` refuses one for a -child (`EOPNOTSUPP`, std's `Unsupported`). libc's `posix_spawn` asks for -descriptors 0 to 2 on every spawn (`spawnreq::plan`), so once a C program has -put a served file there, by `dup2` or `freopen`, every `posix_spawn`, -`system` and `popen` it makes is refused. So is an `adddup2` of a served -descriptor and an `addopen` of a served path, and std's `Command` with -inherited stdio in the same case. Before served files were descriptors, no -served file could sit at 0 to 2. Read from the code; no guest case has run it. - -**Exit**: a served file can be given to a child, as PR 4 of the descriptor -work gives served kinds (`EXPORT`), and a guest C case redirects stdout to a -file under `/home`, runs `system`, and reads the child's output back from it. diff --git a/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md b/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md new file mode 100644 index 00000000000..63680450be8 --- /dev/null +++ b/issues/a-small-bcachefs-journal-makes-reclaim-write-nodes-after-nearly-every-transaction.md @@ -0,0 +1,29 @@ +--- +status: open +kind: defect +opened: 2026-11-04 +--- + +# A small bcachefs journal makes reclaim write nodes after nearly every transaction + +**Owner.** The storage track, which owns `lib/bcachefs`. + +Each dirty node reserves its worst-case pointer update in the journal, and an +ordinary transaction is admitted only with a transaction's journal room left +(`lib/bcachefs/src/engine/mount/mod.rs`). Where the journal holds little, as +on the 64 MiB host test images and QEMU-sized DATA, a few dirty nodes fill +it, so reclaim writes nodes after nearly every transaction and the eager +put-in-place of #1152 gains nothing there. mkfs sizes the journal at 1/128 +of the device, at least eight buckets and at most 32 MiB (`JOURNAL_BYTES` +in `mkfs.rs`), so the benches' DATA partitions are far from this; only +DATA mounts this engine read-write. + +**Evidence.** The 64 MiB host images' tests: a crash after 2 000 creates +replays about a thousand journal keys of them +(`tests/admission.rs`, `the_first_transaction_after_a_replay_takes_each_delta_once`), +the rest already written by reclaim. + +**Exit.** A journal reservation for dirty nodes that does not grow with +each one, for instance one pointer update for each reclaim batch rather than +for each node, with the reclaim interval on a small volume measured before +and after. diff --git a/issues/a-swapped-binary-lives-where-any-process-can-rewrite-it.md b/issues/a-swapped-binary-lives-where-any-process-can-rewrite-it.md deleted file mode 100644 index 3652e90b344..00000000000 --- a/issues/a-swapped-binary-lives-where-any-process-can-rewrite-it.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -status: open -kind: defect -severity: [security] -opened: 2026-09-23 ---- - -# A swapped binary lives where any process can rewrite it - -A service swap (`toyos-swap`) writes the binary the supervisor verified to -`/tmp/swap//` and starts the service from that path. `/tmp` -is a tmpfs every process may write, because the filesystem is ambient by the -owner's ruling, so the file the supervisor hashed is not the file the kernel loads by any -guarantee — only by nobody having rewritten it in between. Two windows: - -- **Between the hash and the spawn.** The supervisor reads the staged bytes, verifies - them and writes them to the installed path itself, so the installed file is - the verified bytes when it is written. It is spawned by path once the - requester has hung up, milliseconds to two seconds later; a process that - rewrites the file in that window runs its own bytes holding the service's - manifest row — its device claims included. -- **For the service's life**, closed: a spawn from `/tmp` runs a snapshot of - the file taken at the spawn (`file_cache::Snapshot`), which no later write - reaches. A restore after a failed swap spawns the previous installed path - again, so it is the first window again. - -Nothing in the tree does either; the authority to ask for a swap is -`/system/bin/swap`'s alone, and this is the one step of it the capability -model does not cover. - -**Exit condition.** The spawn takes what the supervisor verified rather than a path: a -spawn from a file handle the supervisor holds, or from bytes the supervisor hands over — -`SpawnArgs::image`, a memory object the kernel pages the child from -(`Image`, `sdk/toyos/src/process.rs`), which a program on a file server -already starts through — or a directory only the supervisor may write, which the ambient-filesystem -track would have to rule on. - -## Narrowed, not closed - -The supervisor now hashes an installed binary again at every start — the swap's and a -restore's — and refuses to start one that no longer holds what its path names -(`toyos_swap::installed_digest`). That moves the first window from "until the -requester hangs up" to the spawn syscall itself, and the snapshot closed the -second. A directory only the supervisor -may write does not exist in this kernel: every writable mount is ambient, and -gating one on the writer's identity would be pid-as-authority. So the exit -condition above stands, and choosing between its three shapes is the owner's. diff --git a/issues/a-symbolic-link-on-tmp-displaces-its-name-and-lists-nowhere.md b/issues/a-symbolic-link-on-tmp-displaces-its-name-and-lists-nowhere.md deleted file mode 100644 index a614807125b..00000000000 --- a/issues/a-symbolic-link-on-tmp-displaces-its-name-and-lists-nowhere.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-10-01 ---- - -# A symbolic link on /tmp displaces its name and lists nowhere - -Read from the code, not run. The kernel's `/tmp` (`kernel/src/tmpfs.rs`) -makes a link over whatever file its name held, where POSIX's `symlink` refuses -a name that exists `EEXIST`, so libc's `symlink` refuses `ENOSYS` for a link -the kernel would hold (`sdk/libc/src/posix_io.rs`). Its `list` answers files -alone, so no `readdir` of a directory shows a link it holds. - -libc's `symlink` already sends a link a file server is to hold to that -server, through `toyos::files` as std's does, which refuses a name that -exists, answered `EEXIST`, and `readdir` there names the link; corpus case -`211_home_files` asserts both under `/home`. What is left is `/tmp`. - -**Exit**: `/tmp` is a file server's, so every `symlink` is one, sent with -nothing asked first and answered `EEXIST` for a name that exists, and libc's -`readdir` of `/tmp` names the link, each asserted by a test. The kernel's `SYS_SYMLINK` -and its symlink code (`kernel/src/vfs.rs`, `kernel/src/tmpfs.rs`) go wherever -nothing calls them. That `EEXIST` is how LLVM's `LockFileManager` takes its -lock (`create_link`, `::symlink` in `llvm/lib/Support/Unix/Path.inc`). An LLVM -build on ToyOS reaches `symlink` too: -`LLVM_USE_SYMLINKS` is on for a UNIX host, so `add_llvm_tool_symlink` -(`llvm/cmake/modules/AddLLVM.cmake`) makes each tool's aliases with CMake's -`create_symlink`. diff --git a/issues/a-sync-commits-the-entry-of-a-file-whose-pages-were-refused.md b/issues/a-sync-commits-the-entry-of-a-file-whose-pages-were-refused.md new file mode 100644 index 00000000000..bcca2298d20 --- /dev/null +++ b/issues/a-sync-commits-the-entry-of-a-file-whose-pages-were-refused.md @@ -0,0 +1,29 @@ +--- +status: open +kind: defect +opened: 2026-11-01 +--- + +# A sync commits the entry of a file whose pages the volume refused + +`Files::sync` (`system/fileserver/src/pages.rs`) writes every dirty file's +pages back and then calls the volume's `sync` even where a file's write-back +was refused. DATA's `sync` writes every open file's entry, that file's +among them, so the commit makes durable an entry whose new extents name +blocks that never received its pages. In the same boot the file still reads +right: its pages stay dirty in its object and the next sync tries them +again. After a stop before that, it reads what those blocks last held, +which may be a deleted file's bytes +(`issues/a-reallocated-extent-on-data-keeps-the-deleted-files-bytes.md`). + +An fsync does not: a carried file whose pages are refused refuses the +commit (`Files::fsync`). + +Found by reading, while making an fsync commit one file; no test drives a +refused write-back into a stop. + +**Exit**: a sync's commit carries no entry of a file whose pages were +refused, with a host test that refuses one file's write-back, stops the +volume after the sync and reads that file as its last commit left it. + +**Owner**: the storage track (`issues/storage-is-layers-and-a-role-is-a-filesystem.md`). diff --git a/issues/a-tmpfs-metadata-call-costs-toyos-up-to-six-times-linuxs.md b/issues/a-tmpfs-metadata-call-costs-toyos-up-to-six-times-linuxs.md index 30f31aa391b..69afb79af4b 100644 --- a/issues/a-tmpfs-metadata-call-costs-toyos-up-to-six-times-linuxs.md +++ b/issues/a-tmpfs-metadata-call-costs-toyos-up-to-six-times-linuxs.md @@ -7,8 +7,8 @@ opened: 2026-10-28 # A tmpfs metadata call costs ToyOS up to six times Linux's `osbench files` runs the same operations under each system's memory -filesystem: ToyOS's `/tmp`, which the kernel serves through syscalls today, -and Linux's `/dev/shm`, tmpfs through the VFS. +filesystem: ToyOS's `/tmp`, which the kernel served through syscalls when +this was measured, and Linux's `/dev/shm`, tmpfs through the VFS. From the 2026-10-28 scoreboard (evidence branch `scoreboard-2026-10-28`): ToyOS at 069cbbeb8, four boots off the internal NVMe, against five runs of @@ -31,24 +31,18 @@ On the T14 ToyOS loses all eight, 1.7x to 6.5x, each resolved. On the AMD it loses `open_close` (2.1x) and `rename` (1.4x), both resolved, wins `tree.100`, `create_write_unlink` and `write_4k_at`, and is unresolved on the other three. There the HPET explains Linux's spread: its TSC runs read -`open_close` at 2.9 µs and its HPET runs at 4.3 µs. The listing -(`fs.mem.readdir.1000`) is `issues/a-listing-still-scans-the-whole-mount.md`'s, -and the sequential copies are +`open_close` at 2.9 µs and its HPET runs at 4.3 µs. The sequential copies are `issues/user-memory-hot-paths-are-slower-than-2-mib-pages.md`'s. -**The kernel's tmpfs is not where this is fixed.** Step 9 of -`issues/the-kernel-is-small-interrupts-post-and-threads-wait.md`, under the -owner's rulings of 2026-09-25 and 2026-10-03, deletes `tmpfs.rs`, -`fs_rename.rs`, `file_cache.rs` and `vfs.rs`, and moves `/tmp` to -userland per session. An `fs.mem` call then becomes a file server request, -which costs 3 to 12 times a kernel file call today +`/tmp` is now the MEMORY file server's (`system/fileserver/src/memory.rs`), so +each `fs.mem` call is a file server request, which cost 3 to 12 times a kernel +file call when both existed (`issues/a-file-server-request-costs-three-to-twelve-times-a-kernel-file-call.md`). -So the move makes this gap larger, unless the file server's round trip -shrinks first. +These rows have not been measured since the move; the gap is expected to have +grown. -**Exit**: once `/tmp` is served from userland, these eight rows within -Linux's on both benches in a labelled current-state comparison, or each -remaining difference measured to its parts and justified. +**Exit**: these eight rows within Linux's on both benches in a labelled +current-state comparison, or each remaining difference measured to its parts +and justified. -**Owner**: whatever serves `/tmp` through step 9's migration. Until then, -nothing in the kernel's tmpfs is owed this work. +**Owner**: the file server. diff --git a/issues/an-fsync-of-a-served-file-syncs-its-whole-volume.md b/issues/an-fsync-of-a-served-file-syncs-its-whole-volume.md index e955882008a..c47c6158405 100644 --- a/issues/an-fsync-of-a-served-file-syncs-its-whole-volume.md +++ b/issues/an-fsync-of-a-served-file-syncs-its-whole-volume.md @@ -6,10 +6,14 @@ opened: 2026-09-27 # An fsync of a served file syncs its whole volume -The file server answers `FSYNC` by syncing the volume, every open file's +**Built, unmeasured.** `FSYNC` now commits its own file and what the +volume's commit carries (`Files::fsync`, `Volume::carried`), and no longer +syncs the volume. What follows is how it was. + +The file server answered `FSYNC` by syncing the volume, every open file's entry and every dirty block of the cache (`system/fileserver/src/main.rs`, -`FSYNC`, which calls `sync` with the file's node and syncs the whole volume). So an -`fsync` or `File::sync_all` of one file on `/home` syncs all of DATA, and a +`FSYNC`, which called `sync` with the file's node and synced the whole volume). So an +`fsync` or `File::sync_all` of one file on `/home` synced all of DATA, and a 4 KiB create and fsync cost 3.5–3.6 ms p50 on QEMU TCG against the kernel's 1.0–1.3 ms, measured on the branch that moved DATA to the file server. @@ -20,3 +24,5 @@ for no durability. **Exit**: `FSYNC` makes durable the file it names and what that file's entry depends on, with a guest measurement of one file's fsync beside the volume's. +What is owed is that measurement: osbench's +`persist.write_4k_fsync_beside_1m`, base against head on metal. diff --git a/issues/anonymous-memory-swaps-to-an-encrypted-partition.md b/issues/anonymous-memory-swaps-to-an-encrypted-partition.md index 4b47019cc92..c7103f92d2a 100644 --- a/issues/anonymous-memory-swaps-to-an-encrypted-partition.md +++ b/issues/anonymous-memory-swaps-to-an-encrypted-partition.md @@ -351,7 +351,7 @@ Each of these changes at the stage named: (the RAM/8 window cap) becomes the compositor's wire allowance over on-screen buffers. - **Deleted.** `system/fileserver/src/cache.rs` goes at its prerequisite. - `kernel/src/file_cache.rs` behind kernel tmpfs goes at P3's memory format. + `kernel/src/file_cache.rs` went with kernel tmpfs (#1152). - **When text maps its source without the copy** (`issues/process-memory-in-4-kib-pages-still-copies-each-programs-text.md` stage 5), the copied-segment rule goes with the copies. diff --git a/issues/bcachefs-crate-is-not-bcachefs.md b/issues/bcachefs-crate-is-not-bcachefs.md deleted file mode 100644 index 8b58dfe52c0..00000000000 --- a/issues/bcachefs-crate-is-not-bcachefs.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -status: open -kind: track -opened: 2026-08-01 ---- - -# The `lib/bcachefs/` crate does not implement bcachefs - -ToyOS's `lib/bcachefs/` crate implements a ToyOS-native on-disk format written from scratch. -It shares a name with Linux bcachefs and nothing else, and **the magics alone settle -it**: ours is a four-byte `MAGIC = b"BCFS"` plus -`DESIGNATION_MAGIC = b"TOYOS-FORMAT-ME\0"` (`superblock.rs:5,24`) and `NODE_MAGIC = b"BTND"` -(`btree.rs:7`), against upstream's 16-byte UUID `BCHFS_MAGIC`, its per-bset -`BSET_MAGIC ^ sb.uuid` and its `JSET_MAGIC`. Neither implementation could mount -what the other writes, and neither would get past the first block trying. - -That collision has already cost this project once: research into the *upstream* -format was filed under this crate's name and needed a warning at its top to stop -a reader taking it for documentation of what we ship. Warning the reader fixes -the document and not the collision — a crate that does not implement the format -it is named after is a hazard we keep paying for, in exactly this way. Renaming -it is the owner's call, not something to do in a docs pass. - -## Answered by the owner, 2026-08-15 - -> *"bcachefs is the default filesystem for toyos. the crate must be an -> implementation of the spec."* - -The resolution is neither a rename nor a deletion. **`lib/bcachefs/` must become a -real implementation of the bcachefs on-disk format, and that format is ToyOS's -default filesystem.** The name stops being wrong by the crate growing into it. - -This entry therefore stops being a question and becomes work: what is owed is the -implementation, and the gap between the two formats recorded above is the measure -of it. The format is implemented clean-room (root `CLAUDE.md`): an analyst reads -upstream and writes the specification, and a different agent implements from it. - -**Where it stands.** `lib/bcachefs/src/engine` reads single-device 1.39 -volumes, crc32c and ChaCha20/Poly1305, replaying the journal of one not cleanly -unmounted, and formats a clean volume holding a tree -of directories, files, symlinks and hard links (directories alone when -encrypted), with all its allocator and accounting state, that upstream's -`bcachefs fsck -n -K` passes. Its fsck derives -that state from a volume's keys and compares, and the derivation agrees with -what upstream stored in its own volumes, a replayed journal's included; it -checks the journal against the superblock. It mounts a volume read-write -(`Mount`): namespace operations as journaled transactions, nodes appended to or -split by reclaim, a crash replayed and a clean unmount, which upstream's fsck -also passes. It refuses to write a volume of blocks under 4 KiB or nodes under -32 KiB (`issues/the-bcachefs-writer-refuses-small-blocks-and-nodes.md`). It -reuses a btree bucket once the entry that emptied it is durable. File data is -not yet written (B7), so symlinks are not yet created online, and its reclaim reserves are -sized for common rewrites -(`issues/bcachefs-reclaim-reserves-are-sized-for-common-rewrites.md`). It -replaced a read side written from upstream's headers. Its tests read volumes upstream's tools wrote -(`lib/bcachefs/tests/fixtures/`). The build writes ROOT with it, at the -smallest size that leaves no bucket free, and the loader, `imgstat` and the -SYSTEM file server read ROOT through its reader. The rest of the crate is -still the interim format, which DATA uses until the engine replaces it. - -The track itself is not planned here. - -One fact about the tree that a real-bcachefs track inherits whichever way it is -sequenced: the kernel must parse this format to reach `/system/bin/init` at all, so a -machine whose ROOT it cannot mount has no userland. The observation the ruling -inverts is the other half — a home-grown format has no second implementation to -be judged against, and upstream bcachefs is exactly such a judge. - -**How that judge is used, since a judge is not a dependency:** upstream's `bcachefs-tools` is a *development* -instrument, run by a builder on a volume ToyOS wrote, with its `fsck` and its -readback pasted into the pull request. What the suite runs against is committed -fixtures — volumes upstream's tools wrote, recorded in `NOTICE`. Nothing is -fetched and no distribution is depended on. diff --git a/issues/deferred-release-outlives-its-syscall.md b/issues/deferred-release-outlives-its-syscall.md index 5ddbbb1a96d..1bf8cb53150 100644 --- a/issues/deferred-release-outlives-its-syscall.md +++ b/issues/deferred-release-outlives-its-syscall.md @@ -258,9 +258,8 @@ The ruling, in the words it was recorded in, where "wall 4" is this question: The queue landed with #257 (`7ab9367b0`) and left the kernel in `80a1f1ceb`, when nothing the kernel flushed reached a device any more; the conversion it -was sequenced before is planned by nothing. Today `OpenFileState::drop` calls -`file_cache::release` and takes "the file cache's lock and no other", so -`File` needs no release site that parks. +was sequenced before is planned by nothing. `OpenFileState` and the file cache +whose release it ran went in #1152. ## A process's own releases run before its end, ruled 2026-10-13 diff --git a/issues/dropping-an-in-flight-bcachefs-operation-does-not-release-its-grants.md b/issues/dropping-an-in-flight-bcachefs-operation-does-not-release-its-grants.md new file mode 100644 index 00000000000..2802acc2c78 --- /dev/null +++ b/issues/dropping-an-in-flight-bcachefs-operation-does-not-release-its-grants.md @@ -0,0 +1,25 @@ +--- +status: open +kind: finding +opened: 2026-11-02 +--- + +# Dropping an in-flight bcachefs operation does not release its grants + +`Mount::transact` in `lib/bcachefs/src/engine/mount/mod.rs` acquires a +journal/cache grant and then awaits `prepare`. Dropping that future during +device I/O drops the writer slot, but does not call `finish`: the grant, +cached-node holds, and any `Txn::allocated_data` cleanup remain outstanding. +The reclaim path also retains captured or writing nodes if its future is +dropped between planning, node I/O and pointer publication. + +This is a code-inspection finding; cancellation has not been injected. The +fileserver currently runs these operations to completion with `block_on`. +Unused `Reservation` drop and returned operation errors have separate +cleanup paths and are not this finding. + +The owning component is the bcachefs engine. A future cancellation change +must test dropping at each awaited boundary, including already completed +node writes, and establish either retryable ownership or an explicit stopped +mount without leaving admissions waiting forever. It must preserve reserved +DATA slots, reader barriers and the original journal dependencies. diff --git a/issues/every-program-sees-only-the-files-it-was-given.md b/issues/every-program-sees-only-the-files-it-was-given.md index 2951b52c1de..9604831c344 100644 --- a/issues/every-program-sees-only-the-files-it-was-given.md +++ b/issues/every-program-sees-only-the-files-it-was-given.md @@ -100,8 +100,8 @@ nameable by every program and make confused-deputy bugs structural. what it holds, and a package launched from `/apps` is minted its own directory read-only and its own folder of the session's home read-write, `/system` read-only, and nothing else a file server serves - (`toyos_manifest::Program::view`). It still reaches `/tmp`, which the - kernel serves to every program. Every row the image declares still sees + (`toyos_manifest::Program::view`). It still reaches `/tmp`, the one tree + the MEMORY file server serves every view. Every row the image declares still sees the whole tree, `/system` read-only and the rest read-write but `/state`, of which each row holds only its own directory where it keeps state (`service = true` or `state = true`: the daemons' own state is built). diff --git a/issues/four-shares-take-a-file-server.md b/issues/four-shares-take-a-file-server.md index 1128c6c6b60..12e9ca7de59 100644 --- a/issues/four-shares-take-a-file-server.md +++ b/issues/four-shares-take-a-file-server.md @@ -39,6 +39,18 @@ so it waits for one of two exits: a measured parallel build on ToyOS shows which bound it reaches and by how much, or a share's streams are bounded by the bytes they cost its client's job rather than by count. +**`/tmp` now spends the same bound.** Since #1152 `/tmp` is the MEMORY file +server's, and the SDK keeps one connection per directory capability for the +life of the process. So every live process of a share that has touched `/tmp` +holds one of that share's 32 connections there, and the 33rd is refused. Seen +on the T14 at `0507cbe36` (`proctreecase`): a chain of processes each +spawning the next from a copy of its program on `/tmp` stopped at link 32, +refused "the file server serves as many connections as it takes". A parallel +build whose compilers and linkers each write a temporary there reaches it at +32 concurrent processes, before any stream bound. Nothing runs that load now: +`process_tree`'s chain starts each link from an endowed image and touches no +file server, and `bounds.rs`' unit tests hold the count alone. + Owner: the file server. ## Exit condition diff --git a/issues/libc-refuses-what-toyos-cannot-yet-answer.md b/issues/libc-refuses-what-toyos-cannot-yet-answer.md index a4000d786a8..dc2951c8495 100644 --- a/issues/libc-refuses-what-toyos-cannot-yet-answer.md +++ b/issues/libc-refuses-what-toyos-cannot-yet-answer.md @@ -10,10 +10,6 @@ These answer failure in their POSIX form and do nothing (`sdk/libc/src/refused.rs`, `posix_io.rs`), each asserted by `tests/corpus/tinycc/206_libc_refusals.c`. Each waits on what it names: -- `symlink` of a link the kernel would hold, `ENOSYS`: `SYS_SYMLINK` - displaces what its name held - (`issues/a-symbolic-link-on-tmp-displaces-its-name-and-lists-nowhere.md`). A - file server makes one and refuses a name that exists, as POSIX has it. - `chmod`, `fchmod` and `fchmodat`, `ENOSYS`: a file has no mode bits to set. - `statvfs` and `fstatvfs`, `ENOSYS`: no call answers a filesystem's size or free space. diff --git a/issues/ninja-runs-every-command-through-a-bin-sh-toyos-does-not-have.md b/issues/ninja-runs-every-command-through-a-bin-sh-toyos-does-not-have.md index b760c170622..da53b5083db 100644 --- a/issues/ninja-runs-every-command-through-a-bin-sh-toyos-does-not-have.md +++ b/issues/ninja-runs-every-command-through-a-bin-sh-toyos-does-not-have.md @@ -9,8 +9,8 @@ opened: 2026-09-30 Ninja starts each build edge as `/bin/sh -c ` (v1.13.1 `src/subprocess-posix.cc:132-134`), and the command is the build file's own text, written for a POSIX shell; tokio's `tests/process_smoke.rs` runs `sh -c` -too. ToyOS has no `/bin`: `/` holds exactly the names `kernel/src/vfs.rs`'s -`ROOT_ENTRIES` lists. Its shell is `/system/bin/shell`, which takes `-c`; +too. ToyOS has no `/bin`: `/` holds only the directories a process's view +names (`toyos_manifest::ROLES`). Its shell is `/system/bin/shell`, which takes `-c`; nothing establishes that it runs the POSIX shell language build files are written in. diff --git a/issues/one-write-far-past-a-files-end-holds-data-s-server.md b/issues/one-write-far-past-a-files-end-holds-data-s-server.md deleted file mode 100644 index 938a1d0eaed..00000000000 --- a/issues/one-write-far-past-a-files-end-holds-data-s-server.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-09-27 ---- - -# One write far past a file's end holds DATA's server for as long as the gap - -DATA's file server (`system/fileserver`) takes a `WRITE` at any offset up to `toyos::fs::MAX_FILE_BYTES` (16 TiB), -and `Mounted::allocate` bridges the gap from the file's end to it -by allocating and zeroing every block between -(`issues/a-bcachefs-hole-costs-a-block-write-per-page.md` is why). -The server is one thread, so the one request runs until the volume is full or the gap -is bridged, and every other client of `/apps`, `/config`, `/home` and `/state` -waits the whole of it: any program holding `/home` can stop every program's -files with one `seek` and one byte. - -**Exit**: a write whose gap is past what a request may cost is refused by name -before anything is allocated — or the format expresses a hole — with a guest -test that writes one byte a gigabyte past a file's end and has another client -answered meanwhile. diff --git a/issues/probe-mounts-on-a-checksum.md b/issues/probe-mounts-on-a-checksum.md index 78bd535ee05..5450ec055a7 100644 --- a/issues/probe-mounts-on-a-checksum.md +++ b/issues/probe-mounts-on-a-checksum.md @@ -6,57 +6,52 @@ opened: 2026-08-01 # `probe()` mounts on a checksum, and a stamp over a used volume does not reformat -Two things, from reading DATA's probe (`DataVolume::probe` in -`system/fileserver/src/data.rs`, the kernel's `bcachefs_adapter::probe` before it) -against the crate: - -**The threshold does not match the consequence.** `Probed::Mounted` is a -read-write mount: `sync()` rewrites both superblocks, and any file operation -writes the bitmap, btree nodes and data. So mounting a stranger's disk modifies -it, which is a weaker form of the wrong the designation stamp exists to prevent. -Accidental collision is not the risk — random block-0 bytes satisfy 4 bytes of -magic, 4 of version and a 32-bit CRC with probability about 2^-64 — and neither -is a *genuine* upstream bcachefs volume, which does not begin with ASCII `BCFS` -(this crate shares the name and nothing else; `issues/bcachefs-crate-is-not-bcachefs.md`). The risk is a **deliberately -crafted block 0** on a disk somebody hands you, which is the metal track's -situation exactly. - -Recommendation, for the owner to decide: - -- **Done:** `Superblock::check` takes `block_count != device_blocks`, so a - volume image copied onto a different disk stops mounting — the same property - the designation stamp's block count gives a format. It is not authentication: - an attacker who knows the disk size writes the right number. - `a_volume_copied_onto_a_larger_device_does_not_mount` holds it. -- **Done, 2026-08-23:** a file's extents are range-checked against the volume - in `decode_leaf_value`, so "mounting a hostile volume is merely rude" no - longer has an unchecked extent reaching a block read — or a bitmap write — - behind it. The residual bound this left — `read_link` sizing one kernel - allocation from the volume rather than the smaller heap ceiling — was closed - by giving `Mounted::read_link` a `max_len` its caller passes (the file - server's), refused as `FsError::TargetTooLong` before the allocation. -- **The real fix, if the threat model wants one:** read-write requires - something the attacker cannot compute — a keyed MAC, or a designation-like - stamp — and everything else mounts read-only. ToyOS has no key store and no - TPM support, so this is a metal-track decision, not a patch. - -**The owner ruled on 2026-09-01: take the exact check now, authenticate on -the metal track.** The exact check is taken and the first bullet above is -closed. It is not authentication and the entry must keep saying so: an attacker -who knows the disk's size writes the right number. What it removes is every case -where the volume did not come from this disk. - -Authentication — read-write only for something the attacker cannot compute, -read-only for everything else — is deferred to the metal track, because it needs -a place to keep a secret and ToyOS has neither a key store nor TPM support. The -threat this leaves open is stated rather than reduced: a deliberately crafted -block 0 on a disk handed to the machine still earns a read-write mount, and -read-write means written on sight. - -**Separately, and reproduced:** a designation stamp written over a disk that -already held a ToyOS volume does **not** cause a reformat. `designate_for_format` -writes block 0 only, `Superblock::read` falls back to the backup superblock at -the last block when block 0 does not parse, and a stamp does not parse — so -`mount()` succeeds from the backup and `probe()` answers `Mounted`, mounting the old -volume. Harmless for the harness, which stamps freshly created sparse files, but -it means "re-stamp the disk to reformat `/home`" is not a workflow that works. +Two things about DATA's probe (`DataVolume::probe` in +`system/fileserver/src/data.rs`) and the bcachefs engine under it. + +**The threshold does not match the consequence.** A probe that mounts opens +the volume read-write (`Mount::open`): replay, orphan release, the roots it +makes and its first commit all write. So mounting a stranger's disk modifies +it, which is a weaker form of the wrong the designation stamp exists to +prevent. Accidental collision is not the risk: random bytes do not pass a +superblock's magic and checksum. The risk is a volume this machine did not +write on a partition typed as DATA — a **deliberately crafted** one on a disk +somebody hands you, which is the metal track's situation exactly. + +What bounds it today: + +- **The exact check** (owner, 2026-09-01: "take the exact check now"): a + volume mounts only with exactly the buckets mkfs lays out on that partition + (`Mount::spans_its_device`), so a volume copied onto a partition of another + size is refused, not mounted + (`a_volume_copied_onto_a_larger_partition_does_not_mount`). It is not + authentication: whoever knows the partition's size writes a volume of it. +- **Hostile contents are refusals, not reach.** The writer refuses features, + fields and geometries it does not keep (`writable`), every superblock copy, + member and btree pointer is bounded by the device it is read from + (`read_copy`, `read.rs`), a file extent's checksum is verified before its + bytes are used, a symlink target past 4 KiB is refused before it is + allocated (`readlink`), and every block request is bounded to the + partition by the block service. + +**Authentication is deferred to the metal track** by the same ruling: +read-write only for something the attacker cannot compute — a keyed MAC or a +designation-like stamp — and read-only for everything else. It needs a place +to keep a secret, and ToyOS has neither a key store nor TPM support. The +threat this leaves open is stated rather than reduced: a deliberately +crafted volume of the partition's own size still earns a read-write mount, +and read-write means written on sight. + +**Separately: a designation stamp over a used volume does not reformat.** The +stamp is block 0 (`lib/bcachefs/src/designation.rs`), and it zeroes only the +superblock's layout copy at sector 7; the primary copy at sector 8, block 1, +still reads, so the probe mounts the old volume and never looks at the stamp. +A volume whose superblocks all fail, stamped, is refused "an existing volume +cannot be reformatted" while any copy's magic remains (`carries_volume`). +Harmless for the harness, which stamps fresh images, but "re-stamp the disk to +reformat `/home`" is not a workflow that works; `disks format data --commit` +in the preparation image is the way to reformat. + +**Exit**: read-write mount only of a volume this machine can prove it wrote, +by the metal track's secret, with everything else read-only; shown by a test +in which a crafted volume of the partition's size mounts read-only. diff --git a/issues/smaller-data-rows-trail-main-by-up-to-half.md b/issues/smaller-data-rows-trail-main-by-up-to-half.md new file mode 100644 index 00000000000..563d0e18acd --- /dev/null +++ b/issues/smaller-data-rows-trail-main-by-up-to-half.md @@ -0,0 +1,45 @@ +--- +status: open +kind: defect +opened: 2026-11-05 +--- + +# Smaller DATA rows trail main by up to half + +**Owner.** The storage track, which owns `lib/bcachefs` and the file +server's DATA volume. + +osbench at #1152's a1f07627e, one boot a bench, p50, against main +dc1a8642e on its private format (nightly 2026-10-31-2316); T14 / AMD: + +| row | #1152 | main | ratio | +|---|---|---|---| +| `fs.disk.tree.100` | 21.39 / 17.94 ms | 13.96 / 12.73 ms | 1.53 / 1.41 | +| `fs.disk.fstat` | 10.86 / 12.08 us | 8.72 / 9.92 us | 1.25 / 1.22 | +| `fs.disk.direct.fstat` | 10.64 / 11.86 us | 8.45 / 9.75 us | 1.26 / 1.22 | +| `fs.disk.stat` | 13.58 / 14.93 us | 11.63 / 12.54 us | 1.17 / 1.19 | +| `fs.disk.direct.stat` | 12.49 / 13.88 us | 10.66 / 11.54 us | 1.17 / 1.20 | +| `fs.disk.seq_write.4096` | 162 / 164 MB/s | 133 / 211 MB/s | 1.22 / 0.78 | + +The larger gaps have records of their own +(`issues/a-4-kib-overwrite-on-data-costs-about-2-4-times-mains.md`, +`issues/a-data-directory-listing-reads-every-entrys-inode-alone.md`). + +**What is known of the causes.** +- `tree.100` creates, writes and removes 100 small files: each create, + write and unlink is its own journaled transaction of several btrees + (inode, dirent, extent, backpointer, accounting). Which of them makes + the difference is not measured. +- `stat` resolves its path from the server's name cache and reads one + inode (round 5 of #1152: 5.2 us of server time on QEMU/KVM). `fstat` + reads no path. Where their remaining 1.7 to 2.3 us goes is not + established; it was not measured that it is the engine. +- `seq_write.4096` is above main on the T14 and below it on the AMD, after + #1152's write that continues the last extends its reservation. The + AMD's remaining cost is not established. + +Evidence: #1152's evidence, `storage-integration/r10-known-gaps` +(`osbench-a1f07627e/` and `osbench-table-a1f07627e.md`). + +**Exit.** Each row within 1.1 times main's on both benches, or its cause +shown to be inherent. diff --git a/issues/storage-is-layers-and-a-role-is-a-filesystem.md b/issues/storage-is-layers-and-a-role-is-a-filesystem.md index 077dce9a3dd..1e090fa3ecc 100644 --- a/issues/storage-is-layers-and-a-role-is-a-filesystem.md +++ b/issues/storage-is-layers-and-a-role-is-a-filesystem.md @@ -19,12 +19,13 @@ mount protocol — is superseded, and nothing here builds it. ## Block -A block device is a service: one program per controller (`blockd` for NVMe; -the xHCI's own program for USB mass storage, once it has one), serving each -partition as a session over `toyos-blockring`. A partition has one holder, -every request is bounded to it before the device sees it, and a flush answers -for its own writer's writes. GPT is the one partition scheme, read by the -service that drives the disk. Caching is the file server's, never the block +A block device is a service: one driver per controller (`nvme`, and `usbd` +for USB mass storage) offers its disks to the block service (`diskserver`), +which opens a partition for the client its grant admits and hands the session, +over `toyos-blockring`, to the driver that serves it. A partition has one +holder, every request is bounded to it before the device sees it, and a flush +answers for its own writer's writes. GPT is the one partition scheme, read by +the driver of the disk. Caching is the file server's, never the block service's. ## Volume @@ -38,21 +39,21 @@ mounts degraded, or refuses — a member vanishing is never silent. A role on on disk is the one-member case. Two filesystems claiming one unique role are refused rather than guessed, which `boot_partition_identity` already asserts for the boot volume. A blank volume joins the system by the designation stamp -the bcachefs crate already carries (`DESIGNATION_MAGIC`, -`lib/bcachefs/src/superblock.rs`): the probe formats a designated volume and -refuses to reformat a used one (`issues/probe-mounts-on-a-checksum.md` +the bcachefs crate already carries (`lib/bcachefs/src/designation.rs`): the +probe formats a designated volume and refuses to reformat a used one +(`issues/probe-mounts-on-a-checksum.md` is what that refusal still owes). Microsoft's basic-data type is a foreign volume, handed to whichever server recognises its superblock. | partition | filesystem | mounts as | writable | |---|---|---|---| -| ESP | FAT32, firmware's rule | `/boot` | kernel only | +| ESP | FAT32, firmware's rule | `/boot` | no; BOOT's server serves it read-only | | ROOT | bcachefs image the build writes | `/system` | no; signed per release | | DATA | bcachefs, formatted on first boot | `/apps`, `/config`, `/home`, `/state` | yes | | LOG | FAT32 while a Mac has to read the dev stick | `/log` | yes | -`/tmp` has no backing. The dev loop keeps the ESP and ROOT on the stick and -DATA on the internal NVMe; the installed product carries all four on one disk. +`/tmp` is the MEMORY server's, in RAM, with no partition. The dev loop keeps +the ESP and ROOT on the stick and DATA on the internal NVMe; the installed product carries all four on one disk. Roles come from partitions, so both shapes are the same code. ## Filesystem @@ -67,16 +68,17 @@ and the updater write it offline, checkpoint it and mark the journal clean; a server that finds a dirty journal under `/system` refuses by name, because replay is a write-path job and a system image needing one is a broken install. -**Every filesystem is a file server per role** — SYSTEM, LOG, DATA, BOOT — -SYSTEM over ROOT's bytes in memory and every other over its block service's -sessions, linking the whole crate it serves: a server that crashes loses its -own role and nothing else, and no filesystem parser runs in the kernel. +**Every filesystem is a file server per role** — SYSTEM, LOG, DATA, BOOT, +MEMORY — SYSTEM over ROOT's bytes in memory, MEMORY over paged objects alone +and every other over its block service's sessions, linking the whole crate it +serves: a server that crashes loses its own role and nothing else, and no +filesystem parser runs in the kernel. **Multi-device is a filesystem property, not a block-layer subsystem.** Pooling across disks, replicas per subvolume, tiering with a fast foreground device, erasure coding, snapshots, checksums and scrub, compression, encryption, subvolume per user and per app: every one is a feature of upstream bcachefs's -format, which is why `issues/bcachefs-crate-is-not-bcachefs.md`'s +format, which is why `issues/the-bcachefs-crate-must-implement-bcachefs.md`'s ruling — the crate becomes a real implementation — is what makes them reachable. The block layer stays dumb. LVM- and md-shaped aggregation is refused: it cannot tell metadata from data, place a file's replicas on @@ -95,12 +97,10 @@ NTFS by the same shape if wanted. ## Paths The tree, its names and how a program finds them are -`issues/where-everything-lives.md`. What this track owes it is the -mount structure: a mount point is exactly one top-level name -(`kernel/src/vfs.rs`, `ROOT_ENTRIES` and the array indexed by it), so +`issues/where-everything-lives.md`. A client routes a path to the directory +capability with the longest matching prefix (`toyos::files`), so `/media/