diff --git a/issues/a-data-directory-listing-is-read-whole-for-every-readdir-request.md b/issues/a-data-directory-listing-is-read-whole-for-every-readdir-request.md new file mode 100644 index 00000000000..2c25afe62ac --- /dev/null +++ b/issues/a-data-directory-listing-is-read-whole-for-every-readdir-request.md @@ -0,0 +1,34 @@ +--- +status: open +kind: defect +opened: 2026-11-11 +--- + +# A DATA directory listing is read whole for every READDIR request + +**Owner.** The storage track, which owns the file server's DATA volume and +`lib/bcachefs`. + +osbench's `fs.disk.readdir.1000`, listing a directory of 1 000 files, at +#1152's a1f07627e: 2.86 ms on the T14 and 1.99 ms on the AMD, p50, one boot +each; main dc1a8642e 0.91 and 0.69 ms (nightly 2026-10-31-2316), about 3x. + +**Cause.** Every READDIR request lists the whole directory again before +taking its batch from the cursor (`system/fileserver/src/main.rs`). A client +asks in 64 KiB batches, so 1 000 short names are one or two requests. On the +host model of the T14's DATA (CPU only), one whole listing of 1 000 entries +cost 534 us at a1f07627e: the dirents 39 us, an inode lookup from the root +for each entry 416 us, sorting and encoding the rest (#1152's evidence, +`storage-integration/r10-known-gaps`, `perf.rs`'s `listing_a_thousand`). + +The inode lookups are gone: a listing reads its inodes in one pass over each +run of them (`Mount::listing`), held by `lib/bcachefs`'s +`a_listing_reads_its_inodes_in_a_pass_over_each_run_of_them`. On the T14, +four boots a side (#1236's evidence), that took `readdir.1000` from 2.87 to +2.91 ms to 2.38 to 2.40 ms, 0.83 times: far less than the host model's +416 us of 534 promised. Where the remaining 2.4 ms goes is not measured; the +AMD bench is not measured. + +**Exit.** `readdir.1000` within 1.5 times main's on both benches; if the +single pass does not reach it, the listing kept across a directory's READDIR +batches. diff --git a/issues/a-data-directory-listing-reads-every-entrys-inode-alone.md b/issues/a-data-directory-listing-reads-every-entrys-inode-alone.md deleted file mode 100644 index 867cc65a1dc..00000000000 --- a/issues/a-data-directory-listing-reads-every-entrys-inode-alone.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -status: open -kind: defect -opened: 2026-11-05 ---- - -# A DATA directory listing reads every entry's inode alone - -**Owner.** The storage track, which owns the file server's DATA volume and -`lib/bcachefs`. - -osbench's `fs.disk.readdir.1000`, listing a directory of 1 000 files, at -#1152's a1f07627e: 2.86 ms on the T14 and 1.99 ms on the AMD, p50, one boot -each; main dc1a8642e 0.91 and 0.69 ms (nightly 2026-10-31-2316), about 3x. - -**Cause.** A listing entry carries its file's size, so `DataVolume::list` -(`system/fileserver/src/data.rs`) reads each entry's inode with a lookup of -its own, from the root of the inodes btree, after one readdir of the -dirents; and every READDIR request lists the whole directory again before -taking its batch from the cursor. On the host model of the T14's DATA (CPU -only), one whole listing of 1 000 entries costs 534 us: the dirents 39 us, -the 1 000 inode lookups 416 us, sorting and encoding the rest (#1152's -evidence, `storage-integration/r10-known-gaps`, `perf.rs`'s -`listing_a_thousand`). A client asks in 64 KiB batches, so 1 000 short -names are one or two requests. - -**Exit.** The inodes of a listing read in one ordered pass over the inodes -btree, or the listing kept across a directory's READDIR batches, with -`readdir.1000` within 1.5 times main's on both benches. diff --git a/issues/smaller-data-rows-trail-main-by-up-to-half.md b/issues/smaller-data-rows-trail-main-by-up-to-half.md index 563d0e18acd..4dc6813bc6c 100644 --- a/issues/smaller-data-rows-trail-main-by-up-to-half.md +++ b/issues/smaller-data-rows-trail-main-by-up-to-half.md @@ -23,7 +23,7 @@ dc1a8642e on its private format (nightly 2026-10-31-2316); T14 / AMD: The larger gaps have records of their own (`issues/a-4-kib-overwrite-on-data-costs-about-2-4-times-mains.md`, -`issues/a-data-directory-listing-reads-every-entrys-inode-alone.md`). +`issues/a-data-directory-listing-is-read-whole-for-every-readdir-request.md`). **What is known of the causes.** - `tree.100` creates, writes and removes 100 small files: each create, diff --git a/lib/bcachefs/src/engine/mount/ops.rs b/lib/bcachefs/src/engine/mount/ops.rs index bddf763de61..55e6f10837c 100644 --- a/lib/bcachefs/src/engine/mount/ops.rs +++ b/lib/bcachefs/src/engine/mount/ops.rs @@ -151,6 +151,11 @@ impl Mount { Source::readdir(self, dir, cursor, max).await } + /// Every entry of `dir` with its inode, by inode number. + pub async fn listing(&self, dir: u64) -> Result> { + Source::listing(self, dir).await + } + /// A new file or directory `name` in directory `dir`, owned by `owner`, /// its times `now`, in nanoseconds since the Unix epoch. Its number is /// the inode cursor's next free one (A8.7), its name in the first free diff --git a/lib/bcachefs/src/engine/read.rs b/lib/bcachefs/src/engine/read.rs index 8344ec989cf..668d5101353 100644 --- a/lib/bcachefs/src/engine/read.rs +++ b/lib/bcachefs/src/engine/read.rs @@ -84,6 +84,12 @@ pub struct DirEntry { pub next: u64, } +/// How far apart two inode numbers may be for the second to be read by +/// stepping over the keys between them: far fewer keys than a leaf holds, +/// which should cost less than finding the second from the root. Reasoned, +/// not measured. +const NEAR: u64 = 32; + /// Entries [`Volume::names`] asks of one directory at a time. const LISTING: usize = 256; @@ -279,12 +285,57 @@ pub(crate) trait Source { /// The inode numbered `ino`. async fn inode(&self, ino: u64) -> Result { let (key, value) = self.get(Btree::Inodes, Bpos::new(0, ino, ROOT_SNAPSHOT)).await?.ok_or(Error::NotFound)?; + self.decode_inode(ino, &key, &value) + } + + /// The inodes numbered `inos`, in that order, each [`Source::inode`]'s + /// answer. Ascending numbers within [`NEAR`] of each other are read in + /// one pass over their leaves, not each from the root. + async fn inodes(&self, inos: &[u64]) -> Result> { + let mut found: Vec = Vec::with_capacity(inos.len()); + while let Some(&first) = inos.get(found.len()) { + let before = found.len(); + self.walk(Btree::Inodes, Bpos::new(0, first, ROOT_SNAPSHOT), |key, value| { + let Some(&ino) = inos.get(found.len()) else { return Ok(false) }; + let wanted = Bpos::new(0, ino, ROOT_SNAPSHOT); + if key.pos < wanted { + return Ok(true); + } + if key.pos > wanted { + return Err(Error::NotFound); + } + let inode = self.decode_inode(ino, key, value)?; + // Two names of one inode ask for it twice. + while inos.get(found.len()) == Some(&ino) { + found.push(inode.clone()); + } + Ok(inos.get(found.len()).is_some_and(|&next| next.checked_sub(ino).is_some_and(|gap| gap <= NEAR))) + }) + .await?; + if found.len() == before { + return Err(Error::NotFound); + } + } + Ok(found) + } + + /// Every entry of `dir` with its inode, by inode number: the names in + /// one pass over the dirents, then their inodes by [`Source::inodes`]. + async fn listing(&self, dir: u64) -> Result> { + let (mut entries, _) = self.readdir(dir, 0, usize::MAX).await?; + entries.sort_unstable_by_key(|e| e.ino); + let inos: Vec = entries.iter().map(|e| e.ino).collect(); + Ok(entries.into_iter().zip(self.inodes(&inos).await?).collect()) + } + + /// The inode numbered `ino`, from its key in the inodes btree. + fn decode_inode(&self, ino: u64, key: &Key, value: &[u8]) -> Result { match key.kind { kind::INODE_V3 => {} kind::INODE | kind::INODE_V2 => return Err(Error::Unsupported("inode in a format before inode_v3")), _ => return Err(Error::NotFound), } - let v = InodeValue::decode(&value)?; + let v = InodeValue::decode(value)?; let mode = v.mode(); let stored = v.field(inode_field::NLINK) as u32; let nlink = if v.flags & inode_flag::UNLINKED != 0 { diff --git a/lib/bcachefs/src/engine/tests/ops.rs b/lib/bcachefs/src/engine/tests/ops.rs index 5a4c8cf8b12..c21aefa31a2 100644 --- a/lib/bcachefs/src/engine/tests/ops.rs +++ b/lib/bcachefs/src/engine/tests/ops.rs @@ -296,3 +296,102 @@ fn a_rename_across_many_leaves_dirties_no_more_nodes_than_its_grant() { let v = checked(&dev); assert!(names_of(&v, b).contains(&name) && !names_of(&v, a).contains(&name)); } + +/// A source that counts the walks asked of it, each of which starts from a +/// btree's root. +struct Walks<'a, S: read::Source> { + of: &'a S, + walks: std::cell::Cell, +} + +impl read::Source for Walks<'_, S> { + type Dev = S::Dev; + + fn dev(&self) -> &S::Dev { + self.of.dev() + } + fn superblock(&self) -> &superblock::Superblock { + self.of.superblock() + } + fn master_key(&self) -> Option<&crypt::MasterKey> { + self.of.master_key() + } + fn root_dir(&self) -> u64 { + self.of.root_dir() + } + fn root_of(&self, btree: Btree) -> Option { + self.of.root_of(btree) + } + async fn walk(&self, btree: Btree, from: Bpos, visit: impl FnMut(&Key, &[u8]) -> Result) -> Result<()> { + self.walks.set(self.walks.get() + 1); + self.of.walk(btree, from, visit).await + } +} + +/// A thousand files made one after another, a second name for one of them, +/// and a file made after forty elsewhere: the directory's inode, its names, +/// and the inodes in two passes, where an inode read alone for each name is +/// a thousand and two. +#[test] +fn a_listing_reads_its_inodes_in_a_pass_over_each_run_of_them() { + use read::Source; + let dev = formatted(64 << 20, None); + mounted(&dev, async |m: &Mount<&Image>| { + let d = m.create(m.root(), b"d", New::Dir, OWNER, NOW).await.unwrap(); + let mut first = 0; + for i in 0..1000 { + let f = m.create(d.ino, format!("f{i}").as_bytes(), New::File, OWNER, NOW).await.unwrap(); + if i == 0 { + first = f.ino; + } + } + m.link(first, d.ino, b"again", Link::New, NOW).await.unwrap(); + for i in 0..40 { + m.create(m.root(), format!("elsewhere{i}").as_bytes(), New::File, OWNER, NOW).await.unwrap(); + } + m.create(d.ino, b"late", New::File, OWNER, NOW).await.unwrap(); + + let counted = Walks { of: m, walks: std::cell::Cell::new(0) }; + let listed = counted.listing(d.ino).await.unwrap(); + assert_eq!(counted.walks.get(), 4); + assert_eq!(listed.len(), 1002); + let (mut entries, more) = m.readdir(d.ino, 0, usize::MAX).await.unwrap(); + assert!(!more); + entries.sort_unstable_by_key(|e| e.ino); + for ((entry, inode), want) in listed.iter().zip(&entries) { + assert_eq!(entry.ino, want.ino); + assert_eq!(*inode, m.inode(entry.ino).await.unwrap(), "{:?}", String::from_utf8_lossy(&entry.name)); + } + let mut names: Vec<&[u8]> = listed.iter().map(|(e, _)| &e.name[..]).collect(); + names.sort_unstable(); + let mut want: Vec<&[u8]> = entries.iter().map(|e| &e.name[..]).collect(); + want.sort_unstable(); + assert_eq!(names, want); + }); +} + +/// Inodes asked for in any order, one twice, are each the answer of asking +/// for it alone; a number no inode has refuses them all, wherever it falls. +#[test] +fn inodes_read_together_are_those_read_alone() { + use read::Source; + let dev = formatted(64 << 20, None); + mounted(&dev, async |m: &Mount<&Image>| { + let mut made = Vec::new(); + for i in 0..8 { + made.push(m.create(m.root(), format!("f{i}").as_bytes(), New::File, OWNER, NOW).await.unwrap().ino); + } + m.unlink(m.root(), b"f3", false, NOW).await.unwrap(); + m.release(made[3]).await.unwrap(); + let asked = [made[5], made[0], made[0], made[7], made[1]]; + let got = m.inodes(&asked).await.unwrap(); + assert_eq!(got.len(), asked.len()); + for (inode, ino) in got.iter().zip(asked) { + assert_eq!(*inode, m.inode(ino).await.unwrap()); + } + assert_eq!(m.inodes(&[]).await, Ok(Vec::new())); + for absent in [[made[2], made[3], made[4]], [made[3], made[4], made[5]], [made[6], made[7], u64::MAX]] { + assert_eq!(m.inodes(&absent).await, Err(Error::NotFound), "{absent:?}"); + } + }); +} diff --git a/system/fileserver/src/data.rs b/system/fileserver/src/data.rs index ef20eb3545e..1ed8cb9a1f4 100644 --- a/system/fileserver/src/data.rs +++ b/system/fileserver/src/data.rs @@ -372,25 +372,14 @@ impl Volume for DataVolume { if !inode.is_dir() { return Err(Error::NotADirectory); } - let mut cursor = 0; - let mut listed = Vec::new(); - loop { - let (entries, more) = mapped("readdir", self.cache.block_on(self.fs.readdir(inode.ino, cursor, 256)))?; - if entries.is_empty() && more { - return Err(Error::Io); - } - for entry in entries { - cursor = entry.next; - let inode = mapped("directory entry", self.cache.block_on(self.fs.inode(entry.ino)))?; - listed.push(Listed { - name: String::from_utf8(entry.name).map_err(|_| Error::Io)?, - meta: self.meta(&inode)?, - identity: entry.ino, - }); - } - if !more { - break; - } + let entries = mapped("readdir", self.cache.block_on(self.fs.listing(inode.ino)))?; + let mut listed = Vec::with_capacity(entries.len()); + for (entry, inode) in entries { + listed.push(Listed { + name: String::from_utf8(entry.name).map_err(|_| Error::Io)?, + meta: self.meta(&inode)?, + identity: entry.ino, + }); } listed.sort_unstable_by(|a, b| a.name.cmp(&b.name)); Ok(listed) @@ -887,6 +876,27 @@ mod tests { let mut v = reopen(v); assert_eq!(v.read_link("home/moved/link").unwrap(), "file"); } + /// Each entry of a listing is what its own path answers. + #[test] + fn a_listing_says_of_each_entry_what_its_path_does() { + let mut v = volume(); + v.mkdir("home/dir").unwrap(); + for (name, len) in [("b", 3), ("a", 5000), ("c", 0)] { + let f = v.open(&format!("home/{name}"), CREATE).unwrap(); + v.write(f, 0, &vec![7; len]).unwrap(); + v.close(f).unwrap(); + } + v.symlink("home/link", "a").unwrap(); + v.sync().unwrap(); + let mut v = reopen(v); + let listed = v.list("home").unwrap(); + assert_eq!(listed.iter().map(|e| e.name.as_str()).collect::>(), ["a", "b", "c", "dir", "link"]); + for entry in &listed { + let path = format!("home/{}", entry.name); + assert_eq!(entry.meta, v.lstat(&path).unwrap(), "{path}"); + assert_eq!(entry.identity, v.identity(&path).unwrap(), "{path}"); + } + } #[test] fn cancelling_one_reservation_keeps_an_earlier_dirty_page() { let mut v = volume();