diff --git a/library/std/src/os/unix/fs.rs b/library/std/src/os/unix/fs.rs index 7a9524054ef7b..1764ad83c617c 100644 --- a/library/std/src/os/unix/fs.rs +++ b/library/std/src/os/unix/fs.rs @@ -250,9 +250,10 @@ pub trait FileExt { /// /// # Bug /// On some systems, `write_at` utilises [`pwrite64`] to write to files. - /// However, this syscall has a [bug] where files opened with the `O_APPEND` + /// However, on linux this syscall has a [bug] where files opened with the `O_APPEND` /// flag fail to respect the offset parameter, always appending to the end - /// of the file instead. + /// of the file instead. When available `pwritev2(..., RWF_NOAPPEND)` will + /// be used instead to avoid this bug. /// /// It is possible to inadvertently set this flag, like in the example below. /// Therefore, it is important to be vigilant while changing options to mitigate diff --git a/library/std/src/os/unix/fs/tests.rs b/library/std/src/os/unix/fs/tests.rs index 52e0fe940a81f..8c6fcd0bb05db 100644 --- a/library/std/src/os/unix/fs/tests.rs +++ b/library/std/src/os/unix/fs/tests.rs @@ -1,4 +1,9 @@ +use core::assert_matches; + use super::*; +use crate::byte_str::ByteStr; +use crate::io::Seek; +use crate::test_helpers::tmpdir; #[test] fn read_vectored_at() { @@ -33,19 +38,25 @@ fn write_vectored_at() { let msg = b"pwritev is not working!"; let dir = crate::test_helpers::tmpdir(); + let use_append = write_at_append_should_work(); + let filename = dir.join("preadv.txt"); { let mut file = fs::File::create(&filename).unwrap(); file.write_all(msg).unwrap(); } let expected = { - let file = fs::File::options().write(true).open(&filename).unwrap(); + // Open in append mode to test that positioned writes bypass O_APPEND. + let mut file = + fs::File::options().write(!use_append).append(use_append).open(&filename).unwrap(); let buf0 = b" "; let buf1 = b"great "; let iovec = [io::IoSlice::new(buf0), io::IoSlice::new(buf1)]; + assert_eq!(file.stream_position().unwrap(), 0); let n = file.write_vectored_at(&iovec, 11).unwrap(); + assert_eq!(file.stream_position().unwrap(), 0); assert!(n == 4 || n == 11); @@ -53,7 +64,26 @@ fn write_vectored_at() { }; let content = fs::read(&filename).unwrap(); - assert_eq!(&content, expected); + assert_eq!(ByteStr::new(&content), ByteStr::new(expected)); +} + +#[test] +fn file_test_append_write_at() { + use crate::os::unix::fs::FileExt; + + let use_append = write_at_append_should_work(); + + let tmpdir = tmpdir(); + let filename = tmpdir.join("file_test_append_write_at.txt"); + let msg = b"it's not working!"; + assert_matches!(fs::write(&filename, &msg), Ok(())); + let mut f = fs::File::options().write(!use_append).append(use_append).open(&filename).unwrap(); + assert_matches!(f.stream_position(), Ok(0)); + assert_matches!(f.write_at(b" ", 5), Ok(3)); + assert_matches!(f.stream_position(), Ok(0)); + + let content = fs::read(&filename).unwrap(); + assert_eq!(ByteStr::new(&content), ByteStr::new(b"it's working!")); } #[test] @@ -102,3 +132,39 @@ fn test_mkfifo() { assert_eq!(s, "hello, world!"); } + +/// Returns true if write_at is expected to work on O_APPEND files. +fn write_at_append_should_work() -> bool { + #[cfg(target_os = "linux")] + unsafe { + use crate::ffi::CStr; + use crate::mem; + + // SAFETY: the struct is all chars, so zeroing is fine. + let mut buf: libc::utsname = mem::zeroed(); + if libc::uname(&mut buf) != 0 { + panic!("uname failed"); + } + + // SAFETY: uname promises to return null-terminated strings + let release = CStr::from_ptr(buf.release.as_ptr()).to_str().unwrap_or(""); + + // RWF_NOAPPEND has been observed to not work properly on GHA runners + // at least under ubuntu 24.04 and 26.04. + // Perhaps some virtualization bug. + if release.contains("azure") { + return false; + } + + let mut parts = release.split("."); + let major = parts.next().and_then(|s| s.parse::().ok()).unwrap_or(0); + let minor = parts.next().and_then(|s| s.parse::().ok()).unwrap_or(0); + + // RWF_NOAPPEND was introduced in 6.9 + if (major, minor) >= (6, 9) { + return true; + } + } + + true +} diff --git a/library/std/src/sys/fd/unix.rs b/library/std/src/sys/fd/unix.rs index c46ca0cd4d591..2cdd4402bd3cc 100644 --- a/library/std/src/sys/fd/unix.rs +++ b/library/std/src/sys/fd/unix.rs @@ -44,10 +44,10 @@ cfg_select! { use crate::cmp; use crate::io::{self, BorrowedCursor, IoSlice, IoSliceMut, Read}; use crate::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; -#[cfg(all(target_os = "android", target_pointer_width = "64"))] +#[cfg(all(any(target_os = "android", target_os = "linux"), target_pointer_width = "64"))] use crate::sys::pal::weak::syscall; #[cfg(any( - all(target_os = "android", target_pointer_width = "32"), + all(target_os = "android", target_os = "linux", target_pointer_width = "32"), all(target_vendor = "apple", not(all(target_os = "macos", target_arch = "aarch64"))) ))] use crate::sys::pal::weak::weak; @@ -412,6 +412,16 @@ impl FileDesc { } pub fn write_at(&self, buf: &[u8], offset: u64) -> io::Result { + // Work around linux deviating from POSIX where it ignores the + // offset of pwrite when the file was opened with O_APPEND. + #[cfg(any(target_os = "linux", target_os = "android"))] + { + let iov = [IoSlice::new(buf)]; + if let Some(ret) = self.pwritev2(&iov, offset) { + return ret; + } + } + unsafe { cvt(pwrite64( self.as_raw_fd(), @@ -437,6 +447,13 @@ impl FileDesc { all(target_os = "macos", target_arch = "aarch64"), ))] pub fn write_vectored_at(&self, bufs: &[IoSlice<'_>], offset: u64) -> io::Result { + // Work around linux deviating from POSIX where it ignores the + // offset of pwrite when the file was opened with O_APPEND. + #[cfg(any(target_os = "linux", target_os = "android"))] + if let Some(ret) = self.pwritev2(bufs, offset) { + return ret; + } + let ret = cvt(unsafe { libc::pwritev( self.as_raw_fd(), @@ -645,6 +662,70 @@ impl FileDesc { pub fn duplicate(&self) -> io::Result { Ok(Self(self.0.try_clone()?)) } + + #[cfg(any(target_os = "linux", target_os = "android"))] + fn pwritev2(&self, bufs: &[IoSlice<'_>], offset: u64) -> Option> { + // FIXME: remove once https://github.com/rust-lang/miri/pull/5602 is synced + if cfg!(miri) { + return None; + } + + #[cfg(target_pointer_width = "64")] + syscall!( + fn pwritev2( + fd: libc::c_int, + iovec: *const libc::iovec, + n_iovec: libc::c_int, + offset: off64_t, + flags: libc::c_int, + ) -> isize; + ); + #[cfg(target_pointer_width = "32")] + let pwritev2 = { + use crate::sys::weak::weak; + weak!( + fn pwritev64v2( + fd: libc::c_int, + iovec: *const libc::iovec, + n_iovec: libc::c_int, + offset: off64_t, + flags: libc::c_int, + ) -> isize; + ); + let Some(pwritev2) = pwritev64v2.get() else { + return None; + }; + pwritev2 + }; + + use core::sync::atomic::AtomicBool; + + static NOAPPEND_SUPPORTED: AtomicBool = AtomicBool::new(true); + if NOAPPEND_SUPPORTED.load(core::sync::atomic::Ordering::Relaxed) { + let r = unsafe { + cvt(pwritev2( + self.as_raw_fd(), + bufs.as_ptr() as *const libc::iovec, + cmp::min(bufs.len(), max_iov()) as libc::c_int, + offset as off64_t, + libc::RWF_NOAPPEND, + )) + }; + match r { + Ok(ret) => return Some(Ok(ret as usize)), + Err(e) + if let Some(err) = e.raw_os_error() + && (err == libc::EOPNOTSUPP || err == libc::ENOSYS) => + { + NOAPPEND_SUPPORTED.store(false, core::sync::atomic::Ordering::Relaxed); + return None; + } + Err(e) => return Some(Err(e)), + } + } + + None + } } impl<'a> Read for &'a FileDesc {