diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ed3c9d2..eb6bd17 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -37,6 +37,12 @@ jobs: - name: Run tests (Linux 5.7 support) run: cargo test --verbose --features linux5_7 + # The linux6_7 feature implies every lower level, so this single run covers them all. It + # needs kernel headers and a kernel of at least 6.7, which only ubuntu-latest provides. + - name: Run tests (Linux 6.7 support) + if: ${{ matrix.runner == 'ubuntu-latest' }} + run: cargo test --verbose --features linux6_7 + audit: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f4b23f..ccb1eee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +### Unreleased + +- Add support for `UFFD_FEATURE_WP_UNPOPULATED` under the new `linux6_4` feature. +- Add support for `UFFD_FEATURE_WP_ASYNC` under the new `linux6_7` feature. +- Add `Uffd::copy_with_mode` and `CopyMode`, exposing `UFFDIO_COPY_MODE_WP` so that copied pages + can be mapped write-protected. `Uffd::copy` is unchanged. +- The `linux6_4` and `linux6_7` features enable every lower feature level, so a single feature + selects everything up to that kernel version. Existing feature levels are unchanged. + ### 0.9.0 - Add support for `UFFDIO_CONTINUE` and `UFFDIO_REGISTER_MODE_MINOR` under the new `linux5_13` feature. diff --git a/Cargo.toml b/Cargo.toml index 9e18a46..3a9cb5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,3 +24,5 @@ default = [] linux4_14 = ["userfaultfd-sys/linux4_14", "nix/process"] linux5_7 = ["userfaultfd-sys/linux5_7"] linux5_13 = ["userfaultfd-sys/linux5_13"] +linux6_4 = ["linux4_14", "linux5_7", "linux5_13", "userfaultfd-sys/linux6_4"] +linux6_7 = ["linux6_4", "userfaultfd-sys/linux6_7"] diff --git a/src/builder.rs b/src/builder.rs index 7b2a599..59704f9 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -10,7 +10,42 @@ use std::os::fd::AsRawFd; const UFFD_DEVICE_PATH: &str = "/dev/userfaultfd"; cfg_if::cfg_if! { - if #[cfg(any(feature = "linux5_7", feature = "linux4_14"))] { + if #[cfg(feature = "linux6_7")] { + bitflags! { + /// Used with `UffdBuilder` to determine which features are available in the current kernel. + #[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + pub struct FeatureFlags: u64 { + const PAGEFAULT_FLAG_WP = raw::UFFD_FEATURE_PAGEFAULT_FLAG_WP; + const EVENT_FORK = raw::UFFD_FEATURE_EVENT_FORK; + const EVENT_REMAP = raw::UFFD_FEATURE_EVENT_REMAP; + const EVENT_REMOVE = raw::UFFD_FEATURE_EVENT_REMOVE; + const MISSING_HUGETLBFS = raw::UFFD_FEATURE_MISSING_HUGETLBFS; + const MISSING_SHMEM = raw::UFFD_FEATURE_MISSING_SHMEM; + const EVENT_UNMAP = raw::UFFD_FEATURE_EVENT_UNMAP; + const SIGBUS = raw::UFFD_FEATURE_SIGBUS; + const THREAD_ID = raw::UFFD_FEATURE_THREAD_ID; + const WP_UNPOPULATED = raw::UFFD_FEATURE_WP_UNPOPULATED; + const WP_ASYNC = raw::UFFD_FEATURE_WP_ASYNC; + } + } + } else if #[cfg(feature = "linux6_4")] { + bitflags! { + /// Used with `UffdBuilder` to determine which features are available in the current kernel. + #[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + pub struct FeatureFlags: u64 { + const PAGEFAULT_FLAG_WP = raw::UFFD_FEATURE_PAGEFAULT_FLAG_WP; + const EVENT_FORK = raw::UFFD_FEATURE_EVENT_FORK; + const EVENT_REMAP = raw::UFFD_FEATURE_EVENT_REMAP; + const EVENT_REMOVE = raw::UFFD_FEATURE_EVENT_REMOVE; + const MISSING_HUGETLBFS = raw::UFFD_FEATURE_MISSING_HUGETLBFS; + const MISSING_SHMEM = raw::UFFD_FEATURE_MISSING_SHMEM; + const EVENT_UNMAP = raw::UFFD_FEATURE_EVENT_UNMAP; + const SIGBUS = raw::UFFD_FEATURE_SIGBUS; + const THREAD_ID = raw::UFFD_FEATURE_THREAD_ID; + const WP_UNPOPULATED = raw::UFFD_FEATURE_WP_UNPOPULATED; + } + } + } else if #[cfg(any(feature = "linux5_7", feature = "linux4_14"))] { bitflags! { /// Used with `UffdBuilder` to determine which features are available in the current kernel. #[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] diff --git a/src/lib.rs b/src/lib.rs index 3ce9bb5..6a45882 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -93,6 +93,19 @@ bitflags! { } } +bitflags! { + /// The mode used when copying pages into a registered range with `Uffd::copy_with_mode`. + #[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + pub struct CopyMode: u64 { + /// Do not wake up the thread waiting for page fault resolution on the memory range. + const DONTWAKE = raw::UFFDIO_COPY_MODE_DONTWAKE; + /// Map the copied pages write-protected, so that the next write to them raises a + /// write-protect fault. The range must be registered with `RegisterMode::WRITE_PROTECT`. + #[cfg(feature = "linux5_7")] + const WRITE_PROTECT = raw::UFFDIO_COPY_MODE_WP; + } +} + impl Uffd { /// Register a memory address range with the userfaultfd object, and returns the `IoctlFlags` /// that are available for the selected range. @@ -147,16 +160,37 @@ impl Uffd { dst: *mut c_void, len: usize, wake: bool, + ) -> Result { + let mode = if wake { + CopyMode::empty() + } else { + CopyMode::DONTWAKE + }; + self.copy_with_mode(src, dst, len, mode) + } + + /// Atomically copy a continuous memory chunk into the userfaultfd-registered range using the + /// given `mode`, and return the number of bytes that were successfully copied. + /// + /// Unless `CopyMode::DONTWAKE` is set, wake up the thread waiting for page fault resolution on + /// the memory range. + /// + /// # Safety + /// + /// `src` must be valid for reads of `len` bytes, and `dst` must point to a page-aligned + /// range of `len` bytes that is registered with this userfaultfd object. + pub unsafe fn copy_with_mode( + &self, + src: *const c_void, + dst: *mut c_void, + len: usize, + mode: CopyMode, ) -> Result { let mut copy = raw::uffdio_copy { src: src as u64, dst: dst as u64, len: len as u64, - mode: if wake { - 0 - } else { - raw::UFFDIO_COPY_MODE_DONTWAKE - }, + mode: mode.bits(), copy: 0, }; @@ -692,6 +726,8 @@ mod test { uffd.write_protect(mapping, PAGE_SIZE)?; uffd.wake(mapping, PAGE_SIZE)?; } + #[cfg(feature = "linux5_13")] + FaultKind::Minor => panic!("unexpected minor fault"), }, _ => panic!("unexpected event"), } @@ -708,4 +744,245 @@ mod test { Ok(()) } + + /// Allocate an anonymous page-aligned mapping of `size` bytes. + unsafe fn anon_mapping(size: usize) -> *mut c_void { + let mapping = libc::mmap( + ptr::null_mut(), + size, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANON, + -1, + 0, + ); + assert_ne!(mapping, libc::MAP_FAILED); + mapping + } + + #[test] + fn test_copy_with_mode_dontwake() -> Result<()> { + const PAGE_SIZE: usize = 4096; + const FILL: u8 = 0xAB; + + unsafe { + let uffd = UffdBuilder::new().close_on_exec(true).create()?; + + let mapping = anon_mapping(PAGE_SIZE); + let src = anon_mapping(PAGE_SIZE); + ptr::write_bytes(src as *mut u8, FILL, PAGE_SIZE); + + uffd.register(mapping, PAGE_SIZE)?; + + let ptr = mapping as usize; + let thread = thread::spawn(move || { + let ptr = ptr as *mut u8; + *ptr = 1; + }); + + match uffd.read_event()? { + Some(Event::Pagefault { + kind: FaultKind::Missing, + rw: ReadWrite::Write, + addr, + .. + }) => { + assert_eq!(addr, mapping); + let copied = uffd.copy_with_mode(src, addr, PAGE_SIZE, CopyMode::DONTWAKE)?; + assert_eq!(copied, PAGE_SIZE); + + // The page is populated with the source contents, but the faulting thread has + // not been woken yet, so it has not had a chance to perform its write. + assert_eq!(*(addr as *const u8), FILL); + assert_eq!(*(addr as *const u8).add(PAGE_SIZE - 1), FILL); + + uffd.wake(addr, PAGE_SIZE)?; + } + _ => panic!("unexpected event"), + } + + thread.join().expect("failed to join thread"); + + assert_eq!(*(mapping as *const u8), 1); + assert_eq!(*(mapping as *const u8).add(1), FILL); + + uffd.unregister(mapping, PAGE_SIZE)?; + + assert_eq!(libc::munmap(mapping, PAGE_SIZE), 0); + assert_eq!(libc::munmap(src, PAGE_SIZE), 0); + } + + Ok(()) + } + + #[cfg(feature = "linux5_7")] + #[test] + fn test_copy_with_mode_write_protect() -> Result<()> { + const PAGE_SIZE: usize = 4096; + const FILL: u8 = 0xAB; + + unsafe { + let uffd = UffdBuilder::new() + .require_features(FeatureFlags::PAGEFAULT_FLAG_WP) + .close_on_exec(true) + .create()?; + + let mapping = anon_mapping(PAGE_SIZE); + let src = anon_mapping(PAGE_SIZE); + ptr::write_bytes(src as *mut u8, FILL, PAGE_SIZE); + + assert!(uffd + .register_with_mode( + mapping, + PAGE_SIZE, + RegisterMode::MISSING | RegisterMode::WRITE_PROTECT + )? + .contains(IoctlFlags::WRITE_PROTECT)); + + let ptr = mapping as usize; + let thread = thread::spawn(move || { + let ptr = ptr as *mut u8; + *ptr = 1; + }); + + let mut seen_missing = false; + loop { + match uffd.read_event()? { + Some(Event::Pagefault { + kind, + rw: ReadWrite::Write, + addr, + .. + }) => match kind { + FaultKind::Missing => { + assert_eq!(addr, mapping); + assert!(!seen_missing); + seen_missing = true; + + // Resolve the missing fault with a write-protected copy. Waking the + // thread makes it retry its write, which must now raise a + // write-protect fault instead of succeeding. + let copied = + uffd.copy_with_mode(src, addr, PAGE_SIZE, CopyMode::WRITE_PROTECT)?; + assert_eq!(copied, PAGE_SIZE); + } + FaultKind::WriteProtected => { + assert_eq!(addr, mapping); + assert!(seen_missing); + // The write has not landed yet; the page holds the copied data. + assert_eq!(*(addr as *const u8), FILL); + uffd.remove_write_protection(mapping, PAGE_SIZE, true)?; + break; + } + #[cfg(feature = "linux5_13")] + FaultKind::Minor => panic!("unexpected minor fault"), + }, + _ => panic!("unexpected event"), + } + } + + thread.join().expect("failed to join thread"); + + assert_eq!(*(mapping as *const u8), 1); + assert_eq!(*(mapping as *const u8).add(1), FILL); + + uffd.unregister(mapping, PAGE_SIZE)?; + + assert_eq!(libc::munmap(mapping, PAGE_SIZE), 0); + assert_eq!(libc::munmap(src, PAGE_SIZE), 0); + } + + Ok(()) + } + + #[cfg(feature = "linux6_4")] + #[test] + fn test_write_protect_unpopulated() -> Result<()> { + const PAGE_SIZE: usize = 4096; + + unsafe { + let uffd = UffdBuilder::new() + .require_features(FeatureFlags::PAGEFAULT_FLAG_WP | FeatureFlags::WP_UNPOPULATED) + .close_on_exec(true) + .create()?; + + let mapping = anon_mapping(PAGE_SIZE); + + // Only register for write-protect faults: without WP_UNPOPULATED, write-protecting + // a page that was never touched is a no-op and the write below would simply populate + // the page without any fault being reported. + assert!(uffd + .register_with_mode(mapping, PAGE_SIZE, RegisterMode::WRITE_PROTECT)? + .contains(IoctlFlags::WRITE_PROTECT)); + uffd.write_protect(mapping, PAGE_SIZE)?; + + let ptr = mapping as usize; + let thread = thread::spawn(move || { + let ptr = ptr as *mut u8; + *ptr = 1; + }); + + match uffd.read_event()? { + Some(Event::Pagefault { + kind: FaultKind::WriteProtected, + rw: ReadWrite::Write, + addr, + .. + }) => { + assert_eq!(addr, mapping); + uffd.remove_write_protection(mapping, PAGE_SIZE, true)?; + } + _ => panic!("unexpected event"), + } + + thread.join().expect("failed to join thread"); + + assert_eq!(*(mapping as *const u8), 1); + + uffd.unregister(mapping, PAGE_SIZE)?; + + assert_eq!(libc::munmap(mapping, PAGE_SIZE), 0); + } + + Ok(()) + } + + #[cfg(feature = "linux6_7")] + #[test] + fn test_write_protect_async() -> Result<()> { + const PAGE_SIZE: usize = 4096; + + unsafe { + let uffd = UffdBuilder::new() + .require_features(FeatureFlags::PAGEFAULT_FLAG_WP | FeatureFlags::WP_ASYNC) + .close_on_exec(true) + .non_blocking(true) + .create()?; + + let mapping = anon_mapping(PAGE_SIZE); + + assert!(uffd + .register_with_mode(mapping, PAGE_SIZE, RegisterMode::WRITE_PROTECT)? + .contains(IoctlFlags::WRITE_PROTECT)); + uffd.write_protect(mapping, PAGE_SIZE)?; + + // In async mode the kernel resolves the write-protect fault itself by clearing the + // protection, so the write succeeds from this very thread with no handler involved + // and no event is ever queued. + *(mapping as *mut u8) = 1; + assert_eq!(*(mapping as *const u8), 1); + assert!(uffd.read_event()?.is_none()); + + // Protecting the now-populated page again behaves the same way. + uffd.write_protect(mapping, PAGE_SIZE)?; + *(mapping as *mut u8) = 2; + assert_eq!(*(mapping as *const u8), 2); + assert!(uffd.read_event()?.is_none()); + + uffd.unregister(mapping, PAGE_SIZE)?; + + assert_eq!(libc::munmap(mapping, PAGE_SIZE), 0); + } + + Ok(()) + } } diff --git a/userfaultfd-sys/Cargo.toml b/userfaultfd-sys/Cargo.toml index 89be5cb..2f270e8 100644 --- a/userfaultfd-sys/Cargo.toml +++ b/userfaultfd-sys/Cargo.toml @@ -21,3 +21,5 @@ default = [] linux4_14 = [] linux5_7 = ["linux4_14"] linux5_13 = ["linux5_7"] +linux6_4 = ["linux5_13"] +linux6_7 = ["linux6_4"] diff --git a/userfaultfd-sys/src/lib.rs b/userfaultfd-sys/src/lib.rs index 37d80ed..4250b63 100644 --- a/userfaultfd-sys/src/lib.rs +++ b/userfaultfd-sys/src/lib.rs @@ -18,8 +18,18 @@ mod linux5_7; #[cfg(feature = "linux5_13")] mod linux5_13; +#[cfg(feature = "linux6_4")] +mod linux6_4; + +#[cfg(feature = "linux6_7")] +mod linux6_7; + cfg_if! { - if #[cfg(feature = "linux5_13")] { + if #[cfg(feature = "linux6_7")] { + pub use crate::linux6_7::*; + } else if #[cfg(feature = "linux6_4")] { + pub use crate::linux6_4::*; + } else if #[cfg(feature = "linux5_13")] { pub use crate::linux5_13::*; } else if #[cfg(feature = "linux5_7")] { pub use crate::linux5_7::*; diff --git a/userfaultfd-sys/src/linux6_4.rs b/userfaultfd-sys/src/linux6_4.rs new file mode 100644 index 0000000..22f76ea --- /dev/null +++ b/userfaultfd-sys/src/linux6_4.rs @@ -0,0 +1,12 @@ +use super::*; + +pub use linux5_13::{ + UFFDIO_API, UFFDIO_CONTINUE, UFFDIO_CONTINUE_MODE_DONTWAKE, UFFDIO_COPY, + UFFDIO_COPY_MODE_DONTWAKE, UFFDIO_COPY_MODE_WP, UFFDIO_REGISTER, UFFDIO_REGISTER_MODE_MINOR, + UFFDIO_REGISTER_MODE_MISSING, UFFDIO_REGISTER_MODE_WP, UFFDIO_UNREGISTER, UFFDIO_WAKE, + UFFDIO_WRITEPROTECT, UFFDIO_WRITEPROTECT_MODE_DONTWAKE, UFFDIO_WRITEPROTECT_MODE_WP, + UFFDIO_ZEROPAGE, UFFDIO_ZEROPAGE_MODE_DONTWAKE, UFFD_API, UFFD_API_IOCTLS, + UFFD_API_RANGE_IOCTLS, UFFD_API_RANGE_IOCTLS_BASIC, +}; + +pub const UFFD_API_FEATURES: u64 = linux5_13::UFFD_API_FEATURES | UFFD_FEATURE_WP_UNPOPULATED; diff --git a/userfaultfd-sys/src/linux6_7.rs b/userfaultfd-sys/src/linux6_7.rs new file mode 100644 index 0000000..e0194e8 --- /dev/null +++ b/userfaultfd-sys/src/linux6_7.rs @@ -0,0 +1,5 @@ +use super::*; + +pub use linux6_4::*; + +pub const UFFD_API_FEATURES: u64 = linux6_4::UFFD_API_FEATURES | UFFD_FEATURE_WP_ASYNC;