Add Linux 6.4/6.7 write-protect features and Uffd::copy_with_mode

.github/workflows/rust.yml

@@ -37,6 +37,12 @@ jobs:
     - name: Run tests (Linux 5.7 support)
       run: cargo test --verbose --features linux5_7
 
+    # The linux6_7 feature implies every lower level, so this single run covers them all. It
+    # needs kernel headers and a kernel of at least 6.7, which only ubuntu-latest provides.
+    - name: Run tests (Linux 6.7 support)
+      if: ${{ matrix.runner == 'ubuntu-latest' }}
+      run: cargo test --verbose --features linux6_7
+
   audit:
 
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -1,3 +1,12 @@
+### Unreleased
+
+- Add support for `UFFD_FEATURE_WP_UNPOPULATED` under the new `linux6_4` feature.
+- Add support for `UFFD_FEATURE_WP_ASYNC` under the new `linux6_7` feature.
+- Add `Uffd::copy_with_mode` and `CopyMode`, exposing `UFFDIO_COPY_MODE_WP` so that copied pages
+  can be mapped write-protected. `Uffd::copy` is unchanged.
+- The `linux6_4` and `linux6_7` features enable every lower feature level, so a single feature
+  selects everything up to that kernel version. Existing feature levels are unchanged.
+
 ### 0.9.0
 
 - Add support for `UFFDIO_CONTINUE` and `UFFDIO_REGISTER_MODE_MINOR` under the new `linux5_13` feature.

Cargo.toml

@@ -24,3 +24,5 @@ default = []
 linux4_14 = ["userfaultfd-sys/linux4_14", "nix/process"]
 linux5_7 = ["userfaultfd-sys/linux5_7"]
 linux5_13 = ["userfaultfd-sys/linux5_13"]
+linux6_4 = ["linux4_14", "linux5_7", "linux5_13", "userfaultfd-sys/linux6_4"]
+linux6_7 = ["linux6_4", "userfaultfd-sys/linux6_7"]

src/builder.rs

@@ -10,7 +10,42 @@ use std::os::fd::AsRawFd;
 const UFFD_DEVICE_PATH: &str = "/dev/userfaultfd";
 
 cfg_if::cfg_if! {
-    if #[cfg(any(feature = "linux5_7", feature = "linux4_14"))] {
+    if #[cfg(feature = "linux6_7")] {
+        bitflags! {
+            /// Used with `UffdBuilder` to determine which features are available in the current kernel.
+            #[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
+            pub struct FeatureFlags: u64 {
+                const PAGEFAULT_FLAG_WP = raw::UFFD_FEATURE_PAGEFAULT_FLAG_WP;
+                const EVENT_FORK = raw::UFFD_FEATURE_EVENT_FORK;
+                const EVENT_REMAP = raw::UFFD_FEATURE_EVENT_REMAP;
+                const EVENT_REMOVE = raw::UFFD_FEATURE_EVENT_REMOVE;
+                const MISSING_HUGETLBFS = raw::UFFD_FEATURE_MISSING_HUGETLBFS;
+                const MISSING_SHMEM = raw::UFFD_FEATURE_MISSING_SHMEM;
+                const EVENT_UNMAP = raw::UFFD_FEATURE_EVENT_UNMAP;
+                const SIGBUS = raw::UFFD_FEATURE_SIGBUS;
+                const THREAD_ID = raw::UFFD_FEATURE_THREAD_ID;
+                const WP_UNPOPULATED = raw::UFFD_FEATURE_WP_UNPOPULATED;
+                const WP_ASYNC = raw::UFFD_FEATURE_WP_ASYNC;
+            }
+        }
+    } else if #[cfg(feature = "linux6_4")] {
+        bitflags! {
+            /// Used with `UffdBuilder` to determine which features are available in the current kernel.
+            #[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
+            pub struct FeatureFlags: u64 {
+                const PAGEFAULT_FLAG_WP = raw::UFFD_FEATURE_PAGEFAULT_FLAG_WP;
+                const EVENT_FORK = raw::UFFD_FEATURE_EVENT_FORK;
+                const EVENT_REMAP = raw::UFFD_FEATURE_EVENT_REMAP;
+                const EVENT_REMOVE = raw::UFFD_FEATURE_EVENT_REMOVE;
+                const MISSING_HUGETLBFS = raw::UFFD_FEATURE_MISSING_HUGETLBFS;
+                const MISSING_SHMEM = raw::UFFD_FEATURE_MISSING_SHMEM;
+                const EVENT_UNMAP = raw::UFFD_FEATURE_EVENT_UNMAP;
+                const SIGBUS = raw::UFFD_FEATURE_SIGBUS;
+                const THREAD_ID = raw::UFFD_FEATURE_THREAD_ID;
+                const WP_UNPOPULATED = raw::UFFD_FEATURE_WP_UNPOPULATED;
+            }
+        }
+    } else if #[cfg(any(feature = "linux5_7", feature = "linux4_14"))] {
         bitflags! {
             /// Used with `UffdBuilder` to determine which features are available in the current kernel.
             #[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]

src/lib.rs

@@ -93,6 +93,19 @@ bitflags! {
     }
 }
 
+bitflags! {
+    /// The mode used when copying pages into a registered range with `Uffd::copy_with_mode`.
+    #[derive(Copy, Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
+    pub struct CopyMode: u64 {
+        /// Do not wake up the thread waiting for page fault resolution on the memory range.
+        const DONTWAKE = raw::UFFDIO_COPY_MODE_DONTWAKE;
+        /// Map the copied pages write-protected, so that the next write to them raises a
+        /// write-protect fault. The range must be registered with `RegisterMode::WRITE_PROTECT`.
+        #[cfg(feature = "linux5_7")]
+        const WRITE_PROTECT = raw::UFFDIO_COPY_MODE_WP;
+    }
+}
+
 impl Uffd {
     /// Register a memory address range with the userfaultfd object, and returns the `IoctlFlags`
     /// that are available for the selected range.
@@ -147,16 +160,37 @@ impl Uffd {
         dst: *mut c_void,
         len: usize,
         wake: bool,
+    ) -> Result<usize> {
+        let mode = if wake {
+            CopyMode::empty()
+        } else {
+            CopyMode::DONTWAKE
+        };
+        self.copy_with_mode(src, dst, len, mode)
+    }
+
+    /// Atomically copy a continuous memory chunk into the userfaultfd-registered range using the
+    /// given `mode`, and return the number of bytes that were successfully copied.
+    ///
+    /// Unless `CopyMode::DONTWAKE` is set, wake up the thread waiting for page fault resolution on
+    /// the memory range.
+    ///
+    /// # Safety
+    ///
+    /// `src` must be valid for reads of `len` bytes, and `dst` must point to a page-aligned
+    /// range of `len` bytes that is registered with this userfaultfd object.
+    pub unsafe fn copy_with_mode(
+        &self,
+        src: *const c_void,
+        dst: *mut c_void,
+        len: usize,
+        mode: CopyMode,
     ) -> Result<usize> {
         let mut copy = raw::uffdio_copy {
             src: src as u64,
             dst: dst as u64,
             len: len as u64,
-            mode: if wake {
-                0
-            } else {
-                raw::UFFDIO_COPY_MODE_DONTWAKE
-            },
+            mode: mode.bits(),
             copy: 0,
         };
 
@@ -692,6 +726,8 @@ mod test {
                             uffd.write_protect(mapping, PAGE_SIZE)?;
                             uffd.wake(mapping, PAGE_SIZE)?;
                         }
+                        #[cfg(feature = "linux5_13")]
+                        FaultKind::Minor => panic!("unexpected minor fault"),
                     },
                     _ => panic!("unexpected event"),
                 }
@@ -708,4 +744,245 @@ mod test {
 
         Ok(())
     }
+
+    /// Allocate an anonymous page-aligned mapping of `size` bytes.
+    unsafe fn anon_mapping(size: usize) -> *mut c_void {
+        let mapping = libc::mmap(
+            ptr::null_mut(),
+            size,
+            libc::PROT_READ | libc::PROT_WRITE,
+            libc::MAP_PRIVATE | libc::MAP_ANON,
+            -1,
+            0,
+        );
+        assert_ne!(mapping, libc::MAP_FAILED);
+        mapping
+    }
+
+    #[test]
+    fn test_copy_with_mode_dontwake() -> Result<()> {
+        const PAGE_SIZE: usize = 4096;
+        const FILL: u8 = 0xAB;
+
+        unsafe {
+            let uffd = UffdBuilder::new().close_on_exec(true).create()?;
+
+            let mapping = anon_mapping(PAGE_SIZE);
+            let src = anon_mapping(PAGE_SIZE);
+            ptr::write_bytes(src as *mut u8, FILL, PAGE_SIZE);
+
+            uffd.register(mapping, PAGE_SIZE)?;
+
+            let ptr = mapping as usize;
+            let thread = thread::spawn(move || {
+                let ptr = ptr as *mut u8;
+                *ptr = 1;
+            });
+
+            match uffd.read_event()? {
+                Some(Event::Pagefault {
+                    kind: FaultKind::Missing,
+                    rw: ReadWrite::Write,
+                    addr,
+                    ..
+                }) => {
+                    assert_eq!(addr, mapping);
+                    let copied = uffd.copy_with_mode(src, addr, PAGE_SIZE, CopyMode::DONTWAKE)?;
+                    assert_eq!(copied, PAGE_SIZE);
+
+                    // The page is populated with the source contents, but the faulting thread has
+                    // not been woken yet, so it has not had a chance to perform its write.
+                    assert_eq!(*(addr as *const u8), FILL);
+                    assert_eq!(*(addr as *const u8).add(PAGE_SIZE - 1), FILL);
+
+                    uffd.wake(addr, PAGE_SIZE)?;
+                }
+                _ => panic!("unexpected event"),
+            }
+
+            thread.join().expect("failed to join thread");
+
+            assert_eq!(*(mapping as *const u8), 1);
+            assert_eq!(*(mapping as *const u8).add(1), FILL);
+
+            uffd.unregister(mapping, PAGE_SIZE)?;
+
+            assert_eq!(libc::munmap(mapping, PAGE_SIZE), 0);
+            assert_eq!(libc::munmap(src, PAGE_SIZE), 0);
+        }
+
+        Ok(())
+    }
+
+    #[cfg(feature = "linux5_7")]
+    #[test]
+    fn test_copy_with_mode_write_protect() -> Result<()> {
+        const PAGE_SIZE: usize = 4096;
+        const FILL: u8 = 0xAB;
+
+        unsafe {
+            let uffd = UffdBuilder::new()
+                .require_features(FeatureFlags::PAGEFAULT_FLAG_WP)
+                .close_on_exec(true)
+                .create()?;
+
+            let mapping = anon_mapping(PAGE_SIZE);
+            let src = anon_mapping(PAGE_SIZE);
+            ptr::write_bytes(src as *mut u8, FILL, PAGE_SIZE);
+
+            assert!(uffd
+                .register_with_mode(
+                    mapping,
+                    PAGE_SIZE,
+                    RegisterMode::MISSING | RegisterMode::WRITE_PROTECT
+                )?
+                .contains(IoctlFlags::WRITE_PROTECT));
+
+            let ptr = mapping as usize;
+            let thread = thread::spawn(move || {
+                let ptr = ptr as *mut u8;
+                *ptr = 1;
+            });
+
+            let mut seen_missing = false;
+            loop {
+                match uffd.read_event()? {
+                    Some(Event::Pagefault {
+                        kind,
+                        rw: ReadWrite::Write,
+                        addr,
+                        ..
+                    }) => match kind {
+                        FaultKind::Missing => {
+                            assert_eq!(addr, mapping);
+                            assert!(!seen_missing);
+                            seen_missing = true;
+
+                            // Resolve the missing fault with a write-protected copy. Waking the
+                            // thread makes it retry its write, which must now raise a
+                            // write-protect fault instead of succeeding.
+                            let copied =
+                                uffd.copy_with_mode(src, addr, PAGE_SIZE, CopyMode::WRITE_PROTECT)?;
+                            assert_eq!(copied, PAGE_SIZE);
+                        }
+                        FaultKind::WriteProtected => {
+                            assert_eq!(addr, mapping);
+                            assert!(seen_missing);
+                            // The write has not landed yet; the page holds the copied data.
+                            assert_eq!(*(addr as *const u8), FILL);
+                            uffd.remove_write_protection(mapping, PAGE_SIZE, true)?;
+                            break;
+                        }
+                        #[cfg(feature = "linux5_13")]
+                        FaultKind::Minor => panic!("unexpected minor fault"),
+                    },
+                    _ => panic!("unexpected event"),
+                }
+            }
+
+            thread.join().expect("failed to join thread");
+
+            assert_eq!(*(mapping as *const u8), 1);
+            assert_eq!(*(mapping as *const u8).add(1), FILL);
+
+            uffd.unregister(mapping, PAGE_SIZE)?;
+
+            assert_eq!(libc::munmap(mapping, PAGE_SIZE), 0);
+            assert_eq!(libc::munmap(src, PAGE_SIZE), 0);
+        }
+
+        Ok(())
+    }
+
+    #[cfg(feature = "linux6_4")]
+    #[test]
+    fn test_write_protect_unpopulated() -> Result<()> {
+        const PAGE_SIZE: usize = 4096;
+
+        unsafe {
+            let uffd = UffdBuilder::new()
+                .require_features(FeatureFlags::PAGEFAULT_FLAG_WP | FeatureFlags::WP_UNPOPULATED)
+                .close_on_exec(true)
+                .create()?;
+
+            let mapping = anon_mapping(PAGE_SIZE);
+
+            // Only register for write-protect faults: without WP_UNPOPULATED, write-protecting
+            // a page that was never touched is a no-op and the write below would simply populate
+            // the page without any fault being reported.
+            assert!(uffd
+                .register_with_mode(mapping, PAGE_SIZE, RegisterMode::WRITE_PROTECT)?
+                .contains(IoctlFlags::WRITE_PROTECT));
+            uffd.write_protect(mapping, PAGE_SIZE)?;
+
+            let ptr = mapping as usize;
+            let thread = thread::spawn(move || {
+                let ptr = ptr as *mut u8;
+                *ptr = 1;
+            });
+
+            match uffd.read_event()? {
+                Some(Event::Pagefault {
+                    kind: FaultKind::WriteProtected,
+                    rw: ReadWrite::Write,
+                    addr,
+                    ..
+                }) => {
+                    assert_eq!(addr, mapping);
+                    uffd.remove_write_protection(mapping, PAGE_SIZE, true)?;
+                }
+                _ => panic!("unexpected event"),
+            }
+
+            thread.join().expect("failed to join thread");
+
+            assert_eq!(*(mapping as *const u8), 1);
+
+            uffd.unregister(mapping, PAGE_SIZE)?;
+
+            assert_eq!(libc::munmap(mapping, PAGE_SIZE), 0);
+        }
+
+        Ok(())
+    }
+
+    #[cfg(feature = "linux6_7")]
+    #[test]
+    fn test_write_protect_async() -> Result<()> {
+        const PAGE_SIZE: usize = 4096;
+
+        unsafe {
+            let uffd = UffdBuilder::new()
+                .require_features(FeatureFlags::PAGEFAULT_FLAG_WP | FeatureFlags::WP_ASYNC)
+                .close_on_exec(true)
+                .non_blocking(true)
+                .create()?;
+
+            let mapping = anon_mapping(PAGE_SIZE);
+
+            assert!(uffd
+                .register_with_mode(mapping, PAGE_SIZE, RegisterMode::WRITE_PROTECT)?
+                .contains(IoctlFlags::WRITE_PROTECT));
+            uffd.write_protect(mapping, PAGE_SIZE)?;
+
+            // In async mode the kernel resolves the write-protect fault itself by clearing the
+            // protection, so the write succeeds from this very thread with no handler involved
+            // and no event is ever queued.
+            *(mapping as *mut u8) = 1;
+            assert_eq!(*(mapping as *const u8), 1);
+            assert!(uffd.read_event()?.is_none());
+
+            // Protecting the now-populated page again behaves the same way.
+            uffd.write_protect(mapping, PAGE_SIZE)?;
+            *(mapping as *mut u8) = 2;
+            assert_eq!(*(mapping as *const u8), 2);
+            assert!(uffd.read_event()?.is_none());
+
+            uffd.unregister(mapping, PAGE_SIZE)?;
+
+            assert_eq!(libc::munmap(mapping, PAGE_SIZE), 0);
+        }
+
+        Ok(())
+    }
 }

userfaultfd-sys/Cargo.toml

@@ -21,3 +21,5 @@ default = []
 linux4_14 = []
 linux5_7 = ["linux4_14"]
 linux5_13 = ["linux5_7"]
+linux6_4 = ["linux5_13"]
+linux6_7 = ["linux6_4"]