From 21a94314dff90e66a8eb1f70ec6eb60c49c45a37 Mon Sep 17 00:00:00 2001 From: Yunjeong Mun Date: Thu, 30 Jul 2026 09:42:08 +0900 Subject: [PATCH] Pre-fault destination buffer to respect NUMA mempolicy DSA page faults triggered by descriptors submitted from user space are serviced in kworker context, where the allocating task's mempolicy is not applied. When running under numactl, this causes destination pages to be allocated from whatever node the kworker happens to run on, silently ignoring the requested policy. Add dsa_prefault_buffer() and call it on the destination buffer before DSA submission in dto_memset() and dto_memcpymove(). The helper uses madvise(MADV_POPULATE_WRITE) to populate the pages in the calling task's context, where numactl's mempolicy is honored. On older kernels without MADV_POPULATE_*, fall back to a manual read-modify-write sweep over the pages. Before (numactl --interleave 2,3 ignored, pages land on node 0/1): $ numactl -i 2,3 ./memcpy_test --size 2g --thread 10 $ numastat -c -p memcpy_test Per-node process memory usage (in MBs) PID Node 0 Node 1 Node 2 Node 3 Total --------------- ------ ------ ------ ------ ----- 33358 (watch) 1 3 0 0 3 54868 (memcpy_te 8200 12282 10240 10240 40963 55112 (watch) 0 1 0 0 1 55113 (sh) 0 2 0 0 2 --------------- ------ ------ ------ ------ ----- Total 8202 12287 10240 10240 40969 After (numactl --interleave 2,3 respected, pages only on node 2/3): $ numactl -i 2,3 ./memcpy_test --size 2g --thread 10 $ numastat -c -p memcpy_test Per-node process memory usage (in MBs) PID Node 0 Node 1 Node 2 Node 3 Total --------------- ------ ------ ------ ------ ----- 33358 (watch) 1 3 0 0 3 54436 (memcpy_te 0 2 20480 20480 40963 54547 (watch) 0 1 0 0 1 54548 (sh) 0 2 0 0 2 --------------- ------ ------ ------ ------ ----- Total 1 8 20480 20480 40969 Suggested-by: Honggyu Kim Signed-off-by: Yunjeong Mun --- dto.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/dto.c b/dto.c index 48ef3a7..7c57bea 100644 --- a/dto.c +++ b/dto.c @@ -30,6 +30,9 @@ #define likely(x) __builtin_expect((x), 1) #define unlikely(x) __builtin_expect((x), 0) +#define PAGE_ALIGN_DOWN(x) ((unsigned long)(x) & ~(getpagesize() - 1)) +#define PAGE_ALIGN_UP(x) (((unsigned long)(x) + getpagesize() - 1) & ~(getpagesize() - 1)) + // DSA capabilities #define GENCAP_CC_MEMORY 0x4 @@ -63,6 +66,8 @@ static __thread struct dsa_hw_desc thr_desc; static __thread struct dsa_completion_record thr_comp __attribute__((aligned(32))); static __thread uint64_t thr_bytes_completed; +static void dsa_prefault_buffer(void *buf, size_t len, int write); + // original std memory functions static void * (*orig_memset)(void *s, int c, size_t n); static void * (*orig_memcpy)(void *dest, const void *src, size_t n); @@ -1586,6 +1591,9 @@ static void dto_memset(void *s, int c, size_t n, int *result) { uint64_t memset_pattern; size_t cpu_size, dsa_size; + + dsa_prefault_buffer(s, n, 1); + struct dto_wq *wq = get_wq(s); for (int i = 0; i < 8; ++i) @@ -1689,6 +1697,7 @@ static bool dto_memcpymove(void *dest, const void *src, size_t n, bool is_memcpy } dsa_size = n - cpu_size; + dsa_prefault_buffer(dest, n, 1); wq = get_wq(dest); thr_desc.opcode = DSA_OPCODE_MEMMOVE; @@ -2080,3 +2089,24 @@ int memcmp(const void *s1, const void *s2, size_t n) } return ret; } + +static void dsa_prefault_buffer(void *buf, size_t len, int write) +{ + long page_size = sysconf(_SC_PAGESIZE); + unsigned long addr = (unsigned long)buf; + unsigned long start = PAGE_ALIGN_DOWN(addr); + size_t aligned_len = PAGE_ALIGN_UP(len + (addr - start)); + + int ret = madvise(start, aligned_len, write ? MADV_POPULATE_WRITE : MADV_POPULATE_READ); + + if (ret < 0) { + // Fallback + volatile char *p = (volatile char *)buf; + for (size_t i = 0; i < len; i += page_size) { + if (write) + p[i] = p[i]; // read-modify-write + else + (void)p[i]; // read only + } + } +}