From a4a6922de6d6269d28ce51065074e029d8c274e8 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Thu, 19 Nov 2026 16:29:17 -0800 Subject: [PATCH 01/13] fix(cuda.core): share ownership of repeat imports of an IPC memory pool (#3510) Every import of a pool in a process returns the same CUmemoryPool and adds a driver reference, while access settings belong to the pool. Releasing one import revoked device access for all of them, which broke a surviving pinned pool import. Imports of a pool now share one owner that revokes access once and destroys every import when the last reference is released; an import racing that release waits for the revocation. Concurrent imports of one UUID return the registered resource instead of raising. --- cuda_core/cuda/core/_cpp/rt/api.hpp | 5 +- cuda_core/cuda/core/_cpp/rt/memory.cpp | 84 ++++++++- cuda_core/cuda/core/_memory/_ipc.pyx | 21 ++- cuda_core/docs/source/release/1.4.0-notes.rst | 13 ++ .../test_ipc_pool_duplicate_import.py | 174 ++++++++++++++++++ 5 files changed, 285 insertions(+), 12 deletions(-) create mode 100644 cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py diff --git a/cuda_core/cuda/core/_cpp/rt/api.hpp b/cuda_core/cuda/core/_cpp/rt/api.hpp index 40642ab2f26..80b8ff78514 100644 --- a/cuda_core/cuda/core/_cpp/rt/api.hpp +++ b/cuda_core/cuda/core/_cpp/rt/api.hpp @@ -184,7 +184,10 @@ MemoryPoolHandle get_device_mempool(int device_id); // Create an owning memory pool handle from an IPC import. // The file descriptor is NOT owned by this handle (caller manages FD separately). -// When the last reference is released, cuMemPoolDestroy is called automatically. +// Repeat imports of one pool share one handle, because access settings belong to +// the pool rather than to an import (#3510). When the last reference is released, +// access is revoked and cuMemPoolDestroy is called once for every import. An +// import that races the release of the last handle waits until access is revoked. // Returns empty handle on error (caller must check). MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type); diff --git a/cuda_core/cuda/core/_cpp/rt/memory.cpp b/cuda_core/cuda/core/_cpp/rt/memory.cpp index 3cf92649dc4..c11952a7313 100644 --- a/cuda_core/cuda/core/_cpp/rt/memory.cpp +++ b/cuda_core/cuda/core/_cpp/rt/memory.cpp @@ -136,16 +136,94 @@ get_device_mempool(int device_id) return create_mempool_handle_ref(pool); } +// Importing a pool that is already imported in this process returns the same +// CUmemoryPool and adds a driver reference: each import needs its own +// cuMemPoolDestroy, and one made while allocations imported from the pool are +// outstanding fails without dropping its reference. Access settings belong to +// the pool, not to an import, so revoking them on behalf of one import +// (nvbug 5698116) would remove access that another import relies on (#3510). +// All imports of a pool therefore share one owner. It counts the imports and, +// when its last reference is released, revokes access once and then destroys +// every import. +namespace { +struct IpcMemoryPoolBox : MemoryPoolBox { + // Set once the box is registered in ipc_pool_cache; an unregistered box + // owns nothing. + bool owner = false; + IpcMemoryPoolBox() : MemoryPoolBox{} {} + IpcMemoryPoolBox(const IpcMemoryPoolBox &) = delete; + IpcMemoryPoolBox &operator=(const IpcMemoryPoolBox &) = delete; + IpcMemoryPoolBox(IpcMemoryPoolBox &&) = delete; + IpcMemoryPoolBox &operator=(IpcMemoryPoolBox &&) = delete; + ~IpcMemoryPoolBox(); +}; + +struct IpcPoolEntry { + std::weak_ptr handle; + const IpcMemoryPoolBox *box; + std::size_t imports; +}; +} // namespace + +// Guarded by ipc_pool_mutex. An entry outlives its handle's last reference: +// only the owner's destructor erases it, after revoking access. An importer +// that finds an expired entry waits on ipc_pool_release_cv, so the revocation +// cannot remove access granted through a newer owner. No driver call runs +// under the mutex. +static std::unordered_map ipc_pool_cache; +static std::mutex ipc_pool_mutex; +static std::condition_variable ipc_pool_release_cv; + +IpcMemoryPoolBox::~IpcMemoryPoolBox() +{ + if (!owner) { + return; + } + GILReleaseGuard gil; + clear_mempool_peer_access(resource, -1); + std::size_t imports = 1; + { + std::scoped_lock lock(ipc_pool_mutex); + auto entry = ipc_pool_cache.find(resource); + if (entry != ipc_pool_cache.end() && entry->second.box == this) { + imports = entry->second.imports; + ipc_pool_cache.erase(entry); + } + } + ipc_pool_release_cv.notify_all(); + for (; imports > 0; --imports) { + pw_cuMemPoolDestroy(resource); + } + g_live_owned_mempools.fetch_sub(1, std::memory_order_relaxed); +} + MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type) { GILReleaseGuard gil; - CUmemoryPool pool; + // Allocate before importing and before locking: a failed allocation then + // has nothing to undo and cannot run a destructor under the mutex. + auto box = std::make_shared(); auto *handle_ptr = reinterpret_cast(static_cast(fd)); - if (CUDA_SUCCESS != (err = DRIVER_CALL(cuMemPoolImportFromShareableHandle, &pool, handle_ptr, handle_type, 0))) { + if (CUDA_SUCCESS != + (err = DRIVER_CALL(cuMemPoolImportFromShareableHandle, &box->resource, handle_ptr, handle_type, 0))) { return {}; } - return wrap_mempool_owned(pool, -1); + + std::unique_lock lock(ipc_pool_mutex); + for (auto it = ipc_pool_cache.find(box->resource); it != ipc_pool_cache.end(); + it = ipc_pool_cache.find(box->resource)) { + if (MemoryPoolHandle h = it->second.handle.lock()) { + ++it->second.imports; + return h; + } + ipc_pool_release_cv.wait(lock); + } + MemoryPoolHandle h(box, &box->resource); + ipc_pool_cache.emplace(box->resource, IpcPoolEntry{h, box.get(), 1}); + box->owner = true; + g_live_owned_mempools.fetch_add(1, std::memory_order_relaxed); + return h; } // ============================================================================ diff --git a/cuda_core/cuda/core/_memory/_ipc.pyx b/cuda_core/cuda/core/_memory/_ipc.pyx index b75160d59d8..1f45aafe048 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pyx +++ b/cuda_core/cuda/core/_memory/_ipc.pyx @@ -538,6 +538,13 @@ cdef Buffer Buffer_from_ipc_descriptor( # _MemPool IPC Implementation # --------------------------- +cdef inline _check_registered_type(mr, cls, uuid): + if not isinstance(mr, cls): + raise TypeError( + f"Registry contains a {type(mr).__name__} for uuid " + f"{uuid}, but {cls.__name__} was requested") + + cdef _MemPool MP_from_allocation_handle(cls, alloc_handle, MP_import_init_t init=NULL): if isinstance(alloc_handle, IPCAllocationHandle): IPCAllocationHandle_check_open(alloc_handle) @@ -546,10 +553,7 @@ cdef _MemPool MP_from_allocation_handle(cls, alloc_handle, MP_import_init_t init uuid = getattr(alloc_handle, 'uuid', None) # no-cython-lint mr = registry.get(uuid) if mr is not None: - if not isinstance(mr, cls): - raise TypeError( - f"Registry contains a {type(mr).__name__} for uuid " - f"{uuid}, but {cls.__name__} was requested") + _check_registered_type(mr, cls, uuid) MP_check_open(<_MemPool>mr) return mr @@ -587,10 +591,11 @@ cdef _MemPool MP_from_allocation_handle(cls, alloc_handle, MP_import_init_t init if uuid is not None: registered = self.register(uuid) if registered is not self: - raise RuntimeError( - "internal cuda.core error, please report: register() returned a " - "different memory resource than the one being registered" - ) + # A concurrent import of the same UUID registered first. Both + # imports share one pool handle, so dropping this one only drops a + # reference. + _check_registered_type(registered, cls, uuid) + return registered return self diff --git a/cuda_core/docs/source/release/1.4.0-notes.rst b/cuda_core/docs/source/release/1.4.0-notes.rst index 338e14d49e3..bcff47653e7 100644 --- a/cuda_core/docs/source/release/1.4.0-notes.rst +++ b/cuda_core/docs/source/release/1.4.0-notes.rst @@ -23,3 +23,16 @@ Fixes and enhancements supporting device. (`#3415 `__, `#3501 `__) + +- Importing the same memory pool more than once in one process, for example + by passing a raw file descriptor to + :meth:`PinnedMemoryResource.from_allocation_handle` twice, no longer breaks + the other imports when one of them is closed. The driver returns the same + pool for every import, and closing one import used to revoke device access + to the pool for all of them, so device access to a pinned pool failed with + ``CUDA_ERROR_INVALID_VALUE``. All imports of a pool now share one owner, + which revokes access and destroys the pool when the last of them is + released. Concurrent imports of the same allocation handle from several + threads now return the same memory resource instead of raising an internal + error. + (`#3510 `__) diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py new file mode 100644 index 00000000000..7ef3a88ac57 --- /dev/null +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Imports of one memory pool in one process share ownership (#3510). + +The driver returns the same ``CUmemoryPool`` for every import of a pool in a +process, and the pool's access settings are shared by all of them. Releasing +one import used to revoke device access for the pool (the nvbug 5698116 +workaround), which broke the imports that were still open; for a pinned pool +the next device access failed with ``CUDA_ERROR_INVALID_VALUE``. All imports +now share one owner, which revokes access and destroys the pool only when the +last of them is released. +""" + +import contextlib +import multiprocessing as mp +import threading + +import pytest +from helpers.child_processes import child_timeout_sec, kill_subprocesses + +from cuda.bindings import driver +from cuda.core import Buffer, Device, DeviceMemoryResource, PinnedMemoryResource, VirtualMemoryResource +from cuda.core._memory._buffer import _live_owned_mempool_count +from cuda.core._utils.cuda_utils import handle_return + +CHILD_TIMEOUT_SEC = child_timeout_sec() +NBYTES = 64 +THREADS = 4 +ITERATIONS = 20 + +# these tests spawn new processes and files which fails for very many threads; +# child processes and IPC fixtures are shared across workers (#2784); cap at two threads +pytestmark = pytest.mark.parallel_threads_limit(2) + + +def _import(mr_type, device, alloc_handle): + if mr_type is DeviceMemoryResource: + return DeviceMemoryResource.from_allocation_handle(device, alloc_handle) + return PinnedMemoryResource.from_allocation_handle(alloc_handle) + + +def _location(device): + location = driver.CUmemLocation() + location.type = driver.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE + location.id = device.device_id + return location + + +def _grant_device_access(mr, device): + # An imported pinned pool is not device accessible until access is granted; + # an imported device pool is always accessible from its own device, which + # the driver reports as CUDA_ERROR_INVALID_DEVICE. + desc = driver.CUmemAccessDesc() + desc.location = _location(device) + desc.flags = driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + (err,) = driver.cuMemPoolSetAccess(driver.CUmemoryPool(int(mr.handle)), [desc], 1) + assert err in (driver.CUresult.CUDA_SUCCESS, driver.CUresult.CUDA_ERROR_INVALID_DEVICE), err + + +def _device_access(mr, device): + return handle_return(driver.cuMemPoolGetAccess(driver.CUmemoryPool(int(mr.handle)), _location(device))) + + +def child_close_one_import(queue): + device = Device() + device.set_current() + stream = device.default_stream + mr_type, alloc_handle, descriptor = queue.get() + baseline = _live_owned_mempool_count() + + # A raw file descriptor carries no UUID, so the registry cannot dedupe. + first = _import(mr_type, device, int(alloc_handle)) + second = _import(mr_type, device, int(alloc_handle)) + assert first is not second + assert int(first.handle) == int(second.handle) + assert _live_owned_mempool_count() == baseline + 1 + + _grant_device_access(second, device) + first.close() + assert _device_access(second, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + + buffer = Buffer.from_ipc_descriptor(second, descriptor, stream=stream) + buffer.fill(7, stream=stream) + stream.sync() + buffer.close() + second.close() + assert _live_owned_mempool_count() == baseline + + +def _import_concurrently(mr_type, device, alloc_handle): + barrier = threading.Barrier(THREADS) + results = [None] * THREADS + errors = [] + + def importer(index): + try: + Device().set_current() + barrier.wait() + results[index] = _import(mr_type, device, alloc_handle) + except Exception as e: + errors.append(e) + + threads = [threading.Thread(target=importer, args=(i,)) for i in range(THREADS)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + if errors: + raise errors[0] + return results + + +def child_concurrent_uuid_imports(queue): + device = Device() + device.set_current() + mr_type, alloc_handle = queue.get() + assert alloc_handle.uuid is not None + baseline = _live_owned_mempool_count() + + for _ in range(ITERATIONS): + results = _import_concurrently(mr_type, device, alloc_handle) + # Every import of the UUID returns the registered resource, whichever + # thread registered it. + assert all(mr is results[0] for mr in results) + assert _live_owned_mempool_count() == baseline + 1 + results[0].close() + del results + assert _live_owned_mempool_count() == baseline + + +def _run_child(target, *items): + queue = mp.Queue() + process = mp.Process(target=target, args=(queue,)) + process.start() + queue.put(items) + process.join(timeout=CHILD_TIMEOUT_SEC) + survivors = kill_subprocesses(process) + assert not survivors, "child did not exit within timeout" + assert process.exitcode == 0, f"child process failed with exit code {process.exitcode}" + + +class TestIpcPoolDuplicateImport: + """Repeat imports of one pool in a process share the pool and its access settings.""" + + @pytest.fixture(autouse=True) + def _set_start_method(self): + # Ensure spawn is used for multiprocessing + with contextlib.suppress(RuntimeError): + mp.set_start_method("spawn", force=True) + + @pytest.fixture + def pool_resource(self, ipc_memory_resource): + if isinstance(ipc_memory_resource, VirtualMemoryResource): + pytest.skip("a VirtualMemoryResource has no memory pool to import") + return ipc_memory_resource + + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_closing_one_import_keeps_the_other_usable(self, ipc_device, pool_resource): + ipc_device.set_current() + stream = ipc_device.default_stream + buffer = pool_resource.allocate(NBYTES, stream=stream) + stream.sync() + try: + _run_child( + child_close_one_import, type(pool_resource), pool_resource.allocation_handle, buffer.ipc_descriptor + ) + finally: + buffer.close(stream) + + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_concurrent_imports_of_one_uuid_share_a_resource(self, ipc_device, pool_resource): + ipc_device.set_current() + _run_child(child_concurrent_uuid_imports, type(pool_resource), pool_resource.allocation_handle) From 4a0537f1730ec17a8600e4554a4bbe769ffeb225 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Thu, 19 Nov 2026 17:32:12 -0800 Subject: [PATCH 02/13] fix(cuda.core): roll back an IPC pool import whose cache insertion fails Also cover closing both imports while a buffer imported through one of them is still alive, as #3510 asks, and run the children through run_child. --- cuda_core/cuda/core/_cpp/rt/memory.cpp | 12 ++- .../test_ipc_pool_duplicate_import.py | 91 ++++++++++++------- 2 files changed, 71 insertions(+), 32 deletions(-) diff --git a/cuda_core/cuda/core/_cpp/rt/memory.cpp b/cuda_core/cuda/core/_cpp/rt/memory.cpp index c11952a7313..5affd085a67 100644 --- a/cuda_core/cuda/core/_cpp/rt/memory.cpp +++ b/cuda_core/cuda/core/_cpp/rt/memory.cpp @@ -220,7 +220,17 @@ create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type) ipc_pool_release_cv.wait(lock); } MemoryPoolHandle h(box, &box->resource); - ipc_pool_cache.emplace(box->resource, IpcPoolEntry{h, box.get(), 1}); + try { + ipc_pool_cache.emplace(box->resource, IpcPoolEntry{h, box.get(), 1}); + } + catch (...) { + // The unregistered box does not destroy its import, and the pool's + // access settings may belong to an import registered meanwhile, so + // only this import's driver reference is dropped. + lock.unlock(); + pw_cuMemPoolDestroy(box->resource); + throw; + } box->owner = true; g_live_owned_mempools.fetch_add(1, std::memory_order_relaxed); return h; diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py index 7ef3a88ac57..7d6faa17c36 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -17,14 +17,14 @@ import threading import pytest -from helpers.child_processes import child_timeout_sec, kill_subprocesses +from helpers.child_processes import run_child +from helpers.contexts import assert_no_cuda_warning from cuda.bindings import driver from cuda.core import Buffer, Device, DeviceMemoryResource, PinnedMemoryResource, VirtualMemoryResource from cuda.core._memory._buffer import _live_owned_mempool_count from cuda.core._utils.cuda_utils import handle_return -CHILD_TIMEOUT_SEC = child_timeout_sec() NBYTES = 64 THREADS = 4 ITERATIONS = 20 @@ -62,18 +62,22 @@ def _device_access(mr, device): return handle_return(driver.cuMemPoolGetAccess(driver.CUmemoryPool(int(mr.handle)), _location(device))) -def child_close_one_import(queue): - device = Device() - device.set_current() - stream = device.default_stream - mr_type, alloc_handle, descriptor = queue.get() - baseline = _live_owned_mempool_count() - +def _import_twice(mr_type, device, alloc_handle): # A raw file descriptor carries no UUID, so the registry cannot dedupe. first = _import(mr_type, device, int(alloc_handle)) second = _import(mr_type, device, int(alloc_handle)) assert first is not second assert int(first.handle) == int(second.handle) + return first, second + + +def child_close_one_import(mr_type, alloc_handle, descriptor): + device = Device() + device.set_current() + stream = device.default_stream + baseline = _live_owned_mempool_count() + + first, second = _import_twice(mr_type, device, alloc_handle) assert _live_owned_mempool_count() == baseline + 1 _grant_device_access(second, device) @@ -88,6 +92,32 @@ def child_close_one_import(queue): assert _live_owned_mempool_count() == baseline +def child_close_both_imports_with_outstanding_buffer(mr_type, alloc_handle, descriptor): + device = Device() + device.set_current() + stream = device.default_stream + baseline = _live_owned_mempool_count() + + with assert_no_cuda_warning(): + first, second = _import_twice(mr_type, device, alloc_handle) + _grant_device_access(second, device) + # The buffer keeps the pool alive after both resources are closed. + outstanding = Buffer.from_ipc_descriptor(first, descriptor, stream=stream) + first.close() + buffer = Buffer.from_ipc_descriptor(second, descriptor, stream=stream) + buffer.fill(7, stream=stream) + stream.sync() + buffer.close() + second.close() + assert _live_owned_mempool_count() == baseline + 1 + + outstanding.fill(9, stream=stream) + stream.sync() + outstanding.close() + stream.sync() + assert _live_owned_mempool_count() == baseline + + def _import_concurrently(mr_type, device, alloc_handle): barrier = threading.Barrier(THREADS) results = [None] * THREADS @@ -111,10 +141,9 @@ def importer(index): return results -def child_concurrent_uuid_imports(queue): +def child_concurrent_uuid_imports(mr_type, alloc_handle): device = Device() device.set_current() - mr_type, alloc_handle = queue.get() assert alloc_handle.uuid is not None baseline = _live_owned_mempool_count() @@ -129,17 +158,6 @@ def child_concurrent_uuid_imports(queue): assert _live_owned_mempool_count() == baseline -def _run_child(target, *items): - queue = mp.Queue() - process = mp.Process(target=target, args=(queue,)) - process.start() - queue.put(items) - process.join(timeout=CHILD_TIMEOUT_SEC) - survivors = kill_subprocesses(process) - assert not survivors, "child did not exit within timeout" - assert process.exitcode == 0, f"child process failed with exit code {process.exitcode}" - - class TestIpcPoolDuplicateImport: """Repeat imports of one pool in a process share the pool and its access settings.""" @@ -155,20 +173,31 @@ def pool_resource(self, ipc_memory_resource): pytest.skip("a VirtualMemoryResource has no memory pool to import") return ipc_memory_resource - @pytest.mark.agent_authored(model="claude-opus-5-5") - def test_closing_one_import_keeps_the_other_usable(self, ipc_device, pool_resource): + @pytest.fixture + def exported_buffer(self, ipc_device, pool_resource): ipc_device.set_current() stream = ipc_device.default_stream buffer = pool_resource.allocate(NBYTES, stream=stream) stream.sync() - try: - _run_child( - child_close_one_import, type(pool_resource), pool_resource.allocation_handle, buffer.ipc_descriptor - ) - finally: - buffer.close(stream) + yield buffer + buffer.close(stream) + + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_closing_one_import_keeps_the_other_usable(self, pool_resource, exported_buffer): + run_child( + child_close_one_import, type(pool_resource), pool_resource.allocation_handle, exported_buffer.ipc_descriptor + ) + + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_closing_both_imports_with_an_outstanding_buffer(self, pool_resource, exported_buffer): + run_child( + child_close_both_imports_with_outstanding_buffer, + type(pool_resource), + pool_resource.allocation_handle, + exported_buffer.ipc_descriptor, + ) @pytest.mark.agent_authored(model="claude-opus-5-5") def test_concurrent_imports_of_one_uuid_share_a_resource(self, ipc_device, pool_resource): ipc_device.set_current() - _run_child(child_concurrent_uuid_imports, type(pool_resource), pool_resource.allocation_handle) + run_child(child_concurrent_uuid_imports, type(pool_resource), pool_resource.allocation_handle) From 8a5f50dc859f2a820c43d886b78ca0b156ae797f Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Thu, 19 Nov 2026 17:37:23 -0800 Subject: [PATCH 03/13] test(cuda.core): reuse shared IPC test helpers in the duplicate-import tests Use import_pool_resource, pool_access_flags, and is_pool_resource from helpers.memory, and rely on the device access each import now grants (#3509) instead of granting it in the test. --- .../test_ipc_pool_duplicate_import.py | 44 +++---------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py index 7d6faa17c36..18ed0549561 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -19,11 +19,11 @@ import pytest from helpers.child_processes import run_child from helpers.contexts import assert_no_cuda_warning +from helpers.memory import import_pool_resource, is_pool_resource, pool_access_flags from cuda.bindings import driver -from cuda.core import Buffer, Device, DeviceMemoryResource, PinnedMemoryResource, VirtualMemoryResource +from cuda.core import Buffer, Device from cuda.core._memory._buffer import _live_owned_mempool_count -from cuda.core._utils.cuda_utils import handle_return NBYTES = 64 THREADS = 4 @@ -34,38 +34,10 @@ pytestmark = pytest.mark.parallel_threads_limit(2) -def _import(mr_type, device, alloc_handle): - if mr_type is DeviceMemoryResource: - return DeviceMemoryResource.from_allocation_handle(device, alloc_handle) - return PinnedMemoryResource.from_allocation_handle(alloc_handle) - - -def _location(device): - location = driver.CUmemLocation() - location.type = driver.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE - location.id = device.device_id - return location - - -def _grant_device_access(mr, device): - # An imported pinned pool is not device accessible until access is granted; - # an imported device pool is always accessible from its own device, which - # the driver reports as CUDA_ERROR_INVALID_DEVICE. - desc = driver.CUmemAccessDesc() - desc.location = _location(device) - desc.flags = driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - (err,) = driver.cuMemPoolSetAccess(driver.CUmemoryPool(int(mr.handle)), [desc], 1) - assert err in (driver.CUresult.CUDA_SUCCESS, driver.CUresult.CUDA_ERROR_INVALID_DEVICE), err - - -def _device_access(mr, device): - return handle_return(driver.cuMemPoolGetAccess(driver.CUmemoryPool(int(mr.handle)), _location(device))) - - def _import_twice(mr_type, device, alloc_handle): # A raw file descriptor carries no UUID, so the registry cannot dedupe. - first = _import(mr_type, device, int(alloc_handle)) - second = _import(mr_type, device, int(alloc_handle)) + first = import_pool_resource(mr_type, device, int(alloc_handle)) + second = import_pool_resource(mr_type, device, int(alloc_handle)) assert first is not second assert int(first.handle) == int(second.handle) return first, second @@ -80,9 +52,8 @@ def child_close_one_import(mr_type, alloc_handle, descriptor): first, second = _import_twice(mr_type, device, alloc_handle) assert _live_owned_mempool_count() == baseline + 1 - _grant_device_access(second, device) first.close() - assert _device_access(second, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + assert pool_access_flags(second, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE buffer = Buffer.from_ipc_descriptor(second, descriptor, stream=stream) buffer.fill(7, stream=stream) @@ -100,7 +71,6 @@ def child_close_both_imports_with_outstanding_buffer(mr_type, alloc_handle, desc with assert_no_cuda_warning(): first, second = _import_twice(mr_type, device, alloc_handle) - _grant_device_access(second, device) # The buffer keeps the pool alive after both resources are closed. outstanding = Buffer.from_ipc_descriptor(first, descriptor, stream=stream) first.close() @@ -127,7 +97,7 @@ def importer(index): try: Device().set_current() barrier.wait() - results[index] = _import(mr_type, device, alloc_handle) + results[index] = import_pool_resource(mr_type, device, alloc_handle) except Exception as e: errors.append(e) @@ -169,7 +139,7 @@ def _set_start_method(self): @pytest.fixture def pool_resource(self, ipc_memory_resource): - if isinstance(ipc_memory_resource, VirtualMemoryResource): + if not is_pool_resource(ipc_memory_resource): pytest.skip("a VirtualMemoryResource has no memory pool to import") return ipc_memory_resource From 96a6a5c928f957e921755b2957eec0ca41701623 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Thu, 19 Nov 2026 17:54:18 -0800 Subject: [PATCH 04/13] test(cuda.core): check pool access after losing concurrent imports and after closing one import The concurrent-import test now asserts that dropping the losing imports leaves the winner's device access intact, which the old per-import ownership broke for pinned pools. The outstanding-buffer test also checks the shared owner count and the access left after closing one import. --- cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py index 18ed0549561..47596ad66b0 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -71,9 +71,11 @@ def child_close_both_imports_with_outstanding_buffer(mr_type, alloc_handle, desc with assert_no_cuda_warning(): first, second = _import_twice(mr_type, device, alloc_handle) + assert _live_owned_mempool_count() == baseline + 1 # The buffer keeps the pool alive after both resources are closed. outstanding = Buffer.from_ipc_descriptor(first, descriptor, stream=stream) first.close() + assert pool_access_flags(second, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE buffer = Buffer.from_ipc_descriptor(second, descriptor, stream=stream) buffer.fill(7, stream=stream) stream.sync() @@ -123,6 +125,8 @@ def child_concurrent_uuid_imports(mr_type, alloc_handle): # thread registered it. assert all(mr is results[0] for mr in results) assert _live_owned_mempool_count() == baseline + 1 + # Dropping the losing imports must not revoke the winner's access. + assert pool_access_flags(results[0], device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE results[0].close() del results assert _live_owned_mempool_count() == baseline From ffe0ab0adf4d75e1b0d52e167eb3df7705d00c99 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Thu, 19 Nov 2026 19:13:15 -0800 Subject: [PATCH 05/13] fix(cuda.core): make IPC pool registration atomic on free-threaded builds MP_register looked up the UUID and then inserted into the registry as two separate steps. Without the GIL, two threads importing the same UUID could both miss and register different resources, contradicting register()'s documented behavior and failing the concurrent-import test on every Linux free-threaded CI job. Serialize the lookup and insertion with a lock. --- cuda_core/cuda/core/_memory/_ipc.pyx | 29 ++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/cuda_core/cuda/core/_memory/_ipc.pyx b/cuda_core/cuda/core/_memory/_ipc.pyx index 1f45aafe048..1d70193c321 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pyx +++ b/cuda_core/cuda/core/_memory/_ipc.pyx @@ -30,6 +30,7 @@ import multiprocessing import operator import os import platform +import threading import uuid import weakref from typing import TYPE_CHECKING, Iterable @@ -41,6 +42,7 @@ __all__ = [] cdef object registry = weakref.WeakValueDictionary() +cdef object _registry_lock = threading.Lock() # serializes MP_register cdef cydriver.CUmemAllocationHandleType IPC_HANDLE_TYPE = \ @@ -612,18 +614,21 @@ cdef _MemPool MP_from_registry(uuid): cdef _MemPool MP_register(_MemPool self, uuid): MP_check_open(self) - existing = registry.get(uuid) - if existing is not None: - MP_check_open(<_MemPool>existing) - return existing - if not self.is_ipc_enabled: - raise RuntimeError("Memory resource is not IPC-enabled") - if self.uuid is not None and self.uuid != uuid: - raise ValueError( - f"Cannot register memory resource with UUID {uuid}: " - f"the resource already has UUID {self.uuid}" - ) - registry[uuid] = self + # The lookup and the insertion must be one step: without the GIL, two + # threads can both miss and register different resources for one UUID. + with _registry_lock: + existing = registry.get(uuid) + if existing is not None: + MP_check_open(<_MemPool>existing) + return existing + if not self.is_ipc_enabled: + raise RuntimeError("Memory resource is not IPC-enabled") + if self.uuid is not None and self.uuid != uuid: + raise ValueError( + f"Cannot register memory resource with UUID {uuid}: " + f"the resource already has UUID {self.uuid}" + ) + registry[uuid] = self self._ipc_data._alloc_handle._uuid = uuid return self From ec5a3a93a1b08ac77fc11bc510a2b0b04d62b19a Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 20 Nov 2026 14:20:24 -0800 Subject: [PATCH 06/13] refactor(cuda.core): find the owner of an imported pool through HandleRegistry Replace the hand-rolled IPC pool cache with a HandleRegistry keyed by the imported CUmemoryPool, as REGISTRY_DESIGN.md describes for raw driver handles that cuda.core already manages. The owner's box counts the imports it holds and, when its last reference is released, unregisters, revokes access once and destroys every import, as before. Drop the wait for an in-flight release: concurrency.rst makes releasing a pool while another thread imports it the application's responsibility. A comment at the registry lookup asks whether a wait is needed there, as in deviceptr_import_ipc. Add HandleRegistry::unregister_expired, so that a deleter never erases the entry of a newer owner registered under the same key. --- .../cuda/core/_cpp/rt/REGISTRY_DESIGN.md | 6 +- cuda_core/cuda/core/_cpp/rt/api.hpp | 3 +- cuda_core/cuda/core/_cpp/rt/internal.hpp | 12 ++ cuda_core/cuda/core/_cpp/rt/memory.cpp | 118 ++++++++---------- 4 files changed, 66 insertions(+), 73 deletions(-) diff --git a/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md b/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md index 003887f296c..b5ee8681b8a 100644 --- a/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md +++ b/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md @@ -30,8 +30,10 @@ Without this level, a round-tripped handle would produce a new Box with default metadata, losing information that was set at creation. Instances: `context_registry`, `stream_registry`, `event_registry`, -`kernel_registry`, and, per `GraphBox`, one registry for its direct child -graphs and one for its nodes. +`kernel_registry`, `ipc_pool_registry`, and, per `GraphBox`, one registry for +its direct child graphs and one for its nodes. `ipc_pool_registry` maps an +imported `CUmemoryPool` to its owner, because every import of a pool in a +process returns the same `CUmemoryPool` (#3510). ### Reused graph handles diff --git a/cuda_core/cuda/core/_cpp/rt/api.hpp b/cuda_core/cuda/core/_cpp/rt/api.hpp index 80b8ff78514..7457d745998 100644 --- a/cuda_core/cuda/core/_cpp/rt/api.hpp +++ b/cuda_core/cuda/core/_cpp/rt/api.hpp @@ -186,8 +186,7 @@ MemoryPoolHandle get_device_mempool(int device_id); // The file descriptor is NOT owned by this handle (caller manages FD separately). // Repeat imports of one pool share one handle, because access settings belong to // the pool rather than to an import (#3510). When the last reference is released, -// access is revoked and cuMemPoolDestroy is called once for every import. An -// import that races the release of the last handle waits until access is revoked. +// access is revoked and cuMemPoolDestroy is called once for every import. // Returns empty handle on error (caller must check). MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type); diff --git a/cuda_core/cuda/core/_cpp/rt/internal.hpp b/cuda_core/cuda/core/_cpp/rt/internal.hpp index bc5e4cadade..952a52dc877 100644 --- a/cuda_core/cuda/core/_cpp/rt/internal.hpp +++ b/cuda_core/cuda/core/_cpp/rt/internal.hpp @@ -193,6 +193,18 @@ class HandleRegistry { map_.erase(key); } + // Erase the entry only if its handle has expired, so that a deleter never + // erases the entry of a newer handle registered under the same key. + void + unregister_expired(const Key &key) noexcept + { + std::scoped_lock lock(mutex_); + auto it = map_.find(key); + if (it != map_.end() && it->second.expired()) { + map_.erase(it); + } + } + Handle lookup(const Key &key) { diff --git a/cuda_core/cuda/core/_cpp/rt/memory.cpp b/cuda_core/cuda/core/_cpp/rt/memory.cpp index 5affd085a67..275168bba12 100644 --- a/cuda_core/cuda/core/_cpp/rt/memory.cpp +++ b/cuda_core/cuda/core/_cpp/rt/memory.cpp @@ -142,98 +142,78 @@ get_device_mempool(int device_id) // outstanding fails without dropping its reference. Access settings belong to // the pool, not to an import, so revoking them on behalf of one import // (nvbug 5698116) would remove access that another import relies on (#3510). -// All imports of a pool therefore share one owner. It counts the imports and, -// when its last reference is released, revokes access once and then destroys -// every import. +// All imports of a pool therefore share one owner, found through +// ipc_pool_registry. It counts the imports and, when its last reference is +// released, revokes access once and then destroys every import. namespace { -struct IpcMemoryPoolBox : MemoryPoolBox { - // Set once the box is registered in ipc_pool_cache; an unregistered box - // owns nothing. - bool owner = false; - IpcMemoryPoolBox() : MemoryPoolBox{} {} - IpcMemoryPoolBox(const IpcMemoryPoolBox &) = delete; - IpcMemoryPoolBox &operator=(const IpcMemoryPoolBox &) = delete; - IpcMemoryPoolBox(IpcMemoryPoolBox &&) = delete; - IpcMemoryPoolBox &operator=(IpcMemoryPoolBox &&) = delete; - ~IpcMemoryPoolBox(); -}; - -struct IpcPoolEntry { - std::weak_ptr handle; - const IpcMemoryPoolBox *box; - std::size_t imports; +struct IpcMemoryPoolBox { + CUmemoryPool resource; + // The imports this box owns: 0 until it becomes the pool's owner, then + // one for every import of the pool. + mutable std::atomic imports{0}; }; -} // namespace -// Guarded by ipc_pool_mutex. An entry outlives its handle's last reference: -// only the owner's destructor erases it, after revoking access. An importer -// that finds an expired entry waits on ipc_pool_release_cv, so the revocation -// cannot remove access granted through a newer owner. No driver call runs -// under the mutex. -static std::unordered_map ipc_pool_cache; -static std::mutex ipc_pool_mutex; -static std::condition_variable ipc_pool_release_cv; - -IpcMemoryPoolBox::~IpcMemoryPoolBox() +const IpcMemoryPoolBox * +get_ipc_box(const MemoryPoolHandle &h) { - if (!owner) { - return; - } - GILReleaseGuard gil; - clear_mempool_peer_access(resource, -1); - std::size_t imports = 1; - { - std::scoped_lock lock(ipc_pool_mutex); - auto entry = ipc_pool_cache.find(resource); - if (entry != ipc_pool_cache.end() && entry->second.box == this) { - imports = entry->second.imports; - ipc_pool_cache.erase(entry); - } - } - ipc_pool_release_cv.notify_all(); - for (; imports > 0; --imports) { - pw_cuMemPoolDestroy(resource); - } - g_live_owned_mempools.fetch_sub(1, std::memory_order_relaxed); + const CUmemoryPool *p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(IpcMemoryPoolBox, resource) + ); } +} // namespace + +// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) +static HandleRegistry ipc_pool_registry; MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type) { GILReleaseGuard gil; - // Allocate before importing and before locking: a failed allocation then - // has nothing to undo and cannot run a destructor under the mutex. - auto box = std::make_shared(); + // Allocate before importing: a failed allocation then has nothing to undo. + // clang-format off + auto box = std::shared_ptr( + new IpcMemoryPoolBox{}, + [](const IpcMemoryPoolBox *b) { + // Shared ownership synchronizes the increments with this load. + std::size_t imports = b->imports.load(std::memory_order_relaxed); + if (imports > 0) { + GILReleaseGuard gil; + ipc_pool_registry.unregister_expired(b->resource); + clear_mempool_peer_access(b->resource, -1); + for (; imports > 0; --imports) { + pw_cuMemPoolDestroy(b->resource); + } + g_live_owned_mempools.fetch_sub(1, std::memory_order_relaxed); + } + delete b; + } + ); + // clang-format on auto *handle_ptr = reinterpret_cast(static_cast(fd)); if (CUDA_SUCCESS != (err = DRIVER_CALL(cuMemPoolImportFromShareableHandle, &box->resource, handle_ptr, handle_type, 0))) { return {}; } - std::unique_lock lock(ipc_pool_mutex); - for (auto it = ipc_pool_cache.find(box->resource); it != ipc_pool_cache.end(); - it = ipc_pool_cache.find(box->resource)) { - if (MemoryPoolHandle h = it->second.handle.lock()) { - ++it->second.imports; - return h; - } - ipc_pool_release_cv.wait(lock); - } MemoryPoolHandle h(box, &box->resource); + MemoryPoolHandle owner; try { - ipc_pool_cache.emplace(box->resource, IpcPoolEntry{h, box.get(), 1}); + // Should an import wait here while the previous owner's release is in flight, as deviceptr_import_ipc does? + owner = ipc_pool_registry.get_or_create(box->resource, [&h]() noexcept { return h; }); } catch (...) { - // The unregistered box does not destroy its import, and the pool's - // access settings may belong to an import registered meanwhile, so - // only this import's driver reference is dropped. - lock.unlock(); + // The box owns no import yet, so its deleter would not drop this one. pw_cuMemPoolDestroy(box->resource); throw; } - box->owner = true; - g_live_owned_mempools.fetch_add(1, std::memory_order_relaxed); - return h; + // Whether this box became the owner or an earlier import's box was found, + // the owner now holds this import's driver reference. + get_ipc_box(owner)->imports.fetch_add(1, std::memory_order_relaxed); + if (owner == h) { + g_live_owned_mempools.fetch_add(1, std::memory_order_relaxed); + } + return owner; } // ============================================================================ From 35e75c87c34e01f9265b8066f10e0b0f5224eba5 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 20 Nov 2026 15:11:00 -0800 Subject: [PATCH 07/13] fix(cuda.core): wait for an in-flight IPC release, and set the device before registration Address review of #3511: - An import that finds the previous owner's release in flight now waits for it. The last reference can be dropped by a buffer, so the importer cannot synchronize with it: the old owner's revocation would remove the access a new owner granted, and its destroys fail once the new owner has imported a buffer. ReleasingHandleRegistry, a HandleRegistry variant whose entry outlives its handle until the owner's deleter calls release(), provides the wait. The pool deleter revokes access and destroys every import before it releases the entry, and reports a failed destroy only afterwards. - The IPC pointer cache uses the same registry instead of its own mutex and condition variable. Its deleter releases the entry from cleanup_in_context's after_cleanup hook, once the mapping is freed. - DeviceMemoryResource.from_allocation_handle resolves the device before the import and sets it in the import hook, before registration. A concurrent import of the same UUID returns the registered resource without writing to it. Add a test that races a re-import against a buffer dropping the last reference to the pool, and check device_id in the concurrent UUID test. --- cuda_core/cuda/core/_cpp/rt/DESIGN.md | 11 +- .../cuda/core/_cpp/rt/REGISTRY_DESIGN.md | 22 ++- cuda_core/cuda/core/_cpp/rt/api.hpp | 3 +- cuda_core/cuda/core/_cpp/rt/internal.hpp | 91 +++++++-- cuda_core/cuda/core/_cpp/rt/memory.cpp | 173 ++++++++---------- .../core/_memory/_device_memory_resource.pyi | 4 +- .../core/_memory/_device_memory_resource.pyx | 18 +- cuda_core/cuda/core/_memory/_ipc.pxd | 7 +- cuda_core/cuda/core/_memory/_ipc.pyi | 2 +- cuda_core/cuda/core/_memory/_ipc.pyx | 8 +- .../core/_memory/_pinned_memory_resource.pyx | 2 +- .../test_ipc_pool_duplicate_import.py | 71 ++++++- cuda_core/tests/test_rt_layout.py | 2 +- 13 files changed, 281 insertions(+), 133 deletions(-) diff --git a/cuda_core/cuda/core/_cpp/rt/DESIGN.md b/cuda_core/cuda/core/_cpp/rt/DESIGN.md index 1d550a4c03f..9973118cb3c 100644 --- a/cuda_core/cuda/core/_cpp/rt/DESIGN.md +++ b/cuda_core/cuda/core/_cpp/rt/DESIGN.md @@ -305,17 +305,18 @@ one-time function-table fill (`ensure_fn_table()`). Neither may run while a C++ lock is held. **The GIL is the outermost lock.** Code that holds a C++ lock (a registry's -mutex, `ipc_import_mutex`, any `std::mutex`) must not acquire or reacquire the +mutex, any `std::mutex`) must not acquire or reacquire the GIL while the lock is held: no `report_*` or `pw_*` calls, no `GILAcquireGuard`, and no `GILReleaseGuard` whose destructor runs inside the locked region. Code that needs a C++ lock and may run with the GIL held releases the GIL first (`GILReleaseGuard` before `lock_guard`). Otherwise a thread blocked on the lock while holding the GIL deadlocks with the lock holder waiting for the GIL (#2840). Collect statuses under the lock and report after it -is released, as `deviceptr_import_ipc` does: `cleanup_in_context` takes an -`after_cleanup` hook that runs once the cleanup is done and before anything that -may run user code, and the deleter passes one that unlocks its -`std::unique_lock`. The registries store `weak_ptr`s, +is released, as `deviceptr_import_ipc` does for a discarded import. A deleter +that must update shared state before user code can run does so in +`cleanup_in_context`'s `after_cleanup` hook, which runs once the cleanup is done +and before anything that may run user code; the IPC pointer deleter releases its +registry entry there. The registries store `weak_ptr`s, so erasing an entry under a registry lock never runs a deleter. ### Static Initialization and Deadlock Hazards diff --git a/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md b/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md index b5ee8681b8a..8fb71e13a9f 100644 --- a/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md +++ b/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md @@ -30,10 +30,24 @@ Without this level, a round-tripped handle would produce a new Box with default metadata, losing information that was set at creation. Instances: `context_registry`, `stream_registry`, `event_registry`, -`kernel_registry`, `ipc_pool_registry`, and, per `GraphBox`, one registry for -its direct child graphs and one for its nodes. `ipc_pool_registry` maps an -imported `CUmemoryPool` to its owner, because every import of a pool in a -process returns the same `CUmemoryPool` (#3510). +`kernel_registry`, and, per `GraphBox`, one registry for its direct child +graphs and one for its nodes. + +### Imports that wait for an in-flight release + +IPC imports use `ReleasingHandleRegistry`, a variant whose entry outlives its +handle's last reference. Only the owner's deleter removes the entry, with +`release()`, after its last driver call on the resource. `get_or_create()` +waits while the entry under its key has expired, so a new owner never overlaps +the previous owner's cleanup, even when the last reference was dropped by a +dependent handle that the importing thread cannot synchronize with. + +Instances: `ipc_pool_registry` maps an imported `CUmemoryPool` to its owner, +because every import of a pool in a process returns the same `CUmemoryPool` +(#3510); the owner's cleanup revokes access and destroys the imports. +`ipc_ptr_cache` maps a `CUmemPoolPtrExportData` descriptor to its imported +pointer, because a re-import before the previous mapping is freed would share +that mapping (nvbug 5570902). ### Reused graph handles diff --git a/cuda_core/cuda/core/_cpp/rt/api.hpp b/cuda_core/cuda/core/_cpp/rt/api.hpp index 7457d745998..c2c0689edb8 100644 --- a/cuda_core/cuda/core/_cpp/rt/api.hpp +++ b/cuda_core/cuda/core/_cpp/rt/api.hpp @@ -186,7 +186,8 @@ MemoryPoolHandle get_device_mempool(int device_id); // The file descriptor is NOT owned by this handle (caller manages FD separately). // Repeat imports of one pool share one handle, because access settings belong to // the pool rather than to an import (#3510). When the last reference is released, -// access is revoked and cuMemPoolDestroy is called once for every import. +// access is revoked and cuMemPoolDestroy is called once for every import. An +// import that races that release waits until it is done. // Returns empty handle on error (caller must check). MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type); diff --git a/cuda_core/cuda/core/_cpp/rt/internal.hpp b/cuda_core/cuda/core/_cpp/rt/internal.hpp index 952a52dc877..355f4445ff6 100644 --- a/cuda_core/cuda/core/_cpp/rt/internal.hpp +++ b/cuda_core/cuda/core/_cpp/rt/internal.hpp @@ -7,6 +7,7 @@ #include "types.hpp" #include "driver_api.hpp" #include "error.hpp" +#include #include #include #include @@ -192,19 +193,6 @@ class HandleRegistry { std::scoped_lock lock(mutex_); map_.erase(key); } - - // Erase the entry only if its handle has expired, so that a deleter never - // erases the entry of a newer handle registered under the same key. - void - unregister_expired(const Key &key) noexcept - { - std::scoped_lock lock(mutex_); - auto it = map_.find(key); - if (it != map_.end() && it->second.expired()) { - map_.erase(it); - } - } - Handle lookup(const Key &key) { @@ -256,4 +244,81 @@ class HandleRegistry { MapType map_; }; +// ============================================================================ +// Handle registry that waits for in-flight releases +// +// Like HandleRegistry, but an entry outlives its handle's last reference: only +// the owner's deleter removes it, by calling release() once its cleanup is +// done. get_or_create() waits while the entry under its key has expired, so a +// new owner is never created while the previous owner is still releasing the +// resource. Used where the driver hands the same resource to repeat imports +// (IPC pools and IPC pointers), so a new owner and an old owner's cleanup +// would act on one driver resource. See REGISTRY_DESIGN.md. +// ============================================================================ + +template > +class ReleasingHandleRegistry { + public: + // Return the live handle registered under key, waiting first while the + // previous owner's release is in flight. Otherwise register and return the + // handle that create() returns; an empty handle is returned unregistered. + // create() runs under the registry's mutex, so it must not acquire the GIL + // or release a handle whose deleter calls release(). + template + Handle + get_or_create(const Key &key, Factory &&create) + { + std::unique_lock lock(mutex_); + for (auto it = map_.find(key); it != map_.end(); it = map_.find(key)) { + if (Handle h = it->second.handle.lock()) { + return h; + } + released_.wait(lock); + } + // Insert before creating, so a failed insertion has nothing to undo. + auto it = map_.try_emplace(key).first; + Handle h; + try { + h = create(); + } + catch (...) { + map_.erase(it); + throw; + } + if (h) { + it->second = Entry{h, h.get()}; + } + else { + map_.erase(it); + } + return h; + } + + // Called by an owner's deleter after its last driver call on the resource: + // erase key's entry if it belongs to the handle that pointed to resource, + // and wake the importers waiting for it. + void + release(const Key &key, const typename Handle::element_type *resource) noexcept + { + { + std::scoped_lock lock(mutex_); + auto it = map_.find(key); + if (it != map_.end() && it->second.resource == resource) { + map_.erase(it); + } + } + released_.notify_all(); + } + + private: + struct Entry { + std::weak_ptr handle; + const typename Handle::element_type *resource = nullptr; + }; + + std::mutex mutex_; + std::condition_variable released_; + std::unordered_map map_; +}; + } // namespace cuda_core::rt::detail diff --git a/cuda_core/cuda/core/_cpp/rt/memory.cpp b/cuda_core/cuda/core/_cpp/rt/memory.cpp index 275168bba12..97481b3c004 100644 --- a/cuda_core/cuda/core/_cpp/rt/memory.cpp +++ b/cuda_core/cuda/core/_cpp/rt/memory.cpp @@ -9,13 +9,10 @@ #include "error.hpp" #include "internal.hpp" #include -#include #include #include #include -#include #include -#include #include #ifndef _WIN32 #include @@ -144,7 +141,11 @@ get_device_mempool(int device_id) // (nvbug 5698116) would remove access that another import relies on (#3510). // All imports of a pool therefore share one owner, found through // ipc_pool_registry. It counts the imports and, when its last reference is -// released, revokes access once and then destroys every import. +// released, revokes access once and then destroys every import. An import that +// finds the owner's release in flight waits for it, even when the last +// reference was dropped by a buffer and not by a resource the importer could +// synchronize with: the revocation would remove access granted by a new owner, +// and a destroy fails once the new owner has imported a buffer. namespace { struct IpcMemoryPoolBox { CUmemoryPool resource; @@ -164,11 +165,12 @@ get_ipc_box(const MemoryPoolHandle &h) } // namespace // See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) -static HandleRegistry ipc_pool_registry; +static ReleasingHandleRegistry ipc_pool_registry; MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type) { + // Released before the registry can wait: the owner's deleter may need it. GILReleaseGuard gil; // Allocate before importing: a failed allocation then has nothing to undo. // clang-format off @@ -179,12 +181,22 @@ create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type) std::size_t imports = b->imports.load(std::memory_order_relaxed); if (imports > 0) { GILReleaseGuard gil; - ipc_pool_registry.unregister_expired(b->resource); clear_mempool_peer_access(b->resource, -1); + CUresult status = CUDA_SUCCESS; for (; imports > 0; --imports) { - pw_cuMemPoolDestroy(b->resource); + if (CUresult s = DRIVER_CALL(cuMemPoolDestroy, b->resource); s != CUDA_SUCCESS) { + status = s; + } } g_live_owned_mempools.fetch_sub(1, std::memory_order_relaxed); + // Wake waiting importers before reporting: a report can run + // user code, which could import this pool again. + ipc_pool_registry.release(b->resource, &b->resource); + if (status != CUDA_SUCCESS) { + char operation[kOperationNameSize]; + format_operation(operation, sizeof(operation), "cuMemPoolDestroy", handle_bits(b->resource)); + report_cuda_error(operation, status); + } } delete b; } @@ -199,20 +211,24 @@ create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type) MemoryPoolHandle h(box, &box->resource); MemoryPoolHandle owner; try { - // Should an import wait here while the previous owner's release is in flight, as deviceptr_import_ipc does? - owner = ipc_pool_registry.get_or_create(box->resource, [&h]() noexcept { return h; }); + owner = ipc_pool_registry.get_or_create(box->resource, [&]() noexcept { + box->imports.store(1, std::memory_order_relaxed); + return h; + }); } catch (...) { // The box owns no import yet, so its deleter would not drop this one. pw_cuMemPoolDestroy(box->resource); throw; } - // Whether this box became the owner or an earlier import's box was found, - // the owner now holds this import's driver reference. - get_ipc_box(owner)->imports.fetch_add(1, std::memory_order_relaxed); if (owner == h) { g_live_owned_mempools.fetch_add(1, std::memory_order_relaxed); } + else { + // An earlier import's box owns the pool; it now holds this import's + // driver reference too. + get_ipc_box(owner)->imports.fetch_add(1, std::memory_order_relaxed); + } return owner; } @@ -589,21 +605,13 @@ struct ExportDataKeyHash { } }; -// The handle imported for one descriptor, and the box it aliases. -struct IpcCacheEntry { - std::weak_ptr handle; - const DevicePtrBox *box; -}; - } // namespace -// Guarded by ipc_import_mutex. An entry outlives its handle's last reference: -// only that handle's deleter erases it, under the mutex, together with the -// free. An importer that finds an expired entry therefore knows a release of -// the same mapping is in flight and waits on ipc_release_cv until it is done. -static std::unordered_map ipc_ptr_cache; -static std::mutex ipc_import_mutex; -static std::condition_variable ipc_release_cv; +// An importer that finds an expired entry waits until the previous handle's +// deleter has freed the mapping: a re-import before then would share that +// mapping, and the deleter's cuMemFreeAsync would unmap it under the new +// handle (nvbug 5570902). +static ReleasingHandleRegistry ipc_ptr_cache; DevicePtrHandle deviceptr_import_ipc(const MemoryPoolHandle &h_pool, const void *export_data, const StreamHandle &h_stream) @@ -611,7 +619,7 @@ deviceptr_import_ipc(const MemoryPoolHandle &h_pool, const void *export_data, co auto *data = const_cast(reinterpret_cast(export_data)); // Resolve the table before you take any lock: a fill acquires the GIL, and - // nothing under ipc_import_mutex may acquire it (#2840). The raw p_ calls + // nothing under the cache's mutex may acquire it (#2840). The raw p_ calls // below rely on this. if (!ensure_fn_table(FnTable::driver)) { report_unavailable_fn(FnTable::driver, "cuMemPoolImportPointer"); @@ -623,80 +631,59 @@ deviceptr_import_ipc(const MemoryPoolHandle &h_pool, const void *export_data, co ExportDataKey key; std::memcpy(&key.data, data, sizeof(key.data)); - // The mutex makes lookup, import and registration one step, so two - // threads cannot import the same descriptor twice. Release the GIL - // before taking it: a thread blocked on the mutex while holding the - // GIL would deadlock with a holder that needs the GIL back (#2840). - // Nothing under the mutex may acquire the GIL, so a failed discard is - // reported only after the lock is released (see DESIGN.md). + // Release the GIL before the cache's mutex is taken: a thread blocked + // on it while holding the GIL would deadlock with a holder that needs + // the GIL back (#2840). GILReleaseGuard gil; - CUresult discard_status = CUDA_SUCCESS; - CUdeviceptr discarded = 0; - { - std::unique_lock lock(ipc_import_mutex); - - for (auto it = ipc_ptr_cache.find(key); it != ipc_ptr_cache.end(); it = ipc_ptr_cache.find(key)) { - if (DevicePtrHandle h = it->second.handle.lock()) { - return h; + // Allocate before taking the mutex: a failed allocation then cannot run + // a deleter under it. The box owns nothing until it holds an import. + // clang-format off + auto box = std::shared_ptr( + new DevicePtrBox{0, DeallocationStream{}}, + [h_pool, key](DevicePtrBox *b) { + if (b->resource != 0) { + GILReleaseGuard gil; + const DeallocationStream& stream = b->deallocation; + // Release the entry once the mapping is freed, and before + // reporting: a report can run user code. + cleanup_in_context( + deallocation_context(stream), "cuMemFreeAsync", handle_bits(b->resource), + [&]() noexcept { + return DRIVER_CALL(cuMemFreeAsync, b->resource, as_cu(stream.h_stream)); + }, + [&]() noexcept { ipc_ptr_cache.release(key, &b->resource); }); } - // Expired but not erased: the last reference is gone and its - // deleter has not freed the mapping yet. A re-import now would - // share that mapping, and the deleter's cuMemFreeAsync would - // unmap it under us (nvbug 5570902). Wait for the release. - ipc_release_cv.wait(lock); + delete b; } - + ); + // clang-format on + DevicePtrHandle h(box, &box->resource); + + // The factory runs under the cache's mutex, which makes lookup, import + // and registration one step: two threads cannot import the same + // descriptor twice. Nothing under the mutex may acquire the GIL, so a + // failed discard is reported only after the lock is released (see + // DESIGN.md). + CUresult discard_status = CUDA_SUCCESS; + CUdeviceptr discarded = 0; + DevicePtrHandle imported = ipc_ptr_cache.get_or_create(key, [&]() noexcept -> DevicePtrHandle { CUdeviceptr ptr; - if (CUDA_SUCCESS != (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { // raw: under ipc_import_mutex + if (CUDA_SUCCESS != + (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { // raw: under the cache's mutex return {}; } - - DeallocationStream ds; - if (make_deallocation_stream(h_stream, ds)) { - // clang-format off - auto box = std::shared_ptr( - new DevicePtrBox{ptr, std::move(ds)}, - [h_pool, key](DevicePtrBox *b) { - // Release the GIL first (the GIL is the outermost lock), then hold the - // mutex across unregister + free: a concurrent import that finds this - // entry expired must wait until the mapping is gone, or it re-imports - // the same allocation and the first cuMemFreeAsync unmaps it for both - // (nvbug 5570902). Nothing under the mutex may acquire the GIL, so the - // deallocation context is resolved before the lock (it may report) and - // the lock is released as soon as the cleanup is done. - GILReleaseGuard gil; - const DeallocationStream& stream = b->deallocation; - ContextHandle h_dealloc = deallocation_context(stream); - // Explicit template argument: MSVC cannot deduce it here (C2955). - std::unique_lock lock(ipc_import_mutex); - // Erase only this box's entry (importers wait while it - // exists, so no other box can own the key yet). - auto entry = ipc_ptr_cache.find(key); - if (entry != ipc_ptr_cache.end() && entry->second.box == b) { - ipc_ptr_cache.erase(entry); - } - cleanup_in_context( - h_dealloc, "cuMemFreeAsync", handle_bits(b->resource), - [&]() noexcept { - return p_cuMemFreeAsync(b->resource, as_cu(stream.h_stream)); // raw: under ipc_import_mutex - }, - [&]() noexcept { - ipc_release_cv.notify_all(); - lock.release()->unlock(); - }); - delete b; - } - ); - // clang-format on - DevicePtrHandle h(box, &box->resource); - ipc_ptr_cache[key] = IpcCacheEntry{h, box.get()}; - return h; + if (!make_deallocation_stream(h_stream, box->deallocation)) { + // No deallocation stream could be recorded: discard the import + // with the raw call (a pw_ report would acquire the GIL). + discard_status = p_cuMemFreeAsync(ptr, as_cu(h_stream)); // raw: under the cache's mutex + discarded = ptr; + return {}; } - - // No deallocation stream could be recorded: discard the import with - // the raw call (a pw_ report would acquire the GIL under the mutex). - discard_status = p_cuMemFreeAsync(ptr, as_cu(h_stream)); // raw: under ipc_import_mutex - discarded = ptr; + box->resource = ptr; + return h; + }); + if (imported) { + return imported; } if (discard_status != CUDA_SUCCESS) { char operation[kOperationNameSize]; diff --git a/cuda_core/cuda/core/_memory/_device_memory_resource.pyi b/cuda_core/cuda/core/_memory/_device_memory_resource.pyi index ff16f8bef92..4dfa8cb57b9 100644 --- a/cuda_core/cuda/core/_memory/_device_memory_resource.pyi +++ b/cuda_core/cuda/core/_memory/_device_memory_resource.pyi @@ -166,7 +166,9 @@ class DeviceMemoryResource(_MemPool): Returns ------- - A new device memory resource instance with the imported handle. + A new device memory resource instance with the imported handle. If + the handle's memory resource is already registered in this process, + that resource is returned with its original device. """ @property def allocation_handle(self) -> IPCAllocationHandle: diff --git a/cuda_core/cuda/core/_memory/_device_memory_resource.pyx b/cuda_core/cuda/core/_memory/_device_memory_resource.pyx index b7df931cdac..a747c5b375d 100644 --- a/cuda_core/cuda/core/_memory/_device_memory_resource.pyx +++ b/cuda_core/cuda/core/_memory/_device_memory_resource.pyx @@ -209,13 +209,14 @@ cdef class DeviceMemoryResource(_MemPool): Returns ------- - A new device memory resource instance with the imported handle. + A new device memory resource instance with the imported handle. If + the handle's memory resource is already registered in this process, + that resource is returned with its original device. """ - cdef DeviceMemoryResource mr = ( - _ipc.MP_from_allocation_handle(cls, alloc_handle)) from .._device import Device - mr._dev_id = Device(device_id).device_id - return mr + dev_id = Device(device_id).device_id + return ( + _ipc.MP_from_allocation_handle(cls, alloc_handle, _DMR_init_imported, dev_id)) @property def allocation_handle(self) -> IPCAllocationHandle: @@ -312,6 +313,13 @@ cdef inline _DMR_init(DeviceMemoryResource self, device_id, options): ) +cdef int _DMR_init_imported(_MemPool mr, object dev_id) except? -1: + # Runs before the resource is registered: a concurrent import of the same + # UUID returns it from the registry and must not write to it. + (mr)._dev_id = dev_id + return 0 + + # Note: this is referenced in instructions to debug nvbug 5698116. cpdef str DMR_mempool_get_access(DeviceMemoryResource dmr, int device_id): """ diff --git a/cuda_core/cuda/core/_memory/_ipc.pxd b/cuda_core/cuda/core/_memory/_ipc.pxd index ddc28ce52be..5fddd4a4562 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pxd +++ b/cuda_core/cuda/core/_memory/_ipc.pxd @@ -79,10 +79,11 @@ cdef Buffer Buffer_from_ipc_descriptor(cls, object, IPCBufferDescriptor, stream) # _MemPool IPC Implementation # --------------------------- -# Initializes a newly imported pool before MP_from_allocation_handle registers it. -ctypedef int (*MP_import_init_t)(_MemPool) except? -1 +# Initializes a newly imported pool before MP_from_allocation_handle registers +# it. The second argument is the init_arg passed to MP_from_allocation_handle. +ctypedef int (*MP_import_init_t)(_MemPool, object) except? -1 -cdef _MemPool MP_from_allocation_handle(cls, alloc_handle, MP_import_init_t init=*) +cdef _MemPool MP_from_allocation_handle(cls, alloc_handle, MP_import_init_t init=*, object init_arg=*) cdef _MemPool MP_from_registry(uuid) cdef _MemPool MP_register(_MemPool, uuid) cdef IPCAllocationHandle MP_export_mempool(_MemPool) diff --git a/cuda_core/cuda/core/_memory/_ipc.pyi b/cuda_core/cuda/core/_memory/_ipc.pyi index ff871073e62..ebeb839b058 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pyi +++ b/cuda_core/cuda/core/_memory/_ipc.pyi @@ -8,7 +8,7 @@ from cuda.core.typing import VirtualMemoryHandleType from typing_extensions import TypeAlias __all__ = [] -MP_import_init_t: TypeAlias = Callable[[_MemPool], int] +MP_import_init_t: TypeAlias = Callable[[_MemPool, object], int] class IPCDataForBuffer: """Data members related to sharing memory buffers via IPC.""" diff --git a/cuda_core/cuda/core/_memory/_ipc.pyx b/cuda_core/cuda/core/_memory/_ipc.pyx index 1d70193c321..3cf08c7692b 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pyx +++ b/cuda_core/cuda/core/_memory/_ipc.pyx @@ -547,7 +547,8 @@ cdef inline _check_registered_type(mr, cls, uuid): f"{uuid}, but {cls.__name__} was requested") -cdef _MemPool MP_from_allocation_handle(cls, alloc_handle, MP_import_init_t init=NULL): +cdef _MemPool MP_from_allocation_handle( + cls, alloc_handle, MP_import_init_t init=NULL, object init_arg=None): if isinstance(alloc_handle, IPCAllocationHandle): IPCAllocationHandle_check_open(alloc_handle) @@ -585,9 +586,10 @@ cdef _MemPool MP_from_allocation_handle(cls, alloc_handle, MP_import_init_t init self._ipc_data = IPCDataForMR(alloc_handle, True) # Initialize before registering: a registry hit in another thread must not - # return a pool whose initialization is in progress or has failed. + # return a pool whose initialization is in progress or has failed. A + # resource returned from the registry is never initialized again. if init != NULL: - init(self) + init(self, init_arg) # Register it. if uuid is not None: diff --git a/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx b/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx index 33a591dc07f..08bdc8b09cf 100644 --- a/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx +++ b/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx @@ -349,7 +349,7 @@ cdef inline int _PMR_grant_device_access(PinnedMemoryResource self, bint host_nu return 0 -cdef int _PMR_init_imported(_MemPool mr) except? -1: +cdef int _PMR_init_imported(_MemPool mr, object unused) except? -1: # IPC-enabled pinned pools are always placed on a host NUMA node. return _PMR_grant_device_access(mr, True) diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py index 47596ad66b0..64c83914513 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -22,7 +22,7 @@ from helpers.memory import import_pool_resource, is_pool_resource, pool_access_flags from cuda.bindings import driver -from cuda.core import Buffer, Device +from cuda.core import Buffer, Device, DeviceMemoryResource from cuda.core._memory._buffer import _live_owned_mempool_count NBYTES = 64 @@ -90,6 +90,62 @@ def child_close_both_imports_with_outstanding_buffer(mr_type, alloc_handle, desc assert _live_owned_mempool_count() == baseline +def _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor): + stream = device.default_stream + # The buffer holds the last reference to the pool once its resource is + # closed, so closing the buffer releases the pool. + mr = import_pool_resource(mr_type, device, int(alloc_handle)) + outstanding = Buffer.from_ipc_descriptor(mr, descriptor, stream=stream) + mr.close() + del mr + + barrier = threading.Barrier(2) + errors = [] + + def release(): + try: + barrier.wait() + outstanding.close() + except Exception as e: + errors.append(e) + + def reimport(): + try: + Device().set_current() + barrier.wait() + # A raw file descriptor carries no UUID, so this is a real import, + # through an object the releasing thread never sees. + mr = import_pool_resource(mr_type, device, int(alloc_handle)) + assert pool_access_flags(mr, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + buffer = Buffer.from_ipc_descriptor(mr, descriptor, stream=stream) + buffer.fill(7, stream=stream) + stream.sync() + buffer.close() + mr.close() + except Exception as e: + errors.append(e) + + threads = [threading.Thread(target=release), threading.Thread(target=reimport)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + stream.sync() + if errors: + raise errors[0] + + +def child_reimport_races_last_release(mr_type, alloc_handle, descriptor): + device = Device() + device.set_current() + baseline = _live_owned_mempool_count() + + with assert_no_cuda_warning(): + for _ in range(ITERATIONS): + _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor) + assert _live_owned_mempool_count() == baseline + + def _import_concurrently(mr_type, device, alloc_handle): barrier = threading.Barrier(THREADS) results = [None] * THREADS @@ -122,8 +178,10 @@ def child_concurrent_uuid_imports(mr_type, alloc_handle): for _ in range(ITERATIONS): results = _import_concurrently(mr_type, device, alloc_handle) # Every import of the UUID returns the registered resource, whichever - # thread registered it. + # thread registered it, with its device set before it was registered. assert all(mr is results[0] for mr in results) + if mr_type is DeviceMemoryResource: + assert results[0].device_id == device.device_id assert _live_owned_mempool_count() == baseline + 1 # Dropping the losing imports must not revoke the winner's access. assert pool_access_flags(results[0], device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE @@ -171,6 +229,15 @@ def test_closing_both_imports_with_an_outstanding_buffer(self, pool_resource, ex exported_buffer.ipc_descriptor, ) + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_reimport_racing_the_last_release_keeps_access(self, pool_resource, exported_buffer): + run_child( + child_reimport_races_last_release, + type(pool_resource), + pool_resource.allocation_handle, + exported_buffer.ipc_descriptor, + ) + @pytest.mark.agent_authored(model="claude-opus-5-5") def test_concurrent_imports_of_one_uuid_share_a_resource(self, ipc_device, pool_resource): ipc_device.set_current() diff --git a/cuda_core/tests/test_rt_layout.py b/cuda_core/tests/test_rt_layout.py index 649d990997c..29f6fc59da7 100644 --- a/cuda_core/tests/test_rt_layout.py +++ b/cuda_core/tests/test_rt_layout.py @@ -138,7 +138,7 @@ def test_driver_function_table_matches_the_cuda_bindings_loader(): def test_driver_calls_go_through_the_table(): """Every driver call uses DRIVER_CALL or a pw_ wrapper, which resolves the table on first use and never dereferences null. The only raw p_ calls are - the table's own machinery and the sites under ipc_import_mutex. Those sites + the table's own machinery and the sites under the IPC pointer cache's mutex. Those sites resolve the table before the lock and carry the `// raw:` mark.""" machinery = {"driver_api.hpp", "driver_api.cpp", "py_driver_fns.cpp", "internal.hpp"} raw_call = re.compile(r"\bp_(cu|nv)\w+\b") # calls and null checks alike From 8de80bbfec2717513b5c1e30d6635319e834dd67 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 20 Nov 2026 15:15:49 -0800 Subject: [PATCH 08/13] test(cuda.core): make the IPC release race deterministic with a test hook A re-import rarely lands between an owner's last reference and its cleanup, so the race test passed without the wait as well. Add _set_ipc_pool_release_delay_for_testing, which makes the owner of an imported pool sleep after its last reference is released and before it revokes access. With a 50 ms delay, the test re-imports inside the release and checks the new import after it: without the wait, device pools report failed cuMemPoolDestroy calls and pinned pools lose device access. --- cuda_core/cuda/core/_cpp/rt/api.hpp | 5 +++ cuda_core/cuda/core/_cpp/rt/memory.cpp | 14 ++++++++ cuda_core/cuda/core/_rt.pyi | 7 ++++ cuda_core/cuda/core/_rt.pyx | 12 +++++++ .../test_ipc_pool_duplicate_import.py | 34 +++++++++++++------ 5 files changed, 62 insertions(+), 10 deletions(-) diff --git a/cuda_core/cuda/core/_cpp/rt/api.hpp b/cuda_core/cuda/core/_cpp/rt/api.hpp index c2c0689edb8..97355aea4a8 100644 --- a/cuda_core/cuda/core/_cpp/rt/api.hpp +++ b/cuda_core/cuda/core/_cpp/rt/api.hpp @@ -309,6 +309,11 @@ size_t va_reservation_size(const VaReservationHandle &h) noexcept; long long live_owned_mempool_count() noexcept; long long live_va_reservation_count() noexcept; +// Test hook: the owner of an imported pool sleeps this long after its last +// reference is released and before it revokes access, so that a re-import can +// be made to race the release. 0 (the default) disables the delay. +void set_ipc_pool_release_delay_for_testing(unsigned int microseconds) noexcept; + // Map the whole allocation at ptr inside the reservation via cuMemMap and // apply the allocation's access descriptors. The mapping structurally depends // on both handles. When the last reference is released, cuMemUnmap is called diff --git a/cuda_core/cuda/core/_cpp/rt/memory.cpp b/cuda_core/cuda/core/_cpp/rt/memory.cpp index 97481b3c004..831d6baed10 100644 --- a/cuda_core/cuda/core/_cpp/rt/memory.cpp +++ b/cuda_core/cuda/core/_cpp/rt/memory.cpp @@ -9,10 +9,12 @@ #include "error.hpp" #include "internal.hpp" #include +#include #include #include #include #include +#include #include #ifndef _WIN32 #include @@ -37,8 +39,17 @@ struct MemoryPoolBox { // Read by the test suite to gate the per-test gc.collect(); atomic for // free-threaded builds. std::atomic g_live_owned_mempools{0}; + +// See set_ipc_pool_release_delay_for_testing. +std::atomic g_ipc_pool_release_delay_us{0}; } // namespace +void +set_ipc_pool_release_delay_for_testing(unsigned int microseconds) noexcept +{ + g_ipc_pool_release_delay_us.store(microseconds, std::memory_order_relaxed); +} + // Helper to clear peer access before destroying a memory pool. // Works around nvbug 5698116: recycled pool handles inherit peer access state. // The driver validates a request as a whole and rejects it if any entry cannot @@ -181,6 +192,9 @@ create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type) std::size_t imports = b->imports.load(std::memory_order_relaxed); if (imports > 0) { GILReleaseGuard gil; + if (unsigned int delay = g_ipc_pool_release_delay_us.load(std::memory_order_relaxed)) { + std::this_thread::sleep_for(std::chrono::microseconds(delay)); + } clear_mempool_peer_access(b->resource, -1); CUresult status = CUDA_SUCCESS; for (; imports > 0; --imports) { diff --git a/cuda_core/cuda/core/_rt.pyi b/cuda_core/cuda/core/_rt.pyi index fc1c7dd4c69..ad50dd4bd1e 100644 --- a/cuda_core/cuda/core/_rt.pyi +++ b/cuda_core/cuda/core/_rt.pyi @@ -57,6 +57,13 @@ def _set_context_restore_fault_for_testing(status: int): injected failure leaves the target context current, exactly as a failing ``cuCtxSetCurrent`` would, so callers must restore the context themselves. """ +def _set_ipc_pool_release_delay_for_testing(microseconds: int): + """Delay the release of an imported pool's owner by ``microseconds``. + + Test hook: once the owner's last reference is released, it sleeps before it + revokes access and destroys the pool's imports, so that a re-import can be + made to race the release. 0 disables the delay. + """ def _attach_rollback_failure_for_testing(status: int): """Attach a failed CUDA call to the exception being handled, or report it. diff --git a/cuda_core/cuda/core/_rt.pyx b/cuda_core/cuda/core/_rt.pyx index ee7db861d6e..c8f165203c3 100644 --- a/cuda_core/cuda/core/_rt.pyx +++ b/cuda_core/cuda/core/_rt.pyx @@ -194,6 +194,8 @@ cdef extern from "_cpp/rt/rt.hpp" namespace "cuda_core::rt": const VaReservationHandle& h) noexcept nogil long long live_owned_mempool_count "cuda_core::rt::live_owned_mempool_count" () noexcept nogil long long live_va_reservation_count "cuda_core::rt::live_va_reservation_count" () noexcept nogil + void set_ipc_pool_release_delay_for_testing "cuda_core::rt::set_ipc_pool_release_delay_for_testing" ( + unsigned int microseconds) noexcept nogil VaMappingHandle create_va_mapping_handle "cuda_core::rt::create_va_mapping_handle" ( cydriver.CUdeviceptr ptr, const MemAllocationHandle& h_alloc, const VaReservationHandle& h_res) except+ nogil @@ -357,6 +359,16 @@ def _set_context_restore_fault_for_testing(int status): set_context_restore_fault_for_testing(status) +def _set_ipc_pool_release_delay_for_testing(unsigned int microseconds): + """Delay the release of an imported pool's owner by ``microseconds``. + + Test hook: once the owner's last reference is released, it sleeps before it + revokes access and destroys the pool's imports, so that a re-import can be + made to race the release. 0 disables the delay. + """ + set_ipc_pool_release_delay_for_testing(microseconds) + + def _attach_rollback_failure_for_testing(int status): """Attach a failed CUDA call to the exception being handled, or report it. diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py index 64c83914513..dd1d9a5de16 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -15,6 +15,7 @@ import contextlib import multiprocessing as mp import threading +import time import pytest from helpers.child_processes import run_child @@ -24,10 +25,14 @@ from cuda.bindings import driver from cuda.core import Buffer, Device, DeviceMemoryResource from cuda.core._memory._buffer import _live_owned_mempool_count +from cuda.core._rt import _set_ipc_pool_release_delay_for_testing NBYTES = 64 THREADS = 4 ITERATIONS = 20 +# The owner's release sleeps this long in the race test, so the re-import overlaps it. +RELEASE_DELAY_US = 50_000 +RACE_ITERATIONS = 3 # these tests spawn new processes and files which fails for very many threads; # child processes and IPC fixtures are shared across workers (#2784); cap at two threads @@ -100,6 +105,7 @@ def _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor): del mr barrier = threading.Barrier(2) + reimported = [] errors = [] def release(): @@ -113,15 +119,12 @@ def reimport(): try: Device().set_current() barrier.wait() + # Start once the release is under way, inside its delay. + time.sleep(RELEASE_DELAY_US / 4 / 1e6) # A raw file descriptor carries no UUID, so this is a real import, # through an object the releasing thread never sees. mr = import_pool_resource(mr_type, device, int(alloc_handle)) - assert pool_access_flags(mr, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - buffer = Buffer.from_ipc_descriptor(mr, descriptor, stream=stream) - buffer.fill(7, stream=stream) - stream.sync() - buffer.close() - mr.close() + reimported.append((mr, Buffer.from_ipc_descriptor(mr, descriptor, stream=stream))) except Exception as e: errors.append(e) @@ -130,19 +133,30 @@ def reimport(): thread.start() for thread in threads: thread.join() - stream.sync() if errors: raise errors[0] + # The release has finished: its cleanup must have left the new import alone. + ((mr, buffer),) = reimported + assert pool_access_flags(mr, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + buffer.fill(7, stream=stream) + stream.sync() + buffer.close() + mr.close() + def child_reimport_races_last_release(mr_type, alloc_handle, descriptor): device = Device() device.set_current() baseline = _live_owned_mempool_count() - with assert_no_cuda_warning(): - for _ in range(ITERATIONS): - _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor) + _set_ipc_pool_release_delay_for_testing(RELEASE_DELAY_US) + try: + with assert_no_cuda_warning(): + for _ in range(RACE_ITERATIONS): + _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor) + finally: + _set_ipc_pool_release_delay_for_testing(0) assert _live_owned_mempool_count() == baseline From 6b11e5e237a704b3ef5253485ce443b96708ae96 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 20 Nov 2026 15:18:24 -0800 Subject: [PATCH 09/13] docs(cuda.core): cover the device pool and release race fixes in the #3510 release note --- cuda_core/docs/source/release/1.4.0-notes.rst | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/cuda_core/docs/source/release/1.4.0-notes.rst b/cuda_core/docs/source/release/1.4.0-notes.rst index bcff47653e7..ed5588a63c1 100644 --- a/cuda_core/docs/source/release/1.4.0-notes.rst +++ b/cuda_core/docs/source/release/1.4.0-notes.rst @@ -24,15 +24,18 @@ Fixes and enhancements (`#3415 `__, `#3501 `__) -- Importing the same memory pool more than once in one process, for example - by passing a raw file descriptor to - :meth:`PinnedMemoryResource.from_allocation_handle` twice, no longer breaks - the other imports when one of them is closed. The driver returns the same - pool for every import, and closing one import used to revoke device access - to the pool for all of them, so device access to a pinned pool failed with - ``CUDA_ERROR_INVALID_VALUE``. All imports of a pool now share one owner, - which revokes access and destroys the pool when the last of them is - released. Concurrent imports of the same allocation handle from several - threads now return the same memory resource instead of raising an internal - error. +- Repeat imports of a memory pool in one process, for example passing a raw + file descriptor to :meth:`PinnedMemoryResource.from_allocation_handle` + twice, now share one owner, which releases the pool with the last of them. + Closing one import no longer removes device access to a pinned pool from + the others, or emits a ``CUDAWarning`` from ``cuMemPoolDestroy`` for a + device pool while a buffer imported through another is alive. An import + that overlaps the release of the last one waits for it, and concurrent + imports of one allocation handle return the same resource instead of + raising an internal error. + (`#3510 `__) + +- :meth:`DeviceMemoryResource.from_allocation_handle` returns a resource + already registered in the process with its original ``device_id``, instead + of changing it to the requested device. (`#3510 `__) From c09896157adbef8083db66c809252f63bcab4323 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 20 Nov 2026 15:39:41 -0800 Subject: [PATCH 10/13] fix(cuda.core): let an IPC pointer import's allocation failure propagate, and address review nits The pointer-import factory was noexcept but calls make_deallocation_stream, which may throw std::bad_alloc. Drop noexcept; on a throw, discard the import under the cache's mutex and report a failed discard after it is released. Also report how many cuMemPoolDestroy calls failed when an IPC pool owner held several imports, abort the race tests' barriers when a worker fails, and use an explicit stream for the buffer freed on another thread. --- cuda_core/cuda/core/_cpp/rt/memory.cpp | 84 ++++++++++++------- .../test_ipc_pool_duplicate_import.py | 9 +- 2 files changed, 64 insertions(+), 29 deletions(-) diff --git a/cuda_core/cuda/core/_cpp/rt/memory.cpp b/cuda_core/cuda/core/_cpp/rt/memory.cpp index 831d6baed10..53653963447 100644 --- a/cuda_core/cuda/core/_cpp/rt/memory.cpp +++ b/cuda_core/cuda/core/_cpp/rt/memory.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -196,20 +197,26 @@ create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type) std::this_thread::sleep_for(std::chrono::microseconds(delay)); } clear_mempool_peer_access(b->resource, -1); - CUresult status = CUDA_SUCCESS; + const std::size_t total = imports; + std::size_t failed = 0; + CUresult first_failure = CUDA_SUCCESS; for (; imports > 0; --imports) { if (CUresult s = DRIVER_CALL(cuMemPoolDestroy, b->resource); s != CUDA_SUCCESS) { - status = s; + if (failed++ == 0) { + first_failure = s; + } } } g_live_owned_mempools.fetch_sub(1, std::memory_order_relaxed); // Wake waiting importers before reporting: a report can run // user code, which could import this pool again. ipc_pool_registry.release(b->resource, &b->resource); - if (status != CUDA_SUCCESS) { + if (failed > 0) { char operation[kOperationNameSize]; format_operation(operation, sizeof(operation), "cuMemPoolDestroy", handle_bits(b->resource)); - report_cuda_error(operation, status); + char detail[kErrorMessageSize]; + std::snprintf(detail, sizeof(detail), "failed %zu of %zu times, first with", failed, total); + report_cuda_error(operation, first_failure, total > 1 ? detail : nullptr); } } delete b; @@ -680,35 +687,56 @@ deviceptr_import_ipc(const MemoryPoolHandle &h_pool, const void *export_data, co // DESIGN.md). CUresult discard_status = CUDA_SUCCESS; CUdeviceptr discarded = 0; - DevicePtrHandle imported = ipc_ptr_cache.get_or_create(key, [&]() noexcept -> DevicePtrHandle { - CUdeviceptr ptr; - if (CUDA_SUCCESS != - (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { // raw: under the cache's mutex - return {}; + auto report_failed_discard = [&]() noexcept { + if (discard_status != CUDA_SUCCESS) { + char operation[kOperationNameSize]; + format_operation(operation, sizeof(operation), "cuMemFreeAsync", discarded); + report_cuda_error( + operation, + discard_status, + "failed while discarding an IPC import that could not record a " + "deallocation stream; the mapping leaked" + ); } - if (!make_deallocation_stream(h_stream, box->deallocation)) { + }; + DevicePtrHandle imported; + try { + imported = ipc_ptr_cache.get_or_create(key, [&]() -> DevicePtrHandle { + CUdeviceptr ptr; + if (CUDA_SUCCESS != + (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { // raw: under the cache's mutex + return {}; + } // No deallocation stream could be recorded: discard the import - // with the raw call (a pw_ report would acquire the GIL). - discard_status = p_cuMemFreeAsync(ptr, as_cu(h_stream)); // raw: under the cache's mutex - discarded = ptr; - return {}; - } - box->resource = ptr; - return h; - }); + // with the raw call (a pw_ report would acquire the GIL) before + // the mutex is released, or a concurrent import would share the + // mapping this free unmaps. + bool recorded; + try { + recorded = make_deallocation_stream(h_stream, box->deallocation); + } + catch (...) { + discard_status = p_cuMemFreeAsync(ptr, as_cu(h_stream)); // raw: under the cache's mutex + discarded = ptr; + throw; + } + if (!recorded) { + discard_status = p_cuMemFreeAsync(ptr, as_cu(h_stream)); // raw: under the cache's mutex + discarded = ptr; + return {}; + } + box->resource = ptr; + return h; + }); + } + catch (...) { + report_failed_discard(); + throw; + } if (imported) { return imported; } - if (discard_status != CUDA_SUCCESS) { - char operation[kOperationNameSize]; - format_operation(operation, sizeof(operation), "cuMemFreeAsync", discarded); - report_cuda_error( - operation, - discard_status, - "failed while discarding an IPC import that could not record a " - "deallocation stream; the mapping leaked" - ); - } + report_failed_discard(); return {}; } GILReleaseGuard gil; diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py index dd1d9a5de16..d007082bba6 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -96,7 +96,9 @@ def child_close_both_imports_with_outstanding_buffer(mr_type, alloc_handle, desc def _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor): - stream = device.default_stream + # The buffer is freed on another thread, which a per-thread default stream + # would report as a cross-thread free. + stream = device.create_stream() # The buffer holds the last reference to the pool once its resource is # closed, so closing the buffer releases the pool. mr = import_pool_resource(mr_type, device, int(alloc_handle)) @@ -114,6 +116,7 @@ def release(): outstanding.close() except Exception as e: errors.append(e) + barrier.abort() def reimport(): try: @@ -127,6 +130,7 @@ def reimport(): reimported.append((mr, Buffer.from_ipc_descriptor(mr, descriptor, stream=stream))) except Exception as e: errors.append(e) + barrier.abort() threads = [threading.Thread(target=release), threading.Thread(target=reimport)] for thread in threads: @@ -143,6 +147,8 @@ def reimport(): stream.sync() buffer.close() mr.close() + stream.sync() + stream.close() def child_reimport_races_last_release(mr_type, alloc_handle, descriptor): @@ -172,6 +178,7 @@ def importer(index): results[index] = import_pool_resource(mr_type, device, alloc_handle) except Exception as e: errors.append(e) + barrier.abort() threads = [threading.Thread(target=importer, args=(i,)) for i in range(THREADS)] for thread in threads: From 8fe6d0549f2e3ad4ee51e2c1e50d4ea12e99b72a Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 20 Nov 2026 16:21:39 -0800 Subject: [PATCH 11/13] fix(cuda.core): retry the pinned pool import grant once on CUDA_ERROR_ALREADY_MAPPED On drivers 580 and 595, re-importing a pinned pool whose previous owner has revoked access and destroyed its import, while a free of an imported pointer was not synchronized, returns the same CUmemoryPool and fails the first cuMemPoolSetAccess with CUDA_ERROR_ALREADY_MAPPED. An identical retry succeeds. Retry once, on that status only and only on the import path. --- cuda_core/cuda/core/_memory/_memory_pool.pxd | 3 ++- cuda_core/cuda/core/_memory/_memory_pool.pyx | 13 ++++++++++++- .../cuda/core/_memory/_pinned_memory_resource.pyx | 9 +++++---- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/cuda_core/cuda/core/_memory/_memory_pool.pxd b/cuda_core/cuda/core/_memory/_memory_pool.pxd index f2117b4034d..a27a6e46a40 100644 --- a/cuda_core/cuda/core/_memory/_memory_pool.pxd +++ b/cuda_core/cuda/core/_memory/_memory_pool.pxd @@ -37,7 +37,8 @@ cdef int MP_init_current_pool( cdef int MP_raise_release_threshold(_MemPool self) except? -1 cdef int MP_set_access( - _MemPool self, const cydriver.CUmemAccessDesc* desc, size_t count) except? -1 + _MemPool self, const cydriver.CUmemAccessDesc* desc, size_t count, + bint retry_already_mapped=*) except? -1 cdef inline int MP_check_open(_MemPool self) except -1: diff --git a/cuda_core/cuda/core/_memory/_memory_pool.pyx b/cuda_core/cuda/core/_memory/_memory_pool.pyx index 679e25b4f3c..398aecdae7a 100644 --- a/cuda_core/cuda/core/_memory/_memory_pool.pyx +++ b/cuda_core/cuda/core/_memory/_memory_pool.pyx @@ -462,19 +462,30 @@ cdef int MP_raise_release_threshold(_MemPool self) except? -1: cdef int MP_set_access( - _MemPool self, const cydriver.CUmemAccessDesc* desc, size_t count) except? -1: + _MemPool self, const cydriver.CUmemAccessDesc* desc, size_t count, + bint retry_already_mapped=False) except? -1: """Set device access to the pool. Runs in relaxed capture mode, like every pool attribute access (see ``_enter_relaxed_capture_mode``): the driver refuses cuMemPoolSetAccess, and invalidates the capture, while a global-mode capture is active on any thread or a thread-local one on the calling thread. + + With ``retry_already_mapped``, an identical second call follows a + CUDA_ERROR_ALREADY_MAPPED. Importing a pool that is already imported in + this process returns the same CUmemoryPool. Once its previous owner has + revoked access and destroyed its import, the driver can still hold the + pool and fail the first grant over the mapping that an unsynchronized + free left behind. The identical retry succeeds (seen on drivers 580 and + 595). Any other status, and a second failure, raise. """ cdef cydriver.CUstreamCaptureMode mode cdef cydriver.CUresult err with nogil: _enter_relaxed_capture_mode(&mode) err = cydriver.cuMemPoolSetAccess(as_cu(self._h_pool), desc, count) + if retry_already_mapped and err == cydriver.CUDA_ERROR_ALREADY_MAPPED: + err = cydriver.cuMemPoolSetAccess(as_cu(self._h_pool), desc, count) _exit_relaxed_capture_mode(err, &mode) return 0 diff --git a/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx b/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx index 08bdc8b09cf..dd1ae11ea70 100644 --- a/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx +++ b/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx @@ -325,10 +325,11 @@ cdef inline _PMR_init(PinnedMemoryResource self, options): max_size, ) - _PMR_grant_device_access(self, numa_id >= 0) + _PMR_grant_device_access(self, numa_id >= 0, False) -cdef inline int _PMR_grant_device_access(PinnedMemoryResource self, bint host_numa) except? -1: +cdef inline int _PMR_grant_device_access( + PinnedMemoryResource self, bint host_numa, bint imported) except? -1: # A pool on a host location is accessible only from the host by default # (see cuMemPoolCreate), and access is per process, so both the creating # and the importing process grant read/write access to every device that @@ -345,13 +346,13 @@ cdef inline int _PMR_grant_device_access(PinnedMemoryResource self, bint host_nu cydriver.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE, dev.device_id) access.push_back(desc) if not access.empty(): - MP_set_access(self, access.data(), access.size()) + MP_set_access(self, access.data(), access.size(), imported) return 0 cdef int _PMR_init_imported(_MemPool mr, object unused) except? -1: # IPC-enabled pinned pools are always placed on a host NUMA node. - return _PMR_grant_device_access(mr, True) + return _PMR_grant_device_access(mr, True, True) def _deep_reduce_pinned_memory_resource(mr: object) -> tuple[object, ...]: From 86b2ac8319b520287b880eae7062b4aa15ed66ce Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 20 Nov 2026 16:34:17 -0800 Subject: [PATCH 12/13] fix(cuda.core): make the IPC registry lock reentrant Inserting into the registry allocates a weak reference, which can start a garbage collection. A callback or finalizer that registers another IPC resource then ran on the thread holding the non-reentrant lock and deadlocked. Use an RLock, and add a test that registers from a GC callback. --- cuda_core/cuda/core/_memory/_ipc.pyx | 5 ++- .../test_ipc_pool_duplicate_import.py | 44 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/cuda_core/cuda/core/_memory/_ipc.pyx b/cuda_core/cuda/core/_memory/_ipc.pyx index 3cf08c7692b..199af8da6cb 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pyx +++ b/cuda_core/cuda/core/_memory/_ipc.pyx @@ -42,7 +42,10 @@ __all__ = [] cdef object registry = weakref.WeakValueDictionary() -cdef object _registry_lock = threading.Lock() # serializes MP_register +# Serializes MP_register. Reentrant: inserting into the registry allocates a +# weak reference, which can run a garbage-collection callback or finalizer that +# registers another resource on the same thread. +cdef object _registry_lock = threading.RLock() cdef cydriver.CUmemAllocationHandleType IPC_HANDLE_TYPE = \ diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py index d007082bba6..2c5769106b7 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -13,9 +13,11 @@ """ import contextlib +import gc import multiprocessing as mp import threading import time +import uuid import pytest from helpers.child_processes import run_child @@ -33,6 +35,7 @@ # The owner's release sleeps this long in the race test, so the re-import overlaps it. RELEASE_DELAY_US = 50_000 RACE_ITERATIONS = 3 +REENTRY_ITERATIONS = 20 # these tests spawn new processes and files which fails for very many threads; # child processes and IPC fixtures are shared across workers (#2784); cap at two threads @@ -166,6 +169,42 @@ def child_reimport_races_last_release(mr_type, alloc_handle, descriptor): assert _live_owned_mempool_count() == baseline +def child_register_reentered_by_gc(mr_type, alloc_handle): + device = Device() + device.set_current() + baseline = _live_owned_mempool_count() + + nested = import_pool_resource(mr_type, device, int(alloc_handle)) + nested_uuid = uuid.uuid4() + nested.register(nested_uuid) + + # Registering inserts a weak reference into the registry, and allocating + # it can start a collection. A callback that registers again then runs on + # the thread that holds the registry lock. + def register_during_collection(phase, info): + if phase == "start": + nested.register(nested_uuid) + + imports = [] + threshold = gc.get_threshold() + gc.callbacks.append(register_during_collection) + try: + gc.set_threshold(1) + for _ in range(REENTRY_ITERATIONS): + # A raw file descriptor carries no UUID, so each import registers a new one. + mr = import_pool_resource(mr_type, device, int(alloc_handle)) + imports.append(mr) + assert mr.register(uuid.uuid4()) is mr + finally: + gc.set_threshold(*threshold) + gc.callbacks.remove(register_during_collection) + + for mr in imports: + mr.close() + nested.close() + assert _live_owned_mempool_count() == baseline + + def _import_concurrently(mr_type, device, alloc_handle): barrier = threading.Barrier(THREADS) results = [None] * THREADS @@ -259,6 +298,11 @@ def test_reimport_racing_the_last_release_keeps_access(self, pool_resource, expo exported_buffer.ipc_descriptor, ) + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_registration_reentered_from_garbage_collection(self, ipc_device, pool_resource): + ipc_device.set_current() + run_child(child_register_reentered_by_gc, type(pool_resource), pool_resource.allocation_handle) + @pytest.mark.agent_authored(model="claude-opus-5-5") def test_concurrent_imports_of_one_uuid_share_a_resource(self, ipc_device, pool_resource): ipc_device.set_current() From dc522f7df002ce8f48ac62ea391e639baea5889e Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Fri, 20 Nov 2026 17:23:57 -0800 Subject: [PATCH 13/13] fix(cuda.core): grant each device separately when retrying the pinned import grant With two devices, one retry of the combined cuMemPoolSetAccess call still failed with CUDA_ERROR_ALREADY_MAPPED on driver 595. Set each entry on its own and retry each once. --- cuda_core/cuda/core/_memory/_memory_pool.pyx | 29 +++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/cuda_core/cuda/core/_memory/_memory_pool.pyx b/cuda_core/cuda/core/_memory/_memory_pool.pyx index 398aecdae7a..4257332dd36 100644 --- a/cuda_core/cuda/core/_memory/_memory_pool.pyx +++ b/cuda_core/cuda/core/_memory/_memory_pool.pyx @@ -471,21 +471,30 @@ cdef int MP_set_access( and invalidates the capture, while a global-mode capture is active on any thread or a thread-local one on the calling thread. - With ``retry_already_mapped``, an identical second call follows a - CUDA_ERROR_ALREADY_MAPPED. Importing a pool that is already imported in - this process returns the same CUmemoryPool. Once its previous owner has - revoked access and destroyed its import, the driver can still hold the - pool and fail the first grant over the mapping that an unsynchronized - free left behind. The identical retry succeeds (seen on drivers 580 and - 595). Any other status, and a second failure, raise. + With ``retry_already_mapped``, each entry is set on its own, and an + identical second call follows a CUDA_ERROR_ALREADY_MAPPED. Importing a + pool that is already imported in this process returns the same + CUmemoryPool. Once its previous owner has revoked access and destroyed its + import, the driver can still hold the pool and fail a grant over the + mapping that an unsynchronized free left behind. The identical retry + succeeds (seen on drivers 580 and 595); with several devices, one retry of + a combined call is not enough. Any other status, and a second failure for + an entry, raise. """ cdef cydriver.CUstreamCaptureMode mode - cdef cydriver.CUresult err + cdef cydriver.CUresult err = cydriver.CUDA_SUCCESS + cdef size_t i with nogil: _enter_relaxed_capture_mode(&mode) - err = cydriver.cuMemPoolSetAccess(as_cu(self._h_pool), desc, count) - if retry_already_mapped and err == cydriver.CUDA_ERROR_ALREADY_MAPPED: + if not retry_already_mapped: err = cydriver.cuMemPoolSetAccess(as_cu(self._h_pool), desc, count) + else: + for i in range(count): + err = cydriver.cuMemPoolSetAccess(as_cu(self._h_pool), desc + i, 1) + if err == cydriver.CUDA_ERROR_ALREADY_MAPPED: + err = cydriver.cuMemPoolSetAccess(as_cu(self._h_pool), desc + i, 1) + if err != cydriver.CUDA_SUCCESS: + break _exit_relaxed_capture_mode(err, &mode) return 0