diff --git a/cuda_core/cuda/core/_cpp/rt/DESIGN.md b/cuda_core/cuda/core/_cpp/rt/DESIGN.md index 1d550a4c03f..9973118cb3c 100644 --- a/cuda_core/cuda/core/_cpp/rt/DESIGN.md +++ b/cuda_core/cuda/core/_cpp/rt/DESIGN.md @@ -305,17 +305,18 @@ one-time function-table fill (`ensure_fn_table()`). Neither may run while a C++ lock is held. **The GIL is the outermost lock.** Code that holds a C++ lock (a registry's -mutex, `ipc_import_mutex`, any `std::mutex`) must not acquire or reacquire the +mutex, any `std::mutex`) must not acquire or reacquire the GIL while the lock is held: no `report_*` or `pw_*` calls, no `GILAcquireGuard`, and no `GILReleaseGuard` whose destructor runs inside the locked region. Code that needs a C++ lock and may run with the GIL held releases the GIL first (`GILReleaseGuard` before `lock_guard`). Otherwise a thread blocked on the lock while holding the GIL deadlocks with the lock holder waiting for the GIL (#2840). Collect statuses under the lock and report after it -is released, as `deviceptr_import_ipc` does: `cleanup_in_context` takes an -`after_cleanup` hook that runs once the cleanup is done and before anything that -may run user code, and the deleter passes one that unlocks its -`std::unique_lock`. The registries store `weak_ptr`s, +is released, as `deviceptr_import_ipc` does for a discarded import. A deleter +that must update shared state before user code can run does so in +`cleanup_in_context`'s `after_cleanup` hook, which runs once the cleanup is done +and before anything that may run user code; the IPC pointer deleter releases its +registry entry there. The registries store `weak_ptr`s, so erasing an entry under a registry lock never runs a deleter. ### Static Initialization and Deadlock Hazards diff --git a/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md b/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md index 003887f296c..8fb71e13a9f 100644 --- a/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md +++ b/cuda_core/cuda/core/_cpp/rt/REGISTRY_DESIGN.md @@ -33,6 +33,22 @@ Instances: `context_registry`, `stream_registry`, `event_registry`, `kernel_registry`, and, per `GraphBox`, one registry for its direct child graphs and one for its nodes. +### Imports that wait for an in-flight release + +IPC imports use `ReleasingHandleRegistry`, a variant whose entry outlives its +handle's last reference. Only the owner's deleter removes the entry, with +`release()`, after its last driver call on the resource. `get_or_create()` +waits while the entry under its key has expired, so a new owner never overlaps +the previous owner's cleanup, even when the last reference was dropped by a +dependent handle that the importing thread cannot synchronize with. + +Instances: `ipc_pool_registry` maps an imported `CUmemoryPool` to its owner, +because every import of a pool in a process returns the same `CUmemoryPool` +(#3510); the owner's cleanup revokes access and destroys the imports. +`ipc_ptr_cache` maps a `CUmemPoolPtrExportData` descriptor to its imported +pointer, because a re-import before the previous mapping is freed would share +that mapping (nvbug 5570902). + ### Reused graph handles A `GraphHandle` weak reference locks whenever any handle of its hierarchy is diff --git a/cuda_core/cuda/core/_cpp/rt/api.hpp b/cuda_core/cuda/core/_cpp/rt/api.hpp index 40642ab2f26..97355aea4a8 100644 --- a/cuda_core/cuda/core/_cpp/rt/api.hpp +++ b/cuda_core/cuda/core/_cpp/rt/api.hpp @@ -184,7 +184,10 @@ MemoryPoolHandle get_device_mempool(int device_id); // Create an owning memory pool handle from an IPC import. // The file descriptor is NOT owned by this handle (caller manages FD separately). -// When the last reference is released, cuMemPoolDestroy is called automatically. +// Repeat imports of one pool share one handle, because access settings belong to +// the pool rather than to an import (#3510). When the last reference is released, +// access is revoked and cuMemPoolDestroy is called once for every import. An +// import that races that release waits until it is done. // Returns empty handle on error (caller must check). MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type); @@ -306,6 +309,11 @@ size_t va_reservation_size(const VaReservationHandle &h) noexcept; long long live_owned_mempool_count() noexcept; long long live_va_reservation_count() noexcept; +// Test hook: the owner of an imported pool sleeps this long after its last +// reference is released and before it revokes access, so that a re-import can +// be made to race the release. 0 (the default) disables the delay. +void set_ipc_pool_release_delay_for_testing(unsigned int microseconds) noexcept; + // Map the whole allocation at ptr inside the reservation via cuMemMap and // apply the allocation's access descriptors. The mapping structurally depends // on both handles. When the last reference is released, cuMemUnmap is called diff --git a/cuda_core/cuda/core/_cpp/rt/internal.hpp b/cuda_core/cuda/core/_cpp/rt/internal.hpp index bc5e4cadade..355f4445ff6 100644 --- a/cuda_core/cuda/core/_cpp/rt/internal.hpp +++ b/cuda_core/cuda/core/_cpp/rt/internal.hpp @@ -7,6 +7,7 @@ #include "types.hpp" #include "driver_api.hpp" #include "error.hpp" +#include #include #include #include @@ -192,7 +193,6 @@ class HandleRegistry { std::scoped_lock lock(mutex_); map_.erase(key); } - Handle lookup(const Key &key) { @@ -244,4 +244,81 @@ class HandleRegistry { MapType map_; }; +// ============================================================================ +// Handle registry that waits for in-flight releases +// +// Like HandleRegistry, but an entry outlives its handle's last reference: only +// the owner's deleter removes it, by calling release() once its cleanup is +// done. get_or_create() waits while the entry under its key has expired, so a +// new owner is never created while the previous owner is still releasing the +// resource. Used where the driver hands the same resource to repeat imports +// (IPC pools and IPC pointers), so a new owner and an old owner's cleanup +// would act on one driver resource. See REGISTRY_DESIGN.md. +// ============================================================================ + +template > +class ReleasingHandleRegistry { + public: + // Return the live handle registered under key, waiting first while the + // previous owner's release is in flight. Otherwise register and return the + // handle that create() returns; an empty handle is returned unregistered. + // create() runs under the registry's mutex, so it must not acquire the GIL + // or release a handle whose deleter calls release(). + template + Handle + get_or_create(const Key &key, Factory &&create) + { + std::unique_lock lock(mutex_); + for (auto it = map_.find(key); it != map_.end(); it = map_.find(key)) { + if (Handle h = it->second.handle.lock()) { + return h; + } + released_.wait(lock); + } + // Insert before creating, so a failed insertion has nothing to undo. + auto it = map_.try_emplace(key).first; + Handle h; + try { + h = create(); + } + catch (...) { + map_.erase(it); + throw; + } + if (h) { + it->second = Entry{h, h.get()}; + } + else { + map_.erase(it); + } + return h; + } + + // Called by an owner's deleter after its last driver call on the resource: + // erase key's entry if it belongs to the handle that pointed to resource, + // and wake the importers waiting for it. + void + release(const Key &key, const typename Handle::element_type *resource) noexcept + { + { + std::scoped_lock lock(mutex_); + auto it = map_.find(key); + if (it != map_.end() && it->second.resource == resource) { + map_.erase(it); + } + } + released_.notify_all(); + } + + private: + struct Entry { + std::weak_ptr handle; + const typename Handle::element_type *resource = nullptr; + }; + + std::mutex mutex_; + std::condition_variable released_; + std::unordered_map map_; +}; + } // namespace cuda_core::rt::detail diff --git a/cuda_core/cuda/core/_cpp/rt/memory.cpp b/cuda_core/cuda/core/_cpp/rt/memory.cpp index 3cf92649dc4..53653963447 100644 --- a/cuda_core/cuda/core/_cpp/rt/memory.cpp +++ b/cuda_core/cuda/core/_cpp/rt/memory.cpp @@ -9,13 +9,13 @@ #include "error.hpp" #include "internal.hpp" #include -#include +#include #include +#include #include #include -#include #include -#include +#include #include #ifndef _WIN32 #include @@ -40,8 +40,17 @@ struct MemoryPoolBox { // Read by the test suite to gate the per-test gc.collect(); atomic for // free-threaded builds. std::atomic g_live_owned_mempools{0}; + +// See set_ipc_pool_release_delay_for_testing. +std::atomic g_ipc_pool_release_delay_us{0}; } // namespace +void +set_ipc_pool_release_delay_for_testing(unsigned int microseconds) noexcept +{ + g_ipc_pool_release_delay_us.store(microseconds, std::memory_order_relaxed); +} + // Helper to clear peer access before destroying a memory pool. // Works around nvbug 5698116: recycled pool handles inherit peer access state. // The driver validates a request as a whole and rejects it if any entry cannot @@ -136,16 +145,112 @@ get_device_mempool(int device_id) return create_mempool_handle_ref(pool); } +// Importing a pool that is already imported in this process returns the same +// CUmemoryPool and adds a driver reference: each import needs its own +// cuMemPoolDestroy, and one made while allocations imported from the pool are +// outstanding fails without dropping its reference. Access settings belong to +// the pool, not to an import, so revoking them on behalf of one import +// (nvbug 5698116) would remove access that another import relies on (#3510). +// All imports of a pool therefore share one owner, found through +// ipc_pool_registry. It counts the imports and, when its last reference is +// released, revokes access once and then destroys every import. An import that +// finds the owner's release in flight waits for it, even when the last +// reference was dropped by a buffer and not by a resource the importer could +// synchronize with: the revocation would remove access granted by a new owner, +// and a destroy fails once the new owner has imported a buffer. +namespace { +struct IpcMemoryPoolBox { + CUmemoryPool resource; + // The imports this box owns: 0 until it becomes the pool's owner, then + // one for every import of the pool. + mutable std::atomic imports{0}; +}; + +const IpcMemoryPoolBox * +get_ipc_box(const MemoryPoolHandle &h) +{ + const CUmemoryPool *p = h.get(); + return reinterpret_cast( + reinterpret_cast(p) - offsetof(IpcMemoryPoolBox, resource) + ); +} +} // namespace + +// See REGISTRY_DESIGN.md (Level 1: Driver Handle -> Resource Handle) +static ReleasingHandleRegistry ipc_pool_registry; + MemoryPoolHandle create_mempool_handle_ipc(int fd, CUmemAllocationHandleType handle_type) { + // Released before the registry can wait: the owner's deleter may need it. GILReleaseGuard gil; - CUmemoryPool pool; + // Allocate before importing: a failed allocation then has nothing to undo. + // clang-format off + auto box = std::shared_ptr( + new IpcMemoryPoolBox{}, + [](const IpcMemoryPoolBox *b) { + // Shared ownership synchronizes the increments with this load. + std::size_t imports = b->imports.load(std::memory_order_relaxed); + if (imports > 0) { + GILReleaseGuard gil; + if (unsigned int delay = g_ipc_pool_release_delay_us.load(std::memory_order_relaxed)) { + std::this_thread::sleep_for(std::chrono::microseconds(delay)); + } + clear_mempool_peer_access(b->resource, -1); + const std::size_t total = imports; + std::size_t failed = 0; + CUresult first_failure = CUDA_SUCCESS; + for (; imports > 0; --imports) { + if (CUresult s = DRIVER_CALL(cuMemPoolDestroy, b->resource); s != CUDA_SUCCESS) { + if (failed++ == 0) { + first_failure = s; + } + } + } + g_live_owned_mempools.fetch_sub(1, std::memory_order_relaxed); + // Wake waiting importers before reporting: a report can run + // user code, which could import this pool again. + ipc_pool_registry.release(b->resource, &b->resource); + if (failed > 0) { + char operation[kOperationNameSize]; + format_operation(operation, sizeof(operation), "cuMemPoolDestroy", handle_bits(b->resource)); + char detail[kErrorMessageSize]; + std::snprintf(detail, sizeof(detail), "failed %zu of %zu times, first with", failed, total); + report_cuda_error(operation, first_failure, total > 1 ? detail : nullptr); + } + } + delete b; + } + ); + // clang-format on auto *handle_ptr = reinterpret_cast(static_cast(fd)); - if (CUDA_SUCCESS != (err = DRIVER_CALL(cuMemPoolImportFromShareableHandle, &pool, handle_ptr, handle_type, 0))) { + if (CUDA_SUCCESS != + (err = DRIVER_CALL(cuMemPoolImportFromShareableHandle, &box->resource, handle_ptr, handle_type, 0))) { return {}; } - return wrap_mempool_owned(pool, -1); + + MemoryPoolHandle h(box, &box->resource); + MemoryPoolHandle owner; + try { + owner = ipc_pool_registry.get_or_create(box->resource, [&]() noexcept { + box->imports.store(1, std::memory_order_relaxed); + return h; + }); + } + catch (...) { + // The box owns no import yet, so its deleter would not drop this one. + pw_cuMemPoolDestroy(box->resource); + throw; + } + if (owner == h) { + g_live_owned_mempools.fetch_add(1, std::memory_order_relaxed); + } + else { + // An earlier import's box owns the pool; it now holds this import's + // driver reference too. + get_ipc_box(owner)->imports.fetch_add(1, std::memory_order_relaxed); + } + return owner; } // ============================================================================ @@ -521,21 +626,13 @@ struct ExportDataKeyHash { } }; -// The handle imported for one descriptor, and the box it aliases. -struct IpcCacheEntry { - std::weak_ptr handle; - const DevicePtrBox *box; -}; - } // namespace -// Guarded by ipc_import_mutex. An entry outlives its handle's last reference: -// only that handle's deleter erases it, under the mutex, together with the -// free. An importer that finds an expired entry therefore knows a release of -// the same mapping is in flight and waits on ipc_release_cv until it is done. -static std::unordered_map ipc_ptr_cache; -static std::mutex ipc_import_mutex; -static std::condition_variable ipc_release_cv; +// An importer that finds an expired entry waits until the previous handle's +// deleter has freed the mapping: a re-import before then would share that +// mapping, and the deleter's cuMemFreeAsync would unmap it under the new +// handle (nvbug 5570902). +static ReleasingHandleRegistry ipc_ptr_cache; DevicePtrHandle deviceptr_import_ipc(const MemoryPoolHandle &h_pool, const void *export_data, const StreamHandle &h_stream) @@ -543,7 +640,7 @@ deviceptr_import_ipc(const MemoryPoolHandle &h_pool, const void *export_data, co auto *data = const_cast(reinterpret_cast(export_data)); // Resolve the table before you take any lock: a fill acquires the GIL, and - // nothing under ipc_import_mutex may acquire it (#2840). The raw p_ calls + // nothing under the cache's mutex may acquire it (#2840). The raw p_ calls // below rely on this. if (!ensure_fn_table(FnTable::driver)) { report_unavailable_fn(FnTable::driver, "cuMemPoolImportPointer"); @@ -555,91 +652,91 @@ deviceptr_import_ipc(const MemoryPoolHandle &h_pool, const void *export_data, co ExportDataKey key; std::memcpy(&key.data, data, sizeof(key.data)); - // The mutex makes lookup, import and registration one step, so two - // threads cannot import the same descriptor twice. Release the GIL - // before taking it: a thread blocked on the mutex while holding the - // GIL would deadlock with a holder that needs the GIL back (#2840). - // Nothing under the mutex may acquire the GIL, so a failed discard is - // reported only after the lock is released (see DESIGN.md). + // Release the GIL before the cache's mutex is taken: a thread blocked + // on it while holding the GIL would deadlock with a holder that needs + // the GIL back (#2840). GILReleaseGuard gil; - CUresult discard_status = CUDA_SUCCESS; - CUdeviceptr discarded = 0; - { - std::unique_lock lock(ipc_import_mutex); - - for (auto it = ipc_ptr_cache.find(key); it != ipc_ptr_cache.end(); it = ipc_ptr_cache.find(key)) { - if (DevicePtrHandle h = it->second.handle.lock()) { - return h; + // Allocate before taking the mutex: a failed allocation then cannot run + // a deleter under it. The box owns nothing until it holds an import. + // clang-format off + auto box = std::shared_ptr( + new DevicePtrBox{0, DeallocationStream{}}, + [h_pool, key](DevicePtrBox *b) { + if (b->resource != 0) { + GILReleaseGuard gil; + const DeallocationStream& stream = b->deallocation; + // Release the entry once the mapping is freed, and before + // reporting: a report can run user code. + cleanup_in_context( + deallocation_context(stream), "cuMemFreeAsync", handle_bits(b->resource), + [&]() noexcept { + return DRIVER_CALL(cuMemFreeAsync, b->resource, as_cu(stream.h_stream)); + }, + [&]() noexcept { ipc_ptr_cache.release(key, &b->resource); }); } - // Expired but not erased: the last reference is gone and its - // deleter has not freed the mapping yet. A re-import now would - // share that mapping, and the deleter's cuMemFreeAsync would - // unmap it under us (nvbug 5570902). Wait for the release. - ipc_release_cv.wait(lock); - } - - CUdeviceptr ptr; - if (CUDA_SUCCESS != (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { // raw: under ipc_import_mutex - return {}; + delete b; } - - DeallocationStream ds; - if (make_deallocation_stream(h_stream, ds)) { - // clang-format off - auto box = std::shared_ptr( - new DevicePtrBox{ptr, std::move(ds)}, - [h_pool, key](DevicePtrBox *b) { - // Release the GIL first (the GIL is the outermost lock), then hold the - // mutex across unregister + free: a concurrent import that finds this - // entry expired must wait until the mapping is gone, or it re-imports - // the same allocation and the first cuMemFreeAsync unmaps it for both - // (nvbug 5570902). Nothing under the mutex may acquire the GIL, so the - // deallocation context is resolved before the lock (it may report) and - // the lock is released as soon as the cleanup is done. - GILReleaseGuard gil; - const DeallocationStream& stream = b->deallocation; - ContextHandle h_dealloc = deallocation_context(stream); - // Explicit template argument: MSVC cannot deduce it here (C2955). - std::unique_lock lock(ipc_import_mutex); - // Erase only this box's entry (importers wait while it - // exists, so no other box can own the key yet). - auto entry = ipc_ptr_cache.find(key); - if (entry != ipc_ptr_cache.end() && entry->second.box == b) { - ipc_ptr_cache.erase(entry); - } - cleanup_in_context( - h_dealloc, "cuMemFreeAsync", handle_bits(b->resource), - [&]() noexcept { - return p_cuMemFreeAsync(b->resource, as_cu(stream.h_stream)); // raw: under ipc_import_mutex - }, - [&]() noexcept { - ipc_release_cv.notify_all(); - lock.release()->unlock(); - }); - delete b; - } + ); + // clang-format on + DevicePtrHandle h(box, &box->resource); + + // The factory runs under the cache's mutex, which makes lookup, import + // and registration one step: two threads cannot import the same + // descriptor twice. Nothing under the mutex may acquire the GIL, so a + // failed discard is reported only after the lock is released (see + // DESIGN.md). + CUresult discard_status = CUDA_SUCCESS; + CUdeviceptr discarded = 0; + auto report_failed_discard = [&]() noexcept { + if (discard_status != CUDA_SUCCESS) { + char operation[kOperationNameSize]; + format_operation(operation, sizeof(operation), "cuMemFreeAsync", discarded); + report_cuda_error( + operation, + discard_status, + "failed while discarding an IPC import that could not record a " + "deallocation stream; the mapping leaked" ); - // clang-format on - DevicePtrHandle h(box, &box->resource); - ipc_ptr_cache[key] = IpcCacheEntry{h, box.get()}; - return h; } - - // No deallocation stream could be recorded: discard the import with - // the raw call (a pw_ report would acquire the GIL under the mutex). - discard_status = p_cuMemFreeAsync(ptr, as_cu(h_stream)); // raw: under ipc_import_mutex - discarded = ptr; + }; + DevicePtrHandle imported; + try { + imported = ipc_ptr_cache.get_or_create(key, [&]() -> DevicePtrHandle { + CUdeviceptr ptr; + if (CUDA_SUCCESS != + (err = p_cuMemPoolImportPointer(&ptr, *h_pool, data))) { // raw: under the cache's mutex + return {}; + } + // No deallocation stream could be recorded: discard the import + // with the raw call (a pw_ report would acquire the GIL) before + // the mutex is released, or a concurrent import would share the + // mapping this free unmaps. + bool recorded; + try { + recorded = make_deallocation_stream(h_stream, box->deallocation); + } + catch (...) { + discard_status = p_cuMemFreeAsync(ptr, as_cu(h_stream)); // raw: under the cache's mutex + discarded = ptr; + throw; + } + if (!recorded) { + discard_status = p_cuMemFreeAsync(ptr, as_cu(h_stream)); // raw: under the cache's mutex + discarded = ptr; + return {}; + } + box->resource = ptr; + return h; + }); + } + catch (...) { + report_failed_discard(); + throw; } - if (discard_status != CUDA_SUCCESS) { - char operation[kOperationNameSize]; - format_operation(operation, sizeof(operation), "cuMemFreeAsync", discarded); - report_cuda_error( - operation, - discard_status, - "failed while discarding an IPC import that could not record a " - "deallocation stream; the mapping leaked" - ); + if (imported) { + return imported; } + report_failed_discard(); return {}; } GILReleaseGuard gil; diff --git a/cuda_core/cuda/core/_memory/_device_memory_resource.pyi b/cuda_core/cuda/core/_memory/_device_memory_resource.pyi index ff16f8bef92..4dfa8cb57b9 100644 --- a/cuda_core/cuda/core/_memory/_device_memory_resource.pyi +++ b/cuda_core/cuda/core/_memory/_device_memory_resource.pyi @@ -166,7 +166,9 @@ class DeviceMemoryResource(_MemPool): Returns ------- - A new device memory resource instance with the imported handle. + A new device memory resource instance with the imported handle. If + the handle's memory resource is already registered in this process, + that resource is returned with its original device. """ @property def allocation_handle(self) -> IPCAllocationHandle: diff --git a/cuda_core/cuda/core/_memory/_device_memory_resource.pyx b/cuda_core/cuda/core/_memory/_device_memory_resource.pyx index b7df931cdac..a747c5b375d 100644 --- a/cuda_core/cuda/core/_memory/_device_memory_resource.pyx +++ b/cuda_core/cuda/core/_memory/_device_memory_resource.pyx @@ -209,13 +209,14 @@ cdef class DeviceMemoryResource(_MemPool): Returns ------- - A new device memory resource instance with the imported handle. + A new device memory resource instance with the imported handle. If + the handle's memory resource is already registered in this process, + that resource is returned with its original device. """ - cdef DeviceMemoryResource mr = ( - _ipc.MP_from_allocation_handle(cls, alloc_handle)) from .._device import Device - mr._dev_id = Device(device_id).device_id - return mr + dev_id = Device(device_id).device_id + return ( + _ipc.MP_from_allocation_handle(cls, alloc_handle, _DMR_init_imported, dev_id)) @property def allocation_handle(self) -> IPCAllocationHandle: @@ -312,6 +313,13 @@ cdef inline _DMR_init(DeviceMemoryResource self, device_id, options): ) +cdef int _DMR_init_imported(_MemPool mr, object dev_id) except? -1: + # Runs before the resource is registered: a concurrent import of the same + # UUID returns it from the registry and must not write to it. + (mr)._dev_id = dev_id + return 0 + + # Note: this is referenced in instructions to debug nvbug 5698116. cpdef str DMR_mempool_get_access(DeviceMemoryResource dmr, int device_id): """ diff --git a/cuda_core/cuda/core/_memory/_ipc.pxd b/cuda_core/cuda/core/_memory/_ipc.pxd index ddc28ce52be..5fddd4a4562 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pxd +++ b/cuda_core/cuda/core/_memory/_ipc.pxd @@ -79,10 +79,11 @@ cdef Buffer Buffer_from_ipc_descriptor(cls, object, IPCBufferDescriptor, stream) # _MemPool IPC Implementation # --------------------------- -# Initializes a newly imported pool before MP_from_allocation_handle registers it. -ctypedef int (*MP_import_init_t)(_MemPool) except? -1 +# Initializes a newly imported pool before MP_from_allocation_handle registers +# it. The second argument is the init_arg passed to MP_from_allocation_handle. +ctypedef int (*MP_import_init_t)(_MemPool, object) except? -1 -cdef _MemPool MP_from_allocation_handle(cls, alloc_handle, MP_import_init_t init=*) +cdef _MemPool MP_from_allocation_handle(cls, alloc_handle, MP_import_init_t init=*, object init_arg=*) cdef _MemPool MP_from_registry(uuid) cdef _MemPool MP_register(_MemPool, uuid) cdef IPCAllocationHandle MP_export_mempool(_MemPool) diff --git a/cuda_core/cuda/core/_memory/_ipc.pyi b/cuda_core/cuda/core/_memory/_ipc.pyi index ff871073e62..ebeb839b058 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pyi +++ b/cuda_core/cuda/core/_memory/_ipc.pyi @@ -8,7 +8,7 @@ from cuda.core.typing import VirtualMemoryHandleType from typing_extensions import TypeAlias __all__ = [] -MP_import_init_t: TypeAlias = Callable[[_MemPool], int] +MP_import_init_t: TypeAlias = Callable[[_MemPool, object], int] class IPCDataForBuffer: """Data members related to sharing memory buffers via IPC.""" diff --git a/cuda_core/cuda/core/_memory/_ipc.pyx b/cuda_core/cuda/core/_memory/_ipc.pyx index b75160d59d8..199af8da6cb 100644 --- a/cuda_core/cuda/core/_memory/_ipc.pyx +++ b/cuda_core/cuda/core/_memory/_ipc.pyx @@ -30,6 +30,7 @@ import multiprocessing import operator import os import platform +import threading import uuid import weakref from typing import TYPE_CHECKING, Iterable @@ -41,6 +42,10 @@ __all__ = [] cdef object registry = weakref.WeakValueDictionary() +# Serializes MP_register. Reentrant: inserting into the registry allocates a +# weak reference, which can run a garbage-collection callback or finalizer that +# registers another resource on the same thread. +cdef object _registry_lock = threading.RLock() cdef cydriver.CUmemAllocationHandleType IPC_HANDLE_TYPE = \ @@ -538,7 +543,15 @@ cdef Buffer Buffer_from_ipc_descriptor( # _MemPool IPC Implementation # --------------------------- -cdef _MemPool MP_from_allocation_handle(cls, alloc_handle, MP_import_init_t init=NULL): +cdef inline _check_registered_type(mr, cls, uuid): + if not isinstance(mr, cls): + raise TypeError( + f"Registry contains a {type(mr).__name__} for uuid " + f"{uuid}, but {cls.__name__} was requested") + + +cdef _MemPool MP_from_allocation_handle( + cls, alloc_handle, MP_import_init_t init=NULL, object init_arg=None): if isinstance(alloc_handle, IPCAllocationHandle): IPCAllocationHandle_check_open(alloc_handle) @@ -546,10 +559,7 @@ cdef _MemPool MP_from_allocation_handle(cls, alloc_handle, MP_import_init_t init uuid = getattr(alloc_handle, 'uuid', None) # no-cython-lint mr = registry.get(uuid) if mr is not None: - if not isinstance(mr, cls): - raise TypeError( - f"Registry contains a {type(mr).__name__} for uuid " - f"{uuid}, but {cls.__name__} was requested") + _check_registered_type(mr, cls, uuid) MP_check_open(<_MemPool>mr) return mr @@ -579,18 +589,20 @@ cdef _MemPool MP_from_allocation_handle(cls, alloc_handle, MP_import_init_t init self._ipc_data = IPCDataForMR(alloc_handle, True) # Initialize before registering: a registry hit in another thread must not - # return a pool whose initialization is in progress or has failed. + # return a pool whose initialization is in progress or has failed. A + # resource returned from the registry is never initialized again. if init != NULL: - init(self) + init(self, init_arg) # Register it. if uuid is not None: registered = self.register(uuid) if registered is not self: - raise RuntimeError( - "internal cuda.core error, please report: register() returned a " - "different memory resource than the one being registered" - ) + # A concurrent import of the same UUID registered first. Both + # imports share one pool handle, so dropping this one only drops a + # reference. + _check_registered_type(registered, cls, uuid) + return registered return self @@ -607,18 +619,21 @@ cdef _MemPool MP_from_registry(uuid): cdef _MemPool MP_register(_MemPool self, uuid): MP_check_open(self) - existing = registry.get(uuid) - if existing is not None: - MP_check_open(<_MemPool>existing) - return existing - if not self.is_ipc_enabled: - raise RuntimeError("Memory resource is not IPC-enabled") - if self.uuid is not None and self.uuid != uuid: - raise ValueError( - f"Cannot register memory resource with UUID {uuid}: " - f"the resource already has UUID {self.uuid}" - ) - registry[uuid] = self + # The lookup and the insertion must be one step: without the GIL, two + # threads can both miss and register different resources for one UUID. + with _registry_lock: + existing = registry.get(uuid) + if existing is not None: + MP_check_open(<_MemPool>existing) + return existing + if not self.is_ipc_enabled: + raise RuntimeError("Memory resource is not IPC-enabled") + if self.uuid is not None and self.uuid != uuid: + raise ValueError( + f"Cannot register memory resource with UUID {uuid}: " + f"the resource already has UUID {self.uuid}" + ) + registry[uuid] = self self._ipc_data._alloc_handle._uuid = uuid return self diff --git a/cuda_core/cuda/core/_memory/_memory_pool.pxd b/cuda_core/cuda/core/_memory/_memory_pool.pxd index f2117b4034d..a27a6e46a40 100644 --- a/cuda_core/cuda/core/_memory/_memory_pool.pxd +++ b/cuda_core/cuda/core/_memory/_memory_pool.pxd @@ -37,7 +37,8 @@ cdef int MP_init_current_pool( cdef int MP_raise_release_threshold(_MemPool self) except? -1 cdef int MP_set_access( - _MemPool self, const cydriver.CUmemAccessDesc* desc, size_t count) except? -1 + _MemPool self, const cydriver.CUmemAccessDesc* desc, size_t count, + bint retry_already_mapped=*) except? -1 cdef inline int MP_check_open(_MemPool self) except -1: diff --git a/cuda_core/cuda/core/_memory/_memory_pool.pyx b/cuda_core/cuda/core/_memory/_memory_pool.pyx index 679e25b4f3c..4257332dd36 100644 --- a/cuda_core/cuda/core/_memory/_memory_pool.pyx +++ b/cuda_core/cuda/core/_memory/_memory_pool.pyx @@ -462,19 +462,39 @@ cdef int MP_raise_release_threshold(_MemPool self) except? -1: cdef int MP_set_access( - _MemPool self, const cydriver.CUmemAccessDesc* desc, size_t count) except? -1: + _MemPool self, const cydriver.CUmemAccessDesc* desc, size_t count, + bint retry_already_mapped=False) except? -1: """Set device access to the pool. Runs in relaxed capture mode, like every pool attribute access (see ``_enter_relaxed_capture_mode``): the driver refuses cuMemPoolSetAccess, and invalidates the capture, while a global-mode capture is active on any thread or a thread-local one on the calling thread. + + With ``retry_already_mapped``, each entry is set on its own, and an + identical second call follows a CUDA_ERROR_ALREADY_MAPPED. Importing a + pool that is already imported in this process returns the same + CUmemoryPool. Once its previous owner has revoked access and destroyed its + import, the driver can still hold the pool and fail a grant over the + mapping that an unsynchronized free left behind. The identical retry + succeeds (seen on drivers 580 and 595); with several devices, one retry of + a combined call is not enough. Any other status, and a second failure for + an entry, raise. """ cdef cydriver.CUstreamCaptureMode mode - cdef cydriver.CUresult err + cdef cydriver.CUresult err = cydriver.CUDA_SUCCESS + cdef size_t i with nogil: _enter_relaxed_capture_mode(&mode) - err = cydriver.cuMemPoolSetAccess(as_cu(self._h_pool), desc, count) + if not retry_already_mapped: + err = cydriver.cuMemPoolSetAccess(as_cu(self._h_pool), desc, count) + else: + for i in range(count): + err = cydriver.cuMemPoolSetAccess(as_cu(self._h_pool), desc + i, 1) + if err == cydriver.CUDA_ERROR_ALREADY_MAPPED: + err = cydriver.cuMemPoolSetAccess(as_cu(self._h_pool), desc + i, 1) + if err != cydriver.CUDA_SUCCESS: + break _exit_relaxed_capture_mode(err, &mode) return 0 diff --git a/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx b/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx index 33a591dc07f..dd1ae11ea70 100644 --- a/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx +++ b/cuda_core/cuda/core/_memory/_pinned_memory_resource.pyx @@ -325,10 +325,11 @@ cdef inline _PMR_init(PinnedMemoryResource self, options): max_size, ) - _PMR_grant_device_access(self, numa_id >= 0) + _PMR_grant_device_access(self, numa_id >= 0, False) -cdef inline int _PMR_grant_device_access(PinnedMemoryResource self, bint host_numa) except? -1: +cdef inline int _PMR_grant_device_access( + PinnedMemoryResource self, bint host_numa, bint imported) except? -1: # A pool on a host location is accessible only from the host by default # (see cuMemPoolCreate), and access is per process, so both the creating # and the importing process grant read/write access to every device that @@ -345,13 +346,13 @@ cdef inline int _PMR_grant_device_access(PinnedMemoryResource self, bint host_nu cydriver.CUmemLocationType.CU_MEM_LOCATION_TYPE_DEVICE, dev.device_id) access.push_back(desc) if not access.empty(): - MP_set_access(self, access.data(), access.size()) + MP_set_access(self, access.data(), access.size(), imported) return 0 -cdef int _PMR_init_imported(_MemPool mr) except? -1: +cdef int _PMR_init_imported(_MemPool mr, object unused) except? -1: # IPC-enabled pinned pools are always placed on a host NUMA node. - return _PMR_grant_device_access(mr, True) + return _PMR_grant_device_access(mr, True, True) def _deep_reduce_pinned_memory_resource(mr: object) -> tuple[object, ...]: diff --git a/cuda_core/cuda/core/_rt.pyi b/cuda_core/cuda/core/_rt.pyi index fc1c7dd4c69..ad50dd4bd1e 100644 --- a/cuda_core/cuda/core/_rt.pyi +++ b/cuda_core/cuda/core/_rt.pyi @@ -57,6 +57,13 @@ def _set_context_restore_fault_for_testing(status: int): injected failure leaves the target context current, exactly as a failing ``cuCtxSetCurrent`` would, so callers must restore the context themselves. """ +def _set_ipc_pool_release_delay_for_testing(microseconds: int): + """Delay the release of an imported pool's owner by ``microseconds``. + + Test hook: once the owner's last reference is released, it sleeps before it + revokes access and destroys the pool's imports, so that a re-import can be + made to race the release. 0 disables the delay. + """ def _attach_rollback_failure_for_testing(status: int): """Attach a failed CUDA call to the exception being handled, or report it. diff --git a/cuda_core/cuda/core/_rt.pyx b/cuda_core/cuda/core/_rt.pyx index ee7db861d6e..c8f165203c3 100644 --- a/cuda_core/cuda/core/_rt.pyx +++ b/cuda_core/cuda/core/_rt.pyx @@ -194,6 +194,8 @@ cdef extern from "_cpp/rt/rt.hpp" namespace "cuda_core::rt": const VaReservationHandle& h) noexcept nogil long long live_owned_mempool_count "cuda_core::rt::live_owned_mempool_count" () noexcept nogil long long live_va_reservation_count "cuda_core::rt::live_va_reservation_count" () noexcept nogil + void set_ipc_pool_release_delay_for_testing "cuda_core::rt::set_ipc_pool_release_delay_for_testing" ( + unsigned int microseconds) noexcept nogil VaMappingHandle create_va_mapping_handle "cuda_core::rt::create_va_mapping_handle" ( cydriver.CUdeviceptr ptr, const MemAllocationHandle& h_alloc, const VaReservationHandle& h_res) except+ nogil @@ -357,6 +359,16 @@ def _set_context_restore_fault_for_testing(int status): set_context_restore_fault_for_testing(status) +def _set_ipc_pool_release_delay_for_testing(unsigned int microseconds): + """Delay the release of an imported pool's owner by ``microseconds``. + + Test hook: once the owner's last reference is released, it sleeps before it + revokes access and destroys the pool's imports, so that a re-import can be + made to race the release. 0 disables the delay. + """ + set_ipc_pool_release_delay_for_testing(microseconds) + + def _attach_rollback_failure_for_testing(int status): """Attach a failed CUDA call to the exception being handled, or report it. diff --git a/cuda_core/docs/source/release/1.4.0-notes.rst b/cuda_core/docs/source/release/1.4.0-notes.rst index 338e14d49e3..ed5588a63c1 100644 --- a/cuda_core/docs/source/release/1.4.0-notes.rst +++ b/cuda_core/docs/source/release/1.4.0-notes.rst @@ -23,3 +23,19 @@ Fixes and enhancements supporting device. (`#3415 `__, `#3501 `__) + +- Repeat imports of a memory pool in one process, for example passing a raw + file descriptor to :meth:`PinnedMemoryResource.from_allocation_handle` + twice, now share one owner, which releases the pool with the last of them. + Closing one import no longer removes device access to a pinned pool from + the others, or emits a ``CUDAWarning`` from ``cuMemPoolDestroy`` for a + device pool while a buffer imported through another is alive. An import + that overlaps the release of the last one waits for it, and concurrent + imports of one allocation handle return the same resource instead of + raising an internal error. + (`#3510 `__) + +- :meth:`DeviceMemoryResource.from_allocation_handle` returns a resource + already registered in the process with its original ``device_id``, instead + of changing it to the requested device. + (`#3510 `__) diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py new file mode 100644 index 00000000000..2c5769106b7 --- /dev/null +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -0,0 +1,309 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Imports of one memory pool in one process share ownership (#3510). + +The driver returns the same ``CUmemoryPool`` for every import of a pool in a +process, and the pool's access settings are shared by all of them. Releasing +one import used to revoke device access for the pool (the nvbug 5698116 +workaround), which broke the imports that were still open; for a pinned pool +the next device access failed with ``CUDA_ERROR_INVALID_VALUE``. All imports +now share one owner, which revokes access and destroys the pool only when the +last of them is released. +""" + +import contextlib +import gc +import multiprocessing as mp +import threading +import time +import uuid + +import pytest +from helpers.child_processes import run_child +from helpers.contexts import assert_no_cuda_warning +from helpers.memory import import_pool_resource, is_pool_resource, pool_access_flags + +from cuda.bindings import driver +from cuda.core import Buffer, Device, DeviceMemoryResource +from cuda.core._memory._buffer import _live_owned_mempool_count +from cuda.core._rt import _set_ipc_pool_release_delay_for_testing + +NBYTES = 64 +THREADS = 4 +ITERATIONS = 20 +# The owner's release sleeps this long in the race test, so the re-import overlaps it. +RELEASE_DELAY_US = 50_000 +RACE_ITERATIONS = 3 +REENTRY_ITERATIONS = 20 + +# these tests spawn new processes and files which fails for very many threads; +# child processes and IPC fixtures are shared across workers (#2784); cap at two threads +pytestmark = pytest.mark.parallel_threads_limit(2) + + +def _import_twice(mr_type, device, alloc_handle): + # A raw file descriptor carries no UUID, so the registry cannot dedupe. + first = import_pool_resource(mr_type, device, int(alloc_handle)) + second = import_pool_resource(mr_type, device, int(alloc_handle)) + assert first is not second + assert int(first.handle) == int(second.handle) + return first, second + + +def child_close_one_import(mr_type, alloc_handle, descriptor): + device = Device() + device.set_current() + stream = device.default_stream + baseline = _live_owned_mempool_count() + + first, second = _import_twice(mr_type, device, alloc_handle) + assert _live_owned_mempool_count() == baseline + 1 + + first.close() + assert pool_access_flags(second, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + + buffer = Buffer.from_ipc_descriptor(second, descriptor, stream=stream) + buffer.fill(7, stream=stream) + stream.sync() + buffer.close() + second.close() + assert _live_owned_mempool_count() == baseline + + +def child_close_both_imports_with_outstanding_buffer(mr_type, alloc_handle, descriptor): + device = Device() + device.set_current() + stream = device.default_stream + baseline = _live_owned_mempool_count() + + with assert_no_cuda_warning(): + first, second = _import_twice(mr_type, device, alloc_handle) + assert _live_owned_mempool_count() == baseline + 1 + # The buffer keeps the pool alive after both resources are closed. + outstanding = Buffer.from_ipc_descriptor(first, descriptor, stream=stream) + first.close() + assert pool_access_flags(second, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + buffer = Buffer.from_ipc_descriptor(second, descriptor, stream=stream) + buffer.fill(7, stream=stream) + stream.sync() + buffer.close() + second.close() + assert _live_owned_mempool_count() == baseline + 1 + + outstanding.fill(9, stream=stream) + stream.sync() + outstanding.close() + stream.sync() + assert _live_owned_mempool_count() == baseline + + +def _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor): + # The buffer is freed on another thread, which a per-thread default stream + # would report as a cross-thread free. + stream = device.create_stream() + # The buffer holds the last reference to the pool once its resource is + # closed, so closing the buffer releases the pool. + mr = import_pool_resource(mr_type, device, int(alloc_handle)) + outstanding = Buffer.from_ipc_descriptor(mr, descriptor, stream=stream) + mr.close() + del mr + + barrier = threading.Barrier(2) + reimported = [] + errors = [] + + def release(): + try: + barrier.wait() + outstanding.close() + except Exception as e: + errors.append(e) + barrier.abort() + + def reimport(): + try: + Device().set_current() + barrier.wait() + # Start once the release is under way, inside its delay. + time.sleep(RELEASE_DELAY_US / 4 / 1e6) + # A raw file descriptor carries no UUID, so this is a real import, + # through an object the releasing thread never sees. + mr = import_pool_resource(mr_type, device, int(alloc_handle)) + reimported.append((mr, Buffer.from_ipc_descriptor(mr, descriptor, stream=stream))) + except Exception as e: + errors.append(e) + barrier.abort() + + threads = [threading.Thread(target=release), threading.Thread(target=reimport)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + if errors: + raise errors[0] + + # The release has finished: its cleanup must have left the new import alone. + ((mr, buffer),) = reimported + assert pool_access_flags(mr, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + buffer.fill(7, stream=stream) + stream.sync() + buffer.close() + mr.close() + stream.sync() + stream.close() + + +def child_reimport_races_last_release(mr_type, alloc_handle, descriptor): + device = Device() + device.set_current() + baseline = _live_owned_mempool_count() + + _set_ipc_pool_release_delay_for_testing(RELEASE_DELAY_US) + try: + with assert_no_cuda_warning(): + for _ in range(RACE_ITERATIONS): + _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor) + finally: + _set_ipc_pool_release_delay_for_testing(0) + assert _live_owned_mempool_count() == baseline + + +def child_register_reentered_by_gc(mr_type, alloc_handle): + device = Device() + device.set_current() + baseline = _live_owned_mempool_count() + + nested = import_pool_resource(mr_type, device, int(alloc_handle)) + nested_uuid = uuid.uuid4() + nested.register(nested_uuid) + + # Registering inserts a weak reference into the registry, and allocating + # it can start a collection. A callback that registers again then runs on + # the thread that holds the registry lock. + def register_during_collection(phase, info): + if phase == "start": + nested.register(nested_uuid) + + imports = [] + threshold = gc.get_threshold() + gc.callbacks.append(register_during_collection) + try: + gc.set_threshold(1) + for _ in range(REENTRY_ITERATIONS): + # A raw file descriptor carries no UUID, so each import registers a new one. + mr = import_pool_resource(mr_type, device, int(alloc_handle)) + imports.append(mr) + assert mr.register(uuid.uuid4()) is mr + finally: + gc.set_threshold(*threshold) + gc.callbacks.remove(register_during_collection) + + for mr in imports: + mr.close() + nested.close() + assert _live_owned_mempool_count() == baseline + + +def _import_concurrently(mr_type, device, alloc_handle): + barrier = threading.Barrier(THREADS) + results = [None] * THREADS + errors = [] + + def importer(index): + try: + Device().set_current() + barrier.wait() + results[index] = import_pool_resource(mr_type, device, alloc_handle) + except Exception as e: + errors.append(e) + barrier.abort() + + threads = [threading.Thread(target=importer, args=(i,)) for i in range(THREADS)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + if errors: + raise errors[0] + return results + + +def child_concurrent_uuid_imports(mr_type, alloc_handle): + device = Device() + device.set_current() + assert alloc_handle.uuid is not None + baseline = _live_owned_mempool_count() + + for _ in range(ITERATIONS): + results = _import_concurrently(mr_type, device, alloc_handle) + # Every import of the UUID returns the registered resource, whichever + # thread registered it, with its device set before it was registered. + assert all(mr is results[0] for mr in results) + if mr_type is DeviceMemoryResource: + assert results[0].device_id == device.device_id + assert _live_owned_mempool_count() == baseline + 1 + # Dropping the losing imports must not revoke the winner's access. + assert pool_access_flags(results[0], device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + results[0].close() + del results + assert _live_owned_mempool_count() == baseline + + +class TestIpcPoolDuplicateImport: + """Repeat imports of one pool in a process share the pool and its access settings.""" + + @pytest.fixture(autouse=True) + def _set_start_method(self): + # Ensure spawn is used for multiprocessing + with contextlib.suppress(RuntimeError): + mp.set_start_method("spawn", force=True) + + @pytest.fixture + def pool_resource(self, ipc_memory_resource): + if not is_pool_resource(ipc_memory_resource): + pytest.skip("a VirtualMemoryResource has no memory pool to import") + return ipc_memory_resource + + @pytest.fixture + def exported_buffer(self, ipc_device, pool_resource): + ipc_device.set_current() + stream = ipc_device.default_stream + buffer = pool_resource.allocate(NBYTES, stream=stream) + stream.sync() + yield buffer + buffer.close(stream) + + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_closing_one_import_keeps_the_other_usable(self, pool_resource, exported_buffer): + run_child( + child_close_one_import, type(pool_resource), pool_resource.allocation_handle, exported_buffer.ipc_descriptor + ) + + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_closing_both_imports_with_an_outstanding_buffer(self, pool_resource, exported_buffer): + run_child( + child_close_both_imports_with_outstanding_buffer, + type(pool_resource), + pool_resource.allocation_handle, + exported_buffer.ipc_descriptor, + ) + + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_reimport_racing_the_last_release_keeps_access(self, pool_resource, exported_buffer): + run_child( + child_reimport_races_last_release, + type(pool_resource), + pool_resource.allocation_handle, + exported_buffer.ipc_descriptor, + ) + + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_registration_reentered_from_garbage_collection(self, ipc_device, pool_resource): + ipc_device.set_current() + run_child(child_register_reentered_by_gc, type(pool_resource), pool_resource.allocation_handle) + + @pytest.mark.agent_authored(model="claude-opus-5-5") + def test_concurrent_imports_of_one_uuid_share_a_resource(self, ipc_device, pool_resource): + ipc_device.set_current() + run_child(child_concurrent_uuid_imports, type(pool_resource), pool_resource.allocation_handle) diff --git a/cuda_core/tests/test_rt_layout.py b/cuda_core/tests/test_rt_layout.py index 649d990997c..29f6fc59da7 100644 --- a/cuda_core/tests/test_rt_layout.py +++ b/cuda_core/tests/test_rt_layout.py @@ -138,7 +138,7 @@ def test_driver_function_table_matches_the_cuda_bindings_loader(): def test_driver_calls_go_through_the_table(): """Every driver call uses DRIVER_CALL or a pw_ wrapper, which resolves the table on first use and never dereferences null. The only raw p_ calls are - the table's own machinery and the sites under ipc_import_mutex. Those sites + the table's own machinery and the sites under the IPC pointer cache's mutex. Those sites resolve the table before the lock and carry the `// raw:` mark.""" machinery = {"driver_api.hpp", "driver_api.cpp", "py_driver_fns.cpp", "internal.hpp"} raw_call = re.compile(r"\bp_(cu|nv)\w+\b") # calls and null checks alike