diff --git a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py index 2c5769106b7..2b19226b73d 100644 --- a/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py +++ b/cuda_core/tests/memory_ipc/test_ipc_pool_duplicate_import.py @@ -36,55 +36,73 @@ RELEASE_DELAY_US = 50_000 RACE_ITERATIONS = 3 REENTRY_ITERATIONS = 20 +REIMPORT_ROUNDS = 3 # these tests spawn new processes and files which fails for very many threads; # child processes and IPC fixtures are shared across workers (#2784); cap at two threads pytestmark = pytest.mark.parallel_threads_limit(2) -def _import_twice(mr_type, device, alloc_handle): +def _enter_import(stack, mr_type, device, alloc_handle): + """Import the pool for ``alloc_handle`` and register it to close on ``stack``.""" + return stack.enter_context(contextlib.closing(import_pool_resource(mr_type, device, alloc_handle))) + + +def _run_threads(threads): + """Start every thread, then join every thread.""" + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + +@contextlib.contextmanager +def _child_device(): + """Set the child's current device, and assert the owned-pool count returns to its baseline.""" + device = Device() + device.set_current() + baseline = _live_owned_mempool_count() + yield device, baseline + assert _live_owned_mempool_count() == baseline + + +def _import_twice(stack, mr_type, device, alloc_handle): # A raw file descriptor carries no UUID, so the registry cannot dedupe. - first = import_pool_resource(mr_type, device, int(alloc_handle)) - second = import_pool_resource(mr_type, device, int(alloc_handle)) + first = _enter_import(stack, mr_type, device, int(alloc_handle)) + second = _enter_import(stack, mr_type, device, int(alloc_handle)) assert first is not second assert int(first.handle) == int(second.handle) return first, second def child_close_one_import(mr_type, alloc_handle, descriptor): - device = Device() - device.set_current() - stream = device.default_stream - baseline = _live_owned_mempool_count() - - first, second = _import_twice(mr_type, device, alloc_handle) - assert _live_owned_mempool_count() == baseline + 1 + # The stack closes the buffer, then both resources; the first is closed + # early on purpose, and closing it again is a no-op. + with _child_device() as (device, baseline), contextlib.ExitStack() as stack: + stream = device.default_stream + first, second = _import_twice(stack, mr_type, device, alloc_handle) + assert _live_owned_mempool_count() == baseline + 1 - first.close() - assert pool_access_flags(second, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + first.close() + assert pool_access_flags(second, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - buffer = Buffer.from_ipc_descriptor(second, descriptor, stream=stream) - buffer.fill(7, stream=stream) - stream.sync() - buffer.close() - second.close() - assert _live_owned_mempool_count() == baseline + buffer = stack.enter_context(Buffer.from_ipc_descriptor(second, descriptor, stream=stream)) + buffer.fill(7, stream=stream) + stream.sync() def child_close_both_imports_with_outstanding_buffer(mr_type, alloc_handle, descriptor): - device = Device() - device.set_current() - stream = device.default_stream - baseline = _live_owned_mempool_count() - - with assert_no_cuda_warning(): - first, second = _import_twice(mr_type, device, alloc_handle) + # The closes below are the sequence under test; the stack repeats them as + # no-ops, and releases whatever a failed assertion leaves open. + with _child_device() as (device, baseline), assert_no_cuda_warning(), contextlib.ExitStack() as stack: + stream = device.default_stream + first, second = _import_twice(stack, mr_type, device, alloc_handle) assert _live_owned_mempool_count() == baseline + 1 # The buffer keeps the pool alive after both resources are closed. - outstanding = Buffer.from_ipc_descriptor(first, descriptor, stream=stream) + outstanding = stack.enter_context(Buffer.from_ipc_descriptor(first, descriptor, stream=stream)) first.close() assert pool_access_flags(second, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - buffer = Buffer.from_ipc_descriptor(second, descriptor, stream=stream) + buffer = stack.enter_context(Buffer.from_ipc_descriptor(second, descriptor, stream=stream)) buffer.fill(7, stream=stream) stream.sync() buffer.close() @@ -95,114 +113,105 @@ def child_close_both_imports_with_outstanding_buffer(mr_type, alloc_handle, desc stream.sync() outstanding.close() stream.sync() - assert _live_owned_mempool_count() == baseline def _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor): # The buffer is freed on another thread, which a per-thread default stream # would report as a cross-thread free. stream = device.create_stream() - # The buffer holds the last reference to the pool once its resource is - # closed, so closing the buffer releases the pool. - mr = import_pool_resource(mr_type, device, int(alloc_handle)) - outstanding = Buffer.from_ipc_descriptor(mr, descriptor, stream=stream) - mr.close() - del mr - - barrier = threading.Barrier(2) - reimported = [] - errors = [] - - def release(): - try: - barrier.wait() - outstanding.close() - except Exception as e: - errors.append(e) - barrier.abort() - - def reimport(): - try: - Device().set_current() - barrier.wait() - # Start once the release is under way, inside its delay. - time.sleep(RELEASE_DELAY_US / 4 / 1e6) - # A raw file descriptor carries no UUID, so this is a real import, - # through an object the releasing thread never sees. - mr = import_pool_resource(mr_type, device, int(alloc_handle)) - reimported.append((mr, Buffer.from_ipc_descriptor(mr, descriptor, stream=stream))) - except Exception as e: - errors.append(e) - barrier.abort() - - threads = [threading.Thread(target=release), threading.Thread(target=reimport)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - if errors: - raise errors[0] - - # The release has finished: its cleanup must have left the new import alone. - ((mr, buffer),) = reimported - assert pool_access_flags(mr, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - buffer.fill(7, stream=stream) - stream.sync() - buffer.close() - mr.close() - stream.sync() - stream.close() + # The stack closes in reverse order: the new buffer and resource, then the + # stream after it has synchronized the pending frees. + with contextlib.ExitStack() as stack: + stack.enter_context(contextlib.closing(stream)) + stack.callback(stream.sync) + + # The buffer holds the last reference to the pool once its resource is + # closed, so closing the buffer releases the pool. + with contextlib.closing(import_pool_resource(mr_type, device, int(alloc_handle))) as mr: + outstanding = stack.enter_context(Buffer.from_ipc_descriptor(mr, descriptor, stream=stream)) + del mr + + barrier = threading.Barrier(2) + reimported = [] + reimported_buffers = [] + errors = [] + + def release(): + try: + barrier.wait() + outstanding.close() + except Exception as e: + errors.append(e) + barrier.abort() + + def reimport(): + try: + Device().set_current() + barrier.wait() + # Start once the release is under way, inside its delay. + time.sleep(RELEASE_DELAY_US / 4 / 1e6) + # A raw file descriptor carries no UUID, so this is a real import, + # through an object the releasing thread never sees. + mr = import_pool_resource(mr_type, device, int(alloc_handle)) + reimported.append(mr) + reimported_buffers.append(Buffer.from_ipc_descriptor(mr, descriptor, stream=stream)) + except Exception as e: + errors.append(e) + barrier.abort() + + _run_threads([threading.Thread(target=release), threading.Thread(target=reimport)]) + # Own whatever the thread made before looking at its errors. + for mr in reimported: + stack.enter_context(contextlib.closing(mr)) + for buffer in reimported_buffers: + stack.enter_context(buffer) + if errors: + raise errors[0] + + # The release has finished: its cleanup must have left the new import alone. + (mr,) = reimported + (buffer,) = reimported_buffers + assert pool_access_flags(mr, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + buffer.fill(7, stream=stream) + stream.sync() def child_reimport_races_last_release(mr_type, alloc_handle, descriptor): - device = Device() - device.set_current() - baseline = _live_owned_mempool_count() - - _set_ipc_pool_release_delay_for_testing(RELEASE_DELAY_US) - try: - with assert_no_cuda_warning(): - for _ in range(RACE_ITERATIONS): - _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor) - finally: - _set_ipc_pool_release_delay_for_testing(0) - assert _live_owned_mempool_count() == baseline + with _child_device() as (device, _baseline): + _set_ipc_pool_release_delay_for_testing(RELEASE_DELAY_US) + try: + with assert_no_cuda_warning(): + for _ in range(RACE_ITERATIONS): + _race_reimport_with_last_release(mr_type, device, alloc_handle, descriptor) + finally: + _set_ipc_pool_release_delay_for_testing(0) def child_register_reentered_by_gc(mr_type, alloc_handle): - device = Device() - device.set_current() - baseline = _live_owned_mempool_count() - - nested = import_pool_resource(mr_type, device, int(alloc_handle)) - nested_uuid = uuid.uuid4() - nested.register(nested_uuid) - - # Registering inserts a weak reference into the registry, and allocating - # it can start a collection. A callback that registers again then runs on - # the thread that holds the registry lock. - def register_during_collection(phase, info): - if phase == "start": - nested.register(nested_uuid) - - imports = [] - threshold = gc.get_threshold() - gc.callbacks.append(register_during_collection) - try: - gc.set_threshold(1) - for _ in range(REENTRY_ITERATIONS): - # A raw file descriptor carries no UUID, so each import registers a new one. - mr = import_pool_resource(mr_type, device, int(alloc_handle)) - imports.append(mr) - assert mr.register(uuid.uuid4()) is mr - finally: - gc.set_threshold(*threshold) - gc.callbacks.remove(register_during_collection) - - for mr in imports: - mr.close() - nested.close() - assert _live_owned_mempool_count() == baseline + # The stack closes the imports, then the nested resource. + with _child_device() as (device, _baseline), contextlib.ExitStack() as stack: + nested = _enter_import(stack, mr_type, device, int(alloc_handle)) + nested_uuid = uuid.uuid4() + nested.register(nested_uuid) + + # Registering inserts a weak reference into the registry, and allocating + # it can start a collection. A callback that registers again then runs on + # the thread that holds the registry lock. + def register_during_collection(phase, info): + if phase == "start": + nested.register(nested_uuid) + + threshold = gc.get_threshold() + gc.callbacks.append(register_during_collection) + try: + gc.set_threshold(1) + for _ in range(REENTRY_ITERATIONS): + # A raw file descriptor carries no UUID, so each import registers a new one. + mr = _enter_import(stack, mr_type, device, int(alloc_handle)) + assert mr.register(uuid.uuid4()) is mr + finally: + gc.set_threshold(*threshold) + gc.callbacks.remove(register_during_collection) def _import_concurrently(mr_type, device, alloc_handle): @@ -219,35 +228,79 @@ def importer(index): errors.append(e) barrier.abort() - threads = [threading.Thread(target=importer, args=(i,)) for i in range(THREADS)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() + _run_threads([threading.Thread(target=importer, args=(i,)) for i in range(THREADS)]) if errors: + # The imports that succeeded would otherwise stay open. + for mr in results: + if mr is not None: + mr.close() raise errors[0] return results def child_concurrent_uuid_imports(mr_type, alloc_handle): - device = Device() - device.set_current() - assert alloc_handle.uuid is not None - baseline = _live_owned_mempool_count() + with _child_device() as (device, baseline): + assert alloc_handle.uuid is not None + for _ in range(ITERATIONS): + with contextlib.ExitStack() as stack: + results = _import_by_path(stack, "concurrent_uuid", mr_type, device, alloc_handle) + # Every import of the UUID returns the registered resource, whichever + # thread registered it, with its device set before it was registered. + assert all(mr is results[0] for mr in results) + if mr_type is DeviceMemoryResource: + assert results[0].device_id == device.device_id + assert _live_owned_mempool_count() == baseline + 1 + # Dropping the losing imports must not revoke the winner's access. + assert ( + pool_access_flags(results[0], device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + ) + del results + assert _live_owned_mempool_count() == baseline + + +def _import_by_path(stack, import_path, mr_type, device, alloc_handle): + """Import the pool as ``import_path`` names; ``stack`` closes every resource the import made.""" + if import_path == "single": + return [_enter_import(stack, mr_type, device, alloc_handle)] + if import_path == "raw_handle": + # A raw file descriptor carries no UUID, so each import is a real import. + return [_enter_import(stack, mr_type, device, int(alloc_handle)) for _ in range(THREADS)] + assert import_path == "concurrent_uuid", import_path + return [stack.enter_context(contextlib.closing(mr)) for mr in _import_concurrently(mr_type, device, alloc_handle)] + + +def _import_fill_and_release_unsynchronized(import_path, mr_type, device, alloc_handle, descriptor, stream): + # The stack closes in reverse order: the buffers, then the resources. + # Closing a buffer enqueues its cuMemFreeAsync without waiting for it, so + # the frees can still be pending when the last resource releases the pool. + with contextlib.ExitStack() as stack: + resources = _import_by_path(stack, import_path, mr_type, device, alloc_handle) + buffers = [stack.enter_context(Buffer.from_ipc_descriptor(mr, descriptor, stream=stream)) for mr in resources] + for buffer in buffers: + buffer.fill(7, stream=stream) + stream.sync() - for _ in range(ITERATIONS): - results = _import_concurrently(mr_type, device, alloc_handle) - # Every import of the UUID returns the registered resource, whichever - # thread registered it, with its device set before it was registered. - assert all(mr is results[0] for mr in results) - if mr_type is DeviceMemoryResource: - assert results[0].device_id == device.device_id - assert _live_owned_mempool_count() == baseline + 1 - # Dropping the losing imports must not revoke the winner's access. - assert pool_access_flags(results[0], device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE - results[0].close() - del results - assert _live_owned_mempool_count() == baseline + +def _reimport_and_use(import_path, mr_type, device, alloc_handle, descriptor, stream): + with contextlib.ExitStack() as stack: + resources = _import_by_path(stack, import_path, mr_type, device, alloc_handle) + for mr in resources: + assert pool_access_flags(mr, device) == driver.CUmemAccess_flags.CU_MEM_ACCESS_FLAGS_PROT_READWRITE + buffer = stack.enter_context(Buffer.from_ipc_descriptor(resources[0], descriptor, stream=stream)) + buffer.fill(9, stream=stream) + stream.sync() + + +def child_reimport_after_release_with_pending_free(mr_type, alloc_handle, descriptor, import_path): + # A failed grant is also reported as a warning by the cleanup that follows it. + with _child_device() as (device, _baseline), assert_no_cuda_warning(): + stream = device.default_stream + for _ in range(REIMPORT_ROUNDS): + # The helpers keep no reference to any resource once they return, so + # the registry entry is gone and the next import is a real one. + _import_fill_and_release_unsynchronized(import_path, mr_type, device, alloc_handle, descriptor, stream) + _reimport_and_use(import_path, mr_type, device, alloc_handle, descriptor, stream) + stream.sync() class TestIpcPoolDuplicateImport: @@ -307,3 +360,17 @@ def test_registration_reentered_from_garbage_collection(self, ipc_device, pool_r def test_concurrent_imports_of_one_uuid_share_a_resource(self, ipc_device, pool_resource): ipc_device.set_current() run_child(child_concurrent_uuid_imports, type(pool_resource), pool_resource.allocation_handle) + + # Releasing the last import of a pool with unsynchronized frees pending + # leaves the driver holding the pool (#3514); the next import must still + # grant device access. + @pytest.mark.parametrize("import_path", ["single", "raw_handle", "concurrent_uuid"]) + @pytest.mark.agent_authored(model="claude-sonnet-5-5") + def test_reimport_after_release_with_pending_free(self, pool_resource, exported_buffer, import_path): + run_child( + child_reimport_after_release_with_pending_free, + type(pool_resource), + pool_resource.allocation_handle, + exported_buffer.ipc_descriptor, + import_path, + )