diff --git a/cuda_core/tests/AGENTS.md b/cuda_core/tests/AGENTS.md index 732ad1baad5..991b697a6a0 100644 --- a/cuda_core/tests/AGENTS.md +++ b/cuda_core/tests/AGENTS.md @@ -43,6 +43,18 @@ DeviceMemoryResource(dev, {"ipc_enabled": True}) # NEW uncapped pool, expensive Do not add `max_size` to a call that currently passes no options: that converts a free default-pool wrapper into a new pool and makes things worse. +### Exception: tests that also use the graph memory pool + +The device default pool and the graph memory pool (used by `GraphMemoryResource` +and graph allocation nodes) each reserve about twice the device memory of +virtual address space for the lifetime of the process, and neither can be +capped. A single test that touches both needs both reservations at once, which +can exceed a small address space on a large-memory GPU. In such a test, prefer +a capped owned pool (`max_size=POOL_SIZE`) over the default-pool wrapper, and +release it at the end of the test. The capped pool's reservation is small and +returned on `close()`. Examples: `test_graph_alloc` and the `sample_buffer` +fixtures in `test_object_protocols.py`. + ### Managed pools are exempt `cuMemPoolCreate` requires `CUmemPoolProps.maxSize` to be zero for managed diff --git a/cuda_core/tests/graph/test_graph_memory_resource.py b/cuda_core/tests/graph/test_graph_memory_resource.py index faf56c061a3..d9d696e478b 100644 --- a/cuda_core/tests/graph/test_graph_memory_resource.py +++ b/cuda_core/tests/graph/test_graph_memory_resource.py @@ -8,11 +8,13 @@ import pytest from helpers.buffers import compare_buffer_to_constant, make_scratch_buffer, set_buffer, thread_unsafe_on_windows +from helpers.constants import POOL_SIZE from cuda.bindings import driver from cuda.core import ( Device, DeviceMemoryResource, + DeviceMemoryResourceOptions, GraphMemoryResource, LaunchConfig, Program, @@ -101,60 +103,66 @@ def test_graph_alloc(mempool_device, mode, action): NBYTES = 64 device = mempool_device stream = device.create_stream() - dmr = DeviceMemoryResource(device) + # A capped pool rather than the device default pool: together with the + # graph memory pool, the two permanent reservations of about twice the + # device memory each can exceed a small virtual address space. + dmr = DeviceMemoryResource(device, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) gmr = GraphMemoryResource(device) - out = dmr.allocate(NBYTES, stream=stream) - - # Get kernels and define the calling sequence. - mod = _common_kernels_alloc() - set_zero = mod.get_kernel("set_zero") - add_one = mod.get_kernel("add_one") - - # Initialize out to zero. - config = LaunchConfig(grid=1, block=1) - launch(stream, config, set_zero, out, NBYTES) - - if action == "incr": - # Increments out by 3 - def apply_kernels(mr, stream, out): - buffer = mr.allocate(NBYTES, stream=stream) - buffer.copy_from(out, stream=stream) - for kernel in [add_one, add_one, add_one]: - launch(stream, config, kernel, buffer, NBYTES) - out.copy_from(buffer, stream=stream) - buffer.close() - elif action == "fill": - # Fills out with 3 - def apply_kernels(mr, stream, out): - buffer = mr.allocate(NBYTES, stream=stream) - buffer.fill(3, stream=stream) - out.copy_from(buffer, stream=stream) - buffer.close() - - # Apply kernels, with or without graph capture. - if mode == "no_graph": - # Do work without graph capture. - apply_kernels(mr=dmr, stream=stream, out=out) - stream.sync() - assert compare_buffer_to_constant(out, 3) - else: - # Capture work, then upload and launch. - gb = stream.create_graph_builder().begin_building(mode) - apply_kernels(mr=gmr, stream=gb, out=out) - graph = gb.end_building().complete() - - # First launch. - graph.upload(stream) - graph.launch(stream) - stream.sync() - assert compare_buffer_to_constant(out, 3) - - # Second launch. - if action == "incr": - graph.upload(stream) - graph.launch(stream) - stream.sync() - assert compare_buffer_to_constant(out, 6) + try: + with dmr.allocate(NBYTES, stream=stream) as out: + # Get kernels and define the calling sequence. + mod = _common_kernels_alloc() + set_zero = mod.get_kernel("set_zero") + add_one = mod.get_kernel("add_one") + + # Initialize out to zero. + config = LaunchConfig(grid=1, block=1) + launch(stream, config, set_zero, out, NBYTES) + + if action == "incr": + # Increments out by 3 + def apply_kernels(mr, stream, out): + buffer = mr.allocate(NBYTES, stream=stream) + buffer.copy_from(out, stream=stream) + for kernel in [add_one, add_one, add_one]: + launch(stream, config, kernel, buffer, NBYTES) + out.copy_from(buffer, stream=stream) + buffer.close() + elif action == "fill": + # Fills out with 3 + def apply_kernels(mr, stream, out): + buffer = mr.allocate(NBYTES, stream=stream) + buffer.fill(3, stream=stream) + out.copy_from(buffer, stream=stream) + buffer.close() + + # Apply kernels, with or without graph capture. + if mode == "no_graph": + # Do work without graph capture. + apply_kernels(mr=dmr, stream=stream, out=out) + stream.sync() + assert compare_buffer_to_constant(out, 3) + else: + # Capture work, then upload and launch. + gb = stream.create_graph_builder().begin_building(mode) + apply_kernels(mr=gmr, stream=gb, out=out) + graph = gb.end_building().complete() + + # First launch. + graph.upload(stream) + graph.launch(stream) + stream.sync() + assert compare_buffer_to_constant(out, 3) + + # Second launch. + if action == "incr": + graph.upload(stream) + graph.launch(stream) + stream.sync() + assert compare_buffer_to_constant(out, 6) + finally: + dmr.close() + device.sync() @pytest.mark.skipif(IS_WINDOWS or IS_WSL, reason="auto_free_on_launch not supported on Windows") diff --git a/cuda_core/tests/test_object_protocols.py b/cuda_core/tests/test_object_protocols.py index 1e1e410f807..e48bc15a4a6 100644 --- a/cuda_core/tests/test_object_protocols.py +++ b/cuda_core/tests/test_object_protocols.py @@ -62,9 +62,27 @@ def sample_context(sample_device): @pytest.fixture -def sample_buffer(sample_device): +def sample_memory_resource(sample_device): + """A capped device pool, or the device's resource where pools are unsupported. + + The device default pool is avoided so that samples combining a Buffer with + an AllocNode do not reserve both the default and the graph memory pool, + which together can exceed a small virtual address space. + """ + if not sample_device.properties.memory_pools_supported: + yield sample_device.memory_resource + return + mr = DeviceMemoryResource(sample_device, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) + yield mr + mr.close() + sample_device.sync() + + +@pytest.fixture +def sample_buffer(sample_device, sample_memory_resource): """A sample Buffer object.""" - return sample_device.allocate(64, stream=sample_device.default_stream) + with sample_memory_resource.allocate(64, stream=sample_device.default_stream) as buffer: + yield buffer @pytest.fixture @@ -191,9 +209,10 @@ def sample_context_alt(sample_device_alt): @pytest.fixture -def sample_buffer_alt(sample_device): +def sample_buffer_alt(sample_device, sample_memory_resource): """An alternate Buffer object.""" - return sample_device.allocate(1024, stream=sample_device.default_stream) + with sample_memory_resource.allocate(1024, stream=sample_device.default_stream) as buffer: + yield buffer @pytest.fixture