From 809855ba6c41359eb14222c52bc4777de7b90617 Mon Sep 17 00:00:00 2001 From: Ralf Juengling Date: Wed, 18 Nov 2026 15:31:53 -0800 Subject: [PATCH] test(cuda.core): keep graph-pool tests off the device default pool The device default mempool and the graph memory pool each reserve about twice the device memory of virtual address space for the lifetime of the process, and neither can be capped (#2381). A test that touches both needs two such reservations at once, which does not fit in a small virtual address space with a large-memory GPU. Measured per test (one pytest process each, probing both pools at session end), only two tests touch both pools: - graph/test_graph_memory_resource.py::test_graph_alloc used DeviceMemoryResource(device), the default pool, for its output buffer. - test_object_protocols.py::test_no_cross_type_equality combined sample_buffer (Device.allocate, default pool) with sample_alloc_node. Both only need some device pool next to the graph pool. Use a pool capped at POOL_SIZE instead. The object-protocol buffer fixtures now share a capped pool fixture that is closed at teardown; on devices without memory pool support it falls back to the device's memory resource as before. --- cuda_core/tests/AGENTS.md | 12 ++ .../tests/graph/test_graph_memory_resource.py | 114 ++++++++++-------- cuda_core/tests/test_object_protocols.py | 27 ++++- 3 files changed, 96 insertions(+), 57 deletions(-) diff --git a/cuda_core/tests/AGENTS.md b/cuda_core/tests/AGENTS.md index 732ad1baad5..991b697a6a0 100644 --- a/cuda_core/tests/AGENTS.md +++ b/cuda_core/tests/AGENTS.md @@ -43,6 +43,18 @@ DeviceMemoryResource(dev, {"ipc_enabled": True}) # NEW uncapped pool, expensive Do not add `max_size` to a call that currently passes no options: that converts a free default-pool wrapper into a new pool and makes things worse. +### Exception: tests that also use the graph memory pool + +The device default pool and the graph memory pool (used by `GraphMemoryResource` +and graph allocation nodes) each reserve about twice the device memory of +virtual address space for the lifetime of the process, and neither can be +capped. A single test that touches both needs both reservations at once, which +can exceed a small address space on a large-memory GPU. In such a test, prefer +a capped owned pool (`max_size=POOL_SIZE`) over the default-pool wrapper, and +release it at the end of the test. The capped pool's reservation is small and +returned on `close()`. Examples: `test_graph_alloc` and the `sample_buffer` +fixtures in `test_object_protocols.py`. + ### Managed pools are exempt `cuMemPoolCreate` requires `CUmemPoolProps.maxSize` to be zero for managed diff --git a/cuda_core/tests/graph/test_graph_memory_resource.py b/cuda_core/tests/graph/test_graph_memory_resource.py index faf56c061a3..d9d696e478b 100644 --- a/cuda_core/tests/graph/test_graph_memory_resource.py +++ b/cuda_core/tests/graph/test_graph_memory_resource.py @@ -8,11 +8,13 @@ import pytest from helpers.buffers import compare_buffer_to_constant, make_scratch_buffer, set_buffer, thread_unsafe_on_windows +from helpers.constants import POOL_SIZE from cuda.bindings import driver from cuda.core import ( Device, DeviceMemoryResource, + DeviceMemoryResourceOptions, GraphMemoryResource, LaunchConfig, Program, @@ -101,60 +103,66 @@ def test_graph_alloc(mempool_device, mode, action): NBYTES = 64 device = mempool_device stream = device.create_stream() - dmr = DeviceMemoryResource(device) + # A capped pool rather than the device default pool: together with the + # graph memory pool, the two permanent reservations of about twice the + # device memory each can exceed a small virtual address space. + dmr = DeviceMemoryResource(device, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) gmr = GraphMemoryResource(device) - out = dmr.allocate(NBYTES, stream=stream) - - # Get kernels and define the calling sequence. - mod = _common_kernels_alloc() - set_zero = mod.get_kernel("set_zero") - add_one = mod.get_kernel("add_one") - - # Initialize out to zero. - config = LaunchConfig(grid=1, block=1) - launch(stream, config, set_zero, out, NBYTES) - - if action == "incr": - # Increments out by 3 - def apply_kernels(mr, stream, out): - buffer = mr.allocate(NBYTES, stream=stream) - buffer.copy_from(out, stream=stream) - for kernel in [add_one, add_one, add_one]: - launch(stream, config, kernel, buffer, NBYTES) - out.copy_from(buffer, stream=stream) - buffer.close() - elif action == "fill": - # Fills out with 3 - def apply_kernels(mr, stream, out): - buffer = mr.allocate(NBYTES, stream=stream) - buffer.fill(3, stream=stream) - out.copy_from(buffer, stream=stream) - buffer.close() - - # Apply kernels, with or without graph capture. - if mode == "no_graph": - # Do work without graph capture. - apply_kernels(mr=dmr, stream=stream, out=out) - stream.sync() - assert compare_buffer_to_constant(out, 3) - else: - # Capture work, then upload and launch. - gb = stream.create_graph_builder().begin_building(mode) - apply_kernels(mr=gmr, stream=gb, out=out) - graph = gb.end_building().complete() - - # First launch. - graph.upload(stream) - graph.launch(stream) - stream.sync() - assert compare_buffer_to_constant(out, 3) - - # Second launch. - if action == "incr": - graph.upload(stream) - graph.launch(stream) - stream.sync() - assert compare_buffer_to_constant(out, 6) + try: + with dmr.allocate(NBYTES, stream=stream) as out: + # Get kernels and define the calling sequence. + mod = _common_kernels_alloc() + set_zero = mod.get_kernel("set_zero") + add_one = mod.get_kernel("add_one") + + # Initialize out to zero. + config = LaunchConfig(grid=1, block=1) + launch(stream, config, set_zero, out, NBYTES) + + if action == "incr": + # Increments out by 3 + def apply_kernels(mr, stream, out): + buffer = mr.allocate(NBYTES, stream=stream) + buffer.copy_from(out, stream=stream) + for kernel in [add_one, add_one, add_one]: + launch(stream, config, kernel, buffer, NBYTES) + out.copy_from(buffer, stream=stream) + buffer.close() + elif action == "fill": + # Fills out with 3 + def apply_kernels(mr, stream, out): + buffer = mr.allocate(NBYTES, stream=stream) + buffer.fill(3, stream=stream) + out.copy_from(buffer, stream=stream) + buffer.close() + + # Apply kernels, with or without graph capture. + if mode == "no_graph": + # Do work without graph capture. + apply_kernels(mr=dmr, stream=stream, out=out) + stream.sync() + assert compare_buffer_to_constant(out, 3) + else: + # Capture work, then upload and launch. + gb = stream.create_graph_builder().begin_building(mode) + apply_kernels(mr=gmr, stream=gb, out=out) + graph = gb.end_building().complete() + + # First launch. + graph.upload(stream) + graph.launch(stream) + stream.sync() + assert compare_buffer_to_constant(out, 3) + + # Second launch. + if action == "incr": + graph.upload(stream) + graph.launch(stream) + stream.sync() + assert compare_buffer_to_constant(out, 6) + finally: + dmr.close() + device.sync() @pytest.mark.skipif(IS_WINDOWS or IS_WSL, reason="auto_free_on_launch not supported on Windows") diff --git a/cuda_core/tests/test_object_protocols.py b/cuda_core/tests/test_object_protocols.py index 1e1e410f807..e48bc15a4a6 100644 --- a/cuda_core/tests/test_object_protocols.py +++ b/cuda_core/tests/test_object_protocols.py @@ -62,9 +62,27 @@ def sample_context(sample_device): @pytest.fixture -def sample_buffer(sample_device): +def sample_memory_resource(sample_device): + """A capped device pool, or the device's resource where pools are unsupported. + + The device default pool is avoided so that samples combining a Buffer with + an AllocNode do not reserve both the default and the graph memory pool, + which together can exceed a small virtual address space. + """ + if not sample_device.properties.memory_pools_supported: + yield sample_device.memory_resource + return + mr = DeviceMemoryResource(sample_device, DeviceMemoryResourceOptions(max_size=POOL_SIZE)) + yield mr + mr.close() + sample_device.sync() + + +@pytest.fixture +def sample_buffer(sample_device, sample_memory_resource): """A sample Buffer object.""" - return sample_device.allocate(64, stream=sample_device.default_stream) + with sample_memory_resource.allocate(64, stream=sample_device.default_stream) as buffer: + yield buffer @pytest.fixture @@ -191,9 +209,10 @@ def sample_context_alt(sample_device_alt): @pytest.fixture -def sample_buffer_alt(sample_device): +def sample_buffer_alt(sample_device, sample_memory_resource): """An alternate Buffer object.""" - return sample_device.allocate(1024, stream=sample_device.default_stream) + with sample_memory_resource.allocate(1024, stream=sample_device.default_stream) as buffer: + yield buffer @pytest.fixture