From 091602d83137e84583644480461f08555096770d Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 13 Nov 2026 16:09:11 -0500 Subject: [PATCH 1/3] Add GraphDefinition.clone() --- .../cuda/core/graph/_graph_definition.pyi | 39 +++++++ .../cuda/core/graph/_graph_definition.pyx | 78 ++++++++++++- cuda_core/docs/source/api.rst | 21 ++++ cuda_core/docs/source/release/1.3.0-notes.rst | 6 + .../graph/test_graph_definition_lifetime.py | 110 ++++++++++++++++++ 5 files changed, 253 insertions(+), 1 deletion(-) diff --git a/cuda_core/cuda/core/graph/_graph_definition.pyi b/cuda_core/cuda/core/graph/_graph_definition.pyi index 9af0b154826..7c350613734 100644 --- a/cuda_core/cuda/core/graph/_graph_definition.pyi +++ b/cuda_core/cuda/core/graph/_graph_definition.pyi @@ -192,6 +192,45 @@ class GraphDefinition: Graph An executable graph that can be launched on a stream. """ + def clone(self) -> GraphDefinition: + """Create an independent copy of this graph definition. + + Nodes, edges, parameters, and nested child graphs are copied. Resources + referenced by nodes (buffers, kernels, events, and callbacks) are shared + and independently retained; their contents are not copied. Subsequent + graph mutations do not affect the other definition or its executables. + + Returns + ------- + GraphDefinition + An independently owned graph hierarchy. Use :meth:`find_node` on + the clone to obtain the counterparts of source nodes. + + Notes + ----- + Attached resources remain alive independently for each graph and are released + at the appropriate lifetime boundary. + Driver errors propagate as :class:`~cuda.core.CUDAError`. + """ + def find_node(self, original: GraphNode) -> GraphNode: + """Find a source node's counterpart in this cloned graph. + + Parameters + ---------- + original : GraphNode + A valid node from the source graph that existed when this graph + was cloned. The corresponding cloned node must still exist. + + Returns + ------- + GraphNode + The corresponding node, with its operation-specific subclass. + + Notes + ----- + Lookup is local to this graph. + Raises :class:`~cuda.core.CUDAError` for nodes without a counterpart. + """ def debug_dot_print(self, path: str, options: GraphDebugPrintOptions | None=None) -> None: """Write a GraphViz DOT representation of the graph to a file. diff --git a/cuda_core/cuda/core/graph/_graph_definition.pyx b/cuda_core/cuda/core/graph/_graph_definition.pyx index 0f642b840b9..6ba59a5c35f 100644 --- a/cuda_core/cuda/core/graph/_graph_definition.pyx +++ b/cuda_core/cuda/core/graph/_graph_definition.pyx @@ -13,14 +13,17 @@ from libcpp.vector cimport vector from cuda.bindings cimport cydriver -from cuda.core.graph._graph_node cimport GraphNode +from cuda.core.graph._graph_node cimport GraphNode, GN_check_valid from cuda.core._rt cimport ( GraphHandle, as_cu, as_intptr, as_py, + attach_rollback_failure, create_graph_handle, create_graph_node_handle, + graph_clone_attachments, + invalidate_root_graph_state, ) from cuda.core._utils.cuda_utils cimport HANDLE_RETURN IF CUDA_CORE_BUILD_MAJOR >= 13: @@ -360,6 +363,79 @@ cdef class GraphDefinition: return _instantiate_graph(self, options) + def clone(self) -> GraphDefinition: + """Create an independent copy of this graph definition. + + Nodes, edges, parameters, and nested child graphs are copied. Resources + referenced by nodes (buffers, kernels, events, and callbacks) are shared + and independently retained; their contents are not copied. Subsequent + graph mutations do not affect the other definition or its executables. + + Returns + ------- + GraphDefinition + An independently owned graph hierarchy. Use :meth:`find_node` on + the clone to obtain the counterparts of source nodes. + + Notes + ----- + Attached resources remain alive independently for each graph and are released + at the appropriate lifetime boundary. + Driver errors propagate as :class:`~cuda.core.CUDAError`. + """ + GD_check_valid(self) + cdef GraphDefinition cloned = GraphDefinition.__new__(GraphDefinition) + cdef cydriver.CUgraph graph = NULL + cdef cydriver.CUresult result + try: + with nogil: + HANDLE_RETURN(cydriver.cuGraphClone(&graph, as_cu(self._h_graph))) + cloned._h_graph = create_graph_handle(graph) + HANDLE_RETURN(graph_clone_attachments(cloned._h_graph, self._h_graph)) + except: + if graph != NULL: + # attempt roollback + with nogil: + result = cydriver.cuGraphDestroy(graph) + if result == cydriver.CUDA_SUCCESS: + if cloned._h_graph: + invalidate_root_graph_state(cloned._h_graph) + else: + attach_rollback_failure( + b"cuGraphDestroy", result, + b"failed while rolling back a cloned graph") + raise + return cloned + + def find_node(self, GraphNode original not None: GraphNode) -> GraphNode: + """Find a source node's counterpart in this cloned graph. + + Parameters + ---------- + original : GraphNode + A valid node from the source graph that existed when this graph + was cloned. The corresponding cloned node must still exist. + + Returns + ------- + GraphNode + The corresponding node, with its operation-specific subclass. + + Notes + ----- + Lookup is local to this graph. + Raises :class:`~cuda.core.CUDAError` for nodes without a counterpart. + """ + GD_check_valid(self) + GN_check_valid(original) + if original._is_entry: + raise ValueError("The entry node has no cloned counterpart") + cdef cydriver.CUgraphNode node = NULL + with nogil: + HANDLE_RETURN(cydriver.cuGraphNodeFindInClone( + &node, as_cu(original._h_node), as_cu(self._h_graph))) + return GraphNode._create(self._h_graph, node) + def debug_dot_print(self, path: str, options: GraphDebugPrintOptions | None = None) -> None: """Write a GraphViz DOT representation of the graph to a file. diff --git a/cuda_core/docs/source/api.rst b/cuda_core/docs/source/api.rst index bfc9bf042f2..d82741e897b 100644 --- a/cuda_core/docs/source/api.rst +++ b/cuda_core/docs/source/api.rst @@ -154,6 +154,27 @@ CPU overhead. Graphs can be constructed in two ways: edges. Both produce an executable :class:`~graph.Graph` that can be launched on a :class:`Stream`. +:meth:`graph.GraphDefinition.clone` copies a definition and its nested child +graphs into an independently owned hierarchy. Node parameters and dependencies +are preserved. Buffers, kernels, events, and callback objects are shared and +retained independently; cloning does not copy their contents. Subsequent +mutations to either definition do not change the other definition or existing +executables. + +Use :meth:`graph.GraphDefinition.find_node` on the clone to reconstruct a +source node's counterpart with its operation-specific subclass:: + + clone = definition.clone() + cloned_node = clone.find_node(original_node) + cloned_child = clone.find_node(original_child_node).child_graph + cloned_inner_node = cloned_child.find_node(original_inner_node) + +Lookup is local to each graph and requires a valid source node that existed at +cloning time, and a corresponding node still present in the clone. The clone +does not retain its source hierarchy for lookup. CUDA does not support cloning +graphs containing allocation, free, or conditional nodes; driver errors +propagate. Synchronize cloning with mutations of the source hierarchy. + With CUDA >= 13.1, graph definitions and executable graphs expose a read-only integer, ``id``. Nodes expose ``local_id`` (unique within their graph) and ``tools_id`` (for profiler correlation). diff --git a/cuda_core/docs/source/release/1.3.0-notes.rst b/cuda_core/docs/source/release/1.3.0-notes.rst index 5ed06b3e373..25890406469 100644 --- a/cuda_core/docs/source/release/1.3.0-notes.rst +++ b/cuda_core/docs/source/release/1.3.0-notes.rst @@ -29,6 +29,12 @@ Highlights New features ------------ +- Added :meth:`graph.GraphDefinition.clone` and + :meth:`graph.GraphDefinition.find_node` to copy graph definitions, including + nested child graphs, and obtain corresponding nodes in the clone. Clones + retain references to underlying resources. + (`#2438 `__) + - Added read-only ``GraphDefinition.id``, ``Graph.id``, ``GraphNode.local_id`` and ``GraphNode.tools_id`` properties (CUDA 13 build and driver 13.1+), and ``KernelAttributes.device_node_update_supported`` (CUDA 13 build and driver 13.3+). diff --git a/cuda_core/tests/graph/test_graph_definition_lifetime.py b/cuda_core/tests/graph/test_graph_definition_lifetime.py index 5a3a99b65ea..dc1d0b7c049 100644 --- a/cuda_core/tests/graph/test_graph_definition_lifetime.py +++ b/cuda_core/tests/graph/test_graph_definition_lifetime.py @@ -1897,3 +1897,113 @@ def test_memcpy_buffer_and_src_owner_rejected(init_cuda): g = GraphDefinition() with pytest.raises(ValueError, match="src_owner cannot be used when src is a Buffer"): g.memcpy(buf, buf, 4, src_owner=object()) + + +@pytest.mark.parametrize("depth", [0, 2]) +@pytest.mark.parametrize("source_first", [False, True]) +def test_public_clone_retains_attachment_until_last_executable(init_cuda, depth, source_first): + """Cloned metadata releases each node's retain, including in nested graphs.""" + called = [] + + def callback(): + called.append(True) + + callback_weak = weakref.ref(callback) + source = GraphDefinition() + source.callback(callback) + for _ in range(depth): + outer = GraphDefinition() + outer.embed(source) + source = outer + del outer + clone = source.clone() + source_view, clone_view = source, clone + for _ in range(depth): + source_child = next(iter(source_view.nodes())) + clone_child = clone_view.find_node(source_child) + source_view = source_child.child_graph + clone_view = clone_child.child_graph + source_node = next(iter(source_view.nodes())) + clone_node = clone_view.find_node(source_node) + assert clone_node.callback is callback + executable = clone.instantiate() + del callback + + first, second = (source_node, clone_node) if source_first else (clone_node, source_node) + first.destroy() + assert callback_weak() is not None + assert second.callback is callback_weak() + second.destroy() + del first, second, source_node, clone_node + # Both definitions have removed the node, but the executable retains it. + assert callback_weak() is not None + stream = init_cuda.create_stream() + # callback has not fired yet + assert called == [] + executable.launch(stream) + stream.sync() + # callback fired finally + assert called == [True] + executable.close() + _wait_until(lambda: callback_weak() is None) + + +def test_public_clone_callback_update_has_independent_owners(init_cuda): + def original(): + pass + + def replacement(): + pass + + original_weak = weakref.ref(original) + replacement_weak = weakref.ref(replacement) + source = GraphDefinition() + source_node = source.callback(original) + clone = source.clone() + clone_node = clone.find_node(source_node) + clone_node.update(replacement) + del original, replacement + + assert source_node.callback is original_weak() + assert clone_node.callback is replacement_weak() + source_node.destroy() + del source_node + # original gets cleaned up + _wait_until(lambda: original_weak() is None) + assert replacement_weak() is not None + clone_node.destroy() + del clone_node + # now replacement gets cleaned up + _wait_until(lambda: replacement_weak() is None) + + +def test_public_clone_kernel_and_arguments_outlive_source(init_cuda): + """An executable retains the clone's kernel and arguments after both definitions die.""" + from cuda.core._utils.cuda_utils import driver, handle_return + + device = init_cuda + stream = device.create_stream() + with _device_memory_resource(device) as mr: + buffer = mr.allocate(ctypes.sizeof(ctypes.c_int), stream=stream) + buffer.fill(0, stream=stream) + stream.sync() + ptr = int(buffer.handle) + buffer_weak = weakref.ref(buffer) + module = compile_common_kernels() + kernel = module.get_kernel("add_one") + source = GraphDefinition() + source.launch(LaunchConfig(grid=1, block=1), kernel, buffer) + clone = source.clone() + executable = clone.instantiate() + del buffer, module, kernel, source, clone + gc.collect() + + assert buffer_weak() is not None # buffer is still referenced + executable.launch(stream) + stream.sync() + result = (ctypes.c_int * 1)(0) + # can still access + handle_return(driver.cuMemcpyDtoH(result, ptr, ctypes.sizeof(result))) + assert result[0] == 1 + executable.close() + _wait_until(lambda: buffer_weak() is None) From 6c2b93982d81ba64a1dd8cb9c4f3d371b9ba849c Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 18 Nov 2026 11:05:39 -0500 Subject: [PATCH 2/3] Add careful tests --- .../cuda/core/graph/_graph_definition.pyi | 10 ++++- .../cuda/core/graph/_graph_definition.pyx | 10 ++++- .../graph/test_graph_definition_lifetime.py | 45 +++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/cuda_core/cuda/core/graph/_graph_definition.pyi b/cuda_core/cuda/core/graph/_graph_definition.pyi index 7c350613734..dce566d09b9 100644 --- a/cuda_core/cuda/core/graph/_graph_definition.pyi +++ b/cuda_core/cuda/core/graph/_graph_definition.pyi @@ -226,10 +226,18 @@ class GraphDefinition: GraphNode The corresponding node, with its operation-specific subclass. + Raises + ------ + ValueError + If ``original`` is the synthetic entry node. + RuntimeError + If either graph definition is invalid or ``original`` was destroyed. + CUDAError + If CUDA cannot find a counterpart for a valid source node. + Notes ----- Lookup is local to this graph. - Raises :class:`~cuda.core.CUDAError` for nodes without a counterpart. """ def debug_dot_print(self, path: str, options: GraphDebugPrintOptions | None=None) -> None: """Write a GraphViz DOT representation of the graph to a file. diff --git a/cuda_core/cuda/core/graph/_graph_definition.pyx b/cuda_core/cuda/core/graph/_graph_definition.pyx index 6ba59a5c35f..a01faf81e84 100644 --- a/cuda_core/cuda/core/graph/_graph_definition.pyx +++ b/cuda_core/cuda/core/graph/_graph_definition.pyx @@ -421,10 +421,18 @@ cdef class GraphDefinition: GraphNode The corresponding node, with its operation-specific subclass. + Raises + ------ + ValueError + If ``original`` is the synthetic entry node. + RuntimeError + If either graph definition is invalid or ``original`` was destroyed. + CUDAError + If CUDA cannot find a counterpart for a valid source node. + Notes ----- Lookup is local to this graph. - Raises :class:`~cuda.core.CUDAError` for nodes without a counterpart. """ GD_check_valid(self) GN_check_valid(original) diff --git a/cuda_core/tests/graph/test_graph_definition_lifetime.py b/cuda_core/tests/graph/test_graph_definition_lifetime.py index dc1d0b7c049..0b55e0fda14 100644 --- a/cuda_core/tests/graph/test_graph_definition_lifetime.py +++ b/cuda_core/tests/graph/test_graph_definition_lifetime.py @@ -1899,6 +1899,51 @@ def test_memcpy_buffer_and_src_owner_rejected(init_cuda): g.memcpy(buf, buf, 4, src_owner=object()) +def test_public_clone_preserves_edges_and_can_change_them_independently(init_cuda): + called = [] + source = GraphDefinition() + first = source.callback(lambda: called.append("first")) + second = source.callback(lambda: called.append("second")) + first.succ.add(second) + + clone = source.clone() + cloned_first = clone.find_node(first) + cloned_second = clone.find_node(second) + assert clone.edges() == {(cloned_first, cloned_second)} + + cloned_first.succ.remove(cloned_second) + cloned_second.succ.add(cloned_first) + assert clone.edges() == {(cloned_second, cloned_first)} + assert source.edges() == {(first, second)} + + stream = init_cuda.create_stream() + for definition, expected in [(source, ["first", "second"]), (clone, ["second", "first"])]: + called.clear() + executable = definition.instantiate() + try: + executable.launch(stream) + stream.sync() + assert called == expected + finally: + executable.close() + + +def test_public_clone_find_node_rejects_invalid_sources(init_cuda): + source = GraphDefinition() + original = source.empty() + clone = source.clone() + late_node = source.empty() + + with pytest.raises(ValueError): + clone.find_node(source._entry) + with pytest.raises(CUDAError): + clone.find_node(late_node) + + original.destroy() + with pytest.raises(RuntimeError): + clone.find_node(original) + + @pytest.mark.parametrize("depth", [0, 2]) @pytest.mark.parametrize("source_first", [False, True]) def test_public_clone_retains_attachment_until_last_executable(init_cuda, depth, source_first): From 7f603e51e8bf1337b4af6ab7f40914de6faf407a Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 18 Nov 2026 14:15:33 -0500 Subject: [PATCH 3/3] Review suggestions --- .../cuda/core/graph/_graph_definition.pyi | 7 +- .../cuda/core/graph/_graph_definition.pyx | 7 +- cuda_core/docs/source/api.rst | 7 +- cuda_core/docs/source/concurrency.rst | 21 +++++- cuda_core/docs/source/release/1.3.0-notes.rst | 6 -- .../graph/test_graph_definition_lifetime.py | 66 +++++++++++-------- 6 files changed, 72 insertions(+), 42 deletions(-) diff --git a/cuda_core/cuda/core/graph/_graph_definition.pyi b/cuda_core/cuda/core/graph/_graph_definition.pyi index dce566d09b9..71a89798fad 100644 --- a/cuda_core/cuda/core/graph/_graph_definition.pyi +++ b/cuda_core/cuda/core/graph/_graph_definition.pyi @@ -208,9 +208,13 @@ class GraphDefinition: Notes ----- + CUDA rejects graphs containing memory allocation, memory free, or + conditional nodes. Attached resources remain alive independently for each graph and are released at the appropriate lifetime boundary. Driver errors propagate as :class:`~cuda.core.CUDAError`. + Serialize this call with all CUDA access to the underlying graph. + See :ref:`cuda-core-graph-thread-safety`. """ def find_node(self, original: GraphNode) -> GraphNode: """Find a source node's counterpart in this cloned graph. @@ -237,7 +241,8 @@ class GraphDefinition: Notes ----- - Lookup is local to this graph. + Lookup is local to this graph. Serialize this call with all CUDA + access to either hierarchy. See :ref:`cuda-core-graph-thread-safety`. """ def debug_dot_print(self, path: str, options: GraphDebugPrintOptions | None=None) -> None: """Write a GraphViz DOT representation of the graph to a file. diff --git a/cuda_core/cuda/core/graph/_graph_definition.pyx b/cuda_core/cuda/core/graph/_graph_definition.pyx index a01faf81e84..d21f4ebc0ad 100644 --- a/cuda_core/cuda/core/graph/_graph_definition.pyx +++ b/cuda_core/cuda/core/graph/_graph_definition.pyx @@ -379,9 +379,13 @@ cdef class GraphDefinition: Notes ----- + CUDA rejects graphs containing memory allocation, memory free, or + conditional nodes. Attached resources remain alive independently for each graph and are released at the appropriate lifetime boundary. Driver errors propagate as :class:`~cuda.core.CUDAError`. + Serialize this call with all CUDA access to the underlying graph. + See :ref:`cuda-core-graph-thread-safety`. """ GD_check_valid(self) cdef GraphDefinition cloned = GraphDefinition.__new__(GraphDefinition) @@ -432,7 +436,8 @@ cdef class GraphDefinition: Notes ----- - Lookup is local to this graph. + Lookup is local to this graph. Serialize this call with all CUDA + access to either hierarchy. See :ref:`cuda-core-graph-thread-safety`. """ GD_check_valid(self) GN_check_valid(original) diff --git a/cuda_core/docs/source/api.rst b/cuda_core/docs/source/api.rst index d82741e897b..b79e07e8579 100644 --- a/cuda_core/docs/source/api.rst +++ b/cuda_core/docs/source/api.rst @@ -146,6 +146,8 @@ avoid recompiling identical source + options + target without writing the CUDA graphs ----------- +.. currentmodule:: cuda.core + A CUDA graph captures a set of GPU operations and their dependencies, allowing them to be defined once and launched repeatedly with minimal CPU overhead. Graphs can be constructed in two ways: @@ -173,7 +175,8 @@ Lookup is local to each graph and requires a valid source node that existed at cloning time, and a corresponding node still present in the clone. The clone does not retain its source hierarchy for lookup. CUDA does not support cloning graphs containing allocation, free, or conditional nodes; driver errors -propagate. Synchronize cloning with mutations of the source hierarchy. +propagate. See :ref:`cuda-core-graph-thread-safety` for the synchronization +requirements of cloning and node lookup. With CUDA >= 13.1, graph definitions and executable graphs expose a read-only integer, ``id``. Nodes expose ``local_id`` @@ -184,8 +187,6 @@ shares ownership of the underlying graph. IDs are ephemeral, driver-assigned. ``kernel.attributes.device_node_update_supported`` reports whether a kernel supports device-side node updates. It requires CUDA >= 13.3. -.. currentmodule:: cuda.core - .. autosummary:: :toctree: generated/ diff --git a/cuda_core/docs/source/concurrency.rst b/cuda_core/docs/source/concurrency.rst index 51a1baa1be9..f0b9614e31e 100644 --- a/cuda_core/docs/source/concurrency.rst +++ b/cuda_core/docs/source/concurrency.rst @@ -6,8 +6,9 @@ Concurrency and Thread Safety ============================= -``cuda.core`` allows concurrent reads of its objects from multiple host -threads, but concurrent *mutation* of the same object is **not supported** -- +Subject to the CUDA driver limitations below, ``cuda.core`` allows concurrent +reads of its objects from multiple host threads. Concurrent *mutation* of the +same object is **not supported** -- for example, adding nodes to the same graph, or closing a resource while another thread is using it. Whenever an object is shared across threads and at least one of them may mutate it, the application is responsible for providing @@ -26,3 +27,19 @@ safe. For example, modifying peer device access from one thread while another thread accesses device memory affected by that change is unsafe, even though the two threads use different objects. The application is responsible for synchronizing operations that concurrently read and modify shared driver state. + +.. _cuda-core-graph-thread-safety: + +CUDA graph access +----------------- + +Applications must serialize CUDA access to a shared graph hierarchy across host +threads, including read-only calls and calls through different views of that +hierarchy. :meth:`graph.GraphDefinition.clone` accesses the source hierarchy. +:meth:`graph.GraphDefinition.find_node` accesses both the source and the clone. +Hold the relevant lock or locks for the entire call. Independent graph +hierarchies can use separate locks. + +See the CUDA Driver API's `Graph object thread safety +`_ +documentation. diff --git a/cuda_core/docs/source/release/1.3.0-notes.rst b/cuda_core/docs/source/release/1.3.0-notes.rst index 25890406469..5ed06b3e373 100644 --- a/cuda_core/docs/source/release/1.3.0-notes.rst +++ b/cuda_core/docs/source/release/1.3.0-notes.rst @@ -29,12 +29,6 @@ Highlights New features ------------ -- Added :meth:`graph.GraphDefinition.clone` and - :meth:`graph.GraphDefinition.find_node` to copy graph definitions, including - nested child graphs, and obtain corresponding nodes in the clone. Clones - retain references to underlying resources. - (`#2438 `__) - - Added read-only ``GraphDefinition.id``, ``Graph.id``, ``GraphNode.local_id`` and ``GraphNode.tools_id`` properties (CUDA 13 build and driver 13.1+), and ``KernelAttributes.device_node_update_supported`` (CUDA 13 build and driver 13.3+). diff --git a/cuda_core/tests/graph/test_graph_definition_lifetime.py b/cuda_core/tests/graph/test_graph_definition_lifetime.py index 01aade498e8..b1c8900df5f 100644 --- a/cuda_core/tests/graph/test_graph_definition_lifetime.py +++ b/cuda_core/tests/graph/test_graph_definition_lifetime.py @@ -9,6 +9,7 @@ import threading import time import weakref +from contextlib import closing import pytest from cuda_python_test_helpers.subprocess_runner import run_python_snippet @@ -1948,6 +1949,7 @@ def test_public_clone_find_node_rejects_invalid_sources(init_cuda): @pytest.mark.parametrize("depth", [0, 2]) @pytest.mark.parametrize("source_first", [False, True]) +@pytest.mark.thread_unsafe(reason="waits for deferred cleanup on the main thread") def test_public_clone_retains_attachment_until_last_executable(init_cuda, depth, source_first): """Cloned metadata releases each node's retain, including in nested graphs.""" called = [] @@ -1974,28 +1976,32 @@ def callback(): clone_node = clone_view.find_node(source_node) assert clone_node.callback is callback executable = clone.instantiate() - del callback - - first, second = (source_node, clone_node) if source_first else (clone_node, source_node) - first.destroy() - assert callback_weak() is not None - assert second.callback is callback_weak() - second.destroy() - del first, second, source_node, clone_node - # Both definitions have removed the node, but the executable retains it. - assert callback_weak() is not None - stream = init_cuda.create_stream() - # callback has not fired yet - assert called == [] - executable.launch(stream) - stream.sync() - # callback fired finally - assert called == [True] - executable.close() + try: + del callback + + first, second = (source_node, clone_node) if source_first else (clone_node, source_node) + first.destroy() + assert callback_weak() is not None + assert second.callback is callback_weak() + second.destroy() + del first, second, source_node, clone_node + # Both definitions have removed the node, but the executable retains it. + assert callback_weak() is not None + stream = init_cuda.create_stream() + assert called == [] + executable.launch(stream) + stream.sync() + assert called == [True] + finally: + executable.close() _wait_until(lambda: callback_weak() is None) +@pytest.mark.thread_unsafe(reason="waits for deferred cleanup on the main thread") def test_public_clone_callback_update_has_independent_owners(init_cuda): + if driver_version() < (12, 2, 0): + pytest.skip("individual graph node updates require CUDA 12.2+") + def original(): pass @@ -2024,13 +2030,14 @@ def replacement(): _wait_until(lambda: replacement_weak() is None) +@pytest.mark.thread_unsafe(reason="waits for deferred cleanup on the main thread") def test_public_clone_kernel_and_arguments_outlive_source(init_cuda): """An executable retains the clone's kernel and arguments after both definitions die.""" from cuda.core._utils.cuda_utils import driver, handle_return device = init_cuda stream = device.create_stream() - with _device_memory_resource(device) as mr: + with closing(_device_memory_resource(device)) as mr: buffer = mr.allocate(ctypes.sizeof(ctypes.c_int), stream=stream) buffer.fill(0, stream=stream) stream.sync() @@ -2042,15 +2049,16 @@ def test_public_clone_kernel_and_arguments_outlive_source(init_cuda): source.launch(LaunchConfig(grid=1, block=1), kernel, buffer) clone = source.clone() executable = clone.instantiate() - del buffer, module, kernel, source, clone - gc.collect() + try: + del buffer, module, kernel, source, clone + gc.collect() - assert buffer_weak() is not None # buffer is still referenced - executable.launch(stream) - stream.sync() - result = (ctypes.c_int * 1)(0) - # can still access - handle_return(driver.cuMemcpyDtoH(result, ptr, ctypes.sizeof(result))) - assert result[0] == 1 - executable.close() + assert buffer_weak() is not None + executable.launch(stream) + stream.sync() + result = (ctypes.c_int * 1)(0) + handle_return(driver.cuMemcpyDtoH(result, ptr, ctypes.sizeof(result))) + assert result[0] == 1 + finally: + executable.close() _wait_until(lambda: buffer_weak() is None)