From 9bf718f66a18a49503ae6ce12a9cfea5f04bfce9 Mon Sep 17 00:00:00 2001 From: XinhaoTheo Date: Tue, 17 Nov 2026 02:26:35 -0500 Subject: [PATCH 1/2] feat(kv): add CUDA VMM KV storage with fixed addresses Add StableKVStorage for RFC #192. It reserves one virtual address range for each KV pool. The tensor of each pool has the maximum block count from the start. commit() adds physical memory to more blocks. It does not change a tensor address, a stride or a stored value. Because of this, a CUDA Graph that was captured before a commit also writes correctly to the new blocks. The driver binding uses ctypes and libcuda.so.1. It does not need a build step or a new dependency. vmm_unavailable_reason() gives the reason when a device cannot use this storage. The engine does not use this storage yet. --- tests/test_kv_storage.py | 108 ++++++++++++ vllm_rlt/core/kv_storage.py | 339 ++++++++++++++++++++++++++++++++++++ 2 files changed, 447 insertions(+) create mode 100644 tests/test_kv_storage.py create mode 100644 vllm_rlt/core/kv_storage.py diff --git a/tests/test_kv_storage.py b/tests/test_kv_storage.py new file mode 100644 index 00000000..cbb5b1c3 --- /dev/null +++ b/tests/test_kv_storage.py @@ -0,0 +1,108 @@ +"""Stable KV storage keeps the tensor addresses when the physical capacity increases.""" + +from types import SimpleNamespace + +import pytest +import torch + +from vllm_rlt.core.kv_storage import PoolSpec, StableKVStorage, vmm_unavailable_reason + +# The granule is not a multiple of either block size. The two pools round up differently. +SPECS = (PoolSpec((3, 16, 4, 64), torch.bfloat16), PoolSpec((3, 16, 4, 64), torch.float32)) +CUDA = torch.device("cuda") + + +def test_cpu_reports_why_vmm_is_unavailable(): + assert "cpu" in vmm_unavailable_reason(torch.device("cpu")) + + +@pytest.mark.gpu +def test_commit_backs_more_blocks_without_moving_or_clearing_them(): + assert vmm_unavailable_reason(CUDA) is None + storage = StableKVStorage(SPECS, max_blocks=4096, initial_blocks=1, device=CUDA) + first = storage.committed_blocks + assert 1 <= first < 4096 + layout = [(t.device, t.data_ptr(), t.shape, t.stride()) for t in storage.tensors] + written = [torch.randn(t[:first].shape, device=CUDA).to(t.dtype) for t in storage.tensors] + for tensor, values in zip(storage.tensors, written): + tensor[:first] = values + + assert storage.commit(3000) >= 3000 + last = storage.committed_blocks - 1 + assert [(t.device, t.data_ptr(), t.shape, t.stride()) for t in storage.tensors] == layout + for tensor, values in zip(storage.tensors, written): + tensor[last] = 7 + assert torch.equal(tensor[:first], values) + assert bool((tensor[last] == 7).all()) + assert storage.commit(1) == last + 1 + + +def index_put(keys, values, block, k, v): + keys[block, 0, 0] = k + values[block, 0, 0] = v + + +def triton_graph_write(keys, values, block, k, v): + from vllm_rlt.kernels.graph_kv import write_graph_kv + + batch = SimpleNamespace( + write_blocks=block, + write_offsets=torch.zeros_like(block), + active_rows=None, + write_mask=None, + ) + write_graph_kv(keys[:, 0], values[:, 0], batch, k, v) + + +@pytest.mark.gpu +@pytest.mark.parametrize("write", [index_put, triton_graph_write]) +def test_graph_captured_before_commit_writes_newly_committed_blocks(write): + spec = SPECS[0] + storage = StableKVStorage([spec, spec], max_blocks=4096, initial_blocks=1, device=CUDA) + keys, values = storage.tensors + block = torch.zeros(1, dtype=torch.long, device=CUDA) + k = torch.ones((1, *spec.block_shape[2:]), dtype=spec.dtype, device=CUDA) + v = torch.full_like(k, 2) + write(keys, values, block, k, v) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + write(keys, values, block, k, v) + + storage.commit(4096) + block.fill_(4095) + k.fill_(3) + v.fill_(4) + graph.replay() + torch.cuda.synchronize() + assert bool((keys[4095, 0, 0] == 3).all()) + assert bool((values[4095, 0, 0] == 4).all()) + + +@pytest.mark.gpu +def test_close_returns_memory_the_torch_allocator_never_counted(): + torch.cuda.synchronize() + reserved = torch.cuda.memory_reserved() + free = torch.cuda.mem_get_info()[0] + storage = StableKVStorage(SPECS, max_blocks=4096, initial_blocks=4096, device=CUDA) + committed = storage.committed_bytes + assert committed >= 4096 * sum(spec.block_bytes for spec in SPECS) + held = torch.cuda.mem_get_info()[0] + assert free - held >= committed + assert torch.cuda.memory_reserved() == reserved + + storage.close() + assert torch.cuda.mem_get_info()[0] - held >= committed + + +@pytest.mark.gpu +def test_views_keep_memory_mapped_after_the_storage_is_dropped(): + storage = StableKVStorage(SPECS[:1], max_blocks=4096, initial_blocks=4096, device=CUDA) + committed = storage.committed_bytes + view = storage.tensors[0][-1] + del storage + view.fill_(5) + assert bool((view == 5).all()) + held = torch.cuda.mem_get_info()[0] + + del view + assert torch.cuda.mem_get_info()[0] - held >= committed diff --git a/vllm_rlt/core/kv_storage.py b/vllm_rlt/core/kv_storage.py new file mode 100644 index 00000000..fdc661f0 --- /dev/null +++ b/vllm_rlt/core/kv_storage.py @@ -0,0 +1,339 @@ +"""KV pools that keep a fixed device address when their physical memory increases. + +A captured CUDA Graph keeps the cache pointers and strides. A bounds-checked +index operation in a graph also keeps the size of the first dimension. Because +of this, each pool starts with its maximum block count. CUDA virtual memory maps +physical pages into this address range when you commit more capacity. Only the +first ``committed_blocks`` blocks have physical memory. Do not access the blocks +after them. +""" + +import ctypes +import math +import sys +import weakref +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from functools import cache + +import torch + +# These values are CUDA driver enums from cuda.h. +_SUCCESS = 0 +_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED = 102 +_ALLOCATION_TYPE_PINNED = 1 +_LOCATION_TYPE_DEVICE = 1 +_GRANULARITY_MINIMUM = 0 +_ACCESS_READWRITE = 3 + +_Address = ctypes.c_ulonglong +_Handle = ctypes.c_ulonglong + + +class _Location(ctypes.Structure): + _fields_ = [("type", ctypes.c_int), ("id", ctypes.c_int)] + + +class _AllocationFlags(ctypes.Structure): + _fields_ = [ + ("compressionType", ctypes.c_ubyte), + ("gpuDirectRDMACapable", ctypes.c_ubyte), + ("usage", ctypes.c_ushort), + ("reserved", ctypes.c_ubyte * 4), + ] + + +class _AllocationProp(ctypes.Structure): + _fields_ = [ + ("type", ctypes.c_int), + ("requestedHandleTypes", ctypes.c_int), + ("location", _Location), + ("win32HandleMetaData", ctypes.c_void_p), + ("allocFlags", _AllocationFlags), + ] + + +class _AccessDesc(ctypes.Structure): + _fields_ = [("location", _Location), ("flags", ctypes.c_int)] + + +class _Driver: + def __init__(self) -> None: + lib = ctypes.CDLL("libcuda.so.1") + size, pointer = ctypes.c_size_t, ctypes.POINTER + for name, argtypes in { + "cuInit": [ctypes.c_uint], + "cuGetErrorString": [ctypes.c_int, pointer(ctypes.c_char_p)], + "cuDeviceGetAttribute": [pointer(ctypes.c_int), ctypes.c_int, ctypes.c_int], + "cuDevicePrimaryCtxRetain": [pointer(ctypes.c_void_p), ctypes.c_int], + "cuCtxPushCurrent_v2": [ctypes.c_void_p], + "cuCtxPopCurrent_v2": [pointer(ctypes.c_void_p)], + "cuMemGetAllocationGranularity": [ + pointer(size), + pointer(_AllocationProp), + ctypes.c_int, + ], + "cuMemAddressReserve": [pointer(_Address), size, size, _Address, ctypes.c_ulonglong], + "cuMemAddressFree": [_Address, size], + "cuMemCreate": [pointer(_Handle), size, pointer(_AllocationProp), ctypes.c_ulonglong], + "cuMemRelease": [_Handle], + "cuMemMap": [_Address, size, size, _Handle, ctypes.c_ulonglong], + "cuMemSetAccess": [_Address, size, pointer(_AccessDesc), size], + "cuMemUnmap": [_Address, size], + }.items(): + function = getattr(lib, name) + function.argtypes = argtypes + function.restype = ctypes.c_int + self._lib = lib + self._contexts: dict[int, ctypes.c_void_p] = {} + self.check(lib.cuInit(0)) + + def check(self, result: int) -> None: + if result != _SUCCESS: + message = ctypes.c_char_p() + self._lib.cuGetErrorString(result, ctypes.byref(message)) + raise RuntimeError(f"CUDA driver error {result}: {(message.value or b'').decode()}") + + def supports_vmm(self, device: int) -> bool: + flag = ctypes.c_int() + self.check( + self._lib.cuDeviceGetAttribute( + ctypes.byref(flag), _VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, device + ) + ) + return bool(flag.value) + + @contextmanager + def context(self, device: int) -> Iterator[None]: + # A VMM call uses the current context of its thread. We push the primary + # context of torch and pop it after the calls, so the thread context does + # not change. + if device not in self._contexts: + context = ctypes.c_void_p() + self.check(self._lib.cuDevicePrimaryCtxRetain(ctypes.byref(context), device)) + self._contexts[device] = context + self.check(self._lib.cuCtxPushCurrent_v2(self._contexts[device])) + try: + yield + finally: + self.check(self._lib.cuCtxPopCurrent_v2(ctypes.byref(ctypes.c_void_p()))) + + @staticmethod + def device_memory(device: int) -> _AllocationProp: + prop = _AllocationProp() + prop.type = _ALLOCATION_TYPE_PINNED + prop.location.type = _LOCATION_TYPE_DEVICE + prop.location.id = device + return prop + + def granularity(self, device: int) -> int: + granularity = ctypes.c_size_t() + self.check( + self._lib.cuMemGetAllocationGranularity( + ctypes.byref(granularity), + ctypes.byref(self.device_memory(device)), + _GRANULARITY_MINIMUM, + ) + ) + return granularity.value + + def reserve(self, size: int) -> int: + address = _Address() + self.check(self._lib.cuMemAddressReserve(ctypes.byref(address), size, 0, 0, 0)) + return address.value + + def free_address(self, address: int, size: int) -> None: + self.check(self._lib.cuMemAddressFree(address, size)) + + def create(self, size: int, device: int) -> int: + handle = _Handle() + self.check( + self._lib.cuMemCreate( + ctypes.byref(handle), size, ctypes.byref(self.device_memory(device)), 0 + ) + ) + return handle.value + + def release(self, handle: int) -> None: + self.check(self._lib.cuMemRelease(handle)) + + def map(self, address: int, size: int, handle: int) -> None: + self.check(self._lib.cuMemMap(address, size, 0, handle, 0)) + + def set_access(self, address: int, size: int, device: int) -> None: + access = _AccessDesc() + access.location.type = _LOCATION_TYPE_DEVICE + access.location.id = device + access.flags = _ACCESS_READWRITE + self.check(self._lib.cuMemSetAccess(address, size, ctypes.byref(access), 1)) + + def unmap(self, address: int, size: int) -> None: + self.check(self._lib.cuMemUnmap(address, size)) + + +@cache +def cuda_driver() -> _Driver: + return _Driver() + + +def device_index(device: torch.device) -> int: + return torch.cuda.current_device() if device.index is None else device.index + + +def round_up(value: int, multiple: int) -> int: + return -(-value // multiple) * multiple + + +def vmm_unavailable_reason(device: torch.device) -> str | None: + """Return why ``StableKVStorage`` cannot operate on ``device``, or None if it can.""" + if device.type != "cuda": + return f"{device.type} devices have no CUDA virtual memory" + if torch.version.hip is not None: + return "ROCm builds do not have the CUDA driver virtual memory API" + if sys.platform != "linux": + return f"the CUDA driver binding operates only on Linux, not on {sys.platform}" + index = device_index(device) + if not cuda_driver().supports_vmm(index): + return f"CUDA device {index} does not support virtual memory management" + return None + + +@dataclass(frozen=True) +class PoolSpec: + """The shape and dtype of one block in one pool, for example the keys of all layers.""" + + block_shape: tuple[int, ...] + dtype: torch.dtype + + @property + def block_bytes(self) -> int: + return math.prod(self.block_shape) * self.dtype.itemsize + + +def free_pool( + driver: _Driver, + device: int, + base: int, + reserved_bytes: int, + chunks: list[tuple[int, int, int]], +) -> None: + # cuMemUnmap does not always wait for the kernels that use the pages. + torch.cuda.synchronize(device) + with driver.context(device): + for address, size, handle in chunks: + driver.unmap(address, size) + driver.release(handle) + driver.free_address(base, reserved_bytes) + + +class _Pool: + def __init__(self, driver: _Driver, device: int, block_bytes: int, reserved_bytes: int) -> None: + self.block_bytes = block_bytes + self.reserved_bytes = reserved_bytes + self.mapped_bytes = 0 + self.base = driver.reserve(reserved_bytes) + self._driver = driver + self._device = device + self._chunks: list[tuple[int, int, int]] = [] + self.free = weakref.finalize( + self, free_pool, driver, device, self.base, reserved_bytes, self._chunks + ) + # At process exit, the driver removes all mappings. A driver call during + # interpreter shutdown can cause an error, because CUDA can stop first. + self.free.atexit = False + + @property + def __cuda_array_interface__(self) -> dict[str, object]: + return { + "shape": (self.reserved_bytes,), + "typestr": "|u1", + "data": (self.base, False), + "version": 3, + } + + def grow(self, mapped_bytes: int) -> None: + if mapped_bytes <= self.mapped_bytes: + return + address, size = self.base + self.mapped_bytes, mapped_bytes - self.mapped_bytes + handle = self._driver.create(size, self._device) + try: + # cuMemMap also allocates page tables. It can stop with an + # out-of-memory error after cuMemCreate is successful. + self._driver.map(address, size, handle) + except RuntimeError: + self._driver.release(handle) + raise + self._chunks.append((address, size, handle)) + self._driver.set_access(address, size, self._device) + self.mapped_bytes = mapped_bytes + + +class StableKVStorage: + """Each pool has one ``[max_blocks, *block_shape]`` tensor at a fixed address. + + ``commit`` adds physical memory to more blocks at the start of each pool. It + does not change an address, a stride or a stored value. Because of this, a + graph that you captured before a commit also operates correctly on new blocks. + + The PyTorch allocator does not count this physical memory. ``close`` releases + it. The memory is also released when no reference to the storage or to its + tensor views remains. Both release paths wait for the device. After the + release, do not replay a graph that uses these tensors. + """ + + def __init__( + self, + specs: Sequence[PoolSpec], + *, + max_blocks: int, + initial_blocks: int, + device: torch.device, + ) -> None: + self.max_blocks = max_blocks + self._device = device_index(device) + driver = cuda_driver() + with driver.context(self._device): + self.granularity = driver.granularity(self._device) + self._pools = [ + _Pool( + driver, + self._device, + spec.block_bytes, + round_up(max_blocks * spec.block_bytes, self.granularity), + ) + for spec in specs + ] + # PyTorch does not accept an interface pointer that has no physical memory. + self.commit(initial_blocks) + self.tensors = tuple( + torch.as_tensor(pool) + .view(spec.dtype)[: max_blocks * math.prod(spec.block_shape)] + .view(max_blocks, *spec.block_shape) + for pool, spec in zip(self._pools, specs) + ) + + @property + def committed_bytes(self) -> int: + return sum(pool.mapped_bytes for pool in self._pools) + + def commit(self, blocks: int) -> int: + """Add physical memory to at least the first ``blocks`` blocks of each pool. + + Return the number of blocks that have physical memory. The capacity can + only increase. ``blocks`` must not be more than ``max_blocks``. + """ + driver = cuda_driver() + with driver.context(self._device): + for pool in self._pools: + pool.grow(round_up(blocks * pool.block_bytes, self.granularity)) + # The granule round-up can add physical memory for more blocks than the request. + self.committed_blocks = min( + self.max_blocks, *(pool.mapped_bytes // pool.block_bytes for pool in self._pools) + ) + return self.committed_blocks + + def close(self) -> None: + """Release the physical memory immediately, also when tensor views remain.""" + for pool in self._pools: + pool.free() From 594dc63306ff7f9338d6d8f82bcc318d3675809b Mon Sep 17 00:00:00 2001 From: XinhaoTheo Date: Tue, 17 Nov 2026 11:50:09 -0500 Subject: [PATCH 2/2] fix(kv): release the KV storage only when no reference remains close() released the memory also when tensor views remained. A remaining view then wrote to unmapped memory without an error. Now the memory is released only when no reference to the storage or to its views remains, as with torch.empty. The release test does not read the free memory of the device, because other processes on a shared device change it. The test checks the pool lifetime, and a failed release in the finalizer fails the test. --- tests/test_kv_storage.py | 28 +++++++++------------------- vllm_rlt/core/kv_storage.py | 18 +++++++----------- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/tests/test_kv_storage.py b/tests/test_kv_storage.py index cbb5b1c3..17950689 100644 --- a/tests/test_kv_storage.py +++ b/tests/test_kv_storage.py @@ -1,5 +1,6 @@ """Stable KV storage keeps the tensor addresses when the physical capacity increases.""" +import weakref from types import SimpleNamespace import pytest @@ -79,30 +80,19 @@ def test_graph_captured_before_commit_writes_newly_committed_blocks(write): @pytest.mark.gpu -def test_close_returns_memory_the_torch_allocator_never_counted(): - torch.cuda.synchronize() +# A release error in a finalizer is only a warning by default. +@pytest.mark.filterwarnings("error::pytest.PytestUnraisableExceptionWarning") +def test_last_reference_releases_memory_the_torch_allocator_never_counted(): reserved = torch.cuda.memory_reserved() - free = torch.cuda.mem_get_info()[0] storage = StableKVStorage(SPECS, max_blocks=4096, initial_blocks=4096, device=CUDA) - committed = storage.committed_bytes - assert committed >= 4096 * sum(spec.block_bytes for spec in SPECS) - held = torch.cuda.mem_get_info()[0] - assert free - held >= committed + assert storage.committed_bytes >= 4096 * sum(spec.block_bytes for spec in SPECS) assert torch.cuda.memory_reserved() == reserved - - storage.close() - assert torch.cuda.mem_get_info()[0] - held >= committed - - -@pytest.mark.gpu -def test_views_keep_memory_mapped_after_the_storage_is_dropped(): - storage = StableKVStorage(SPECS[:1], max_blocks=4096, initial_blocks=4096, device=CUDA) - committed = storage.committed_bytes + pools = [weakref.ref(pool) for pool in storage._pools] view = storage.tensors[0][-1] + del storage view.fill_(5) assert bool((view == 5).all()) - held = torch.cuda.mem_get_info()[0] - + assert [pool() is not None for pool in pools] == [True, False] del view - assert torch.cuda.mem_get_info()[0] - held >= committed + assert pools[0]() is None diff --git a/vllm_rlt/core/kv_storage.py b/vllm_rlt/core/kv_storage.py index fdc661f0..392eb750 100644 --- a/vllm_rlt/core/kv_storage.py +++ b/vllm_rlt/core/kv_storage.py @@ -236,12 +236,12 @@ def __init__(self, driver: _Driver, device: int, block_bytes: int, reserved_byte self._driver = driver self._device = device self._chunks: list[tuple[int, int, int]] = [] - self.free = weakref.finalize( + finalizer = weakref.finalize( self, free_pool, driver, device, self.base, reserved_bytes, self._chunks ) # At process exit, the driver removes all mappings. A driver call during # interpreter shutdown can cause an error, because CUDA can stop first. - self.free.atexit = False + finalizer.atexit = False @property def __cuda_array_interface__(self) -> dict[str, object]: @@ -276,10 +276,11 @@ class StableKVStorage: does not change an address, a stride or a stored value. Because of this, a graph that you captured before a commit also operates correctly on new blocks. - The PyTorch allocator does not count this physical memory. ``close`` releases - it. The memory is also released when no reference to the storage or to its - tensor views remains. Both release paths wait for the device. After the - release, do not replay a graph that uses these tensors. + The PyTorch allocator does not count this physical memory. As with + ``torch.empty``, the memory is released when no reference to the storage or + to its tensor views remains. There is no explicit release, because it would + leave the remaining views on unmapped memory. The release waits for the + device. After the release, do not replay a graph that uses these tensors. """ def __init__( @@ -332,8 +333,3 @@ def commit(self, blocks: int) -> int: self.max_blocks, *(pool.mapped_bytes // pool.block_bytes for pool in self._pools) ) return self.committed_blocks - - def close(self) -> None: - """Release the physical memory immediately, also when tensor views remain.""" - for pool in self._pools: - pool.free()