From 4ce2fef82fb0f92fba208445436fafe181f51ae1 Mon Sep 17 00:00:00 2001 From: Dragonborn <1078757180@qq.com> Date: Fri, 13 Nov 2026 21:23:34 +0800 Subject: [PATCH 1/2] Add Dory hybrid KV storage and SWA reclamation --- docs/dory.md | 68 ++++- tests/test_dory.py | 8 +- tests/test_hybrid_kv.py | 235 +++++++++++++++ tests/test_swa_reclaim.py | 373 +++++++++++++++++++++++ vllm_rlt/config.py | 6 +- vllm_rlt/core/kv_cache_manager.py | 479 ++++++++++++++++++++++++++++-- vllm_rlt/core/memory.py | 63 +++- vllm_rlt/core/scheduler.py | 2 + vllm_rlt/engine/stage_engine.py | 15 + vllm_rlt/models/dory.py | 6 + vllm_rlt/worker/buffers.py | 21 +- vllm_rlt/worker/metadata.py | 58 +++- vllm_rlt/worker/model_runner.py | 60 ++-- 13 files changed, 1318 insertions(+), 76 deletions(-) create mode 100644 tests/test_hybrid_kv.py create mode 100644 tests/test_swa_reclaim.py diff --git a/docs/dory.md b/docs/dory.md index 13f1f94a..2c0d64af 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -61,13 +61,71 @@ count (`DoryConfig.num_kv_layers`), not by all 72 physical sublayers, and declares one. Attention layers therefore reach the cache through a packed layer id rather than their physical sublayer index. +## Grouped storage and optional SWA reclamation + +Dory uses one pool of attention-layer pages, grouped by window policy and +boundary/recurrent plane domain. `reclaim_swa` defaults to `False`. To return +expired sliding-window pages to the pool, enable it together with incremental +allocation on an already loaded model: + +```python +from vllm_rlt import CacheConfig +from vllm_rlt.config import ExecutionConfig +from vllm_rlt.engine.llm_engine import LLMEngine + +engine = LLMEngine( + model, + attention_backend="torch", + execution_config=ExecutionConfig(use_uva=False), + cache_config=CacheConfig( + num_blocks=256, + layout=model.kv_cache_layout, + incremental_allocation=True, + reclaim_swa=True, + ), +) +``` + +`num_blocks` preserves the existing all-layer KV byte budget: the native pool +converts it to `num_blocks * model.config.num_kv_layers` single-layer pages. +Reclamation supports ordinary synchronous fixed-loop `ouro` execution. Each +`per_loop` plane advances independently; shared recurrent history in +`last_loop` advances only after the token's final loop. Pages retire only after +all layers in their group have consumed the range and the submission completes. +Full-attention history remains resident. + +Native SWA attention stages only the needed page-aligned read span. Logical token +positions, write addresses and public layer maps remain absolute; the backend's +read-table context lengths are local to that span. Native attention calls must +match the layer's declared window, including `None` for full-attention layers. + +For read-only group policy, `cache.layer_group(kv_layer_id)` returns +`(group_id, policy)`: a manager-local group ID and the immutable `KVGroup` with +compact layer IDs, window, recurrent domain and plane count. The layer ID is a +compact KV index, not a physical sublayer index. + +For read-only cache inspection, `cache.retained_range(request_id, kv_layer_id, +depth)` returns `(resident_start, written_end)` in absolute token positions. +`cache.layer_block_table(request_id, kv_layer_id, depth)` returns the absolute +logical-page map, with `-1` for reclaimed pages. Here `cache` is +`engine.cache_manager`, and `kv_layer_id` is a compact KV index. Reading a +reclaimed full prefix is rejected; these queries do not retain page references +or expose mutable allocator state. + +Grouped prefix publish, lookup and import remain unsupported. A future prefix +consumer must retain pages through the same manager before publishing them or +allowing their request owner to reclaim them. Reclamation drops the owner's +references; a page becomes reusable only when its final reference reaches zero. +The inspection methods are not a pin API, and no public grouped retain/pin +interface is provided yet. + ## Supported today - Native strict safetensors loading with the artifact's per-projection key names (`q/k/v/g/o_proj`, `gate_proj`/`up_proj`) and the learned scale tensors. There is no fused-key remapping: a checkpoint that packs `qkv_proj` or `gate_up_proj` fails loudly instead of loading approximately. -- Synchronous eager execution: full-prompt prefill followed by ordinary token +- Synchronous eager execution: ordinary bounded-chunk prefill followed by token decoding, with the configured fixed loop count. - `loops=` at load time to fix 1, 2 or 4 loops. - Greedy and sampled decoding, request seeds, EOS (`2`, `11`) and length @@ -96,6 +154,7 @@ These fail fast rather than degrade silently: - WCPB, mixed or staged prefill; packed prefill metadata (`packed_prefill`); PD. - Prefix caching. +- Preemption and native-cache snapshot/restore. - Self-speculation combined with CUDA Graphs, asynchronous scheduling, preemption or `last_loop` KV, and sampled token-pipeline speculation. The synchronous eager form is supported; see below. @@ -103,17 +162,12 @@ These fail fast rather than degrade silently: trained exit gate, so the runtime refuses to invent one, including via `min_loops` with `exit_threshold=0`. -Preemption and snapshots are **not qualified** rather than enforced: the -generic KV snapshot mechanism copies every plane, so it may well work, but no -restoration test exists for Dory's boundary state. Keep `enable_preemption` -off until 2E covers it. - Sliding-window masking exists in the Torch reference backend and in the Triton decode kernel (`window=` on `KVCacheManager.attend`). Backend selection requires the `sliding_window` capability, so a Flash-backed Dory engine fails at construction instead of attending full history. Masking accepts host-planned, graph-private and device-owned metadata; storage reclamation of out-of-window -pages stays with 2B. +pages stays with 2B. Optional physical SWA page reclamation is available on the grouped Torch path described above. ## Synchronous self-speculation diff --git a/tests/test_dory.py b/tests/test_dory.py index 4f47a002..cd3468c1 100644 --- a/tests/test_dory.py +++ b/tests/test_dory.py @@ -128,10 +128,10 @@ def make_cache(model, layout=None, block_size=2): ) -def run_one_token_per_step(model, tokens): +def run_one_token_per_step(model, tokens, cache=None): """Execute the declared reference schedule: one token row at a time.""" config = model.config - cache = make_cache(model) + cache = make_cache(model) if cache is None else cache assert cache.allocate("request", len(tokens)) loops = [[] for _ in range(config.total_ut_steps)] logits = [] @@ -148,10 +148,10 @@ def run_one_token_per_step(model, tokens): return {"logits": torch.stack(logits), "loops": [torch.stack(item) for item in loops]}, cache -def run_all_positions_at_once(model, tokens): +def run_all_positions_at_once(model, tokens, cache=None): """Same schedule, but every stage call carries all causal positions.""" config = model.config - cache = make_cache(model) + cache = make_cache(model) if cache is None else cache assert cache.allocate("request", len(tokens)) positions = list(range(len(tokens))) ids = ["request"] * len(tokens) diff --git a/tests/test_hybrid_kv.py b/tests/test_hybrid_kv.py new file mode 100644 index 00000000..d77b24a0 --- /dev/null +++ b/tests/test_hybrid_kv.py @@ -0,0 +1,235 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Native Dory storage: numerical dataflow, ownership and physical-page budgets.""" + +import pytest +import torch + +from tests.reference import dory as reference +from tests.test_dory import ( + assert_matches_reference, + run_all_positions_at_once, + run_one_token_per_step, + tiny_dory_config, +) +from vllm_rlt import CacheConfig, SamplingParams, SchedulerConfig +from vllm_rlt.config import ExecutionConfig +from vllm_rlt.core.kv_cache_manager import KVCacheManager +from vllm_rlt.core.memory import plan_cache +from vllm_rlt.engine.llm_engine import LLMEngine +from vllm_rlt.models import DoryForCausalLM +from vllm_rlt.worker.metadata import GroupedKVMetadata, MetadataBuilder + + +def native_cache(model, blocks=48): + config = model.config + return KVCacheManager( + num_layers=config.num_kv_layers, + num_kv_heads=config.num_key_value_heads, + head_dim=config.head_dim, + num_blocks=blocks, + block_size=2, + max_loops=config.total_ut_steps, + layout=config.kv_cache_layout, + recurrent_layers=model.recurrent_kv_layers, + layer_windows=model.kv_layer_windows, + ) + + +@pytest.mark.parametrize("mode", ["per_loop", "last_loop"]) +def test_native_dory_matches_unchanged_dense_reference_and_kv(monkeypatch, mode): + torch.manual_seed(42) + model = DoryForCausalLM(tiny_dory_config(recurrent_kv_cache_mode=mode)).eval() + tokens = torch.tensor([4, 7, 9, 3, 12, 5]) + histories = [] + original = reference._History + + class ObservedHistory(original): + def __init__(self): + super().__init__() + histories.append(self) + + # Observe the existing oracle's dense K/V without changing its operations. + monkeypatch.setattr(reference, "_History", ObservedHistory) + expected = reference.dense_dory_reference(model, tokens) + cache = native_cache(model) + actual, cache = run_one_token_per_step(model, tokens, cache=cache) + assert_matches_reference(actual, expected) + for (physical_layer, plane), entries in histories[0].entries.items(): + layer = model.config.kv_layer_index(physical_layer) + keys, values = cache.read(layer, "request", plane) + torch.testing.assert_close( + keys, torch.stack([entries[p][0] for p in range(6)]), atol=3e-6, rtol=3e-5 + ) + torch.testing.assert_close( + values, torch.stack([entries[p][1] for p in range(6)]), atol=3e-6, rtol=3e-5 + ) + slots = 6 if mode == "per_loop" else 4 + assert cache.allocated_blocks("request") == cache.num_used_blocks == 3 * slots + tables = [] + for layer in model.kv_layer_windows: + depths = range(2) if layer in model.recurrent_kv_layers and mode == "per_loop" else (0,) + for depth in depths: + tables.extend(cache.layer_block_table("request", layer, depth)) + assert len(set(tables)) == 3 * slots + assert set(cache.layer_groups) == {0, 1, 2, 3} + assert model.config.attention_layer_ids == (0, 2, 4, 6) + assert len(cache.groups) == 3 # SWA boundary, FA recurrent, SWA recurrent. + with pytest.raises(ValueError, match="no attention KV"): + model.config.kv_layer_index(1) + with pytest.raises(ValueError, match="layer must be"): + cache.read(4, "request", 0) + with pytest.raises(ValueError, match="depth zero"): + cache.read(0, "request", 1) + if mode == "last_loop": + assert cache.layer_block_table("request", 2, 0) == cache.layer_block_table("request", 2, 1) + assert cache.token_written("request", 5, 1) + cache.free("request") + assert cache.num_used_blocks == 0 + + +def test_native_per_loop_batched_queries_and_sliced_metadata(): + torch.manual_seed(42) + model = DoryForCausalLM(tiny_dory_config()).eval() + tokens = torch.tensor([4, 7, 9, 3, 12, 5]) + cache = native_cache(model) + boundary_id, boundary = cache.layer_group(0) + assert cache.layer_group(3) == (boundary_id, boundary) + assert boundary.layers == (0, 3) + assert (boundary.window, boundary.recurrent, boundary.planes) == (4, False, 1) + swa_id, swa = cache.layer_group(2) + assert swa_id != boundary_id + assert (swa.window, swa.recurrent, swa.planes) == (4, True, 2) + assert model.config.kv_layer_index(4) == 2 + assert cache.layer_group(1)[1].window is None + actual, cache = run_all_positions_at_once(model, tokens, cache=cache) + assert_matches_reference(actual, reference.dense_dory_reference(model, tokens)) + batch = cache._prepare_batch(["request"] * 2, [0, 0], [4, 5], padded_rows=3) + assert isinstance(batch, GroupedKVMetadata) + assert not hasattr(batch, "write_blocks") + view = cache.layer_view(batch, 1) + assert cache.layer_view(batch, 1) is view + assert view.write_blocks.tolist() == [cache.layer_block_table("request", 1, 0)[2]] * 2 + assert view.write_offsets.tolist() == [0, 1] + assert view.context_lengths.tolist() == [5, 6, 0] + assert view.position_ids.tolist() == [4, 5, 0] + windowed = cache.layer_view(batch, 0) + table = cache.layer_block_table("request", 0, 0) + assert windowed.context_lengths.tolist() == [5, 4, 0] + assert windowed.position_ids.tolist() == [4, 5, 0] + assert windowed.block_tables.tolist() == [list(table[:3]), [*table[1:3], -1], [0, 0, 0]] + assert windowed.write_blocks.tolist() == [table[2], table[2]] + assert windowed.write_offsets.tolist() == [0, 1] + sliced = MetadataBuilder.slice_rows(batch, 1, 2) + assert not sliced.layer_views and len(sliced.rows) == 1 + assert cache.layer_view(sliced, 2).write_blocks.tolist() == [ + cache.layer_block_table("request", 2, 0)[2] + ] + assert len(MetadataBuilder.without_padding(batch).position_ids) == 2 + with pytest.raises(NotImplementedError, match="ordinary layer views"): + cache.get_block_table("request", 0) + cache.free("request") + assert cache.allocate("request", 6) + with pytest.raises(RuntimeError, match="stale prepared"): + cache.layer_view(batch, 1) + + +def test_native_atomic_growth_failure_free_and_early_exit_copy(): + model = DoryForCausalLM(tiny_dory_config()).eval() + cache = native_cache(model, blocks=18) + assert cache.required_blocks(6) == 18 + assert cache.allocate("a", 6, initial_tokens=2) + assert cache.allocate("b", 6, initial_tokens=2) + assert cache.num_used_blocks == 12 + before = tuple(cache.layer_block_table("a", layer, 0) for layer in model.kv_layer_windows) + assert not cache.ensure_capacity("a", 6) + assert cache.num_used_blocks == 12 + assert before == tuple( + cache.layer_block_table("a", layer, 0) for layer in model.kv_layer_windows + ) + assert cache.ensure_capacity("a", 4) + assert cache.num_used_blocks == 18 + assert not cache.allocate("c", 6, initial_tokens=2) + assert not cache.has_allocation("c") + cache.free("b") + assert cache.ensure_capacity("a", 6) + for layer in model.kv_layer_windows: + values = torch.full((6, 2, 8), float(layer + 1)) + cache.write(layer, ["a"] * 6, [0] * 6, list(range(6)), values, values + 10) + for position in range(6): + cache.finalize_token("a", position, 0) + for layer in model.recurrent_kv_layers: + k0, v0 = cache.read(layer, "a", 0) + k1, v1 = cache.read(layer, "a", 1) + torch.testing.assert_close(k0, k1, atol=0, rtol=0) + torch.testing.assert_close(v0, v1, atol=0, rtol=0) + assert cache.token_written("a", 5, 1) + cache.free("a") + assert cache.num_used_blocks == 0 and cache.num_free_blocks == 18 + assert cache.allocate("c", 6) + assert not cache.token_written("c", 0, 0) + cache.free("c") + assert cache.num_free_blocks == 18 + + +def test_native_planner_preserves_configured_all_layer_byte_budget(): + model = DoryForCausalLM(tiny_dory_config()).eval() + scheduler, execution = SchedulerConfig(), ExecutionConfig() + native = native_cache(model) + page_bytes = 2 * 2 * 2 * 8 * 4 + assert native.bytes_per_block == page_bytes + blocks, plan = plan_cache( + model, CacheConfig(num_blocks=3, block_size=2), scheduler, execution, "torch" + ) + assert model.config.num_kv_layers == 4 < model.config.num_hidden_layers + assert blocks == 3 * model.config.num_kv_layers + assert blocks * plan["bytes_per_block"] == 3 * 4 * page_bytes + assert plan["block_unit"] == "physical_layer_page" and plan["configured_num_blocks"] == 3 + blocks, plan = plan_cache( + model, + CacheConfig(kv_cache_memory_bytes=24 * page_bytes, block_size=2), + scheduler, + execution, + "torch", + ) + assert blocks == 24 and plan["bytes_per_block"] == page_bytes + blocks, _ = plan_cache(model, CacheConfig(block_size=2), scheduler, execution, "torch") + assert blocks == 256 * 4 + + +@pytest.mark.parametrize("mode", ["per_loop", "last_loop"]) +def test_native_engine_generation_uses_grouped_storage(mode): + torch.manual_seed(42) + model = DoryForCausalLM(tiny_dory_config(recurrent_kv_cache_mode=mode)).eval() + engine = LLMEngine( + model, + cache_config=CacheConfig(num_blocks=8, block_size=2, layout=model.config.kv_cache_layout), + scheduler_config=SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=4), + ) + cache = engine.cache_manager + assert cache.grouped and cache.num_blocks == 8 * 4 + assert cache.key_cache.shape[1] == 1 + prompt = [4, 7, 3] + params = SamplingParams(max_tokens=3, min_loops=2, max_loops=2, ignore_eos=True) + engine.add_request("request", prompt, params) + result = None + while engine.has_unfinished_requests(): + for output in engine.step(): + if output.finished: + result = output + assert result is not None + tokens = list(prompt) + for _ in range(3): + logits = reference.dense_dory_reference(model, torch.tensor(tokens))["logits"] + tokens.append(int(logits[-1].argmax())) + assert result.token_ids == tokens[len(prompt) :] + assert cache.num_used_blocks == 0 + + +def test_native_engine_refuses_preemption_before_running(): + model = DoryForCausalLM(tiny_dory_config()).eval() + with pytest.raises(NotImplementedError, match="preemption snapshots"): + LLMEngine( + model, + cache_config=CacheConfig(num_blocks=8, block_size=2), + scheduler_config=SchedulerConfig(enable_preemption=True), + ) diff --git a/tests/test_swa_reclaim.py b/tests/test_swa_reclaim.py new file mode 100644 index 00000000..11dc274a --- /dev/null +++ b/tests/test_swa_reclaim.py @@ -0,0 +1,373 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Ordinary fixed-loop SWA reclamation, including shared final-depth readers.""" + +import math + +import pytest +import torch + +from tests.reference.dory import dense_dory_reference +from tests.test_dory import tiny_dory_config +from vllm_rlt import CacheConfig, SamplingParams, SchedulerConfig +from vllm_rlt.core.kv_cache_manager import KVCacheManager +from vllm_rlt.engine.llm_engine import LLMEngine +from vllm_rlt.models import DoryForCausalLM +from vllm_rlt.request import RequestPhase + + +def live_pages(cache, model, request_id, *, sliding): + pages = set() + for layer, window in model.kv_layer_windows.items(): + if (window is not None) != sliding: + continue + depths = range(cache.storage_depths) if layer in model.recurrent_kv_layers else (0,) + for depth in depths: + pages.update( + page for page in cache.layer_block_table(request_id, layer, depth) if page >= 0 + ) + return pages + + +@pytest.mark.parametrize("mode", ["per_loop", "last_loop"]) +def test_reclaim_engine_crosses_chunks_and_matches_dense_logits(monkeypatch, mode): + torch.manual_seed(42) + model = DoryForCausalLM(tiny_dory_config(recurrent_kv_cache_mode=mode)).eval() + observed = [] + logits = model.logits + + def observe(state): + result = logits(state) + observed.append(result.detach().clone()) + return result + + monkeypatch.setattr(model, "logits", observe) + engine = LLMEngine( + model, + cache_config=CacheConfig( + num_blocks=8, + block_size=2, + layout=model.config.kv_cache_layout, + incremental_allocation=True, + reclaim_swa=True, + ), + scheduler_config=SchedulerConfig( + max_num_seqs=1, max_num_batched_tokens=3, prefill_chunk_size=5 + ), + ) + cache = engine.cache_manager + assert cache.max_submission_tokens == 3 + prompt = [4, 7, 9, 3, 12, 5, 6, 8, 11, 13, 14, 15] + engine.add_request( + "request", prompt, SamplingParams(max_tokens=4, min_loops=2, max_loops=2, ignore_eos=True) + ) + request = engine.scheduler.requests["request"] + assert engine.scheduler._plan_admission(request, 0).initial_tokens == 3 + occupancy = [] + finish = cache.finish_submission + + def record_finish(event=None): + finish(event) + if cache.has_allocation("request"): + occupancy.append( + ( + len(live_pages(cache, model, "request", sliding=True)), + len(live_pages(cache, model, "request", sliding=False)), + ) + ) + + monkeypatch.setattr(cache, "finish_submission", record_finish) + output = None + while engine.has_unfinished_requests(): + for item in engine.step(): + if item.finished: + output = item + assert output is not None and len(observed) == 4 + tokens = list(prompt) + for actual in observed: + expected = dense_dory_reference(model, torch.tensor(tokens))["logits"][-1:] + torch.testing.assert_close(actual, expected, atol=3e-6, rtol=3e-5) + tokens.append(int(expected[-1].argmax())) + assert output.token_ids == tokens[len(prompt) :] + peak_swa_pages = (2 + cache.storage_depths) * math.ceil((4 - 1 + 3 + 2 - 1) / 2) + assert max(swa for swa, _ in occupancy) <= peak_swa_pages + assert max(full for _, full in occupancy) >= 6 * cache.storage_depths + assert any(swa < full for swa, full in occupancy) + assert cache.num_used_blocks == 0 + + +@pytest.mark.parametrize("mode", ["per_loop", "last_loop"]) +def test_reclaim_waits_for_group_reader_reuses_pages_and_rebuilds_or_rejects_views(mode): + torch.manual_seed(42) + model = DoryForCausalLM(tiny_dory_config(recurrent_kv_cache_mode=mode)).eval() + pool_size = 20 if mode == "per_loop" else 13 + cache = KVCacheManager( + num_layers=4, + num_kv_heads=2, + head_dim=8, + num_blocks=pool_size, + layout=model.config.kv_cache_layout, + block_size=2, + max_loops=2, + recurrent_layers=model.recurrent_kv_layers, + layer_windows=model.kv_layer_windows, + incremental_allocation=True, + reclaim_swa=True, + max_submission_tokens=1, + ) + assert cache.allocate("request", 8) + original_swa_pages = live_pages(cache, model, "request", sliding=True) + old = cache._prepare_batch(["request"], [0], [0]) + cache.layer_view(old, 0) + recent = recent_view = None + waits = [] + + class Completion: + def __init__(self): + self.used = cache.num_used_blocks + + def synchronize(self): + # No references or physical mapping may change before completion. + assert cache.num_used_blocks == self.used + assert cache.layer_block_table("request", 0, 0)[0] >= 0 + waits.append(True) + + for position in range(7): + assert cache.ensure_capacity("request", position + 1) + if position == 4: + rebuilt = cache.layer_view(recent, 0) + assert rebuilt is not recent_view # Growth epoch: live rows rebuild safely. + torch.testing.assert_close(rebuilt.write_blocks, recent_view.write_blocks) + if position == 6: + # The tight pool must actually reuse at least one physically retired page. + assert original_swa_pages & live_pages(cache, model, "request", sliding=True) + boundary = cache._prepare_batch(["request"], [0], [position]) + hidden = model.prelude_prepared(torch.tensor([position + 2]), boundary, cache) + for depth in range(2): + core = cache._prepare_batch(["request"], [depth], [position]) + hidden, _ = model.recurrent_prepared(hidden, core, cache) + if mode == "last_loop" and position == 4 and depth == 0: + # q4's next loop still reads key1 from page0: first-loop completion + # cannot advance the shared recurrent consumption frontier. + cache.finish_submission(Completion()) + assert cache.layer_block_table("request", 2, 0)[0] >= 0 + assert cache.retained_range("request", 2, 0)[0] == 0 + assert not waits + if position == 4: + # Input consumed position 4, but output still needs position 1 in page 0. + cache.finish_submission(Completion()) + assert cache.layer_block_table("request", 0, 0)[0] >= 0 + assert cache.layer_block_table("request", 2, 0)[0] == -1 + model.coda_prepared(hidden, boundary, cache) + cache.finish_submission(Completion() if position == 4 else None) + cache.finalize_token("request", position, 1) + if position == 3: + recent = cache._prepare_batch(["request"], [0], [3]) + recent_view = cache.layer_view(recent, 0) + if position == 4: + assert len(waits) == 2 + assert cache.retained_range("request", 0, 0) == (2, 5) + assert cache.retained_range("request", 1, 0) == (0, 5) + with pytest.raises(RuntimeError, match="resident"): + cache.layer_view(old, 0) + with pytest.raises(ValueError, match="window must match the layer policy"): + cache.attend(0, ["request"], [0], [4], torch.zeros(1, 4, 8)) + with pytest.raises(RuntimeError, match="prefix has been reclaimed"): + cache.read(0, "request", 0) + cache.free("request") + assert cache.num_used_blocks == 0 and cache.num_free_blocks == pool_size + + +def test_window_one_page_one_finalization_and_swa_occupancy_plateau(monkeypatch): + torch.manual_seed(42) + model = DoryForCausalLM(tiny_dory_config(sliding_window=1)).eval() + engine = LLMEngine( + model, + cache_config=CacheConfig( + num_blocks=9, block_size=1, incremental_allocation=True, reclaim_swa=True + ), + scheduler_config=SchedulerConfig( + max_num_seqs=1, max_num_batched_tokens=3, prefill_chunk_size=5 + ), + ) + cache = engine.cache_manager + occupancy = [] + finish = cache.finish_submission + + def record_finish(event=None): + finish(event) + if cache.has_allocation("request"): + occupancy.append( + ( + len(live_pages(cache, model, "request", sliding=True)), + len(live_pages(cache, model, "request", sliding=False)), + ) + ) + + monkeypatch.setattr(cache, "finish_submission", record_finish) + prompt = [4, 7, 9, 3, 12, 5, 6, 8, 11] + engine.add_request( + "request", prompt, SamplingParams(max_tokens=3, min_loops=2, max_loops=2, ignore_eos=True) + ) + output = None + while engine.has_unfinished_requests(): + for item in engine.step(): + if item.finished: + output = item + assert output is not None + tokens = list(prompt) + for _ in range(3): + expected = dense_dory_reference(model, torch.tensor(tokens))["logits"] + tokens.append(int(expected[-1].argmax())) + assert output.token_ids == tokens[len(prompt) :] + # SWA stays within its append bound while the full-attention history grows. + assert max(swa for swa, _ in occupancy) <= 4 * 3 + assert max(full for _, full in occupancy) >= 2 * len(prompt) + assert any(swa == 0 and full > 0 for swa, full in occupancy) + assert cache.num_used_blocks == 0 + + +@pytest.mark.parametrize("mode", ["per_loop", "last_loop"]) +def test_reclaim_multi_request_abort_and_reuse(monkeypatch, mode): + torch.manual_seed(42) + model = DoryForCausalLM(tiny_dory_config(recurrent_kv_cache_mode=mode)).eval() + reference = type(model)(model.config).eval() + reference.load_state_dict(model.state_dict()) + engine = LLMEngine( + model, + cache_config=CacheConfig( + num_blocks=16 if mode == "per_loop" else 10, + block_size=2, + layout=model.config.kv_cache_layout, + incremental_allocation=True, + reclaim_swa=True, + ), + scheduler_config=SchedulerConfig( + max_num_seqs=2, + max_num_batched_tokens=3, + prefill_chunk_size=3, + ), + ) + cache = engine.cache_manager + prompts = { + "long": [4, 7, 9, 3, 12, 5, 6, 8, 11, 13, 14, 15], + "reused": [8, 2, 5, 9, 3, 7, 12], + } + replacement = [21, 6, 17, 5, 14, 2, 19, 4] + for rid, count in (("long", 7), ("reused", 4)): + engine.add_request( + rid, + prompts[rid], + SamplingParams( + max_tokens=count, + min_loops=2, + max_loops=2, + ignore_eos=True, + ), + ) + original_b = engine.scheduler.requests["reused"] + observed, completed = [], {} + original_logits = model.logits + + def observe(state): + values = original_logits(state) + scheduled = engine.last_schedule.requests + assert len(values) == len(scheduled) + for value, item in zip(values, scheduled): + request = item.request + context = tuple(request.prompt_token_ids + request.generated_token_ids) + observed.append( + (request, len(request.generated_token_ids), context, value.detach().clone()) + ) + return values + + monkeypatch.setattr(model, "logits", observe) + first_b_pages, reused_while_b_active = set(), set() + stale = None + aborted = stale_checked = False + for step in range(128): + if not engine.has_unfinished_requests(): + break + for output in engine.step(): + if output.finished: + completed[output.request_id] = output + owned = { + rid: live_pages(cache, model, rid, sliding=True) + | live_pages(cache, model, rid, sliding=False) + for rid in ("long", "reused") + if cache.has_allocation(rid) + } + assert sum(map(len, owned.values())) == cache.num_used_blocks + if len(owned) == 2: + assert owned["long"].isdisjoint(owned["reused"]) + current_b = engine.scheduler.requests.get("reused") + if current_b is original_b and "reused" in owned: + if not first_b_pages: + first_b_pages = { + cache.layer_block_table("reused", layer, depth)[0] + for layer, window in model.kv_layer_windows.items() + if window is not None + for depth in ( + range(cache.storage_depths) if layer in model.recurrent_kv_layers else (0,) + ) + } + assert min(first_b_pages) >= 0 + reused_while_b_active |= first_b_pages & owned.get("long", set()) + if stale is not None and "reused" in owned and not stale_checked: + assert current_b is not original_b + with pytest.raises(RuntimeError, match="stale prepared KV batch"): + cache.layer_view(stale, 0) + stale_checked = True + a = engine.scheduler.requests.get("long") + if ( + not aborted + and reused_while_b_active + and a is not None + and current_b is original_b + and a.num_prefilled_tokens >= 9 + and current_b.num_prefilled_tokens >= 6 + and len(current_b.generated_token_ids) == 1 + and current_b.phase == RequestPhase.RECURRENT + and current_b.loops_done == 0 + ): + position = cache.retained_range("reused", 0, 0)[1] - 1 + stale = cache._prepare_batch(["reused"], [0], [position], for_write=False) + cache.layer_view(stale, 0) # A valid, resident descriptor before cancellation. + engine.abort_request("reused") + engine.add_request( + "reused", + replacement, + SamplingParams( + max_tokens=4, + min_loops=2, + max_loops=2, + ignore_eos=True, + ), + ) + assert engine.scheduler.requests["reused"].num_prefilled_tokens == 0 + prompts["reused"] = replacement + aborted = True + assert not engine.has_unfinished_requests(), "small-pool admission/growth did not complete" + assert aborted and stale_checked and reused_while_b_active + assert set(completed) == {"long", "reused"} + # Device snapshots stay on-device until the whole cancel/reuse scenario ends. + for request, index, context, actual in observed: + assert index == len(context) - len(request.prompt_token_ids) + expected = dense_dory_reference(reference, torch.tensor(context))["logits"][-1] + torch.testing.assert_close(actual.cpu(), expected, atol=3e-6, rtol=3e-5) + assert any(request is original_b for request, _, _, _ in observed) + assert any( + request is not original_b and request.request_id == "reused" + for request, _, _, _ in observed + ) + for rid, count in (("long", 7), ("reused", 4)): + tokens = list(prompts[rid]) + for _ in range(count): + logits = dense_dory_reference(reference, torch.tensor(tokens))["logits"] + tokens.append(int(logits[-1].argmax())) + assert completed[rid].token_ids == tokens[len(prompts[rid]) :] + assert cache.num_used_blocks == 0 and cache.num_free_blocks == cache.num_blocks + print( + f"{mode}: steps={step + 1}, live_cross_request_reused_pages=" + f"{sorted(reused_while_b_active)}, stale_rejected={stale_checked}, " + f"logits_checked={len(observed)}, cleanup=0" + ) diff --git a/vllm_rlt/config.py b/vllm_rlt/config.py index b5b58cb6..cfbdaf1d 100644 --- a/vllm_rlt/config.py +++ b/vllm_rlt/config.py @@ -18,13 +18,17 @@ class CacheConfig: memory_reserve_bytes: int = 256 * 1024 * 1024 enable_prefix_caching: bool = False incremental_allocation: bool = False + # Ordinary fixed-loop grouped KV only; return expired SWA pages to the pool. + reclaim_swa: bool = False # Admission headroom for incremental allocation; otherwise effective watermark is zero. watermark_ratio: float = 0.0 def __post_init__(self): - for name in ("enable_prefix_caching", "incremental_allocation"): + for name in ("enable_prefix_caching", "incremental_allocation", "reclaim_swa"): if type(getattr(self, name)) is not bool: raise ValueError(f"{name} must be a boolean") + if self.reclaim_swa and not self.incremental_allocation: + raise ValueError("reclaim_swa requires incremental_allocation") if not 0 <= self.watermark_ratio < 1: raise ValueError("watermark_ratio must be in [0, 1)") if self.enable_prefix_caching and self.layout != "last_exited": diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 0d1e8641..25cff1cf 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -9,7 +9,7 @@ import hashlib import struct from collections import OrderedDict -from collections.abc import Callable, Sequence +from collections.abc import Callable, Mapping, Sequence from copy import deepcopy from dataclasses import dataclass, field from numbers import Integral @@ -18,14 +18,22 @@ from vllm_rlt.attention import ( AttentionBackend, + AttentionRows, BackendCapabilities, backend_capabilities, create_backend, + plan_attention_metadata, validate_backend_name, ) from vllm_rlt.request import ExecutionBatch from vllm_rlt.worker.buffers import stage_metadata -from vllm_rlt.worker.metadata import DeviceKVMetadata, KVMetadata, MetadataBuilder +from vllm_rlt.worker.metadata import ( + DeviceKVMetadata, + GroupedKVMetadata, + KVLayout, + KVMetadata, + MetadataBuilder, +) @dataclass @@ -55,10 +63,24 @@ class _Allocation: max_tokens: int block_tables: tuple[tuple[int, ...], ...] written: list[list[_WrittenPositions]] + # Native group -> plane -> absolute logical page -> one page per group layer. + group_tables: tuple = () + group_epochs: list[list[int]] = field(default_factory=list) + retired_pages: list[list[int]] = field(default_factory=list) + consumed: list[list[_WrittenPositions]] = field(default_factory=list) + reclaim_pending: bool = False transfer_leases: set[str] = field(default_factory=set) release_requested: bool = False +@dataclass(frozen=True) +class KVGroup: + layers: tuple[int, ...] + window: int | None + recurrent: bool + planes: int + + @dataclass(frozen=True) class KVSnapshot: """What preemption keeps for one request: its pages and its written state. @@ -81,8 +103,9 @@ class KVCacheManager: """Own a fixed physical page pool shared by request/depth allocations. ``num_blocks`` counts total physical blocks across all recurrence depths. - Every physical block contains K and V for all decoder layers. Depth indices - are zero-based throughout the interface. + Legacy blocks contain all decoder layers. With ``layer_windows``, a block + is one physical attention-layer page; groups own independent page bundles. + Depth indices are zero-based throughout the interface. """ def __init__( @@ -103,6 +126,9 @@ def __init__( recurrent_layers: Sequence[int] | None = None, *, attention: AttentionBackend | None = None, + layer_windows: Mapping[int, int | None] | None = None, + reclaim_swa: bool = False, + max_submission_tokens: int = 1, ): for name, value in ( ("num_layers", num_layers), @@ -133,6 +159,47 @@ def __init__( ): raise ValueError("recurrent_layers must be valid cache layer indices") self.storage_depths = max_loops if layout == "last_exited" else 1 + self.grouped = layer_windows is not None + if type(reclaim_swa) is not bool: + raise ValueError("reclaim_swa must be a boolean") + if type(max_submission_tokens) is not int or max_submission_tokens <= 0: + raise ValueError("max_submission_tokens must be a positive integer") + if reclaim_swa and (not self.grouped or not incremental_allocation): + raise ValueError("SWA reclamation requires grouped incremental allocation") + self.reclaim_swa = reclaim_swa + self.max_submission_tokens = max_submission_tokens + self.groups = () + self.layer_groups = {} + if self.grouped: + if enable_prefix_caching: + raise NotImplementedError("grouped KV does not support prefix caching") + policies = dict(layer_windows) + if not policies or any( + type(layer) is not int or not 0 <= layer < num_layers for layer in policies + ): + raise ValueError("layer_windows must name actual attention layer indices") + if not set(self.recurrent_layers) <= policies.keys(): + raise ValueError("recurrent layers must be actual attention layers") + groups = {} + for layer, window in sorted(policies.items()): + if window is not None and (type(window) is not int or window <= 0): + raise ValueError("layer windows must be positive integers or None") + recurrent = layer in self.recurrent_layers + groups.setdefault((window, recurrent), []).append(layer) + self.groups = tuple( + KVGroup(tuple(layers), window, recurrent, self.storage_depths if recurrent else 1) + for (window, recurrent), layers in groups.items() + ) + self.layer_groups = { + layer: (group_id, local) + for group_id, group in enumerate(self.groups) + for local, layer in enumerate(group.layers) + } + self.pages_per_token_block = ( + sum(len(group.layers) * group.planes for group in self.groups) + if self.grouped + else self.storage_depths + ) self.device = torch.device(device) self.dtype = dtype self.attention = attention or create_backend( @@ -140,7 +207,7 @@ def __init__( ) self.attention_info = self.attention.info self.backend = self.attention_info["selected_backend"] - shape = (num_blocks, num_layers, block_size, num_kv_heads, head_dim) + shape = (num_blocks, 1 if self.grouped else num_layers, block_size, num_kv_heads, head_dim) self.key_cache = torch.empty(shape, device=self.device, dtype=dtype) self.value_cache = torch.empty_like(self.key_cache) # Resolve an implicit CUDA index to the actual storage device once. @@ -176,7 +243,7 @@ def num_used_blocks(self) -> int: def bytes_per_block(self) -> int: return ( 2 - * self.num_layers + * (1 if self.grouped else self.num_layers) * self.block_size * self.num_kv_heads * self.head_dim @@ -186,7 +253,22 @@ def bytes_per_block(self) -> int: def required_blocks(self, max_tokens: int) -> int: if not isinstance(max_tokens, Integral) or isinstance(max_tokens, bool) or max_tokens <= 0: raise ValueError("max_tokens must be a positive integer") - return ((max_tokens + self.block_size - 1) // self.block_size) * self.storage_depths + pages = (max_tokens + self.block_size - 1) // self.block_size + if self.reclaim_swa: + # W-1 retained keys plus one append S, including worst page alignment. + return sum( + min( + pages, + (group.window - 1 + self.max_submission_tokens + 2 * self.block_size - 2) + // self.block_size, + ) + * len(group.layers) + * group.planes + if group.window is not None + else pages * len(group.layers) * group.planes + for group in self.groups + ) + return pages * self.pages_per_token_block def _drop_refs(self, blocks): for b in blocks: @@ -287,10 +369,41 @@ def allocate(self, request_id: str, max_tokens: int, *, initial_tokens=None, pre raise ValueError( f"request needs {required} physical KV blocks, but the pool has {self.num_blocks}" ) - initial_tokens = max_tokens if initial_tokens is None else initial_tokens + if initial_tokens is None: + initial_tokens = ( + min(max_tokens, self.max_submission_tokens) if self.reclaim_swa else max_tokens + ) + if self.reclaim_swa and initial_tokens > self.max_submission_tokens: + raise ValueError("initial KV frontier exceeds the configured submission bound") if not 0 < initial_tokens <= max_tokens: raise ValueError("invalid initial KV frontier") pages = (initial_tokens + self.block_size - 1) // self.block_size + if self.grouped: + if prefix: + raise NotImplementedError("grouped KV does not support prefix imports") + fresh = self._claim(pages * self.pages_per_token_block) + if fresh is None: + return False + self._allocations[request_id] = _Allocation( + int(max_tokens), + (), + [ + [_WrittenPositions() for _ in range(self.num_layers)] + for _ in range(self.storage_depths) + ], + group_tables=self._group_tables(fresh, pages), + group_epochs=[[0] * group.planes for group in self.groups], + retired_pages=[[0] * group.planes for group in self.groups], + consumed=( + [ + [_WrittenPositions() for _ in range(self.num_layers)] + for _ in range(self.storage_depths) + ] + if self.reclaim_swa + else [] + ), + ) + return True if len(prefix) > pages: raise ValueError("prefix exceeds initial KV frontier") claimed = [b for group in prefix for b in group] @@ -323,6 +436,23 @@ def ensure_capacity(self, request_id, tokens): if not 0 < tokens <= allocation.max_tokens: raise ValueError("KV growth exceeds logical capacity") pages = (tokens + self.block_size - 1) // self.block_size + if self.grouped: + current = len(allocation.group_tables[0][0]) + extra = pages - current + if extra <= 0: + return True + fresh = self._claim(extra * self.pages_per_token_block) + if fresh is None: + return False + added = self._group_tables(fresh, extra) + allocation.group_tables = tuple( + tuple(table + tail for table, tail in zip(planes, new_planes)) + for planes, new_planes in zip(allocation.group_tables, added) + ) + for epochs in allocation.group_epochs: + for plane in range(len(epochs)): + epochs[plane] += 1 + return True extra = pages - len(allocation.block_tables[0]) if extra <= 0: return True @@ -342,6 +472,8 @@ def truncate_suffix(self, request_id: str, frontier: int) -> None: admission reservation; stale bytes are hidden by written/context lengths. Shared prompt pages must never be truncated or subsequently overwritten. """ + if self.grouped: + raise NotImplementedError("grouped KV supports ordinary layer views only") allocation = self._get_allocation(request_id) if type(frontier) is not int or not 0 <= frontier <= allocation.max_tokens: raise ValueError("invalid KV truncation frontier") @@ -362,9 +494,21 @@ def free(self, request_id: str) -> None: return allocation = self._allocations.pop(request_id, None) if allocation is not None: - self._drop_refs(b for table in allocation.block_tables for b in reversed(table)) + if self.grouped: + self._drop_refs( + b + for planes in allocation.group_tables + for table in planes + for bundle in reversed(table) + if bundle is not None + for b in reversed(bundle) + ) + else: + self._drop_refs(b for table in allocation.block_tables for b in reversed(table)) def pin_transfer(self, request_id: str, transfer_id: str): + if self.grouped: + raise NotImplementedError("grouped KV supports ordinary layer views only") allocation = self._get_allocation(request_id) if allocation.release_requested: raise RuntimeError("cannot transfer KV scheduled for release") @@ -378,6 +522,8 @@ def unpin_transfer(self, request_id: str, transfer_id: str): def mark_imported_prefix(self, request_id: str, length: int, *, start=0): """Publish received KV only after the connector confirms remote writes are complete.""" + if self.grouped: + raise NotImplementedError("grouped KV supports ordinary layer views only") allocation = self._get_allocation(request_id) if type(length) is not int or not 0 < length <= allocation.max_tokens: raise ValueError("imported prefix exceeds reserved capacity") @@ -397,9 +543,20 @@ def has_allocation(self, request_id: str) -> bool: def allocated_blocks(self, request_id: str) -> int: """How many physical blocks this request holds, counting every plane.""" - return len(self._get_allocation(request_id).block_tables[0]) * self.storage_depths + allocation = self._get_allocation(request_id) + if self.grouped: + return sum( + len(bundle) + for planes in allocation.group_tables + for table in planes + for bundle in table + if bundle is not None + ) + return len(allocation.block_tables[0]) * self.storage_depths def get_block_table(self, request_id: str, depth: int) -> tuple[int, ...]: + if self.grouped: + raise NotImplementedError("grouped KV supports ordinary layer views only") self._validate_depth(depth) return self._get_allocation(request_id).block_tables[self._plane(depth)] @@ -410,6 +567,8 @@ def plane_block_tables(self, request_id: str) -> tuple[tuple[int, ...], ...]: ``max_loops`` planes and SHARED has one, so ``tables[2]`` fails under SHARED while ``get_block_table(rid, 2)`` works under both layouts. """ + if self.grouped: + raise NotImplementedError("grouped KV supports ordinary layer views only") return self._get_allocation(request_id).block_tables def has_transfer_lease(self, request_id: str) -> bool: @@ -433,6 +592,12 @@ def token_written(self, request_id: str, position: int, depth: int) -> bool: def _token_written(self, allocation: _Allocation, position: int, depth: int) -> bool: plane = allocation.written[self._plane(depth)] + if self.grouped: + return all( + position in allocation.written[self._plane(depth) if group.recurrent else 0][layer] + for group in self.groups + for layer in group.layers + ) return all(position in plane[layer] for layer in range(self.num_layers)) def exit_written(self, request_id: str, position: int, depth: int) -> bool: @@ -485,6 +650,8 @@ def snapshot(self, request_id: str) -> KVSnapshot: Copying page by page avoids a request-sized temporary on a device that is already full, which is exactly when preemption happens. """ + if self.grouped: + raise NotImplementedError("grouped KV supports ordinary layer views only") allocation = self._get_allocation(request_id) if allocation.transfer_leases: raise RuntimeError("cannot snapshot KV during a transfer") @@ -509,6 +676,8 @@ def restore(self, request_id: str, snapshot: KVSnapshot) -> None: decision belongs to the scheduler. The copies go on the current stream; the caller decides when other streams may read them. """ + if self.grouped: + raise NotImplementedError("grouped KV supports ordinary layer views only") allocation = self._get_allocation(request_id) if allocation.max_tokens != snapshot.max_tokens: raise ValueError("snapshot capacity does not match the allocation") @@ -522,6 +691,197 @@ def restore(self, request_id: str, snapshot: KVSnapshot) -> None: self.value_cache[block].copy_(snapshot.values[row]) allocation.written = deepcopy(snapshot.written) + def _group_tables(self, blocks, pages): + """Partition one atomic claim into group/plane/logical-page bundles.""" + cursor, result = 0, [] + for group in self.groups: + planes = [] + width = len(group.layers) + for _ in range(group.planes): + table = [] + for _ in range(pages): + table.append(tuple(blocks[cursor : cursor + width])) + cursor += width + planes.append(tuple(table)) + result.append(tuple(planes)) + assert cursor == len(blocks) + return tuple(result) + + def _group_table(self, allocation, layer, depth): + group_id, local = self.layer_groups[layer] + group = self.groups[group_id] + if not group.recurrent and depth != 0: + raise ValueError("boundary attention layers only own depth zero") + plane = self._plane(depth) if group.recurrent else 0 + return allocation.group_tables[group_id][plane], local + + def _layer_table(self, allocation, layer, depth): + bundles, local = self._group_table(allocation, layer, depth) + return tuple(-1 if bundle is None else bundle[local] for bundle in bundles) + + def layer_group(self, layer: int) -> tuple[int, KVGroup]: + """Read-only group policy for a compact KV layer; this does not pin pages. + + Group IDs are local to this manager. The returned policy is immutable. + """ + self._validate_layer(layer) + if not self.grouped: + raise NotImplementedError("layer groups require grouped KV storage") + group_id, _ = self.layer_groups[layer] + return group_id, self.groups[group_id] + + def layer_block_table(self, request_id, layer, depth): + """Read-only absolute logical-page map for one actual attention layer.""" + self._validate_layer(layer) + self._validate_depth(depth) + allocation = self._get_allocation(request_id) + if not self.grouped: + return allocation.block_tables[self._plane(depth)] + return self._layer_table(allocation, layer, depth) + + def layer_view(self, batch, layer): + """Resolve native ownership; SWA read tables use a page-aligned local span. + + Rows, RoPE positions and write addresses retain their absolute meaning. + Only the attention planner sees positions relative to each row's span. + """ + self._validate_layer(layer) + self._require_live_batch(batch) + if not isinstance(batch, GroupedKVMetadata): + raise TypeError("grouped KV requires a logical grouped descriptor") + group_id, _ = self.layer_groups[layer] + group = self.groups[group_id] + epochs = tuple( + a.group_epochs[group_id][self._plane(d) if group.recurrent else 0] + for a, d, _ in batch.rows + ) + cached = batch.layer_views.get(layer) + if cached is not None and cached[0] == epochs: + return cached[1] + tables, blocks, read_positions = [], [], [] + positions = [p for _, _, p in batch.rows] + for allocation, depth, position in batch.rows: + bundles, local = self._group_table(allocation, layer, depth) + page = position // self.block_size + if page >= len(bundles): + raise ValueError("KV row exceeds the acquired physical capacity") + bundle = bundles[page] + blocks.append(-1 if bundle is None else bundle[local]) + first = ( + 0 + if group.window is None + else max(0, position - group.window + 1) // self.block_size + ) + tables.append(tuple(-1 if b is None else b[local] for b in bundles[first : page + 1])) + read_positions.append(position - first * self.block_size) + offsets = [p % self.block_size for p in positions] + if any(block < 0 for block in blocks): + raise RuntimeError("current KV write/query page is no longer resident") + if self.reclaim_swa: + self._require_resident_rows(batch, layer, group.window) + if batch.writable and len(set(zip(blocks, offsets))) != len(blocks): + raise ValueError("duplicate physical KV write addresses") + attention = plan_attention_metadata( + AttentionRows(block_tables=tables, positions=read_positions), + block_size=self.block_size, + ) + layout = KVLayout( + self, + batch.rows, + batch.allocations, + positions, + blocks, + offsets, + attention, + batch.writable, + ) + view = stage_metadata((layout,), self._stage, padded_rows=len(batch.position_ids))[0] + batch.layer_views[layer] = (epochs, view) + return view + + def retained_range(self, request_id, layer, depth): + """Absolute resident start and logical written end for a layer/plane.""" + self._validate_layer(layer) + self._validate_depth(depth) + allocation = self._get_allocation(request_id) + start = 0 + if self.grouped: + self._group_table(allocation, layer, depth) + group_id, _ = self.layer_groups[layer] + group = self.groups[group_id] + plane = self._plane(depth) if group.recurrent else 0 + start = allocation.retired_pages[group_id][plane] * self.block_size + return start, allocation.written[self._plane(depth)][layer].prefix + + def _require_resident_rows(self, batch, layer, window): + for allocation, depth, position in batch.rows: + bundles, _ = self._group_table(allocation, layer, depth) + start = 0 if window is None else max(0, position - window + 1) + if position // self.block_size >= len(bundles): + raise ValueError("KV row exceeds the acquired physical capacity") + if any( + bundles[page] is None + for page in range(start // self.block_size, position // self.block_size + 1) + ): + raise RuntimeError("required KV history is no longer resident") + + def _record_consumed(self, layer, batch): + group_id, _ = self.layer_groups[layer] + if self.groups[group_id].window is None: + return + group = self.groups[group_id] + for allocation, depth, position in batch.rows: + # Shared recurrent history is still read by later loops of this token. + if self.layout == "shared" and group.recurrent and depth != self.max_loops - 1: + continue + allocation.consumed[self._plane(depth)][layer].add(position) + allocation.reclaim_pending = True + + def finish_submission(self, event=None): + """Return whole expired bundles only after this submission's final reader. + + Consumption candidates are host progress, not device completion. Reuse + the ordinary runner's event and wait only when physical pages retire. + """ + if not self.reclaim_swa: + return + plans = [] + pending = [a for a in self._allocations.values() if a.reclaim_pending] + for allocation in pending: + for group_id, group in enumerate(self.groups): + if group.window is None: + continue + for plane, table in enumerate(allocation.group_tables[group_id]): + frontier = min( + allocation.consumed[plane][layer].prefix for layer in group.layers + ) + stop = max(0, frontier - group.window + 1) // self.block_size + start = allocation.retired_pages[group_id][plane] + if stop > start: + plans.append((allocation, group_id, plane, start, stop)) + if plans: + if event is None and self.device.type == "cuda": + raise ValueError("CUDA reclamation requires the submission completion event") + if event is not None: + event.synchronize() + for allocation, group_id, plane, start, stop in plans: + groups = list(allocation.group_tables) + planes = list(groups[group_id]) + table = list(planes[plane]) + released = [] + for page in range(start, stop): + if table[page] is not None: + released.extend(table[page]) + table[page] = None + planes[plane] = tuple(table) + groups[group_id] = tuple(planes) + allocation.group_tables = tuple(groups) + allocation.retired_pages[group_id][plane] = stop + allocation.group_epochs[group_id][plane] += 1 + self._drop_refs(released) + for allocation in pending: + allocation.reclaim_pending = False + def _plane(self, depth: int) -> int: return depth if self.layout == "last_exited" else 0 @@ -538,6 +898,8 @@ def _validate_layer(self, layer: int) -> None: or not 0 <= layer < self.num_layers ): raise ValueError(f"layer must be in [0, {self.num_layers})") + if self.grouped and layer not in self.layer_groups: + raise ValueError("layer has no attention KV storage") def _validate_depth(self, depth: int) -> None: if ( @@ -574,7 +936,9 @@ def _validate_rows(self, request_ids, depths, positions): def metadata_builder(self): return MetadataBuilder(self) - def prepare_recurrent(self, batch: ExecutionBatch, *, packed=False) -> KVMetadata: + def prepare_recurrent( + self, batch: ExecutionBatch, *, packed=False + ) -> KVMetadata | GroupedKVMetadata: """Compatibility entry for standalone cache consumers.""" ids, depths, positions = batch.addresses() return self._prepare_batch( @@ -594,7 +958,7 @@ def _prepare_batch( packed_prefill: bool = False, stage: Callable[[Sequence[int], torch.dtype], torch.Tensor] | None = None, padded_rows: int | None = None, - ) -> KVMetadata: + ) -> KVMetadata | GroupedKVMetadata: """Build layer-independent addresses once; do not initialize any KV slot.""" return self._prepare_batches( request_ids, @@ -616,7 +980,7 @@ def _prepare_batches( packed_prefill: bool = False, stage: Callable[[Sequence[int], torch.dtype], torch.Tensor] | None = None, padded_rows: int | None = None, - ) -> tuple[KVMetadata, ...]: + ) -> tuple[KVMetadata | GroupedKVMetadata, ...]: """Convenience API; layout construction belongs to MetadataBuilder.""" layouts = self.metadata_builder.build( request_ids, @@ -633,8 +997,10 @@ def _stage(self, values, dtype): host = torch.tensor(values, dtype=dtype, pin_memory=self.device.type == "cuda") return host.to(self.device, non_blocking=True) - def _require_live_batch(self, batch: KVMetadata | DeviceKVMetadata) -> None: - if isinstance(batch, KVMetadata) and batch.owner is not self: + def _require_live_batch(self, batch: KVMetadata | GroupedKVMetadata | DeviceKVMetadata) -> None: + if self.grouped and isinstance(batch, DeviceKVMetadata): + raise NotImplementedError("grouped KV does not support device-owned metadata") + if isinstance(batch, (KVMetadata, GroupedKVMetadata)) and batch.owner is not self: raise ValueError("prepared KV batch belongs to a different cache manager") for request_id, allocation in batch.allocations: if self._allocations.get(request_id) is not allocation: @@ -674,7 +1040,11 @@ def write( @torch.no_grad() def _write_prepared( - self, layer: int, batch: KVMetadata | DeviceKVMetadata, k: torch.Tensor, v: torch.Tensor + self, + layer: int, + batch: KVMetadata | GroupedKVMetadata | DeviceKVMetadata, + k: torch.Tensor, + v: torch.Tensor, ) -> None: self._validate_layer(layer) self._require_live_batch(batch) @@ -689,14 +1059,21 @@ def _write_prepared( self.key_cache[batch.write_blocks, layer, batch.write_offsets] = k self.value_cache[batch.write_blocks, layer, batch.write_offsets] = v return + if self.grouped: + batch = self.layer_view(batch, layer) if not batch.writable: raise ValueError("a read-only prepared KV batch cannot be written") self._validate_tensor(k, len(batch.position_ids), "k") self._validate_tensor(v, len(batch.position_ids), "v") if not batch.rows: return - self.key_cache[batch.write_blocks, layer, batch.write_offsets] = k[: len(batch.rows)] - self.value_cache[batch.write_blocks, layer, batch.write_offsets] = v[: len(batch.rows)] + storage_layer = 0 if self.grouped else layer + self.key_cache[batch.write_blocks, storage_layer, batch.write_offsets] = k[ + : len(batch.rows) + ] + self.value_cache[batch.write_blocks, storage_layer, batch.write_offsets] = v[ + : len(batch.rows) + ] for allocation, depth, position in batch.rows: allocation.written[self._plane(depth)][layer].add(position) @@ -729,7 +1106,7 @@ def attend( def _attend_prepared( self, layer: int, - batch: KVMetadata | DeviceKVMetadata, + batch: KVMetadata | GroupedKVMetadata | DeviceKVMetadata, q: torch.Tensor, *, window: int | None = None, @@ -737,8 +1114,17 @@ def _attend_prepared( self._validate_layer(layer) self._require_live_batch(batch) self._validate_tensor(q, len(batch.position_ids), "q", query=True) + logical = batch + if self.grouped: + group_id, _ = self.layer_groups[layer] + if window != self.groups[group_id].window: + raise ValueError("native attention window must match the layer policy") + batch = self.layer_view(batch, layer) if window is not None: - return self._attend_windowed(layer, batch, q, window) + output = self._attend_windowed(layer, batch, q, window) + if self.reclaim_swa: + self._record_consumed(layer, logical) + return output if isinstance(batch, DeviceKVMetadata): return self.attention( q, @@ -754,8 +1140,8 @@ def _attend_prepared( if batch.cu_seqlens_q is not None: return self.attention.prefill( q, - self.key_cache[:, layer], - self.value_cache[:, layer], + self.key_cache[:, 0 if self.grouped else layer], + self.value_cache[:, 0 if self.grouped else layer], batch.block_tables, batch.context_lengths, batch.cu_seqlens_q, @@ -763,8 +1149,8 @@ def _attend_prepared( ) return self.attention( q, - self.key_cache[:, layer], - self.value_cache[:, layer], + self.key_cache[:, 0 if self.grouped else layer], + self.value_cache[:, 0 if self.grouped else layer], batch.block_tables, batch.context_lengths, ) @@ -795,8 +1181,8 @@ def _attend_windowed( self._require_prefix(allocation, layer, depth, position + 1) return self.attention.windowed( q, - self.key_cache[:, layer], - self.value_cache[:, layer], + self.key_cache[:, 0 if self.grouped else layer], + self.value_cache[:, 0 if self.grouped else layer], batch.block_tables, batch.context_lengths, window, @@ -805,10 +1191,29 @@ def _attend_windowed( @torch.no_grad() def finalize_token(self, request_id: str, position: int, exit_depth: int) -> None: """Propagate the final executed depth's K/V into every unexecuted depth.""" + if self.reclaim_swa and exit_depth != self.max_loops - 1: + raise NotImplementedError( + "SWA reclamation currently requires full fixed-loop execution" + ) allocation = self._require_exit_written(request_id, position, exit_depth) - if self.layout == "shared": + if self.layout == "shared" or exit_depth == self.max_loops - 1: return logical_page, offset = divmod(position, self.block_size) + if self.grouped: + for layer in self.recurrent_layers: + source = self.layer_block_table(request_id, layer, exit_depth)[logical_page] + if source < 0: + raise RuntimeError("cannot finalize reclaimed KV") + for depth in range(exit_depth + 1, self.max_loops): + destination = self.layer_block_table(request_id, layer, depth)[logical_page] + if destination < 0: + raise RuntimeError("cannot finalize into reclaimed KV") + self.key_cache[destination, 0, offset].copy_(self.key_cache[source, 0, offset]) + self.value_cache[destination, 0, offset].copy_( + self.value_cache[source, 0, offset] + ) + self._record_finalized(allocation, position, exit_depth) + return source = allocation.block_tables[self._plane(exit_depth)][logical_page] destinations = [ allocation.block_tables[self._plane(depth)][logical_page] @@ -836,6 +1241,8 @@ def read(self, layer: int, request_id: str, depth: int, length: int | None = Non self._validate_layer(layer) self._validate_depth(depth) allocation = self._get_allocation(request_id) + if self.grouped: + self._layer_table(allocation, layer, depth) if length is None: length = allocation.written[self._plane(depth)][layer].prefix if ( @@ -845,10 +1252,24 @@ def read(self, layer: int, request_id: str, depth: int, length: int | None = Non ): raise ValueError("read length exceeds the request's reserved token budget") self._require_prefix(allocation, layer, depth, length) + if self.grouped and length and self.retained_range(request_id, layer, depth)[0]: + raise RuntimeError( + "KV prefix has been reclaimed; use retained_range and layer_block_table" + ) positions = torch.arange(length, device=self.device) table = torch.tensor( - allocation.block_tables[self._plane(depth)], device=self.device, dtype=torch.long + ( + self._layer_table(allocation, layer, depth) + if self.grouped + else allocation.block_tables[self._plane(depth)] + ), + device=self.device, + dtype=torch.long, ) blocks = table[positions // self.block_size] offsets = positions % self.block_size - return self.key_cache[blocks, layer, offsets], self.value_cache[blocks, layer, offsets] + storage_layer = 0 if self.grouped else layer + return ( + self.key_cache[blocks, storage_layer, offsets], + self.value_cache[blocks, storage_layer, offsets], + ) diff --git a/vllm_rlt/core/memory.py b/vllm_rlt/core/memory.py index 1f7ed1e0..8dc5275f 100644 --- a/vllm_rlt/core/memory.py +++ b/vllm_rlt/core/memory.py @@ -59,21 +59,40 @@ def plan_cache( # Models whose physical stack mixes KV-owning and KV-free sublayers (Dory) # expose the compact count; everything else keeps the layer count. kv_layers = getattr(config, "num_kv_layers", config.num_hidden_layers) + layer_windows = getattr(model, "kv_layer_windows", None) + native = layer_windows is not None + unit = "physical_layer_page" if native else "all_layer_block" + recurrent = set(getattr(model, "recurrent_kv_layers", range(kv_layers))) + depths = config.total_ut_steps if cache.layout == "last_exited" else 1 + page_factor = ( + sum(depths if layer in recurrent else 1 for layer in layer_windows) if native else depths + ) per_block = ( 2 - * kv_layers + * (1 if native else kv_layers) * cache.block_size * config.num_key_value_heads * config.head_dim * parameter.element_size() ) if cache.num_blocks is not None: - return cache.num_blocks, {"source": "blocks", "bytes_per_block": per_block} + blocks = cache.num_blocks * (kv_layers if native else 1) + return blocks, { + "source": "blocks", + "bytes_per_block": per_block, + "block_unit": unit, + "configured_num_blocks": cache.num_blocks, + } if cache.kv_cache_memory_bytes is not None: blocks = budget_blocks(cache.kv_cache_memory_bytes, per_block) - return blocks, {"source": "bytes", "bytes_per_block": per_block} + return blocks, {"source": "bytes", "bytes_per_block": per_block, "block_unit": unit} if device.type != "cuda": - return 256, {"source": "cpu_default", "bytes_per_block": per_block} + return 256 * (kv_layers if native else 1), { + "source": "cpu_default", + "bytes_per_block": per_block, + "block_unit": unit, + "configured_num_blocks": 256, + } torch.cuda.synchronize(device) # Return unused allocator segments before measuring driver-free memory. # In particular, a previous engine may have left its KV pool cached here. @@ -99,12 +118,14 @@ def plan_cache( head_dim=config.head_dim, max_loops=config.total_ut_steps, num_blocks=sum(math.ceil(length / cache.block_size) for length in lengths) - * config.total_ut_steps, + * (page_factor if native else config.total_ut_steps), + layout=cache.layout if native else "last_exited", block_size=cache.block_size, device=device, dtype=parameter.dtype, backend=backend, recurrent_layers=getattr(model, "recurrent_kv_layers", None), + layer_windows=layer_windows, ) ids, positions = [], [] for index, length in enumerate(lengths): @@ -167,15 +188,37 @@ def plan_cache( ) # More blocks than all concurrent full-context requests can use only waste # memory, particularly for tiny-model GPU tests with very small pages. - useful_blocks = ( - math.ceil(config.max_position_embeddings / cache.block_size) - * (config.total_ut_steps if cache.layout == "last_exited" else 1) - * scheduler.max_num_seqs - ) + if native and cache.reclaim_swa: + submission = min(scheduler.prefill_chunk_size, scheduler.max_num_batched_tokens) + pages = math.ceil(config.max_position_embeddings / cache.block_size) + useful_blocks = ( + sum( + ( + min( + pages, + math.ceil( + (window - 1 + submission + cache.block_size - 1) / cache.block_size + ), + ) + if window is not None + else pages + ) + * (depths if layer in recurrent else 1) + for layer, window in layer_windows.items() + ) + * scheduler.max_num_seqs + ) + else: + useful_blocks = ( + math.ceil(config.max_position_embeddings / cache.block_size) + * page_factor + * scheduler.max_num_seqs + ) blocks = min(budget_blocks(budget, per_block), useful_blocks) return blocks, dict( source="cuda_profile", bytes_per_block=per_block, + block_unit=unit, profile_peak_bytes=peak, concurrent_peak_bytes=concurrent_peak, max_useful_blocks=useful_blocks, diff --git a/vllm_rlt/core/scheduler.py b/vllm_rlt/core/scheduler.py index 14d95950..09e28c55 100644 --- a/vllm_rlt/core/scheduler.py +++ b/vllm_rlt/core/scheduler.py @@ -391,6 +391,8 @@ def _plan_admission(self, request: Request, active_count: int) -> AdmissionPlan: initial_tokens = min( len(request.prompt_token_ids), cached_tokens + self.config.prefill_chunk_size ) + if cache.reclaim_swa: + initial_tokens = min(initial_tokens, cache.max_submission_tokens) if not cache.incremental_allocation: initial_tokens = capacity reserve_outputs = cache.incremental_allocation and self.suspend_callback is None diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index 1cae4096..2df7e5ff 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -181,6 +181,10 @@ def __init__( staged_prefill=staged_prefill, sliding_window=getattr(model, "requires_sliding_window", False), ) + if cache_config.reclaim_swa and ( + not getattr(model, "fixed_loop_count", False) or self.exit_config.mode != "ouro" + ): + raise ValueError("SWA reclamation requires ordinary fixed-loop ouro execution") attention = create_backend( attention_backend, parameter.device, @@ -212,6 +216,11 @@ def __init__( speculative=speculative_config, staged_prefill=staged_prefill, ) + if ( + getattr(model, "kv_layer_windows", None) is not None + and scheduler_config.enable_preemption + ): + raise NotImplementedError("grouped KV does not support preemption snapshots") self.cache_manager = KVCacheManager( num_layers=getattr(config, "num_kv_layers", config.num_hidden_layers), num_kv_heads=config.num_key_value_heads, @@ -228,6 +237,11 @@ def __init__( incremental_allocation=cache_config.incremental_allocation, watermark_ratio=cache_config.watermark_ratio, recurrent_layers=getattr(model, "recurrent_kv_layers", None), + layer_windows=getattr(model, "kv_layer_windows", None), + reclaim_swa=cache_config.reclaim_swa, + max_submission_tokens=min( + scheduler_config.prefill_chunk_size, scheduler_config.max_num_batched_tokens + ), ) self.scheduler = Scheduler( scheduler_config, @@ -521,6 +535,7 @@ def _step(self) -> list[RequestOutput]: return [] try: ticket = self.executor.submit(batch) + self.cache_manager.finish_submission(ticket.event) # Only host-dependent decisions collect now. Prompt/PRELUDE submissions # advance logically under their event without a readback wait. result = ticket.collect() if ticket.has_result else None diff --git a/vllm_rlt/models/dory.py b/vllm_rlt/models/dory.py index 8e6a388e..da3a8b52 100644 --- a/vllm_rlt/models/dory.py +++ b/vllm_rlt/models/dory.py @@ -512,6 +512,12 @@ def __init__(self, config: DoryConfig) -> None: self.coda_kv_layers = tuple( config.kv_layer_index(index) for index in config.attention_layers("output") ) + self.kv_layer_windows = { + config.kv_layer_index(index): ( + config.sliding_window if config.layer_types[index] == "sliding_attention" else None + ) + for index in config.attention_layer_ids + } self.apply(self._initialize) self.requires_grad_(False) self.eval() diff --git a/vllm_rlt/worker/buffers.py b/vllm_rlt/worker/buffers.py index 621e1715..8a7836f0 100644 --- a/vllm_rlt/worker/buffers.py +++ b/vllm_rlt/worker/buffers.py @@ -5,7 +5,12 @@ import torch -from vllm_rlt.worker.metadata import KVLayout, KVMetadata +from vllm_rlt.worker.metadata import ( + GroupedKVLayout, + GroupedKVMetadata, + KVLayout, + KVMetadata, +) from vllm_rlt.worker.submission_queue import MAX_CONCURRENT_SUBMISSIONS, SubmissionQueue from vllm_rlt.worker.tensors import UvaBackedTensor, copy_tensor, use_uva_on @@ -95,6 +100,8 @@ def record_done(self): def store_metadata(self, layout: KVLayout, size): """Fill fixed host buffers from a storage-independent layout.""" + if isinstance(layout, GroupedKVLayout): + raise NotImplementedError("grouped KV does not support UVA metadata workspaces") n = len(layout.rows) attention = layout.attention if attention.cu_seqlens_q is not None: @@ -155,6 +162,18 @@ def tokens(self, ids, size): def stage_metadata(layouts, stage, *, padded_rows=None): """Store computed layouts in two typed buffers using caller-owned staging.""" first = layouts[0] + if isinstance(first, GroupedKVLayout): + n = len(first.rows) + size = n if padded_rows is None else padded_rows + if size < n: + raise ValueError("padding requires at least the real row count") + positions = stage(first.positions + [0] * (size - n), torch.int64) + return tuple( + GroupedKVMetadata( + layout.owner, layout.rows, layout.allocations, positions, layout.writable + ) + for layout in layouts + ) n = len(first.rows) size = n if padded_rows is None else padded_rows if size < n or (size != n and first.attention.cu_seqlens_q is not None): diff --git a/vllm_rlt/worker/metadata.py b/vllm_rlt/worker/metadata.py index 95fde7bb..5617fd31 100644 --- a/vllm_rlt/worker/metadata.py +++ b/vllm_rlt/worker/metadata.py @@ -7,7 +7,7 @@ from __future__ import annotations from collections.abc import Sequence -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING import torch @@ -26,6 +26,8 @@ class KVMetadata: no write address. Packed query boundaries describe layout, not request stage. Host rows and allocation identities are captured independently of caller lists. Device tensors are private, read-only inputs to the cache operations. + Read tables and context lengths are backend-local spans: native SWA views + may start at a window page boundary. Rows and position_ids remain absolute. """ owner: KVCacheManager @@ -41,6 +43,31 @@ class KVMetadata: max_seqlen_q: int = 1 +@dataclass(frozen=True) +class GroupedKVLayout: + """Logical rows only; physical addresses depend on the attention layer.""" + + owner: KVCacheManager + rows: tuple[tuple[_Allocation, int, int], ...] + allocations: tuple[tuple[str, _Allocation], ...] + positions: list[int] + writable: bool + + +@dataclass(frozen=True, eq=False) +class GroupedKVMetadata: + """Borrowed logical traversal with descriptor-local resolved layer views.""" + + owner: KVCacheManager + rows: tuple[tuple[_Allocation, int, int], ...] + allocations: tuple[tuple[str, _Allocation], ...] + position_ids: torch.Tensor + writable: bool + layer_views: dict[int, tuple[tuple[int, ...], KVMetadata]] = field(default_factory=dict) + cu_seqlens_q: None = None + max_seqlen_q: int = 1 + + @dataclass(frozen=True, eq=False) class DeviceKVMetadata: """Stream-ordered speculative metadata with device-owned positions. @@ -107,6 +134,8 @@ def prepare_device( until GPU completion. This does not advance host written-position state. Allocation identity also rejects descriptors from another cache manager. """ + if self.cache.grouped: + raise NotImplementedError("grouped KV does not support device-owned metadata") batch = DeviceKVMetadata( allocations=tuple(allocations), position_ids=position_ids, @@ -165,6 +194,8 @@ def _write_addresses(self, rows, *, offsets=None, for_write=True): def resolve_rows(self, ids, depths, positions): """Validate host-known rows for device routing without building host tables.""" + if self.cache.grouped: + raise NotImplementedError("grouped KV does not support device routing") rows = tuple(self.cache._validate_rows(ids, depths, positions)) self._write_addresses(rows) return rows @@ -177,9 +208,11 @@ def build( *, packed=False, for_write=True, - ) -> tuple[KVLayout, ...]: + ) -> tuple[KVLayout | GroupedKVLayout, ...]: """Calculate all requested depth layouts without selecting a storage path.""" cache = self.cache + if cache.grouped and packed: + raise NotImplementedError("grouped KV supports ordinary host-planned rows only") if not depth_sets: raise ValueError("at least one depth assignment is required") if packed and ( @@ -204,6 +237,12 @@ def build( allocations = tuple(dict(zip(request_ids, (a for a, _, _ in first))).items()) layouts = [] for rows in row_sets: + if cache.grouped: + identities = [(id(a), cache._plane(d), p) for a, d, p in rows] + if for_write and len(set(identities)) != len(rows): + raise ValueError("duplicate logical KV write rows") + layouts.append(GroupedKVLayout(cache, rows, allocations, positions, for_write)) + continue blocks, _ = self._write_addresses(rows, offsets=offsets, for_write=for_write) attention = plan_attention_metadata( AttentionRows( @@ -283,6 +322,8 @@ def view(tensor): tensor.shape, tensor.stride(), tensor.storage_offset() ) + if isinstance(metadata, GroupedKVMetadata): + return replace(metadata, position_ids=view(metadata.position_ids), layer_views={}) return replace( metadata, position_ids=view(metadata.position_ids), @@ -296,6 +337,13 @@ def view(tensor): @staticmethod def slice_rows(metadata, start, end): """Select contiguous single-query rows, e.g. a dependent coda subset.""" + if isinstance(metadata, GroupedKVMetadata): + return replace( + metadata, + rows=metadata.rows[start:end], + position_ids=metadata.position_ids[start:end], + layer_views={}, + ) if metadata.cu_seqlens_q is not None: raise ValueError("packed query boundaries must be rebuilt when selecting rows") return replace( @@ -310,6 +358,10 @@ def slice_rows(metadata, start, end): @staticmethod def without_padding(metadata): + if isinstance(metadata, GroupedKVMetadata): + return replace( + metadata, position_ids=metadata.position_ids[: len(metadata.rows)], layer_views={} + ) if metadata.cu_seqlens_q is not None: return metadata count = len(metadata.rows) @@ -333,6 +385,8 @@ def prepare_shared_rows( loop_count, ): """Reuse one shared-plane table across token groups and loop assignments.""" + if isinstance(base, GroupedKVMetadata): + raise NotImplementedError("grouped KV does not support shared device row routing") count = len(spans) rows = tuple( (base.rows[row][0], span.depth, positions[row]) for row, span in enumerate(spans) diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index 78f78a78..84eec76b 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -649,12 +649,16 @@ def _prepare_shared_loop_inputs(self, batch): cache = self.cache_manager spans = sorted(batch.spans, key=lambda span: span.token_count, reverse=True) ids = [span.request.request_id for span in spans] - base = self._stage_metadata( - ids, - [[span.depth for span in spans]], - [span.token_start + span.token_count - 1 for span in spans], - padded_rows=self._size(len(spans)), - )[0] + base = ( + None + if cache.grouped + else self._stage_metadata( + ids, + [[span.depth for span in spans]], + [span.token_start + span.token_count - 1 for span in spans], + padded_rows=self._size(len(spans)), + )[0] + ) groups, positions, blocks, tokens, contexts = [], [], [], [], [] for offset in range(spans[0].token_count): active = [span for span in spans if offset < span.token_count] @@ -662,13 +666,15 @@ def _prepare_shared_loop_inputs(self, batch): for row, span in enumerate(active): position = span.token_start + offset positions.append(position) - blocks.append(base.rows[row][0].block_tables[0][position // cache.block_size]) + if not cache.grouped: + blocks.append(base.rows[row][0].block_tables[0][position // cache.block_size]) tokens.append(span.input_tokens[offset]) contexts.append(position + 1) size = self._size(len(active)) padding = size - len(active) positions.extend([0] * padding) - blocks.extend([0] * padding) + if not cache.grouped: + blocks.extend([0] * padding) tokens.extend([0] * padding) contexts.extend([0] * padding) outputs = tuple( @@ -678,24 +684,33 @@ def _prepare_shared_loop_inputs(self, batch): ) groups.append((start, size, active, outputs)) n = len(positions) - wide = self.input_buffers[torch.int64].append( - positions + blocks + [p % cache.block_size for p in positions] - ) - lengths = self.input_buffers[torch.int32].append(contexts) + if not cache.grouped: + wide = self.input_buffers[torch.int64].append( + positions + blocks + [p % cache.block_size for p in positions] + ) + lengths = self.input_buffers[torch.int32].append(contexts) token_ids = self.input_buffers[torch.int64].append(tokens) inputs = [] for start, size, active, outputs in groups: count, end = len(active), start + size - metadata = self.metadata.prepare_shared_rows( - base, - active, - positions[start : start + count], - position_ids=wide[start:end], - write_blocks=wide[n + start : n + start + count], - write_offsets=wide[2 * n + start : 2 * n + start + count], - context_lengths=lengths[start:end], - loop_count=batch.loop_count, - ) + if cache.grouped: + metadata = self._stage_metadata( + ids[:count], + [[span.depth + loop for span in active] for loop in range(batch.loop_count)], + positions[start : start + count], + padded_rows=size, + ) + else: + metadata = self.metadata.prepare_shared_rows( + base, + active, + positions[start : start + count], + position_ids=wide[start:end], + write_blocks=wide[n + start : n + start + count], + write_offsets=wide[2 * n + start : 2 * n + start + count], + context_lengths=lengths[start:end], + loop_count=batch.loop_count, + ) kv = metadata[0] requests = tuple(span.request for span in active) coda, coda_rows = self._prepare_prefill_coda( @@ -942,6 +957,7 @@ def _prepare_cpu_inputs(self, batch: ExecutionBatch): prepared = replace(prepared, workspace=workspace) if ( workspace is not None + and not self.cache_manager.grouped and batch.operation == Operation.RECURRENT and not (batch.emit_gate and self.graphs is not None) and batch.loop_count == 1 From 0837c4564a32349c4e18c26423cfc0a4cff977ad Mon Sep 17 00:00:00 2001 From: HowardYangAI <13777842855@163.com> Date: Mon, 16 Nov 2026 16:06:50 +0800 Subject: [PATCH 2/2] feat(core): enable Dory per-loop prefix reuse Publish complete prefix pages using boundary and recurrent plane ownership, retain cached pages after request release, and restore coverage on hits. Use incremental prefix hashing and completion-aware publication, releasing pending records with their allocation. Keep compatible all-layer storage for graph, async and speculative execution, and harden grouped allocation, padding, reclamation and initialization paths. Preserve CacheConfig positional compatibility and document CPU/GPU validation. This commit squashes only the 12 HowardYangAI commits. The prerequisite 2B storage/reclamation change remains a separate commit by its original author. --- docs/dory.md | 137 +++++++++++--- tests/test_backend_selection.py | 10 +- tests/test_dory.py | 236 ++++++++++++++++++++++- tests/test_dory_gpu.py | 182 +++++++++++++++++- tests/test_dory_speculative.py | 50 +++++ tests/test_engine.py | 7 + tests/test_hybrid_kv.py | 116 +++++++++++- tests/test_prefix_growth.py | 225 ++++++++++++++++++++++ tests/test_swa_reclaim.py | 94 +++++++++- vllm_rlt/config.py | 4 +- vllm_rlt/core/kv_cache_manager.py | 301 +++++++++++++++++++----------- vllm_rlt/core/memory.py | 14 +- vllm_rlt/core/scheduler.py | 2 +- vllm_rlt/engine/stage_engine.py | 32 +++- vllm_rlt/models/dory.py | 3 + vllm_rlt/worker/model_runner.py | 3 +- 16 files changed, 1243 insertions(+), 173 deletions(-) diff --git a/docs/dory.md b/docs/dory.md index 2c0d64af..d8e91d04 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -52,8 +52,9 @@ to one cache layout: Boundary stages always write depth 0, so input and output KV are never replicated per loop. In `last_loop` mode a token's current-step KV is overwritten by the next loop, so results depend on the chunk schedule; the qualified -schedule is one token per step. `per_loop` gives the same result for -token-at-a-time and all-positions-at-once execution. +schedule is one token per step. `per_loop` preserves the same mathematical +computation for token-at-a-time and all-positions-at-once execution; floating-point +rounding can still vary with chunk and batch shapes. Only the 36 attention sublayers own KV. The cache pool is sized by that compact count (`DoryConfig.num_kv_layers`), not by all 72 physical sublayers, and @@ -63,10 +64,10 @@ layer id rather than their physical sublayer index. ## Grouped storage and optional SWA reclamation -Dory uses one pool of attention-layer pages, grouped by window policy and -boundary/recurrent plane domain. `reclaim_swa` defaults to `False`. To return -expired sliding-window pages to the pool, enable it together with incremental -allocation on an already loaded model: +Dory's ordinary synchronous eager execution uses one pool of attention-layer +pages, grouped by window policy and boundary/recurrent plane domain. +`reclaim_swa` defaults to `False`. To return expired sliding-window pages to the +pool, enable it together with incremental allocation on an already loaded model: ```python from vllm_rlt import CacheConfig @@ -94,6 +95,11 @@ Reclamation supports ordinary synchronous fixed-loop `ouro` execution. Each all layers in their group have consumed the range and the submission completes. Full-attention history remains resident. +CUDA Graphs and asynchronous scheduling borrow device-owned KV metadata; +self-speculation needs suffix truncation after a rejection. Grouped pages do not +provide these operations yet. For all three modes, the engine uses all-layer +blocks, including for prefix caching, and rejects `reclaim_swa`. + Native SWA attention stages only the needed page-aligned read span. Logical token positions, write addresses and public layer maps remain absolute; the backend's read-table context lengths are local to that span. Native attention calls must @@ -112,12 +118,12 @@ logical-page map, with `-1` for reclaimed pages. Here `cache` is reclaimed full prefix is rejected; these queries do not retain page references or expose mutable allocator state. -Grouped prefix publish, lookup and import remain unsupported. A future prefix -consumer must retain pages through the same manager before publishing them or -allowing their request owner to reclaim them. Reclamation drops the owner's -references; a page becomes reusable only when its final reference reaches zero. -The inspection methods are not a pin API, and no public grouped retain/pin -interface is provided yet. +Grouped prefix caching publishes one entry per token page holding every +owned (group, plane, layer) page, ordered as the allocator splits a claim. +The entry takes its own reference, so the publishing request can finish or +release its pages without freeing the cached ones. Publication stops at the +first token page with any owned page missing. Prefix caching together with +`reclaim_swa` is rejected until hit-path reclamation is qualified. ## Supported today @@ -147,27 +153,41 @@ interface is provided yet. - The official tokenizer, including the Hub's `tokenizer.json` and chat template, resolved without requiring transformers to accept the checkpoint config (its generic validator rejects Dory's `mlp` sublayer entries). +- Prefix caching (`enable_prefix_caching=True`) in `per_loop` mode. A + published page carries boundary KV from plane 0 and recurrent KV from every + plane; boundary layers never write deeper planes, so they are not required + there. A hit restores exactly that coverage and the scheduler resumes prefill + from the cached offset. `last_loop` keeps one shared plane and stays rejected. + Prefix keys hash tokens only. That is sound because the loop count is fixed + when the model loads and the layout is fixed by `CacheConfig`, and each + engine owns its own KV manager, so pages from different loop counts or + layouts never share a cache. A cache shared across loop counts would have to + add both to the key. Evicted prefixes fall back to a full cold prefill. ## Rejected today These fail fast rather than degrade silently: - WCPB, mixed or staged prefill; packed prefill metadata (`packed_prefill`); PD. -- Prefix caching. -- Preemption and native-cache snapshot/restore. +- Prefix caching in `last_loop` mode. +- Preemption and snapshot/restore with grouped KV. - Self-speculation combined with CUDA Graphs, asynchronous scheduling, - preemption or `last_loop` KV, and sampled token-pipeline speculation. The - synchronous eager form is supported; see below. + preemption, `reclaim_swa` or `last_loop` KV, and sampled token-pipeline + speculation. The synchronous eager form is supported; see below. - Per-request loop counts other than the configured full depth: Dory has no trained exit gate, so the runtime refuses to invent one, including via `min_loops` with `exit_threshold=0`. +For ordinary decoding, the all-layer fallback used with CUDA Graphs or +asynchronous scheduling accepts `enable_preemption`. Dory preemption in these +modes has not been qualified on GPU. + Sliding-window masking exists in the Torch reference backend and in the Triton decode kernel (`window=` on `KVCacheManager.attend`). Backend selection requires the `sliding_window` capability, so a Flash-backed Dory engine fails at construction instead of attending full history. Masking accepts host-planned, -graph-private and device-owned metadata; storage reclamation of out-of-window -pages stays with 2B. Optional physical SWA page reclamation is available on the grouped Torch path described above. +graph-private and device-owned metadata. Physical reclamation of out-of-window +pages is the optional grouped path described above. ## Synchronous self-speculation @@ -241,19 +261,76 @@ views; all views are bound before execution. `tests/test_dory.py` and `tests/test_dory_gpu.py` hold the regression suites; `tests/test_dory_real_checkpoint.py` holds the pinned-weight run. Measured results are below, including what they do not cover. + ### CPU suite -`tests/test_dory.py` (44 cases) covers configuration validation, strict loading +`tests/test_dory.py` covers configuration validation, strict loading with single-file and sharded checkpoints, stage/KV contracts, sliding-window masking, KV layer compaction, engine scheduling, EOS stopping, and numerical agreement with an independent dense oracle (`tests/reference/dory.py`) for both -KV modes — including greedy generation against that oracle. The full CPU run is -945 passed with only the pre-existing failures of the reviewed baseline -(`test_ouro` packed prefill, one `test_pd`, eleven `test_serving`). +KV modes — including greedy generation against that oracle. The full CPU run on +2026-11-16 passed 1109 tests with no failures; GPU-marked tests were skipped. + +### Prefix reuse + +Prefix reuse is checked at loops 1, 2 and 4: cold and cache-hit prefill +logits match the oracle, cached prefixes longer than the sliding window, +block-aligned prompts, diverging suffixes, concurrent sharers and a +re-request after its prefix was evicted all reproduce the cold greedy tokens, +and publication refuses pages missing any owned plane. Prefix pages stay +pinned for the life of the cache entry. Prefix tests run on both the legacy +all-layer blocks and grouped storage. On a small FP32 model, prefix hits and +diverging branches match the oracle in sync eager, sync Graph, async eager and +async Graph modes. + +Real checkpoint, one L20, two loops, 16 greedy tokens, `prefill_chunk_size=256`, +prompts cut from the model card. Each cell compares prefix caching on and off in +a fresh engine; time to first token in seconds: + +| Prompt tokens | Request | TTFT off | TTFT on | Tokens equal (BF16) | +| --- | --- | --- | --- | --- | +| 1100 | repeat | 15.3 | 0.30 | yes | +| 1100 | branch | 15.2 | 0.60 | yes | +| 4096 | repeat | 60.4 | 0.41 | yes | +| 4096 | branch | 60.3 | 0.78 | yes | + +An evicted prefix falls back to a cold prefill with equal tokens, and a control +without eviction hits 1088 tokens. In BF16, one of four concurrent sharers and a +seeded sampled (temperature 0.8) repeat differed from the uncached run; in FP32 +all of them match. A hit resumes prefill at the cached offset, which changes +the chunk split, and with caching off, changing `prefill_chunk_size` alone +already changes BF16 greedy tokens. FP32 tokens matched in these checks; BF16 +agreement is not promised across chunk splits. These are single runs on a +shared host. + +The same checks were repeated on one RTX PRO 5000 Blackwell (torch 2.13+cu130, +Torch backend), BF16, at loops 1, 2 and 4: + +| Loops | Prompt tokens | TTFT off | TTFT on (repeat) | +| --- | --- | --- | --- | +| 1 | 1100 | 6.6 | 0.13 | +| 1 | 4096 | 27.9 | 0.20 | +| 2 | 1100 | 9.3 | 0.18 | +| 2 | 4096 | 40.9 | 0.29 | +| 4 | 1100 | 16.4 | 0.31 | +| 4 | 4096 | 69.9 | 0.49 | + +Repeat tokens were equal everywhere; eviction and its 1088-token control +behaved as on the L20. At two loops and 1100 tokens in FP32, all eight +concurrent requests, the seeded sampled repeat, every `prefill_chunk_size` +from 16 to 256 and the cache hit produced identical tokens. + +With the Triton backend at two loops and 1100 tokens, sync eager, sync Graph, +async eager and async Graph each produced the uncached tokens for the cold, +repeat and branch requests, with the hit cutting time to first token from +0.54-0.79 s to 0.06-0.10 s. Through the OpenAI-compatible HTTP server (1516 +prompt tokens), every request succeeded and hits cut time to first token from +1.5-7 s to 0.09-0.35 s; the branch and one concurrent response differed in +BF16 text, which is the chunk-split effect above. ### CUDA -`tests/test_dory_gpu.py` (66 cases) checks the Triton backend against the Torch +`tests/test_dory_gpu.py` checks the Triton backend against the Torch reference: windowed decode for `head_dim` 8/128/256, windows 1/3/16/64 and lengths 1/7/63/64/65; unwindowed decode unchanged; both KV modes against the dense oracle; a greedy engine run whose tokens match the oracle; and @@ -286,7 +363,7 @@ to be complete. ### Execution modes -`tests/test_dory_gpu.py` adds 36 cases on a small FP32 model. For each of sync +`tests/test_dory_gpu.py` checks a small FP32 model. For each of sync eager, sync Graph, async eager and async Graph, with 1, 2 and 3 concurrent requests of different prompt lengths, the generated tokens equal the dense oracle's, and the Graph modes must replay the input-stage, recurrent and @@ -326,12 +403,12 @@ BF16 the four modes are identical up to 16 requests and at every prompt length measured there; from 32 requests on, sync eager, sync Graph and async eager still agree while async Graph differs, because it merges the two prefill cohorts into one decode wave and BF16 greedy tokens depend on batch -composition. FP32 is therefore the exact check; BF16 agreement is not promised -across batch shapes. +composition. FP32 tokens matched in these measured workloads; BF16 agreement is +not promised across batch shapes. ### Self-speculation -`tests/test_dory_speculative.py` (54 CPU cases) checks block and token-pipeline +`tests/test_dory_speculative.py` checks block and token-pipeline speculation for the depth pairs 1→2, 1→4 and 2→4 against the dense oracle: greedy tokens for ragged batches, a forced rejection at every draft position, all-accepted rounds with the bonus token, EOS inside an accepted prefix, output @@ -360,9 +437,9 @@ same eight BF16 requests without speculation in waves of four or one instead of eight also leaves only 2–4 of 8 requests token-identical, while the FP32 baseline is identical across wave sizes (32/32). Speculation changes the batch shapes of the same per-row arithmetic, BF16 rounding then differs, and greedy -near-ties flip; every later token of that request diverges. FP32 is therefore -the exact check, and BF16 output is a valid greedy continuation up to rounding, -not a bit-reproduction of the baseline. +near-ties flip; later tokens of that request can then diverge. FP32 tokens matched +in these measured workloads; BF16 does not reproduce the baseline token for token +across these batch shapes. Acceptance (accepted / drafted, BF16): 1→2 0.78–0.93, 2→4 0.90–0.98, 1→4 0.72–0.93. diff --git a/tests/test_backend_selection.py b/tests/test_backend_selection.py index 882b7d43..b9637162 100644 --- a/tests/test_backend_selection.py +++ b/tests/test_backend_selection.py @@ -270,7 +270,15 @@ class ProfileReached(Exception): pass def plan_cache( - model, cache, scheduler, execution, selected, *, speculative=None, staged_prefill=None + model, + cache, + scheduler, + execution, + selected, + *, + speculative=None, + staged_prefill=None, + layer_windows=None, ): assert selected == "triton" # FA2 cannot use the configured 16-token pages. assert staged_prefill is False diff --git a/tests/test_dory.py b/tests/test_dory.py index cd3468c1..bad8f137 100644 --- a/tests/test_dory.py +++ b/tests/test_dory.py @@ -111,9 +111,11 @@ def model(): return DoryForCausalLM(tiny_dory_config()).eval() -def make_cache(model, layout=None, block_size=2): +def make_cache(model, layout=None, block_size=2, enable_prefix_caching=False, grouped=False): config = model.config return KVCacheManager( + layer_windows=model.kv_layer_windows if grouped else None, + enable_prefix_caching=enable_prefix_caching, num_layers=config.num_kv_layers, num_kv_heads=config.num_key_value_heads, head_dim=config.head_dim, @@ -670,3 +672,235 @@ def test_exit_written_ignores_boundary_layers(model): # The output stage has not run, so "every layer" is still false. assert not cache.token_written("request", 0, model.config.total_ut_steps - 1) cache.free("request") + + +def prefill_rows(model, cache, request_id, tokens, start): + """Run positions ``start:`` one row at a time on an allocated request.""" + config = model.config + logits = [] + for position in range(start, len(tokens)): + boundary = cache._prepare_batch([request_id], [0], [position]) + hidden = model.prelude_prepared(tokens[position : position + 1], boundary, cache) + for depth in range(config.total_ut_steps): + core = cache._prepare_batch([request_id], [depth], [position]) + hidden, _ = model.recurrent_prepared(hidden, core, cache) + logits.append(model.coda_prepared(hidden, boundary, cache)[0]) + cache.finalize_token(request_id, position, config.total_ut_steps - 1) + return torch.stack(logits) + + +@pytest.mark.parametrize("grouped", [False, True], ids=["legacy", "grouped"]) +@pytest.mark.parametrize("loops", [1, 2, 4]) +def test_prefix_hit_resumes_boundary_and_recurrent_kv(loops, grouped): + torch.manual_seed(42) + model = DoryForCausalLM(tiny_dory_config(n_recurrent_loops=loops)).eval() + cache = make_cache(model, enable_prefix_caching=True, grouped=grouped) + # The cached prefix exceeds the four-token sliding attention window. + tokens = torch.tensor([4, 7, 3, 9, 5, 2, 8, 6, 3]) + expected = dense_dory_reference(model, tokens)["logits"] + + assert cache.allocate("cold", len(tokens)) + torch.testing.assert_close( + prefill_rows(model, cache, "cold", tokens, 0), expected, atol=3e-6, rtol=3e-5 + ) + cache.publish_prefix("cold", tokens.tolist(), len(tokens)) + # The last prompt token is always recomputed, so four whole pages publish. + assert len(cache._prefixes) == 4 + cache.free("cold") + + prefix = cache.lookup_prefix(tokens.tolist()) + assert len(prefix) == 4 + assert cache.allocate("hit", len(tokens), prefix=prefix) + written = cache._get_allocation("hit").written + for plane, layers in enumerate(written): + for layer, positions in enumerate(layers): + owns = plane == 0 or layer in model.recurrent_kv_layers + # Boundary KV is restored only where the cold run wrote it. + assert positions.prefix == (8 if owns else 0) + torch.testing.assert_close( + prefill_rows(model, cache, "hit", tokens, 8), expected[8:], atol=3e-6, rtol=3e-5 + ) + cache.free("hit") + assert cache.num_used_blocks == 0 + + +@pytest.mark.parametrize("grouped", [False, True], ids=["legacy", "grouped"]) +@pytest.mark.parametrize("missing", ["boundary_plane0", "recurrent_plane0", "recurrent_deep_plane"]) +def test_prefix_publication_requires_every_owned_plane(model, missing, grouped): + cache = make_cache(model, enable_prefix_caching=True, grouped=grouped) + tokens = torch.tensor([4, 7, 3, 9, 5]) + assert cache.allocate("request", len(tokens)) + prefill_rows(model, cache, "request", tokens, 0) + layers = cache._get_allocation("request").written + boundary = model.prelude_kv_layers[0] + recurrent = model.recurrent_kv_layers[0] + plane, layer = { + "boundary_plane0": (0, boundary), + "recurrent_plane0": (0, recurrent), + "recurrent_deep_plane": (cache.storage_depths - 1, recurrent), + }[missing] + # Mark page 1 (positions 2-3) incomplete in one plane: only page 0 may publish. + layers[plane][layer].prefix = 2 + cache.publish_prefix("request", tokens.tolist(), len(tokens)) + assert len(cache._prefixes) == 1 + + +def test_grouped_prefix_pages_outlive_the_publishing_request(model): + cache = make_cache(model, enable_prefix_caching=True, grouped=True) + tokens = torch.tensor([4, 7, 3, 9, 5]) + expected = dense_dory_reference(model, tokens)["logits"] + assert cache.allocate("cold", len(tokens)) + prefill_rows(model, cache, "cold", tokens, 0) + cache.publish_prefix("cold", tokens.tolist(), len(tokens)) + published = {b for blocks in cache._prefixes.values() for b in blocks} + # Two token pages, each one physical page per (group, plane, layer). + assert len(published) == 2 * cache.pages_per_token_block + cache.free("cold") + # Allocate another request; the published pages must not be handed out. + assert cache.allocate("filler", len(tokens)) + assert not published & set( + b + for layer in cache.layer_groups + for depth in range(cache.max_loops if cache.layer_group(layer)[1].recurrent else 1) + for b in cache.layer_block_table("filler", layer, depth) + ) + cache.free("filler") + prefix = cache.lookup_prefix(tokens.tolist()) + assert cache.allocate("hit", len(tokens), prefix=prefix) + for layer in cache.layer_groups: + assert cache.retained_range("hit", layer, 0) == (0, 4) + torch.testing.assert_close( + prefill_rows(model, cache, "hit", tokens, 4), expected[4:], atol=3e-6, rtol=3e-5 + ) + cache.free("hit") + assert cache.num_used_blocks == 0 + + +def test_grouped_prefix_caching_rejects_swa_reclamation(model): + config = model.config + with pytest.raises(NotImplementedError, match="SWA reclamation"): + KVCacheManager( + num_layers=config.num_kv_layers, + num_kv_heads=config.num_key_value_heads, + head_dim=config.head_dim, + num_blocks=256, + block_size=2, + max_loops=config.total_ut_steps, + device="cpu", + dtype=torch.float32, + backend="torch", + layout=config.kv_cache_layout, + recurrent_layers=model.recurrent_kv_layers, + layer_windows=model.kv_layer_windows, + enable_prefix_caching=True, + incremental_allocation=True, + reclaim_swa=True, + ) + + +def test_engine_rejects_prefix_caching_with_swa_reclamation_before_planning(model, monkeypatch): + def unexpected_plan(*args, **kwargs): + pytest.fail("prefix caching with SWA reclamation must be rejected before cache planning") + + monkeypatch.setattr("vllm_rlt.engine.stage_engine.plan_cache", unexpected_plan) + with pytest.raises(NotImplementedError, match="prefix caching with SWA reclamation"): + LLMEngine( + model, + cache_config=CacheConfig( + block_size=2, + enable_prefix_caching=True, + incremental_allocation=True, + reclaim_swa=True, + ), + ) + + +def run_engine(engine, requests, params): + for request_id, prompt in requests: + engine.add_request(request_id, prompt, params) + finished = {} + while engine.has_unfinished_requests(): + for output in engine.step(): + if output.finished: + finished[output.request_id] = output.token_ids + return finished + + +def greedy_reference(model, prompt, steps): + tokens = list(prompt) + for _ in range(steps): + logits = dense_dory_reference(model, torch.tensor(tokens))["logits"] + tokens.append(int(logits[-1].argmax())) + return tokens[len(prompt) :] + + +@pytest.mark.parametrize("loops", [1, 2, 4]) +def test_engine_prefix_hits_match_cold_greedy_reference(loops): + torch.manual_seed(42) + model = DoryForCausalLM(tiny_dory_config(n_recurrent_loops=loops)).eval() + engine = LLMEngine( + model, + cache_config=CacheConfig(num_blocks=512, block_size=2, enable_prefix_caching=True), + scheduler_config=SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=4), + ) + steps = model.config.total_ut_steps + params = SamplingParams(max_tokens=3, min_loops=steps, max_loops=steps, ignore_eos=True) + shared = [4, 7, 3, 9, 5, 2, 8, 6] + prompts = { + "cold": shared + [3], + # Block-aligned: the last full page is still recomputed. + "aligned": shared + [3, 5], + "branch": shared + [10, 11, 12], + "same": shared + [3], + } + cache = engine.cache_manager + assert run_engine(engine, [("cold", prompts["cold"])], params) == { + "cold": greedy_reference(model, prompts["cold"], 3) + } + assert cache.prefix_hits == 0 + # Run the remaining requests concurrently over the shared published prefix. + finished = run_engine( + engine, [(rid, prompts[rid]) for rid in ("aligned", "branch", "same")], params + ) + for rid in ("aligned", "branch", "same"): + assert finished[rid] == greedy_reference(model, prompts[rid], 3), rid + assert cache.prefix_hits >= 3 * len(shared) + assert cache.num_used_blocks == 0 + + +@pytest.mark.parametrize("loops", [1, 2, 4]) +def test_evicted_prefix_falls_back_to_cold_prefill(loops): + torch.manual_seed(42) + model = DoryForCausalLM(tiny_dory_config(n_recurrent_loops=loops)).eval() + steps = model.config.total_ut_steps + # Grouped KV stores one layer per physical page: boundary layers own one + # plane, recurrent layers own ``steps``. A configured block is + # ``num_kv_layers`` physical pages. Each request reserves eleven tokens, + # spanning six token pages; those plus four cached pages exceed the pool's eight. + recurrent = set(model.recurrent_kv_layers) + pages_per_token_page = sum( + steps if layer in recurrent else 1 for layer in model.kv_layer_windows + ) + engine = LLMEngine( + model, + cache_config=CacheConfig( + num_blocks=8 * pages_per_token_page // model.config.num_kv_layers, + block_size=2, + enable_prefix_caching=True, + ), + scheduler_config=SchedulerConfig(max_num_seqs=1, max_num_batched_tokens=4), + ) + cache = engine.cache_manager + params = SamplingParams(max_tokens=3, min_loops=steps, max_loops=steps, ignore_eos=True) + prompt = [4, 7, 3, 9, 5, 2, 8, 6, 3] + other = [10, 11, 12, 13, 14, 15, 16, 17, 18] + expected = greedy_reference(model, prompt, 3) + + assert run_engine(engine, [("cold", prompt)], params) == {"cold": expected} + assert len(cache.lookup_prefix(prompt)) == 4 + run_engine(engine, [("other", other)], params) + assert cache.lookup_prefix(prompt) == () + hits = cache.prefix_hits + assert run_engine(engine, [("again", prompt)], params) == {"again": expected} + assert cache.prefix_hits == hits + assert cache.num_used_blocks == 0 diff --git a/tests/test_dory_gpu.py b/tests/test_dory_gpu.py index 7941ca00..d70cc690 100644 --- a/tests/test_dory_gpu.py +++ b/tests/test_dory_gpu.py @@ -10,7 +10,13 @@ from tests.reference.dory import dense_dory_reference from tests.test_dory import tiny_dory_config -from vllm_rlt import CacheConfig, ExecutionConfig, SamplingParams, SchedulerConfig +from vllm_rlt import ( + CacheConfig, + ExecutionConfig, + SamplingParams, + SchedulerConfig, + SpeculativeConfig, +) from vllm_rlt.attention import create_backend from vllm_rlt.core.kv_cache_manager import KVCacheManager from vllm_rlt.core.memory import plan_cache @@ -169,13 +175,10 @@ def test_plan_cache_profiles_dory_on_cuda(): # The probe ran the real staged path, including windowed boundary attention. assert plan["source"] == "cuda_profile" config = model.config + # Grouped storage budgets one physical attention-layer page per block. + assert plan["block_unit"] == "physical_layer_page" assert plan["bytes_per_block"] == ( - 2 - * config.num_kv_layers - * cache.block_size - * config.num_key_value_heads - * config.head_dim - * 4 + 2 * cache.block_size * config.num_key_value_heads * config.head_dim * 4 ) @@ -288,3 +291,168 @@ def failing(batch): engine.add_request("a", PROMPTS["a"], params) assert drain(engine)["a"].token_ids == greedy_reference(reference, PROMPTS["a"], 4) assert engine.cache_manager.num_used_blocks == 0 + + +@pytest.mark.parametrize("mode", EXECUTION_MODES) +def test_dory_prefix_hits_match_cpu_oracle_in_every_execution_mode(mode): + """Prefix reuse composes with Graph replay and async submission: a cache hit + and a diverging branch serve the same greedy tokens as the dense oracle.""" + reference, model = reference_pair() + engine = LLMEngine( + model, + attention_backend="triton", + cache_config=CacheConfig(num_blocks=512, block_size=2, enable_prefix_caching=True), + scheduler_config=SchedulerConfig(max_num_seqs=3, max_num_batched_tokens=16), + execution_config=ExecutionConfig(**EXECUTION_MODES[mode]), + ) + params = SamplingParams(max_tokens=6, min_loops=2, max_loops=2, ignore_eos=True) + prompt = PROMPTS["c"] + branch = prompt[:6] + [4, 9, 1] + cache = engine.cache_manager + try: + assert cache.grouped is (mode == "sync_eager") + expected = greedy_reference(reference, prompt, 6) + engine.add_request("cold", prompt, params) + assert drain(engine)["cold"].token_ids == expected + assert cache.prefix_hits == 0 + graphs = ( + ( + engine.model_runner.prelude_graphs, + engine.model_runner.graphs, + engine.model_runner.coda_graphs, + ) + if "graph" in mode + else () + ) + replays = [graph.replays for graph in graphs] + + engine.add_request("hit", prompt, params) + engine.add_request("branch", branch, params) + actual = drain(engine) + assert actual["hit"].token_ids == expected + assert actual["branch"].token_ids == greedy_reference(reference, branch, 6) + assert cache.prefix_hits == 14 # Eight shared tokens for hit, six for branch. + assert all(graph.replays > before for graph, before in zip(graphs, replays)) + assert cache.num_used_blocks == 0 + finally: + engine.close() + + +@pytest.mark.parametrize("speculative_mode", ["block", "pipeline"]) +def test_dory_speculative_prefix_hits_match_cpu_oracle(speculative_mode): + reference, model = reference_pair() + engine = LLMEngine( + model, + attention_backend="triton", + cache_config=CacheConfig(num_blocks=512, block_size=2, enable_prefix_caching=True), + scheduler_config=SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=16), + speculative_config=SpeculativeConfig( + 2, draft_loops=1, target_loops=2, mode=speculative_mode + ), + ) + prompt = PROMPTS["c"] + branch = prompt[:6] + [4, 9, 1] + params = SamplingParams(max_tokens=4, min_loops=2, max_loops=2, ignore_eos=True) + try: + cache = engine.cache_manager + assert not cache.grouped + expected = greedy_reference(reference, prompt, 4) + engine.add_request("cold", prompt, params) + assert drain(engine)["cold"].token_ids == expected + assert cache.prefix_hits == 0 + rounds = engine.speculative_runner.stats.rounds + + engine.add_request("hit", prompt, params) + engine.add_request("branch", branch, params) + actual = drain(engine) + assert actual["hit"].token_ids == expected + assert actual["branch"].token_ids == greedy_reference(reference, branch, 4) + assert cache.prefix_hits == 14 + assert engine.speculative_runner.stats.rounds > rounds + assert cache.num_used_blocks == 0 + finally: + engine.close() + + +def test_grouped_static_buffers_keep_three_decode_rows_aligned(): + reference, model = reference_pair() + engine = LLMEngine( + model, + attention_backend="triton", + cache_config=CacheConfig(num_blocks=64, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=3, max_num_batched_tokens=3), + execution_config=ExecutionConfig(static_buffers=True, pad_to_power_of_two=True), + ) + prompts = {"a": [4], "b": [7], "c": [3]} + params = SamplingParams(max_tokens=3, min_loops=2, max_loops=2, ignore_eos=True) + try: + assert engine.cache_manager.grouped + assert engine.model_runner.requestspaces + for request_id, prompt in prompts.items(): + engine.add_request(request_id, prompt, params) + actual = drain(engine) + assert set(actual) == set(prompts) + for request_id, prompt in prompts.items(): + assert actual[request_id].token_ids == greedy_reference(reference, prompt, 3) + assert engine.cache_manager.num_used_blocks == 0 + finally: + engine.close() + + +@pytest.mark.parametrize("kv_mode", ["per_loop", "last_loop"]) +def test_grouped_swa_reclamation_uses_cuda_completion_and_reuses_pages(kv_mode, monkeypatch): + model = make_gpu_model(recurrent_kv_cache_mode=kv_mode) + reference = DoryForCausalLM(tiny_dory_config(recurrent_kv_cache_mode=kv_mode)).eval() + reference.load_state_dict({name: value.cpu() for name, value in model.state_dict().items()}) + engine = LLMEngine( + model, + attention_backend="triton", + cache_config=CacheConfig( + num_blocks=8, + block_size=2, + layout=model.kv_cache_layout, + incremental_allocation=True, + reclaim_swa=True, + ), + # One token per chunk preserves last_loop's token-then-loop ordering. + scheduler_config=SchedulerConfig( + max_num_seqs=1, max_num_batched_tokens=1, prefill_chunk_size=1 + ), + ) + prompt = [4, 7, 9, 3, 12, 5, 6, 8, 11, 13, 14, 15] + cache = engine.cache_manager + retired, reused = set(), set() + finish = cache.finish_submission + + def resident_pages(): + return { + page + for layer in model.kv_layer_windows + for depth in ( + range(cache.storage_depths) if layer in model.recurrent_kv_layers else (0,) + ) + for page in cache.layer_block_table("request", layer, depth) + if page >= 0 + } + + def observe_completion(event=None): + assert isinstance(event, torch.cuda.Event) + before = resident_pages() + reused.update(before & retired) + finish(event) + retired.update(before - resident_pages()) + + monkeypatch.setattr(cache, "finish_submission", observe_completion) + try: + assert cache.grouped and cache.reclaim_swa + engine.add_request( + "request", + prompt, + SamplingParams(max_tokens=4, min_loops=2, max_loops=2, ignore_eos=True), + ) + actual = drain(engine)["request"] + assert actual.token_ids == greedy_reference(reference, prompt, 4) + assert retired and reused + assert cache.num_used_blocks == 0 + finally: + engine.close() diff --git a/tests/test_dory_speculative.py b/tests/test_dory_speculative.py index e52b2832..d22355e1 100644 --- a/tests/test_dory_speculative.py +++ b/tests/test_dory_speculative.py @@ -64,6 +64,56 @@ def greedy_reference(model, prompt, count): return tokens +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +def test_prefix_hits_keep_speculative_outputs_on_all_layer_kv(mode): + model, count = dory(2), 5 + prompt = [4, 7, 3, 9, 5, 2, 8, 6, 3] + branch = prompt[:6] + [10, 11, 12] + sampling = params(model, count) + expected = greedy_reference(model, prompt, count)[len(prompt) :] + e = engine( + model, + 2, + 1, + mode, + cache_config=CacheConfig(num_blocks=256, block_size=2, enable_prefix_caching=True), + scheduler_config=SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=4), + ) + try: + cache = e.cache_manager + assert not cache.grouped + assert e.memory_plan["block_unit"] == "all_layer_block" + e.add_request("cold", prompt, sampling) + assert drain(e)["cold"].token_ids == expected + assert cache.prefix_hits == 0 + + rounds = e.speculative_runner.stats.rounds + e.add_request("hit", prompt, sampling) + e.add_request("branch", branch, sampling) + actual = drain(e) + assert actual["hit"].token_ids == expected + assert actual["branch"].token_ids == greedy_reference(model, branch, count)[len(branch) :] + assert cache.prefix_hits == 14 # Eight shared tokens for hit, six for branch. + assert e.speculative_runner.stats.rounds > rounds + assert cache.num_used_blocks == 0 + finally: + e.close() + + +@pytest.mark.parametrize("mode", ["block", "pipeline"]) +def test_speculation_rejects_swa_reclamation(mode): + with pytest.raises(ValueError, match="without speculation"): + engine( + dory(2), + 2, + 1, + mode, + cache_config=CacheConfig( + num_blocks=256, block_size=2, incremental_allocation=True, reclaim_swa=True + ), + ) + + def test_round_layouts_align_partial_coda_rows_and_upload_once(monkeypatch): model = dory(4) e = engine(model, 3, 2, scheduler_config=SchedulerConfig(max_num_seqs=2)) diff --git a/tests/test_engine.py b/tests/test_engine.py index e265904a..2cf3dd8d 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -312,6 +312,13 @@ def test_invalid_request_id_does_not_poison_admission(request_id): assert drain(engine)[0]["valid"].finished +def test_cache_config_preserves_existing_positional_arguments(): + config = CacheConfig(64, 2, "last_exited", 0.9, None, 256 * 1024 * 1024, False, True, 0.25) + assert config == CacheConfig( + num_blocks=64, block_size=2, incremental_allocation=True, watermark_ratio=0.25 + ) + + @pytest.mark.parametrize("invalid", [1.5, float("nan"), float("inf"), True, 0]) def test_integer_limits_rejected_before_scheduling(invalid): for constructor, name in [ diff --git a/tests/test_hybrid_kv.py b/tests/test_hybrid_kv.py index d77b24a0..aff3c36a 100644 --- a/tests/test_hybrid_kv.py +++ b/tests/test_hybrid_kv.py @@ -133,6 +133,73 @@ def test_native_per_loop_batched_queries_and_sliced_metadata(): cache.layer_view(batch, 1) +@pytest.mark.parametrize("loops", [1, 2, 4]) +@pytest.mark.parametrize("tail_pages", [0, 1, 3]) +def test_grouped_prefix_import_preserves_page_order_and_shared_ownership(loops, tail_pages): + cached_pages, block_size = 64, 2 + recurrent = (2, 3, 4, 5) + # Every group has multiple layers; recurrent groups also have multiple + # planes. Distinct data per layer/plane detects any flattening mismatch. + windows = {layer: (4 if layer % 2 == 0 else None) for layer in range(8)} + pages_per_block = 4 + 4 * loops + pool_pages = max(cached_pages + 1, cached_pages + 2 * tail_pages) + cache = KVCacheManager( + num_layers=8, + num_kv_heads=1, + head_dim=2, + num_blocks=pool_pages * pages_per_block, + block_size=block_size, + max_loops=loops, + recurrent_layers=recurrent, + layer_windows=windows, + enable_prefix_caching=True, + ) + tokens = list(range(cached_pages * block_size + 1)) + assert cache.allocate("source", len(tokens)) + expected_tables = {} + for layer in windows: + for depth in range(loops) if layer in recurrent else (0,): + values = torch.full((len(tokens), 1, 2), float(layer * loops + depth)) + cache.write( + layer, ["source"] * len(tokens), [depth] * len(tokens), tokens, values, values + 100 + ) + expected_tables[layer, depth] = cache.layer_block_table("source", layer, depth)[ + :cached_pages + ] + cache.publish_prefix("source", tokens, len(tokens)) + prefix = cache.lookup_prefix(tokens) + assert len(prefix) == cached_pages + cache.free("source") + + capacity = (cached_pages + tail_pages) * block_size + fresh_pages = set() + for request_id in ("hit", "sharer"): + assert cache.allocate(request_id, capacity, prefix=prefix) + for (layer, depth), expected in expected_tables.items(): + table = cache.layer_block_table(request_id, layer, depth) + assert table[:cached_pages] == expected + assert len(table) == cached_pages + tail_pages + suffix = set(table[cached_pages:]) + assert not suffix & fresh_pages + fresh_pages.update(suffix) + keys, values = cache.read(layer, request_id, depth) + expected_keys = torch.full_like(keys, float(layer * loops + depth)) + torch.testing.assert_close(keys, expected_keys) + torch.testing.assert_close(values, expected_keys + 100) + assert cache.num_used_blocks == (cached_pages + 2 * tail_pages) * pages_per_block + if tail_pages: + # A failed third import must roll back its shared-page references. + assert not cache.allocate("blocked", capacity, prefix=prefix) + assert not cache.has_allocation("blocked") + cache.free("hit") + cache.free("sharer") + assert cache.num_used_blocks == 0 + # Evict every cached page to prove no import left a reference behind. + assert cache.allocate("evict", pool_pages * block_size) + cache.free("evict") + assert cache.num_free_blocks == cache.num_blocks + + def test_native_atomic_growth_failure_free_and_early_exit_copy(): model = DoryForCausalLM(tiny_dory_config()).eval() cache = native_cache(model, blocks=18) @@ -225,11 +292,56 @@ def test_native_engine_generation_uses_grouped_storage(mode): assert cache.num_used_blocks == 0 -def test_native_engine_refuses_preemption_before_running(): +@pytest.mark.parametrize("mode", ["per_loop", "last_loop"]) +@pytest.mark.parametrize("loops", [1, 2, 4]) +def test_grouped_static_buffers_with_three_padded_rows_match_dense_reference(mode, loops): + torch.manual_seed(42) + model = DoryForCausalLM( + tiny_dory_config(recurrent_kv_cache_mode=mode, n_recurrent_loops=loops) + ).eval() + engine = LLMEngine( + model, + cache_config=CacheConfig(num_blocks=64, block_size=2, layout=model.kv_cache_layout), + scheduler_config=SchedulerConfig(max_num_seqs=3, max_num_batched_tokens=3), + execution_config=ExecutionConfig(static_buffers=True, pad_to_power_of_two=True), + ) + prompts = {"a": [4], "b": [7], "c": [3]} + try: + cache = engine.cache_manager + assert cache.grouped + assert engine.model_runner.requestspaces + for request_id, prompt in prompts.items(): + engine.add_request( + request_id, + prompt, + SamplingParams(max_tokens=3, min_loops=loops, max_loops=loops, ignore_eos=True), + ) + completed = {} + while engine.has_unfinished_requests(): + for output in engine.step(): + if output.finished: + completed[output.request_id] = output.token_ids + assert set(completed) == set(prompts) + for request_id, prompt in prompts.items(): + tokens = list(prompt) + for _ in range(3): + logits = reference.dense_dory_reference(model, torch.tensor(tokens))["logits"] + tokens.append(int(logits[-1].argmax())) + assert completed[request_id] == tokens[len(prompt) :] + assert cache.num_used_blocks == 0 + finally: + engine.close() + + +def test_native_engine_refuses_preemption_before_running(monkeypatch): + def unexpected_plan(*args, **kwargs): + pytest.fail("unsupported preemption must be rejected before cache planning") + + monkeypatch.setattr("vllm_rlt.engine.stage_engine.plan_cache", unexpected_plan) model = DoryForCausalLM(tiny_dory_config()).eval() with pytest.raises(NotImplementedError, match="preemption snapshots"): LLMEngine( model, - cache_config=CacheConfig(num_blocks=8, block_size=2), + cache_config=CacheConfig(block_size=2), scheduler_config=SchedulerConfig(enable_preemption=True), ) diff --git a/tests/test_prefix_growth.py b/tests/test_prefix_growth.py index 9fe717c6..4e9062de 100644 --- a/tests/test_prefix_growth.py +++ b/tests/test_prefix_growth.py @@ -1,3 +1,6 @@ +import hashlib +import weakref + import pytest import torch @@ -66,6 +69,228 @@ def test_prefix_refcounts_and_eviction(): assert c.num_free_blocks == 12 +def test_completed_chunks_hash_each_prefix_page_once(monkeypatch): + count, block_size = 8192, 16 + cache = KVCacheManager(1, 1, 4, count // block_size, block_size, 1, enable_prefix_caching=True) + tokens = list(range(count)) + assert cache.allocate("request", count) + cache.mark_imported_prefix("request", count) + hash_calls = 0 + sha256 = hashlib.sha256 + + def counted(data): + nonlocal hash_calls + hash_calls += 1 + return sha256(data) + + monkeypatch.setattr(hashlib, "sha256", counted) + for length in range(128, count + 1, 128): + cache.publish_prefix("request", tokens, length, PrefixEvent(ready=True)) + cache.publish_prefix("request", tokens, count) + assert hash_calls == (count - 1) // block_size + assert len(cache.lookup_prefix(tokens)) == (count - 1) // block_size + + +@pytest.mark.parametrize("missing", ["coverage", "physical_page"]) +def test_prefix_publication_retries_the_first_incomplete_page(missing): + cache = KVCacheManager( + 2, + 1, + 4, + 12, + 2, + 2, + enable_prefix_caching=True, + recurrent_layers=(1,), + layer_windows={0: 4, 1: None}, + ) + tokens = list(range(7)) + assert cache.allocate( + "request", len(tokens), initial_tokens=2 if missing == "physical_page" else 7 + ) + allocation = cache._get_allocation("request") + for plane, layers in enumerate(allocation.written): + for layer, written in enumerate(layers): + if plane == 0 or layer == 1: + written.prefix = len(tokens) + if missing == "coverage": + allocation.written[1][1].prefix = 2 + + cache.publish_prefix("request", tokens, len(tokens)) + assert len(cache.lookup_prefix(tokens)) == 1 + if missing == "physical_page": + assert cache.ensure_capacity("request", len(tokens)) + else: + allocation.written[1][1].prefix = len(tokens) + cache.publish_prefix("request", tokens, len(tokens)) + assert len(cache.lookup_prefix(tokens)) == 3 + + +def test_active_request_republishes_its_prefix_after_lru_eviction(): + cache = KVCacheManager(1, 1, 4, 10, 2, 1, enable_prefix_caching=True) + tokens, other = list(range(7)), list(range(100, 107)) + assert cache.allocate("publisher", len(tokens)) + cache.mark_imported_prefix("publisher", len(tokens)) + cache.publish_prefix("publisher", tokens, len(tokens)) + cache.free("publisher") + assert cache.allocate("source", len(tokens), prefix=cache.lookup_prefix(tokens)) + cache.publish_prefix("source", tokens, len(tokens)) + assert cache.allocate("other", len(other)) + cache.mark_imported_prefix("other", len(other)) + cache.publish_prefix("other", other, len(other)) + cache.free("other") + + # Reaching the evictable pages also removes the older source entries, + # whose physical pages remain pinned by the live request. + assert cache.allocate("pressure", 10) + assert not cache.lookup_prefix(tokens) + cache.publish_prefix("source", tokens, len(tokens)) + assert len(cache.lookup_prefix(tokens)) == 3 + cache.free("source") + cache.free("pressure") + assert cache.num_free_blocks == cache.num_blocks + + +class PrefixEvent: + def __init__(self, ready=False): + self.ready = ready + self.queries = 0 + + def query(self): + self.queries += 1 + return self.ready + + def synchronize(self): + raise AssertionError("prefix publication must not wait for an event") + + +@pytest.mark.parametrize("leased", [False, True]) +def test_free_drops_pending_prefix_without_polling_or_waiting(leased): + cache = KVCacheManager(1, 1, 4, 6, 2, 1, enable_prefix_caching=True) + tokens = [1, 2, 3, 4, 5] + events = [] + for request_id in ("request", "other"): + assert cache.allocate(request_id, len(tokens)) + event = PrefixEvent() + events.append(event) + cache.publish_prefix(request_id, tokens, len(tokens), event) + owner = weakref.ref(cache._get_allocation("request")) + if leased: + cache.pin_transfer("request", "transfer") + cache.free("request") + if leased: + assert cache.has_allocation("request") + assert len(cache._pending_prefixes) == 2 + cache.unpin_transfer("request", "transfer") + assert owner() is None + assert not cache.has_allocation("request") + assert len(cache._pending_prefixes) == 1 + assert cache._pending_prefixes[0][0] == "other" + + assert cache.allocate("request", len(tokens)) + events.append(PrefixEvent()) + cache.publish_prefix("request", tokens, len(tokens), events[-1]) + cache.free("unknown") + assert len(cache._pending_prefixes) == 2 + cache.free("request") + cache.free("request") + assert len(cache._pending_prefixes) == 1 + assert cache._pending_prefixes[0][0] == "other" + cache.free("other") + assert not cache._pending_prefixes + assert all(event.queries == 1 for event in events) + assert cache.num_free_blocks == cache.num_blocks + + +def test_completed_prefix_event_publishes_without_pending_storage(): + cache = KVCacheManager(1, 1, 4, 6, 2, 1, enable_prefix_caching=True) + tokens = [1, 2, 3, 4, 5] + assert cache.allocate("request", len(tokens)) + cache.mark_imported_prefix("request", len(tokens)) + + cache.publish_prefix("request", tokens, len(tokens), PrefixEvent(ready=True)) + + assert not cache._pending_prefixes + assert len(cache.lookup_prefix(tokens)) == 2 + + +def test_pending_prefix_chunks_share_one_prompt_and_do_not_rescan_events(): + count, chunk_size = 8192, 128 + cache = KVCacheManager(1, 1, 4, count // 16, 16, 1, enable_prefix_caching=True) + tokens = list(range(count)) + assert cache.allocate("request", count) + cache.mark_imported_prefix("request", count) + events = [] + for length in range(chunk_size, count + 1, chunk_size): + event = PrefixEvent() + events.append(event) + cache.publish_prefix("request", tokens, length, event) + + assert len(cache._pending_prefixes) == 1 + _, _, snapshot, frontier, pending_events = cache._pending_prefixes[0] + assert snapshot == tuple(tokens) + assert frontier == count + assert len(pending_events) == len(events) + assert all(event.queries == 1 for event in events) + assert not cache._prefixes + + cache.poll_prefixes() + assert cache._pending_prefixes[0][2] is snapshot + assert all(event.queries == 2 for event in events) + for event in events: + event.ready = True + cache.poll_prefixes() + assert not cache._pending_prefixes + assert len(cache.lookup_prefix(tokens)) == (count - 1) // cache.block_size + + +def test_pending_prefix_waits_for_all_events_and_keeps_the_largest_frontier(): + cache = KVCacheManager(1, 1, 4, 8, 2, 1, enable_prefix_caching=True) + tokens = list(range(9)) + assert cache.allocate("request", len(tokens)) + cache.mark_imported_prefix("request", len(tokens)) + earlier, later = PrefixEvent(), PrefixEvent() + cache.publish_prefix("request", tokens, 4, earlier) + cache.publish_prefix("request", tokens, 8, later) + # A completed submission with a smaller frontier must not discard either + # outstanding writer or the larger prefix awaiting publication. + cache.publish_prefix("request", tokens, 2, PrefixEvent(ready=True)) + later.ready = True + cache.poll_prefixes() + assert not cache._prefixes + assert cache._pending_prefixes[0][3] == 8 + assert cache._pending_prefixes[0][4] == [earlier] + + earlier.ready = True + cache.poll_prefixes() + assert not cache._pending_prefixes + assert len(cache.lookup_prefix(tokens)) == 4 + + +def test_pending_prefix_does_not_publish_a_replaced_allocation(): + cache = KVCacheManager(1, 1, 4, 6, 2, 1, enable_prefix_caching=True) + old_tokens, new_tokens = [1, 2, 3, 4, 5], [6, 7, 8, 9, 10] + assert cache.allocate("request", len(old_tokens)) + cache.mark_imported_prefix("request", len(old_tokens)) + old_event = PrefixEvent() + cache.publish_prefix("request", old_tokens, len(old_tokens), old_event) + cache.free("request") + assert cache.allocate("request", len(new_tokens)) + cache.mark_imported_prefix("request", len(new_tokens)) + new_event = PrefixEvent() + cache.publish_prefix("request", new_tokens, len(new_tokens), new_event) + + old_event.ready = True + cache.poll_prefixes() + assert not cache._prefixes + assert len(cache._pending_prefixes) == 1 + assert cache._pending_prefixes[0][2] == tuple(new_tokens) + new_event.ready = True + cache.poll_prefixes() + assert len(cache.lookup_prefix(new_tokens)) == 2 + assert not cache.lookup_prefix(old_tokens) + + def test_incremental_growth_matches_reserved_generation(): m = model() p = SamplingParams(max_tokens=9, exit_threshold=0, ignore_eos=True) diff --git a/tests/test_swa_reclaim.py b/tests/test_swa_reclaim.py index 11dc274a..b29d9dfc 100644 --- a/tests/test_swa_reclaim.py +++ b/tests/test_swa_reclaim.py @@ -167,6 +167,12 @@ def synchronize(self): assert len(waits) == 2 assert cache.retained_range("request", 0, 0) == (2, 5) assert cache.retained_range("request", 1, 0) == (0, 5) + for layer in (*model.prelude_kv_layers, *model.coda_kv_layers): + with pytest.raises( + ValueError, match="boundary attention layers only own depth zero" + ): + cache.retained_range("request", layer, 1) + assert cache.token_written("request", position, 1) with pytest.raises(RuntimeError, match="resident"): cache.layer_view(old, 0) with pytest.raises(ValueError, match="window must match the layer policy"): @@ -226,6 +232,86 @@ def record_finish(event=None): assert cache.num_used_blocks == 0 +@pytest.mark.parametrize("layout", ["last_exited", "shared"]) +def test_long_reclaimed_history_is_not_copied_on_growth_or_retirement(monkeypatch, layout): + history_tokens, extra_tokens, loops = 4096, 8, 2 + planes = loops if layout == "last_exited" else 1 + cache = KVCacheManager( + num_layers=1, + num_kv_heads=1, + head_dim=2, + num_blocks=history_tokens * planes, + block_size=1, + max_loops=loops, + layout=layout, + layer_windows={0: 1}, + incremental_allocation=True, + reclaim_swa=True, + max_submission_tokens=history_tokens, + ) + assert cache.allocate("request", history_tokens + extra_tokens, initial_tokens=history_tokens) + snapshots = tuple(cache.layer_block_table("request", 0, depth) for depth in range(planes)) + # Exercise real coverage and consumption bookkeeping without benchmarking attention. + monkeypatch.setattr(cache.attention, "windowed", lambda query, *args: query) + values = torch.zeros(history_tokens, 1, 2) + for depth in range(loops): + batch = cache._prepare_batch( + ["request"] * history_tokens, [depth] * history_tokens, list(range(history_tokens)) + ) + cache._write_prepared(0, batch, values, values) + cache._attend_prepared(0, batch, values, window=1) + cache.finish_submission() + assert cache.num_used_blocks == 0 + assert all(isinstance(snapshot, tuple) and min(snapshot) >= 0 for snapshot in snapshots) + + class NoHistoryCopy(list): + # A one-page append or retirement must not traverse thousands of dead slots. + def __iter__(self): + raise AssertionError("allocator scanned the full historical page table") + + def __add__(self, other): + raise AssertionError("allocator copied the full historical page table") + + allocation = cache._get_allocation("request") + tables = tuple(NoHistoryCopy(table) for table in allocation.group_tables[0]) + allocation.group_tables = (tables,) + assert cache.allocated_blocks("request") == 0 + epochs = tuple(allocation.group_epochs[0]) + for position in range(history_tokens, history_tokens + extra_tokens): + assert cache.ensure_capacity("request", position + 1) + assert cache.num_used_blocks == planes + assert cache.allocated_blocks("request") == planes + for depth in range(loops): + batch = cache._prepare_batch(["request"], [depth], [position]) + cache._write_prepared(0, batch, values[:1], values[:1]) + cache._attend_prepared(0, batch, values[:1], window=1) + + class Completion: + def synchronize(self): + # Residency and references remain intact until the final reader completes. + assert cache.num_used_blocks == planes + assert all(table[position] is not None for table in tables) + + cache.finish_submission(Completion()) + assert cache.num_used_blocks == 0 + assert cache.allocated_blocks("request") == 0 + assert all(table[position] is None for table in tables) + assert all(current is saved for current, saved in zip(allocation.group_tables[0], tables)) + assert allocation.group_epochs[0] == [epoch + 2 * extra_tokens for epoch in epochs] + # Public inspection still returns immutable snapshots independent of internal mutation. + allocation.group_tables = (tuple(table[:] for table in tables),) + for depth in range(planes): + assert cache.layer_block_table("request", 0, depth) == (-1,) * ( + history_tokens + extra_tokens + ) + assert cache.retained_range("request", 0, depth) == ( + history_tokens + extra_tokens, + history_tokens + extra_tokens, + ) + cache.free("request") + assert cache.num_free_blocks == cache.num_blocks + + @pytest.mark.parametrize("mode", ["per_loop", "last_loop"]) def test_reclaim_multi_request_abort_and_reuse(monkeypatch, mode): torch.manual_seed(42) @@ -284,7 +370,7 @@ def observe(state): first_b_pages, reused_while_b_active = set(), set() stale = None aborted = stale_checked = False - for step in range(128): + for _ in range(128): if not engine.has_unfinished_requests(): break for output in engine.step(): @@ -349,7 +435,6 @@ def observe(state): assert not engine.has_unfinished_requests(), "small-pool admission/growth did not complete" assert aborted and stale_checked and reused_while_b_active assert set(completed) == {"long", "reused"} - # Device snapshots stay on-device until the whole cancel/reuse scenario ends. for request, index, context, actual in observed: assert index == len(context) - len(request.prompt_token_ids) expected = dense_dory_reference(reference, torch.tensor(context))["logits"][-1] @@ -366,8 +451,3 @@ def observe(state): tokens.append(int(logits[-1].argmax())) assert completed[rid].token_ids == tokens[len(prompts[rid]) :] assert cache.num_used_blocks == 0 and cache.num_free_blocks == cache.num_blocks - print( - f"{mode}: steps={step + 1}, live_cross_request_reused_pages=" - f"{sorted(reused_while_b_active)}, stale_rejected={stale_checked}, " - f"logits_checked={len(observed)}, cleanup=0" - ) diff --git a/vllm_rlt/config.py b/vllm_rlt/config.py index cfbdaf1d..58c5afc1 100644 --- a/vllm_rlt/config.py +++ b/vllm_rlt/config.py @@ -18,10 +18,10 @@ class CacheConfig: memory_reserve_bytes: int = 256 * 1024 * 1024 enable_prefix_caching: bool = False incremental_allocation: bool = False - # Ordinary fixed-loop grouped KV only; return expired SWA pages to the pool. - reclaim_swa: bool = False # Admission headroom for incremental allocation; otherwise effective watermark is zero. watermark_ratio: float = 0.0 + # Ordinary fixed-loop grouped KV only; return expired SWA pages to the pool. + reclaim_swa: bool = False def __post_init__(self): for name in ("enable_prefix_caching", "incremental_allocation", "reclaim_swa"): diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 25cff1cf..261049b5 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -63,7 +63,7 @@ class _Allocation: max_tokens: int block_tables: tuple[tuple[int, ...], ...] written: list[list[_WrittenPositions]] - # Native group -> plane -> absolute logical page -> one page per group layer. + # Native group -> plane -> mutable logical-page list -> one page per group layer. group_tables: tuple = () group_epochs: list[list[int]] = field(default_factory=list) retired_pages: list[list[int]] = field(default_factory=list) @@ -71,6 +71,8 @@ class _Allocation: reclaim_pending: bool = False transfer_leases: set[str] = field(default_factory=set) release_requested: bool = False + prefix_keys: list[bytes] = field(default_factory=list) + prefix_epoch: int = 0 @dataclass(frozen=True) @@ -171,8 +173,8 @@ def __init__( self.groups = () self.layer_groups = {} if self.grouped: - if enable_prefix_caching: - raise NotImplementedError("grouped KV does not support prefix caching") + if enable_prefix_caching and reclaim_swa: + raise NotImplementedError("prefix caching with SWA reclamation is not qualified") policies = dict(layer_windows) if not policies or any( type(layer) is not int or not 0 <= layer < num_layers for layer in policies @@ -195,6 +197,7 @@ def __init__( for group_id, group in enumerate(self.groups) for local, layer in enumerate(group.layers) } + # Physical pool pages needed for one token block across all layers and planes. self.pages_per_token_block = ( sum(len(group.layers) * group.planes for group in self.groups) if self.grouped @@ -220,6 +223,7 @@ def __init__( self.watermark_blocks = int(watermark_ratio * num_blocks) if incremental_allocation else 0 self._refs = [0] * num_blocks self._prefixes = OrderedDict() + self._prefix_epoch = 0 self._pending_prefixes = [] self.prefix_hits = self.prefix_queries = 0 if enable_prefix_caching and layout != "last_exited": @@ -290,6 +294,7 @@ def _claim(self, count): return None while len(self._free_blocks) < count and self._prefixes: _, blocks = self._prefixes.popitem(last=False) + self._prefix_epoch += 1 self._drop_refs(blocks) if len(self._free_blocks) < count: return None @@ -299,8 +304,7 @@ def _claim(self, count): self._refs[b] = 1 return blocks - def _prefix_keys(self, tokens): - digest = b"" + def _prefix_keys(self, tokens, digest=b""): for start in range(0, len(tokens) // self.block_size * self.block_size, self.block_size): values = tokens[start : start + self.block_size] digest = hashlib.sha256(digest + struct.pack(f"<{len(values)}q", *values)).digest() @@ -308,11 +312,12 @@ def _prefix_keys(self, tokens): def poll_prefixes(self): pending, self._pending_prefixes = self._pending_prefixes, [] - for request_id, allocation, tokens, length, event in pending: + for request_id, allocation, tokens, length, events in pending: if self._allocations.get(request_id) is not allocation: continue - if event is not None and not event.query(): - self._pending_prefixes.append((request_id, allocation, tokens, length, event)) + events[:] = [event for event in events if not event.query()] + if events: + self._pending_prefixes.append((request_id, allocation, tokens, length, events)) continue self.publish_prefix(request_id, tokens, length) @@ -321,29 +326,99 @@ def publish_prefix(self, request_id, tokens, length, event=None): length is the prefilled extent; len(tokens) is the full prompt length. A supplied event defers publication until the GPU writes complete. + The full prompt stays immutable for the allocation's lifetime. """ if not self.enable_prefix_caching: return allocation = self._get_allocation(request_id) if event is not None: - self._pending_prefixes.append((request_id, allocation, tuple(tokens), length, event)) - return + ready = event.query() + for index, (_, owner, snapshot, frontier, events) in enumerate(self._pending_prefixes): + if owner is allocation: + if not ready: + events.append(event) + # Keep one prompt snapshot per allocation. Retain every + # unfinished event: later submissions may complete first. + self._pending_prefixes[index] = ( + request_id, + allocation, + snapshot, + max(frontier, length), + events, + ) + return + if not ready: + self._pending_prefixes.append( + (request_id, allocation, tuple(tokens), length, [event]) + ) + return # Prefix entries retain KV, not the final hidden state needed by CODA. # Leave at least the last prompt token to recompute on a cache hit; # full-page alignment may leave additional tokens to recompute. publishable_tokens = min(length, len(tokens) - 1) - for index, key in enumerate(self._prefix_keys(tokens[:publishable_tokens])): + if allocation.prefix_epoch != self._prefix_epoch: + allocation.prefix_keys.clear() + allocation.prefix_epoch = self._prefix_epoch + keys = allocation.prefix_keys + start = len(keys) + digest = keys[-1] if keys else b"" + # Eviction invalidates the saved chain's publication frontier. Without + # eviction, hash only new token pages instead of revisiting every chunk. + for index, key in enumerate( + self._prefix_keys(tokens[start * self.block_size : publishable_tokens], digest), + start=start, + ): if key in self._prefixes: + keys.append(key) continue end = (index + 1) * self.block_size - # Every depth and layer must have a contiguous written prefix. - # GPU completion alone does not establish this coverage. - if any(w.prefix < end for plane in allocation.written for w in plane): + # Every layer that attention reads must have a contiguous written + # prefix. GPU completion alone does not establish this coverage. + if any( + written.prefix < end + for plane, layers in enumerate(allocation.written) + for layer, written in enumerate(layers) + if self._plane_owns(plane, layer) + ): + break + blocks = self._prefix_page(allocation, index) + if blocks is None: break - blocks = tuple(t[index] for t in allocation.block_tables) for b in blocks: self._refs[b] += 1 self._prefixes[key] = blocks + keys.append(key) + + def _prefix_page(self, allocation, index): + """Every physical page holding token page ``index``, or None if any is gone. + + Grouped entries are flattened group -> plane -> layer, the order + ``_group_tables`` uses to split one page's claim back into bundles. + """ + if not self.grouped: + return tuple(t[index] for t in allocation.block_tables) + bundles = [ + table[index] if index < len(table) else None + for planes in allocation.group_tables + for table in planes + ] + if any(bundle is None for bundle in bundles): + return None + return tuple(b for bundle in bundles for b in bundle) + + def _plane_owns(self, plane: int, layer: int) -> bool: + """Whether ``layer`` keeps KV history in ``plane``. + + Boundary (non-recurrent) layers run once per token, at depth 0, so + they only ever write plane 0. Recurrent layers own every plane. + Grouped storage takes ownership from the layer's group policy. + """ + if self.grouped: + if layer not in self.layer_groups: + return False + group_id, _ = self.layer_groups[layer] + return plane == 0 or self.groups[group_id].recurrent + return plane == 0 or layer in self.recurrent_layers def lookup_prefix(self, tokens): if not self.enable_prefix_caching: @@ -378,59 +453,61 @@ def allocate(self, request_id: str, max_tokens: int, *, initial_tokens=None, pre if not 0 < initial_tokens <= max_tokens: raise ValueError("invalid initial KV frontier") pages = (initial_tokens + self.block_size - 1) // self.block_size - if self.grouped: - if prefix: - raise NotImplementedError("grouped KV does not support prefix imports") - fresh = self._claim(pages * self.pages_per_token_block) - if fresh is None: - return False - self._allocations[request_id] = _Allocation( - int(max_tokens), - (), - [ - [_WrittenPositions() for _ in range(self.num_layers)] - for _ in range(self.storage_depths) - ], - group_tables=self._group_tables(fresh, pages), - group_epochs=[[0] * group.planes for group in self.groups], - retired_pages=[[0] * group.planes for group in self.groups], - consumed=( - [ - [_WrittenPositions() for _ in range(self.num_layers)] - for _ in range(self.storage_depths) - ] - if self.reclaim_swa - else [] - ), - ) - return True if len(prefix) > pages: raise ValueError("prefix exceeds initial KV frontier") - claimed = [b for group in prefix for b in group] + claimed = [b for bundle in prefix for b in bundle] for b in claimed: self._refs[b] += 1 - fresh = self._claim((pages - len(prefix)) * self.storage_depths) - if fresh is None: + if self.grouped: + tables = self._claim_group_tables(pages - len(prefix), prefix=prefix) + else: + tail = pages - len(prefix) + blocks = self._claim(tail * self.storage_depths) + tables = ( + None + if blocks is None + else tuple( + tuple(g[d] for g in prefix) + tuple(blocks[d * tail : (d + 1) * tail]) + for d in range(self.storage_depths) + ) + ) + if tables is None: self._drop_refs(claimed) return False - tail = pages - len(prefix) - tables = tuple( - tuple(g[d] for g in prefix) + tuple(fresh[d * tail : (d + 1) * tail]) - for d in range(self.storage_depths) - ) length = len(prefix) * self.block_size + # A hit restores exactly the coverage publish_prefix required. + written = [ + [ + _WrittenPositions(prefix=length if self._plane_owns(plane, layer) else 0) + for layer in range(self.num_layers) + ] + for plane in range(self.storage_depths) + ] self._allocations[request_id] = _Allocation( int(max_tokens), - tables, - [ - [_WrittenPositions(prefix=length) for _ in range(self.num_layers)] - for _ in range(self.storage_depths) - ], + () if self.grouped else tables, + written, + group_tables=tables if self.grouped else (), + group_epochs=[[0] * group.planes for group in self.groups], + retired_pages=[[0] * group.planes for group in self.groups], + consumed=( + [ + [_WrittenPositions() for _ in range(self.num_layers)] + for _ in range(self.storage_depths) + ] + if self.reclaim_swa + else [] + ), ) self.prefix_queries += 1 self.prefix_hits += length return True + def _claim_group_tables(self, new_pages, *, prefix=()): + """Append newly claimed pages to ``prefix``, or return None if the pool is short.""" + fresh = self._claim(new_pages * self.pages_per_token_block) + return None if fresh is None else self._group_tables(fresh, new_pages, prefix=prefix) + def ensure_capacity(self, request_id, tokens): allocation = self._get_allocation(request_id) if not 0 < tokens <= allocation.max_tokens: @@ -441,14 +518,12 @@ def ensure_capacity(self, request_id, tokens): extra = pages - current if extra <= 0: return True - fresh = self._claim(extra * self.pages_per_token_block) - if fresh is None: + added = self._claim_group_tables(extra) + if added is None: return False - added = self._group_tables(fresh, extra) - allocation.group_tables = tuple( - tuple(table + tail for table, tail in zip(planes, new_planes)) - for planes, new_planes in zip(allocation.group_tables, added) - ) + for planes, new_planes in zip(allocation.group_tables, added): + for table, tail in zip(planes, new_planes): + table.extend(tail) for epochs in allocation.group_epochs: for plane in range(len(epochs)): epochs[plane] += 1 @@ -472,8 +547,7 @@ def truncate_suffix(self, request_id: str, frontier: int) -> None: admission reservation; stale bytes are hidden by written/context lengths. Shared prompt pages must never be truncated or subsequently overwritten. """ - if self.grouped: - raise NotImplementedError("grouped KV supports ordinary layer views only") + self._require_legacy() allocation = self._get_allocation(request_id) if type(frontier) is not int or not 0 <= frontier <= allocation.max_tokens: raise ValueError("invalid KV truncation frontier") @@ -505,10 +579,12 @@ def free(self, request_id: str) -> None: ) else: self._drop_refs(b for table in allocation.block_tables for b in reversed(table)) + self._pending_prefixes = [ + pending for pending in self._pending_prefixes if pending[1] is not allocation + ] def pin_transfer(self, request_id: str, transfer_id: str): - if self.grouped: - raise NotImplementedError("grouped KV supports ordinary layer views only") + self._require_legacy() allocation = self._get_allocation(request_id) if allocation.release_requested: raise RuntimeError("cannot transfer KV scheduled for release") @@ -522,8 +598,7 @@ def unpin_transfer(self, request_id: str, transfer_id: str): def mark_imported_prefix(self, request_id: str, length: int, *, start=0): """Publish received KV only after the connector confirms remote writes are complete.""" - if self.grouped: - raise NotImplementedError("grouped KV supports ordinary layer views only") + self._require_legacy() allocation = self._get_allocation(request_id) if type(length) is not int or not 0 < length <= allocation.max_tokens: raise ValueError("imported prefix exceeds reserved capacity") @@ -546,17 +621,14 @@ def allocated_blocks(self, request_id: str) -> int: allocation = self._get_allocation(request_id) if self.grouped: return sum( - len(bundle) - for planes in allocation.group_tables - for table in planes - for bundle in table - if bundle is not None + (len(table) - allocation.retired_pages[group_id][plane]) * len(group.layers) + for group_id, group in enumerate(self.groups) + for plane, table in enumerate(allocation.group_tables[group_id]) ) return len(allocation.block_tables[0]) * self.storage_depths def get_block_table(self, request_id: str, depth: int) -> tuple[int, ...]: - if self.grouped: - raise NotImplementedError("grouped KV supports ordinary layer views only") + self._require_legacy() self._validate_depth(depth) return self._get_allocation(request_id).block_tables[self._plane(depth)] @@ -567,8 +639,7 @@ def plane_block_tables(self, request_id: str) -> tuple[tuple[int, ...], ...]: ``max_loops`` planes and SHARED has one, so ``tables[2]`` fails under SHARED while ``get_block_table(rid, 2)`` works under both layouts. """ - if self.grouped: - raise NotImplementedError("grouped KV supports ordinary layer views only") + self._require_legacy() return self._get_allocation(request_id).block_tables def has_transfer_lease(self, request_id: str) -> bool: @@ -591,13 +662,13 @@ def token_written(self, request_id: str, position: int, depth: int) -> bool: return self._token_written(allocation, position, depth) def _token_written(self, allocation: _Allocation, position: int, depth: int) -> bool: - plane = allocation.written[self._plane(depth)] if self.grouped: return all( - position in allocation.written[self._plane(depth) if group.recurrent else 0][layer] + position in allocation.written[self._group_plane(group, depth)][layer] for group in self.groups for layer in group.layers ) + plane = allocation.written[self._plane(depth)] return all(position in plane[layer] for layer in range(self.num_layers)) def exit_written(self, request_id: str, position: int, depth: int) -> bool: @@ -650,8 +721,7 @@ def snapshot(self, request_id: str) -> KVSnapshot: Copying page by page avoids a request-sized temporary on a device that is already full, which is exactly when preemption happens. """ - if self.grouped: - raise NotImplementedError("grouped KV supports ordinary layer views only") + self._require_legacy() allocation = self._get_allocation(request_id) if allocation.transfer_leases: raise RuntimeError("cannot snapshot KV during a transfer") @@ -676,8 +746,7 @@ def restore(self, request_id: str, snapshot: KVSnapshot) -> None: decision belongs to the scheduler. The copies go on the current stream; the caller decides when other streams may read them. """ - if self.grouped: - raise NotImplementedError("grouped KV supports ordinary layer views only") + self._require_legacy() allocation = self._get_allocation(request_id) if allocation.max_tokens != snapshot.max_tokens: raise ValueError("snapshot capacity does not match the allocation") @@ -691,29 +760,35 @@ def restore(self, request_id: str, snapshot: KVSnapshot) -> None: self.value_cache[block].copy_(snapshot.values[row]) allocation.written = deepcopy(snapshot.written) - def _group_tables(self, blocks, pages): - """Partition one atomic claim into group/plane/logical-page bundles.""" - cursor, result = 0, [] + def _group_tables(self, blocks, new_pages, *, prefix=()): + """Build each plane once from cached bundles followed by one fresh claim.""" + block_offset, prefix_offset, result = 0, 0, [] for group in self.groups: planes = [] width = len(group.layers) for _ in range(group.planes): - table = [] - for _ in range(pages): - table.append(tuple(blocks[cursor : cursor + width])) - cursor += width - planes.append(tuple(table)) + # Entries are flattened group -> plane -> layer. + table = [tuple(page[prefix_offset : prefix_offset + width]) for page in prefix] + prefix_offset += width + for _ in range(new_pages): + table.append(tuple(blocks[block_offset : block_offset + width])) + block_offset += width + planes.append(table) result.append(tuple(planes)) - assert cursor == len(blocks) + assert block_offset == len(blocks) + assert all(len(page) == prefix_offset for page in prefix) return tuple(result) + def _group_plane(self, group: KVGroup, depth: int) -> int: + """Storage plane owning ``depth`` for ``group``; boundary layers keep depth 0.""" + return self._plane(depth) if group.recurrent else 0 + def _group_table(self, allocation, layer, depth): group_id, local = self.layer_groups[layer] group = self.groups[group_id] if not group.recurrent and depth != 0: raise ValueError("boundary attention layers only own depth zero") - plane = self._plane(depth) if group.recurrent else 0 - return allocation.group_tables[group_id][plane], local + return allocation.group_tables[group_id][self._group_plane(group, depth)], local def _layer_table(self, allocation, layer, depth): bundles, local = self._group_table(allocation, layer, depth) @@ -752,8 +827,7 @@ def layer_view(self, batch, layer): group_id, _ = self.layer_groups[layer] group = self.groups[group_id] epochs = tuple( - a.group_epochs[group_id][self._plane(d) if group.recurrent else 0] - for a, d, _ in batch.rows + a.group_epochs[group_id][self._group_plane(group, d)] for a, d, _ in batch.rows ) cached = batch.layer_views.get(layer) if cached is not None and cached[0] == epochs: @@ -808,8 +882,7 @@ def retained_range(self, request_id, layer, depth): if self.grouped: self._group_table(allocation, layer, depth) group_id, _ = self.layer_groups[layer] - group = self.groups[group_id] - plane = self._plane(depth) if group.recurrent else 0 + plane = self._group_plane(self.groups[group_id], depth) start = allocation.retired_pages[group_id][plane] * self.block_size return start, allocation.written[self._plane(depth)][layer].prefix @@ -865,17 +938,12 @@ def finish_submission(self, event=None): if event is not None: event.synchronize() for allocation, group_id, plane, start, stop in plans: - groups = list(allocation.group_tables) - planes = list(groups[group_id]) - table = list(planes[plane]) + table = allocation.group_tables[group_id][plane] released = [] for page in range(start, stop): if table[page] is not None: released.extend(table[page]) table[page] = None - planes[plane] = tuple(table) - groups[group_id] = tuple(planes) - allocation.group_tables = tuple(groups) allocation.retired_pages[group_id][plane] = stop allocation.group_epochs[group_id][plane] += 1 self._drop_refs(released) @@ -885,6 +953,15 @@ def finish_submission(self, event=None): def _plane(self, depth: int) -> int: return depth if self.layout == "last_exited" else 0 + def _storage_layer(self, layer: int) -> int: + """Physical page slot holding ``layer``; grouped pages hold exactly one.""" + return 0 if self.grouped else layer + + def _require_legacy(self) -> None: + """Require legacy all-layer KV storage.""" + if self.grouped: + raise NotImplementedError("grouped KV supports ordinary layer views only") + def _get_allocation(self, request_id: str) -> _Allocation: try: return self._allocations[request_id] @@ -1067,7 +1144,7 @@ def _write_prepared( self._validate_tensor(v, len(batch.position_ids), "v") if not batch.rows: return - storage_layer = 0 if self.grouped else layer + storage_layer = self._storage_layer(layer) self.key_cache[batch.write_blocks, storage_layer, batch.write_offsets] = k[ : len(batch.rows) ] @@ -1140,8 +1217,8 @@ def _attend_prepared( if batch.cu_seqlens_q is not None: return self.attention.prefill( q, - self.key_cache[:, 0 if self.grouped else layer], - self.value_cache[:, 0 if self.grouped else layer], + self.key_cache[:, self._storage_layer(layer)], + self.value_cache[:, self._storage_layer(layer)], batch.block_tables, batch.context_lengths, batch.cu_seqlens_q, @@ -1149,8 +1226,8 @@ def _attend_prepared( ) return self.attention( q, - self.key_cache[:, 0 if self.grouped else layer], - self.value_cache[:, 0 if self.grouped else layer], + self.key_cache[:, self._storage_layer(layer)], + self.value_cache[:, self._storage_layer(layer)], batch.block_tables, batch.context_lengths, ) @@ -1181,8 +1258,8 @@ def _attend_windowed( self._require_prefix(allocation, layer, depth, position + 1) return self.attention.windowed( q, - self.key_cache[:, 0 if self.grouped else layer], - self.value_cache[:, 0 if self.grouped else layer], + self.key_cache[:, self._storage_layer(layer)], + self.value_cache[:, self._storage_layer(layer)], batch.block_tables, batch.context_lengths, window, @@ -1242,7 +1319,7 @@ def read(self, layer: int, request_id: str, depth: int, length: int | None = Non self._validate_depth(depth) allocation = self._get_allocation(request_id) if self.grouped: - self._layer_table(allocation, layer, depth) + self._group_table(allocation, layer, depth) if length is None: length = allocation.written[self._plane(depth)][layer].prefix if ( @@ -1268,7 +1345,7 @@ def read(self, layer: int, request_id: str, depth: int, length: int | None = Non ) blocks = table[positions // self.block_size] offsets = positions % self.block_size - storage_layer = 0 if self.grouped else layer + storage_layer = self._storage_layer(layer) return ( self.key_cache[blocks, storage_layer, offsets], self.value_cache[blocks, storage_layer, offsets], diff --git a/vllm_rlt/core/memory.py b/vllm_rlt/core/memory.py index 8dc5275f..c89b991f 100644 --- a/vllm_rlt/core/memory.py +++ b/vllm_rlt/core/memory.py @@ -52,14 +52,24 @@ def graph_memory_reserve(model, scheduler, execution, speculative=None): @torch.inference_mode() def plan_cache( - model, cache, scheduler, execution, backend, *, speculative=None, staged_prefill=None + model, + cache, + scheduler, + execution, + backend, + *, + speculative=None, + staged_prefill=None, + layer_windows=..., ): parameter = next(model.parameters()) config, device = model.config, parameter.device # Models whose physical stack mixes KV-owning and KV-free sublayers (Dory) # expose the compact count; everything else keeps the layer count. kv_layers = getattr(config, "num_kv_layers", config.num_hidden_layers) - layer_windows = getattr(model, "kv_layer_windows", None) + # Omitted uses the model policy; explicit None forces all-layer blocks. + if layer_windows is ...: + layer_windows = getattr(model, "kv_layer_windows", None) native = layer_windows is not None unit = "physical_layer_page" if native else "all_layer_block" recurrent = set(getattr(model, "recurrent_kv_layers", range(kv_layers))) diff --git a/vllm_rlt/core/scheduler.py b/vllm_rlt/core/scheduler.py index 09e28c55..1b8df28c 100644 --- a/vllm_rlt/core/scheduler.py +++ b/vllm_rlt/core/scheduler.py @@ -401,7 +401,7 @@ def _plan_admission(self, request: Request, active_count: int) -> AdmissionPlan: if cache.incremental_allocation and not reserve_outputs else capacity ) - required = cache.required_blocks(budget_tokens) - len(prefix) * cache.storage_depths + required = cache.required_blocks(budget_tokens) - len(prefix) * cache.pages_per_token_block reserved = self._reserved_growth_blocks(reserve_outputs=reserve_outputs) # Allow an otherwise idle engine to start a large request. admission_headroom_blocks = cache.watermark_blocks if active_count else 0 diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index 2df7e5ff..fba18b6c 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -66,7 +66,11 @@ def __init__( required_layout = getattr(model, "kv_cache_layout", "last_exited") if cache_config.layout != required_layout: raise ValueError(f"boundary-KV models require {required_layout} KV") - if cache_config.enable_prefix_caching: + # Only models that verified hit/miss parity opt in: the cache must + # cover boundary KV at depth 0 and recurrent KV at every depth. + if cache_config.enable_prefix_caching and not getattr( + model, "supports_boundary_prefix_caching", False + ): raise ValueError("boundary-KV prefix caching is not yet supported") self.exit_config = exit_config or ExitConfig() self.execution_config = resolve_cuda_graph_rows( @@ -207,6 +211,24 @@ def __init__( self.execution_config, packed_prefill=auto_packed and attention.capabilities.packed_prefill, ) + layer_windows = getattr(model, "kv_layer_windows", None) + if layer_windows is not None and ( + self.execution_config.cuda_graphs + or self.execution_config.async_scheduling + or speculative_config is not None + ): + # Graph, async and speculative paths require all-layer block tables + # until their metadata and rollback support grouped pages. + if cache_config.reclaim_swa: + raise ValueError( + "SWA reclamation requires synchronous eager execution without speculation" + ) + layer_windows = None + if layer_windows is not None: + if scheduler_config.enable_preemption: + raise NotImplementedError("grouped KV does not support preemption snapshots") + if cache_config.enable_prefix_caching and cache_config.reclaim_swa: + raise NotImplementedError("prefix caching with SWA reclamation is not qualified") num_blocks, self.memory_plan = plan_cache( model, cache_config, @@ -215,12 +237,8 @@ def __init__( attention.info["selected_backend"], speculative=speculative_config, staged_prefill=staged_prefill, + layer_windows=layer_windows, ) - if ( - getattr(model, "kv_layer_windows", None) is not None - and scheduler_config.enable_preemption - ): - raise NotImplementedError("grouped KV does not support preemption snapshots") self.cache_manager = KVCacheManager( num_layers=getattr(config, "num_kv_layers", config.num_hidden_layers), num_kv_heads=config.num_key_value_heads, @@ -237,7 +255,7 @@ def __init__( incremental_allocation=cache_config.incremental_allocation, watermark_ratio=cache_config.watermark_ratio, recurrent_layers=getattr(model, "recurrent_kv_layers", None), - layer_windows=getattr(model, "kv_layer_windows", None), + layer_windows=layer_windows, reclaim_swa=cache_config.reclaim_swa, max_submission_tokens=min( scheduler_config.prefill_chunk_size, scheduler_config.max_num_batched_tokens diff --git a/vllm_rlt/models/dory.py b/vllm_rlt/models/dory.py index da3a8b52..43f531d7 100644 --- a/vllm_rlt/models/dory.py +++ b/vllm_rlt/models/dory.py @@ -491,6 +491,9 @@ class DoryForCausalLM(nn.Module): # Synchronous self-speculation with per-loop KV: a draft's loops are the # target's first loops, and boundary KV is committed by the target pass. supports_self_speculation = True + # Complete-state prefix reuse: boundary KV is published from depth 0 and + # recurrent KV from every depth. Prefix caching requires per_loop KV. + supports_boundary_prefix_caching = True def __init__(self, config: DoryConfig) -> None: super().__init__() diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index 84eec76b..e2ed123e 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -1051,10 +1051,11 @@ def _prepare_forward(self, prepared): kv = prepared.kv if kv is not None: metadata = (kv,) + metadata + # Grouped views omit padding even when the hidden workspace includes it. forward = ForwardBatch( Operation.RECURRENT, hidden=hidden[: batch.num_rows] - if graphs is not None or batch.loop_count > 1 + if graphs is not None or batch.loop_count > 1 or self.cache_manager.grouped else hidden, metadata=metadata, graphs=graphs,