diff --git a/docs/dory.md b/docs/dory.md index 13f1f94a..46cfd8d6 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -90,11 +90,51 @@ layer id rather than their physical sublayer index. template, resolved without requiring transformers to accept the checkpoint config (its generic validator rejects Dory's `mlp` sublayer entries). +## Packed Triton prefill + +With `attention_backend="triton"`, the default `packed_prefill=None` automatically +enables packed prefill when execution and backend support it. Set +`ExecutionConfig(packed_prefill=True)` / `--packed-prefill` to require support, +or `packed_prefill=False` / `--no-packed-prefill` to keep per-query prefill. +An unsupported explicit enable request fails at startup. + +This path uses BF16, `per_loop` KV, ordinary prefill, and synchronous eager +single-stage execution. Speculation, preemption, CUDA graphs and asynchronous +scheduling are outside its scope. + +On NVIDIA CUDA, the packed path uses a 32 × 32 tile. At startup, Triton compiles +the full-attention kernel and, if the model uses a sliding window, the +sliding-window kernel. It checks their shared-memory requirements against the +device limit. Automatic mode uses per-query attention if the check fails; +explicit enable requests fail through backend capability validation. +The result is cached by device and attention geometry within the process. +The check runs before KV memory planning and uses tensor metadata only. +The check establishes that the tile fits, not that it is fastest. The fixed +32 × 32 tile was validated on RTX 4060 and RTX 5090; other GPUs and workloads +need measurements to determine their best tile. +The full BF16 checkpoint and long-context service validation require enough +device memory for the model weights, runtime buffers and KV pool. + +A batch uses packed attention across PRELUDE, recurrence and the CODA +writes for prompt history when every prompt is longer than 64 tokens. +The CODA that samples the first token uses per-query attention. A batch containing +a shorter prompt or a 64-token prompt uses per-query attention. The +cutoff uses the full prompt length. Decode keeps the per-query path. + +Packed full and sliding-window attention use the same causal ranges as +per-query attention. The matrix products and softmax reductions use a +different order and intermediate rounding. The value product compensates +for the BF16 rounding of softmax weights. Remaining differences can +propagate through Dory's recurrent layers and change a greedy choice. +For reproducible comparisons, use per-query attention and hold the +batch shape fixed. Changes in arrival timing can change the batch shape +even with per-query attention. + ## Rejected today These fail fast rather than degrade silently: -- WCPB, mixed or staged prefill; packed prefill metadata (`packed_prefill`); PD. +- WCPB, mixed or staged prefill, and PD. - Prefix caching. - Self-speculation combined with CUDA Graphs, asynchronous scheduling, preemption or `last_loop` KV, and sampled token-pipeline speculation. The diff --git a/docs/runtime.md b/docs/runtime.md index 66f515e4..910d3451 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -81,12 +81,17 @@ For a task-assigned GPU, set `CUDA_VISIBLE_DEVICES` and add `--device cuda --att `ExecutionConfig.packed_prefill` controls packed prefill attention and metadata staging. Its default, `None`, selects packed prefill when the selected backend supports it on CUDA with `last_exited` KV and ordinary, non-staged prefill. -Mixed scheduling, wavefront chunked prefill and sliding-window attention keep -their existing paths. These selection rules apply to all models. +Mixed scheduling and wavefront chunked prefill keep their existing paths. +Sliding-window packing additionally requires backend support for both packed +prefill and packed sliding-window attention, with ordinary synchronous eager +single-stage execution, without speculation or preemption. These selection rules +apply to all models. Incompatible automatic configurations keep per-query attention. Set `packed_prefill=True` / `--packed-prefill` to require packed prefill, or `packed_prefill=False` / `--no-packed-prefill` to disable its staging path. -Explicit requests still undergo device, model and backend compatibility checks. +Explicit enable requests fail if execution constraints or required backend +capabilities are unsupported; they do not silently fall back. Explicit disable +does not probe a packed sliding-window tile. `ExecutionConfig.use_uva` independently controls how input buffers are read. `use_uva=False` / `--no-uva` copies them to the GPU and can be combined with diff --git a/tests/test_backend_selection.py b/tests/test_backend_selection.py index 882b7d43..3778ae71 100644 --- a/tests/test_backend_selection.py +++ b/tests/test_backend_selection.py @@ -24,7 +24,7 @@ def import_module(name): if name == "vllm_rlt.kernels.triton_attention": if "triton" not in installed: raise ImportError("triton is not installed") - return SimpleNamespace() + return SimpleNamespace(select_packed_window_tile=lambda *args: None) if name in ("flash_attn", "flash_attn_3.flash_attn_interface", "flash_attn.cute"): if name not in installed: raise ImportError(f"{name} is not installed") @@ -309,6 +309,62 @@ def test_explicit_triton_rejects_unsupported_nvidia_hardware(packages, monkeypat create_backend("triton", torch.device("cuda"), torch.float16, 128, 256) +def test_triton_advertises_packed_window_only_after_selecting_a_tile(packages, monkeypatch): + from vllm_rlt.attention import backend as selection + from vllm_rlt.kernels.paged_attention import PackedWindowShape + + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (8, 9)) + base_import = selection.importlib.import_module + shape = PackedWindowShape(16, 4, 36, 1024) + calls = [] + + def probe(device, dtype, head_dim, block_size, requested_shape): + calls.append((device, dtype, head_dim, block_size, requested_shape)) + return (32, 32) + + def import_module(name): + if name == "vllm_rlt.kernels.triton_attention": + return SimpleNamespace(select_packed_window_tile=probe) + return base_import(name) + + monkeypatch.setattr(selection.importlib, "import_module", import_module) + args = (torch.device("cuda"), torch.bfloat16, 256, 16) + backend = create_backend( + "auto", + *args, + required_capabilities=BackendCapabilities(sliding_window=True), + packed_window_shape=shape, + ) + assert backend.info["selected_backend"] == "triton" + assert backend.info["packed_prefill_tile"] == (32, 32) + assert backend.capabilities.packed_prefill and backend.capabilities.packed_sliding_window + assert not create_backend("triton", *args).capabilities.packed_sliding_window + assert calls == [(*args, shape)] + + +@pytest.mark.parametrize("backend", ["triton", "auto"]) +def test_packed_window_falls_back_to_per_query_when_probe_has_no_tile( + packages, monkeypatch, backend +): + from vllm_rlt.kernels.paged_attention import PackedWindowShape + + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (8, 9)) + shape = PackedWindowShape(16, 4, 36, 1024) + selected = create_backend( + backend, + torch.device("cuda"), + torch.bfloat16, + 256, + 16, + required_capabilities=BackendCapabilities(sliding_window=True), + packed_window_shape=shape, + ) + assert selected.info["selected_backend"] == "triton" + assert selected.info["packed_prefill_tile"] is None + assert not selected.capabilities.packed_prefill + assert not selected.capabilities.packed_sliding_window + + def test_auto_skips_broken_binary_import(packages, monkeypatch): from vllm_rlt.kernels import flash_attention @@ -326,3 +382,170 @@ def import_module(name): assert "incompatible CUDA extension" in selected.info["selection_reason"] with pytest.raises(OSError, match="incompatible CUDA extension"): create_backend("flash_attn_2", torch.device("cuda"), torch.bfloat16, 128, 256) + + +@pytest.mark.parametrize("requested", [None, True, False]) +@pytest.mark.parametrize("tile", [None, (32, 32)]) +@pytest.mark.parametrize("backend", ["triton", "auto"]) +def test_window_engine_resolves_tristate_before_cache_planning( + packages, monkeypatch, requested, tile, backend +): + from tests.test_dory import tiny_dory_config + from vllm_rlt.attention import backend as selection + from vllm_rlt.engine import stage_engine + + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (8, 9)) + original_import = selection.importlib.import_module + probes = [] + + def import_module(name): + if name == "vllm_rlt.kernels.triton_attention": + + def probe(*args): + probes.append(args[-1]) + return tile + + return SimpleNamespace(select_packed_window_tile=probe) + return original_import(name) + + monkeypatch.setattr(selection.importlib, "import_module", import_module) + config = tiny_dory_config() + model = SimpleNamespace( + config=config, + requires_boundary_kv=True, + requires_sliding_window=True, + parameters=lambda: iter( + [SimpleNamespace(device=torch.device("cuda"), dtype=torch.bfloat16)] + ), + ) + + class Planned(Exception): + pass + + def plan_cache(model, cache, scheduler, execution, selected, **kwargs): + assert execution.packed_prefill is (requested is not False and tile is not None) + assert selected == "triton" + raise Planned + + monkeypatch.setattr(stage_engine, "plan_cache", plan_cache) + error = ValueError if requested is True and tile is None else Planned + with pytest.raises(error): + LLMEngine( + model, + attention_backend=backend, + execution_config=ExecutionConfig(packed_prefill=requested), + ) + assert len(probes) == int(requested is not False) + if probes: + assert probes[0].query_heads == config.num_attention_heads + assert probes[0].kv_layers == config.num_kv_layers + assert probes[0].window == config.sliding_window + + +@pytest.mark.parametrize("missing", ["packed_prefill", "packed_sliding_window"]) +@pytest.mark.parametrize("requested", [None, True]) +def test_window_engine_requires_both_packed_capabilities(monkeypatch, missing, requested): + from dataclasses import replace + + from tests.test_dory import tiny_dory_config + from vllm_rlt.attention import AttentionBackend + from vllm_rlt.attention import backend as selection + from vllm_rlt.engine import stage_engine + + capabilities = replace( + BackendCapabilities(packed_prefill=True, sliding_window=True, packed_sliding_window=True), + **{missing: False}, + ) + monkeypatch.setattr( + selection, + "_create_explicit", + lambda *args, **kwargs: AttentionBackend(None, capabilities, {"backend": "triton"}), + ) + model = SimpleNamespace( + config=tiny_dory_config(), + requires_boundary_kv=True, + requires_sliding_window=True, + parameters=lambda: iter( + [SimpleNamespace(device=torch.device("cuda"), dtype=torch.bfloat16)] + ), + ) + + class Planned(Exception): + pass + + def plan_cache(model, cache, scheduler, execution, selected, **kwargs): + assert requested is None + assert execution.packed_prefill is False + raise Planned + + monkeypatch.setattr(stage_engine, "plan_cache", plan_cache) + error = ValueError if requested is True else Planned + with pytest.raises(error, match=missing if requested else None): + LLMEngine( + model, + attention_backend="triton", + execution_config=ExecutionConfig(packed_prefill=requested), + ) + + +@pytest.mark.parametrize("requested", [None, True]) +@pytest.mark.parametrize( + "asynchronous,graphs,preemption,role", + [ + (True, False, False, "prefill_decode"), + (False, True, False, "prefill_decode"), + (False, False, True, "prefill_decode"), + (False, False, False, "decode"), + ], +) +def test_incompatible_window_engine_does_not_probe_packed_tile( + packages, monkeypatch, requested, asynchronous, graphs, preemption, role +): + from tests.test_dory import tiny_dory_config + from vllm_rlt import SchedulerConfig + from vllm_rlt.attention import backend as selection + from vllm_rlt.engine import stage_engine + from vllm_rlt.engine.stage_config import StageConfig + + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (8, 9)) + original_import = selection.importlib.import_module + + def import_module(name): + if name == "vllm_rlt.kernels.triton_attention": + return SimpleNamespace( + select_packed_window_tile=lambda *args: pytest.fail( + "incompatible execution probed a packed tile" + ) + ) + return original_import(name) + + monkeypatch.setattr(selection.importlib, "import_module", import_module) + model = SimpleNamespace( + config=tiny_dory_config(), + requires_boundary_kv=True, + requires_sliding_window=True, + fixed_loop_count=True, + supports_async_scheduling=True, + parameters=lambda: iter( + [SimpleNamespace(device=torch.device("cuda"), dtype=torch.bfloat16)] + ), + ) + + class Planned(Exception): + pass + + def plan_cache(model, cache, scheduler, execution, selected, **kwargs): + assert requested is None and execution.packed_prefill is False + raise Planned + + monkeypatch.setattr(stage_engine, "plan_cache", plan_cache) + with pytest.raises(ValueError if requested is True else Planned): + LLMEngine( + model, + attention_backend="triton", + stage_config=StageConfig(role=role), + scheduler_config=SchedulerConfig(enable_preemption=preemption), + execution_config=ExecutionConfig( + packed_prefill=requested, async_scheduling=asynchronous, cuda_graphs=graphs + ), + ) diff --git a/tests/test_dory.py b/tests/test_dory.py index 4f47a002..45260837 100644 --- a/tests/test_dory.py +++ b/tests/test_dory.py @@ -670,3 +670,23 @@ def test_exit_written_ignores_boundary_layers(model): # The output stage has not run, so "every layer" is still false. assert not cache.token_written("request", 0, model.config.total_ut_steps - 1) cache.free("request") + + +@pytest.mark.parametrize("packed_prefill", [None, False]) +def test_dory_cpu_keeps_per_row_prefill(model, packed_prefill): + engine = LLMEngine(model, execution_config=ExecutionConfig(packed_prefill=packed_prefill)) + assert engine.execution_config.packed_prefill is False + assert not engine.cache_manager.attention_capabilities.packed_sliding_window + + +def test_dory_packed_prefill_requires_cuda(model): + with pytest.raises(ValueError, match="packed_prefill requires CUDA"): + LLMEngine(model, execution_config=ExecutionConfig(packed_prefill=True)) + + +def test_dory_packed_prefill_rejects_graphs(model): + with pytest.raises(ValueError, match="packed prefill requires synchronous eager"): + LLMEngine( + model, + execution_config=ExecutionConfig(packed_prefill=True, cuda_graphs=True), + ) diff --git a/tests/test_dory_gpu.py b/tests/test_dory_gpu.py index 7941ca00..1038a0ba 100644 --- a/tests/test_dory_gpu.py +++ b/tests/test_dory_gpu.py @@ -288,3 +288,219 @@ def failing(batch): engine.add_request("a", PROMPTS["a"], params) assert drain(engine)["a"].token_ids == greedy_reference(reference, PROMPTS["a"], 4) assert engine.cache_manager.num_used_blocks == 0 + + +def _packed_tile(query_heads, kv_heads, kv_layers=1, *, window): + from vllm_rlt.kernels.paged_attention import PackedWindowShape + from vllm_rlt.kernels.triton_attention import select_packed_window_tile + + tile = select_packed_window_tile( + torch.device(DEVICE), + torch.bfloat16, + 256, + 16, + PackedWindowShape(query_heads, kv_heads, kv_layers, window), + ) + if tile is None: + pytest.skip("No supported packed prefill tile for this device") + return tile + + +@pytest.mark.parametrize("window", [0, 32]) +def test_dory_packed_attention_matches_per_row_reference(window): + from vllm_rlt.kernels.triton_attention import paged_attention, paged_prefill_attention + + _packed_tile(16, 4, window=window or None) + # The first chunk is far enough into its history to skip a full SWA tile. + # The two sequences have distinct pages and different causal offsets. + torch.manual_seed(123) + q = torch.randn((42, 16, 256), device=DEVICE, dtype=torch.bfloat16) + k = torch.randn((10, 16, 4, 256), device=DEVICE, dtype=torch.bfloat16) + v = torch.randn_like(k) + tables = torch.tensor( + [[8, 2, 3, 0, 1, 4, 5, 6, 7], [9, -1, -1, -1, -1, -1, -1, -1, -1]], + device=DEVICE, + dtype=torch.int32, + ) + lengths = torch.tensor([132, 15], device=DEVICE, dtype=torch.int32) + offsets = torch.tensor([0, 35, 42], device=DEVICE, dtype=torch.int32) + row_tables = torch.cat((tables[:1].expand(35, -1), tables[1:].expand(7, -1))) + row_lengths = torch.tensor([*range(98, 133), *range(9, 16)], device=DEVICE, dtype=torch.int32) + expected = paged_attention(q, k, v, row_tables, row_lengths, window=window) + actual = paged_prefill_attention( + q, k, v, tables, lengths, offsets, 35, window=window, compensate=True + ) + torch.testing.assert_close(actual, expected, atol=2e-3, rtol=1e-2) + + +@pytest.mark.parametrize("multi_stream", [True, False]) +@pytest.mark.parametrize(("long_tokens", "max_num_seqs"), [(65, 1), (129, 2)]) +@pytest.mark.parametrize("sliding_window", [None, 32], ids=["full", "mixed"]) +def test_dory_packed_prefill_preserves_short_prompt_logits( + monkeypatch, multi_stream, long_tokens, max_num_seqs, sliding_window +): + from vllm_rlt.kernels import triton_attention + + layout = {"layer_types": ("full_attention", "mlp") * 4} if sliding_window is None else {} + model = make_gpu_model( + head_dim=256, max_position_embeddings=256, sliding_window=sliding_window, **layout + ).to(dtype=torch.bfloat16) + tile = _packed_tile( + model.config.num_attention_heads, + model.config.num_key_value_heads, + model.config.num_kv_layers, + window=sliding_window, + ) + prompt = [4, 7, 9, 3, 12, 5, 23] + scheduler = SchedulerConfig( + max_num_seqs=max_num_seqs, + max_num_batched_tokens=128, + prefill_chunk_size=64, + ) + + def run(*, enable, expect_packed): + engine = LLMEngine( + model, + cache_config=CacheConfig(num_blocks=64), + scheduler_config=scheduler, + attention_backend="triton", + execution_config=ExecutionConfig(packed_prefill=enable, multi_stream=multi_stream), + ) + packed_layers = set() + short_logits = [] + sampler = engine.model_runner.sampler + original_sample = sampler.sample + original_attention = engine.cache_manager._attend_prepared + + def sample(logits, params, generator): + if params is short_options: + short_logits.append(logits.detach().cpu()) + return original_sample(logits, params, generator) + + def attention(layer, metadata, q, *, window=None): + if metadata.cu_seqlens_q is not None: + assert all(rid == "long" for rid, _ in metadata.allocations) + packed_layers.add((layer, window is not None)) + return original_attention(layer, metadata, q, window=window) + + short_options = SamplingParams(max_tokens=3, ignore_eos=True) + with monkeypatch.context() as mp: + mp.setattr(sampler, "sample", sample) + mp.setattr(engine.cache_manager, "_attend_prepared", attention) + try: + assert engine.execution_config.packed_prefill is expect_packed + assert ( + engine.cache_manager.attention_capabilities.packed_sliding_window + is expect_packed + ) + assert engine.cache_manager.attention_info["packed_prefill_tile"] == ( + tile if expect_packed else None + ) + engine.add_request( + "long", + (prompt * 19)[:long_tokens], + SamplingParams(max_tokens=3, ignore_eos=True), + ) + engine.add_request("short", (prompt * 10)[:64], short_options) + while engine.has_unfinished_requests(): + engine.step() + assert engine.cache_manager.num_used_blocks == 0 + finally: + engine.close() + return torch.stack(short_logits), packed_layers + + baseline, baseline_packed = run(enable=False, expect_packed=False) + automatic, automatic_packed = run(enable=None, expect_packed=True) + actual, packed = run(enable=True, expect_packed=True) + with monkeypatch.context() as patch: + patch.setattr(triton_attention, "select_packed_window_tile", lambda *args: None) + fallback, fallback_packed = run(enable=None, expect_packed=False) + with pytest.raises(ValueError, match="lacks required capabilities: packed_sliding_window"): + run(enable=True, expect_packed=False) + assert not fallback_packed + assert torch.equal(baseline, fallback) + assert not baseline_packed + assert automatic_packed == packed + assert torch.equal(automatic, actual) + assert {layer for layer, _ in packed} == set(range(model.config.num_kv_layers)) + assert {window for _, window in packed} == ( + {False} if sliding_window is None else {False, True} + ) + assert torch.equal(baseline, actual) + + +def test_prefill_tile_probe_rejects_oversized_window(monkeypatch): + from types import SimpleNamespace + + from vllm_rlt.kernels import triton_attention + from vllm_rlt.kernels.paged_attention import PackedWindowShape + + if torch.version.hip: + pytest.skip("NVIDIA shared-memory resource check") + + index = torch.cuda.current_device() + limit = triton_attention.driver.active.utils.get_device_properties(index)["max_shared_mem"] + + def compile_tile(*args, window, **kwargs): + return SimpleNamespace(metadata=SimpleNamespace(shared=limit + bool(window))) + + monkeypatch.setattr(triton_attention, "_launch_paged_prefill", compile_tile) + probe = triton_attention._select_packed_window_tile.__wrapped__ + assert probe(index, 256, 16, PackedWindowShape(16, 4, 36, 1024)) is None + + +@pytest.mark.parametrize(("window", "expected_windows"), [(None, [0]), (128, [0, 128])]) +def test_prefill_tile_probe_compiles_only_the_default_and_caches_result( + monkeypatch, window, expected_windows +): + from types import SimpleNamespace + + from vllm_rlt.kernels import triton_attention as attention + from vllm_rlt.kernels.paged_attention import PackedWindowShape + + if torch.version.hip: + pytest.skip("NVIDIA packed prefill selection") + + device = torch.device(DEVICE) + index = torch.cuda.current_device() + limit = attention.driver.active.utils.get_device_properties(index)["max_shared_mem"] + windows = [] + monkeypatch.setattr(torch.cuda, "get_device_name", lambda device: "Unlisted GPU") + + def compile_tile(*args, **kwargs): + assert (kwargs["BLOCK_M"], kwargs["BLOCK_N"]) == (32, 32) + assert kwargs["COMPENSATE"] + assert all(arg.device.type == "meta" for arg in args if isinstance(arg, torch.Tensor)) + windows.append(kwargs["WINDOW"]) + return SimpleNamespace(metadata=SimpleNamespace(shared=limit)) + + monkeypatch.setattr(attention._paged_prefill_kernel, "warmup", compile_tile) + args = (device, torch.bfloat16, 64, 16, PackedWindowShape(4, 2, 2, window)) + + attention._select_packed_window_tile.cache_clear() + try: + assert attention.select_packed_window_tile(*args) == (32, 32) + assert windows == expected_windows + assert attention.select_packed_window_tile(*args) == (32, 32) + assert windows == expected_windows + finally: + attention._select_packed_window_tile.cache_clear() + + +def test_dory_packed_prefill_keeps_small_value_residual(): + from vllm_rlt.kernels.triton_attention import paged_attention, paged_prefill_attention + + _packed_tile(1, 1, window=None) + q = torch.zeros((1, 1, 256), device=DEVICE, dtype=torch.bfloat16) + kv = torch.zeros((1, 16, 1, 256), device=DEVICE, dtype=torch.bfloat16) + k, v = kv.clone(), kv.clone() + q[0, 0, 0], k[0, 1, 0, 0] = 16, -0.5 + # Rounding exp(-0.5) to BF16 before multiplying would zero the numerator. + v[0, 0, 0, 0], v[0, 1, 0, 0] = -0.60546875, 1 + table = torch.tensor([[0]], device=DEVICE, dtype=torch.int32) + length = torch.tensor([2], device=DEVICE, dtype=torch.int32) + offset = torch.tensor([0, 1], device=DEVICE, dtype=torch.int32) + expected = paged_attention(q, k, v, table, length) + actual = paged_prefill_attention(q, k, v, table, length, offset, 1, compensate=True) + assert expected[0, 0, 0] > 0.0006 + torch.testing.assert_close(actual, expected, atol=5e-6, rtol=0) diff --git a/tests/test_engine.py b/tests/test_engine.py index e265904a..e738d8f4 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -422,17 +422,28 @@ def test_packed_prefill_resolves_automatically_when_unset(): from vllm_rlt.engine.packed_prefill import packed_prefill_eligible common = dict(layout="last_exited", staged_prefill=False) - # Unset: on for CUDA, last_exited, ordinary prefill without sliding windows. - assert packed_prefill_eligible(None, device_type="cuda", **common) - assert not packed_prefill_eligible(None, device_type="cpu", **common) - assert not packed_prefill_eligible(None, device_type="cuda", **{**common, "layout": "shared"}) - assert not packed_prefill_eligible(None, device_type="cuda", **common, sliding_window=True) + # Unset: prefer packing for compatible CUDA execution, including windows. + assert packed_prefill_eligible(ExecutionConfig(), device_type="cuda", **common) + assert not packed_prefill_eligible(ExecutionConfig(), device_type="cpu", **common) assert not packed_prefill_eligible( - None, device_type="cuda", **{**common, "staged_prefill": True} + ExecutionConfig(), device_type="cuda", **{**common, "layout": "shared"} + ) + assert packed_prefill_eligible( + ExecutionConfig(), device_type="cuda", **common, sliding_window=True + ) + assert not packed_prefill_eligible( + ExecutionConfig(), device_type="cuda", **{**common, "staged_prefill": True} + ) + # Explicit settings override the automatic choice. + assert not packed_prefill_eligible( + ExecutionConfig(packed_prefill=False), device_type="cuda", **common + ) + assert packed_prefill_eligible( + ExecutionConfig(packed_prefill=True), device_type="cuda", **common + ) + assert packed_prefill_eligible( + ExecutionConfig(packed_prefill=True), device_type="cuda", **common, sliding_window=True ) - # Explicit settings win regardless of eligibility. - assert not packed_prefill_eligible(False, device_type="cuda", **common) - assert packed_prefill_eligible(True, device_type="cuda", **common) assert ExecutionConfig().packed_prefill is None with pytest.raises(ValueError, match="packed_prefill"): ExecutionConfig(packed_prefill=1) @@ -443,3 +454,37 @@ def test_packed_prefill_resolves_automatically_when_unset(): LLMEngine(tiny_model(), execution_config=ExecutionConfig(packed_prefill=True)) engine = LLMEngine(tiny_model(), execution_config=ExecutionConfig(packed_prefill=False)) assert engine.execution_config.packed_prefill is False + + +@pytest.mark.parametrize("requested", [None, True, False]) +@pytest.mark.parametrize( + "execution, constraints, error", + [ + ({}, {}, None), + ({}, {"device_type": "cpu"}, "requires CUDA"), + ({}, {"layout": "shared"}, "last_exited"), + ({"async_scheduling": True}, {}, "synchronous eager"), + ({"cuda_graphs": True}, {}, "synchronous eager"), + ({}, {"staged_prefill": True}, "ordinary prefill"), + ({}, {"speculative": True}, "without speculation"), + ({}, {"preemption": True}, "without speculation or preemption"), + ({}, {"stage_role": "prefill"}, "single-stage"), + ({}, {"stage_role": "decode"}, "single-stage"), + ], +) +def test_sliding_window_packed_prefill_tristate(requested, execution, constraints, error): + from vllm_rlt import ExecutionConfig + from vllm_rlt.engine.packed_prefill import packed_prefill_eligible + + options = dict( + device_type="cuda", layout="last_exited", staged_prefill=False, sliding_window=True + ) + options.update(constraints) + config = ExecutionConfig(packed_prefill=requested, **execution) + if requested is True and error is not None: + with pytest.raises(ValueError, match=error): + packed_prefill_eligible(config, **options) + else: + assert packed_prefill_eligible(config, **options) is ( + requested is not False and error is None + ) diff --git a/tests/test_tensor_transfer.py b/tests/test_tensor_transfer.py index 787dd401..83900e4f 100644 --- a/tests/test_tensor_transfer.py +++ b/tests/test_tensor_transfer.py @@ -401,3 +401,18 @@ def test_uva_configuration_and_capability(monkeypatch, flags, packed_prefill): assert not use_uva_on("cpu") monkeypatch.setattr(torch.version, "hip", "test") assert not use_uva_on("cuda") + + +@pytest.mark.parametrize( + "flags", + [ + ("--no-packed-prefill", "--packed-prefill"), + ("--packed-prefill", "--no-packed-prefill"), + ], +) +def test_packed_prefill_flags_are_mutually_exclusive(flags): + parser = argparse.ArgumentParser() + add_runtime_args(parser) + with pytest.raises(SystemExit) as error: + parser.parse_args(flags) + assert error.value.code == 2 diff --git a/vllm_rlt/attention/backend.py b/vllm_rlt/attention/backend.py index ce34d024..fcc58278 100644 --- a/vllm_rlt/attention/backend.py +++ b/vllm_rlt/attention/backend.py @@ -8,7 +8,11 @@ import torch from vllm_rlt.kernels.flash_attention import FLASH_BACKENDS, FlashPagedAttention -from vllm_rlt.kernels.paged_attention import TorchPagedAttention, TritonPagedAttention +from vllm_rlt.kernels.paged_attention import ( + PackedWindowShape, + TorchPagedAttention, + TritonPagedAttention, +) @dataclass(frozen=True) @@ -25,6 +29,7 @@ class BackendCapabilities: device_context_lengths: bool = False # Mask keys outside a per-layer sliding window (not storage reclamation). sliding_window: bool = False + packed_sliding_window: bool = False @dataclass @@ -82,17 +87,35 @@ def validate_backend_name(backend: str) -> None: raise ValueError("unknown attention backend") -def _create_explicit(backend, device, dtype, head_dim, block_size): +def _create_explicit(backend, device, dtype, head_dim, block_size, *, packed, packed_window_shape): if backend == "triton": if device.type != "cuda": raise ValueError("the Triton attention backend requires a CUDA or ROCm device") if head_dim > 256: raise ValueError("the Triton attention backend supports head_dim <= 256") - if not torch.version.hip and torch.cuda.get_device_capability(device)[0] < 8: + sm = None if torch.version.hip else torch.cuda.get_device_capability(device) + if sm is not None and sm[0] < 8: raise ValueError("the Triton attention backend requires NVIDIA Compute Capability 8.0+") - # Probe the actual module, including its dependency imports, before selection. - importlib.import_module("vllm_rlt.kernels.triton_attention") - return AttentionBackend(TritonPagedAttention(), _TRITON_CAPABILITIES, {"backend": backend}) + # Import the implementation before selection and probe the opted-in tile + # before the engine profiles and allocates its KV pool. + implementation = importlib.import_module("vllm_rlt.kernels.triton_attention") + tile = ( + implementation.select_packed_window_tile( + device, dtype, head_dim, block_size, packed_window_shape + ) + if packed_window_shape is not None + else None + ) + compensate = tile is not None + capabilities = replace( + _TRITON_CAPABILITIES, + packed_prefill=packed or compensate, + packed_sliding_window=compensate, + ) + kernel = TritonPagedAttention(compensate=compensate) + return AttentionBackend( + kernel, capabilities, {"backend": backend, "packed_prefill_tile": tile} + ) if backend in FLASH_BACKENDS: kernel = FlashPagedAttention(device, dtype, head_dim, block_size, backend) return AttentionBackend(kernel, _FLASH_CAPABILITIES[kernel.generation], dict(kernel.info)) @@ -108,6 +131,7 @@ def create_backend( *, required_capabilities: BackendCapabilities = BackendCapabilities(), prefer_packed_prefill: bool = False, + packed_window_shape: PackedWindowShape | None = None, ) -> AttentionBackend: """Explicit choices fail without fallback; auto tries FA4/3/2, Triton, then Torch. @@ -117,6 +141,7 @@ def create_backend( filter candidates before memory profiling or device-buffer allocation. Preferred packed prefill enables it where available without filtering or reordering candidates; required packed prefill still rejects unsupported ones. + A packed window shape opts into tile selection and keeps row attention available. """ validate_backend_name(backend) candidates = _AUTO_PRIORITY if backend == "auto" else (backend,) @@ -126,16 +151,15 @@ def create_backend( skipped.append(f"{device.type.upper()} uses Torch reference attention") for candidate in candidates: try: - implementation = _create_explicit(candidate, device, dtype, head_dim, block_size) - if candidate == "triton" and ( - required_capabilities.packed_prefill or prefer_packed_prefill - ): - # Packed BF16 prefill has different arithmetic from per-row decode. - # Opt in for packed_prefill without changing existing split/mixed or - # Huginn boundary attention when that option is disabled. - implementation.capabilities = replace( - implementation.capabilities, packed_prefill=True - ) + implementation = _create_explicit( + candidate, + device, + dtype, + head_dim, + block_size, + packed=required_capabilities.packed_prefill or prefer_packed_prefill, + packed_window_shape=packed_window_shape, + ) missing = [ name for name in ( @@ -144,6 +168,7 @@ def create_backend( "async_scheduling", "device_context_lengths", "sliding_window", + "packed_sliding_window", ) if getattr(required_capabilities, name) and not getattr(implementation.capabilities, name) @@ -168,6 +193,14 @@ def create_backend( implementation.info.update( requested_backend=backend, selected_backend=selected, selection_reason=reason ) + if ( + packed_window_shape is not None + and not implementation.capabilities.packed_sliding_window + ): + implementation.info.update( + packed_prefill_tile=None, + packed_prefill_fallback="no supported packed prefill tile", + ) logging.getLogger(__name__).info("Attention selection: %s", implementation.info) return implementation raise ValueError("no compatible attention backend; " + "; ".join(skipped)) diff --git a/vllm_rlt/config.py b/vllm_rlt/config.py index b5b58cb6..1619ba80 100644 --- a/vllm_rlt/config.py +++ b/vllm_rlt/config.py @@ -152,7 +152,8 @@ class ExecutionConfig: # use_uva independently controls its transfer mechanism. # None resolves at engine construction: on when the attention backend packs # prefill (Triton opts in, FlashAttention-4 natively) with last_exited KV and - # ordinary (non-staged) prefill without sliding-window attention; off otherwise. + # ordinary (non-staged) prefill. Sliding windows additionally require packed-window + # support and compatible synchronous eager execution; off otherwise. packed_prefill: bool | None = None def __post_init__(self): diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 0d1e8641..ec59c9cc 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -738,8 +738,17 @@ def _attend_prepared( self._require_live_batch(batch) self._validate_tensor(q, len(batch.position_ids), "q", query=True) if window is not None: - return self._attend_windowed(layer, batch, q, window) + if batch.cu_seqlens_q is None: + return self._attend_windowed(layer, batch, q, window) + if not self.attention_capabilities.packed_sliding_window: + raise NotImplementedError("sliding-window attention needs one query row per token") + if type(window) is not int or window <= 0: + raise ValueError("window must be a positive integer") if isinstance(batch, DeviceKVMetadata): + if window is not None: + raise NotImplementedError( + "packed sliding-window attention requires host-planned KV" + ) return self.attention( q, self.key_cache[:, layer], @@ -760,6 +769,7 @@ def _attend_prepared( batch.context_lengths, batch.cu_seqlens_q, batch.max_seqlen_q, + **({} if window is None else {"window": window}), ) return self.attention( q, diff --git a/vllm_rlt/engine/packed_prefill.py b/vllm_rlt/engine/packed_prefill.py index 49c49d14..82eec246 100644 --- a/vllm_rlt/engine/packed_prefill.py +++ b/vllm_rlt/engine/packed_prefill.py @@ -2,22 +2,40 @@ def packed_prefill_eligible( - packed_prefill, *, device_type, layout, staged_prefill, sliding_window=False + execution, + *, + device_type, + layout, + staged_prefill, + sliding_window=False, + stage_role="prefill_decode", + speculative=False, + preemption=False, ): - """Validate explicit requests and return whether packing may be enabled. + """Validate execution constraints before filtering by backend capability. - Staged prefill and sliding-window attention retain their existing metadata. - Backend support is checked separately by attention selection. + Sliding-window packing requires host-planned metadata and ordinary + synchronous prefill in a single stage. """ - if packed_prefill and sliding_window: - raise ValueError("packed_prefill does not support sliding-window attention") - if packed_prefill and (device_type != "cuda" or layout != "last_exited"): - raise ValueError("packed_prefill requires CUDA packed prefill with last_exited KV") - if packed_prefill is not None: - return bool(packed_prefill) - return ( - device_type == "cuda" - and layout == "last_exited" - and not staged_prefill - and not sliding_window - ) + requested = execution.packed_prefill + if requested is False: + return False + if sliding_window and ( + execution.async_scheduling + or execution.cuda_graphs + or staged_prefill + or speculative + or preemption + or stage_role != "prefill_decode" + ): + if requested is True: + raise ValueError( + "sliding-window packed prefill requires synchronous eager single-stage execution " + "with ordinary prefill, without speculation or preemption" + ) + return False + if device_type != "cuda" or layout != "last_exited": + if requested is True: + raise ValueError("packed_prefill requires CUDA packed prefill with last_exited KV") + return False + return requested is True or not staged_prefill diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index 1cae4096..2f9fde18 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -18,6 +18,7 @@ from vllm_rlt.engine.packed_prefill import packed_prefill_eligible from vllm_rlt.engine.preemption import PreemptionManager from vllm_rlt.engine.stage_config import StageConfig +from vllm_rlt.kernels.paged_attention import PackedWindowShape from vllm_rlt.profiling import Profiler from vllm_rlt.request import FinishReason, Request, RequestOutput, RequestPhase from vllm_rlt.sampling_params import SamplingParams @@ -75,6 +76,17 @@ def __init__( staged_prefill = requires_staged_prefill( scheduler_config, produces_tokens=self.stage_config.produces_tokens ) + sliding_window = getattr(model, "requires_sliding_window", False) + prefer_packed = packed_prefill_eligible( + self.execution_config, + device_type=parameter.device.type, + layout=cache_config.layout, + staged_prefill=staged_prefill, + sliding_window=sliding_window, + stage_role=self.stage_config.role, + speculative=speculative_config is not None, + preemption=scheduler_config.enable_preemption, + ) if getattr(model, "requires_boundary_kv", False): if staged_prefill: raise ValueError( @@ -174,12 +186,15 @@ def __init__( ) if staged_prefill and cache_config.layout != "last_exited": raise ValueError("staged prefill requires last_exited KV planes") - auto_packed = packed_prefill_eligible( - self.execution_config.packed_prefill, - device_type=parameter.device.type, - layout=cache_config.layout, - staged_prefill=staged_prefill, - sliding_window=getattr(model, "requires_sliding_window", False), + require_packed = self.execution_config.packed_prefill is True + kv_layers = getattr(config, "num_kv_layers", config.num_hidden_layers) + window = getattr(config, "sliding_window", None) + packed_window_shape = ( + PackedWindowShape( + config.num_attention_heads, config.num_key_value_heads, kv_layers, window + ) + if prefer_packed and sliding_window + else None ) attention = create_backend( attention_backend, @@ -188,20 +203,24 @@ def __init__( config.head_dim, cache_config.block_size, required_capabilities=BackendCapabilities( - packed_prefill=bool(self.execution_config.packed_prefill), + packed_prefill=require_packed, + packed_sliding_window=require_packed and sliding_window, cuda_graphs=self.execution_config.cuda_graphs, async_scheduling=( parameter.device.type == "cuda" and self.execution_config.async_scheduling ), device_context_lengths=self.async_speculative, - sliding_window=getattr(model, "requires_sliding_window", False), + sliding_window=sliding_window, ), - prefer_packed_prefill=auto_packed, + prefer_packed_prefill=prefer_packed and not sliding_window, + packed_window_shape=packed_window_shape, ) if self.execution_config.packed_prefill is None: + supported = attention.capabilities.packed_prefill and ( + not sliding_window or attention.capabilities.packed_sliding_window + ) self.execution_config = replace( - self.execution_config, - packed_prefill=auto_packed and attention.capabilities.packed_prefill, + self.execution_config, packed_prefill=prefer_packed and supported ) num_blocks, self.memory_plan = plan_cache( model, @@ -213,7 +232,7 @@ def __init__( staged_prefill=staged_prefill, ) self.cache_manager = KVCacheManager( - num_layers=getattr(config, "num_kv_layers", config.num_hidden_layers), + num_layers=kv_layers, num_kv_heads=config.num_key_value_heads, head_dim=config.head_dim, max_loops=config.total_ut_steps, diff --git a/vllm_rlt/entrypoints/runtime_args.py b/vllm_rlt/entrypoints/runtime_args.py index 0995207f..49d2b13f 100644 --- a/vllm_rlt/entrypoints/runtime_args.py +++ b/vllm_rlt/entrypoints/runtime_args.py @@ -24,14 +24,15 @@ def add_runtime_args(parser): parser.add_argument("--kv-watermark", type=float, default=0.0) parser.add_argument("--scheduling-policy", choices=["fcfs", "priority"], default="fcfs") parser.add_argument("--enable-preemption", action="store_true") - parser.add_argument( + packed_prefill_mode = parser.add_mutually_exclusive_group() + packed_prefill_mode.add_argument( "--packed-prefill", action="store_true", default=None, - help="Force packed prefill metadata staging (requires packed attention); " - "default resolves automatically (on for CUDA with last_exited KV)", + help="Require packed prefill on a compatible backend; fail if unsupported. " + "Defaults to automatic selection when execution and backend support it.", ) - parser.add_argument( + packed_prefill_mode.add_argument( "--no-packed-prefill", action="store_false", dest="packed_prefill", diff --git a/vllm_rlt/kernels/paged_attention.py b/vllm_rlt/kernels/paged_attention.py index dda4b2ef..d92d8279 100644 --- a/vllm_rlt/kernels/paged_attention.py +++ b/vllm_rlt/kernels/paged_attention.py @@ -5,6 +5,7 @@ """ import math +from dataclasses import dataclass import torch @@ -102,20 +103,48 @@ def triton_paged_attention( return paged_attention(q, key_cache, value_cache, block_tables, context_lengths, window=window) +@dataclass(frozen=True) +class PackedWindowShape: + query_heads: int + kv_heads: int + kv_layers: int + window: int | None + + def triton_paged_prefill( - q, key_cache, value_cache, block_tables, context_lengths, cu_seqlens_q, max_seqlen_q + q, + key_cache, + value_cache, + block_tables, + context_lengths, + cu_seqlens_q, + max_seqlen_q, + *, + window=0, + compensate=False, ): """Packed causal prefill with one block table per sequence (see triton_attention).""" from vllm_rlt.kernels.triton_attention import paged_prefill_attention return paged_prefill_attention( - q, key_cache, value_cache, block_tables, context_lengths, cu_seqlens_q, max_seqlen_q + q, + key_cache, + value_cache, + block_tables, + context_lengths, + cu_seqlens_q, + max_seqlen_q, + window=window, + compensate=compensate, ) class TritonPagedAttention: """Triton single-query decode attention plus packed causal prefill.""" + def __init__(self, *, compensate=False): + self.compensate = compensate + def __call__(self, q, key_cache, value_cache, block_tables, context_lengths): return triton_paged_attention(q, key_cache, value_cache, block_tables, context_lengths) @@ -125,4 +154,4 @@ def windowed(self, q, key_cache, value_cache, block_tables, context_lengths, win ) def prefill(self, *args, **kwargs): - return triton_paged_prefill(*args, **kwargs) + return triton_paged_prefill(*args, **kwargs, compensate=self.compensate) diff --git a/vllm_rlt/kernels/triton_attention.py b/vllm_rlt/kernels/triton_attention.py index e8f41288..ae9a8d90 100644 --- a/vllm_rlt/kernels/triton_attention.py +++ b/vllm_rlt/kernels/triton_attention.py @@ -8,9 +8,15 @@ row serialized its whole context in one program and set the decode step time. """ +from functools import lru_cache, partial + import torch import triton import triton.language as tl +from triton.errors import TritonError +from triton.runtime import driver + +_PACKED_WINDOW_TILE = (32, 32) # Splits per row for single-query attention. A power of two (tl.arange) and a # constant, so the grid is static under CUDA graphs and independent of batch. @@ -230,6 +236,8 @@ def _paged_prefill_kernel( BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, PRECISION: tl.constexpr, + WINDOW: tl.constexpr, + COMPENSATE: tl.constexpr, max_q_blocks, ): pid = tl.program_id(0) @@ -261,7 +269,8 @@ def _paged_prefill_kernel( accumulator = tl.zeros((BLOCK_M, BLOCK_D), tl.float32) # Bottom-right causal alignment: the last query row sees every key. end = tl.minimum(kv_len, prefix + (q_block + 1) * BLOCK_M) - for start in range(0, end, BLOCK_N): + first = tl.maximum(0, prefix + q_block * BLOCK_M - WINDOW + 1) if WINDOW else 0 + for start in range(first // BLOCK_N * BLOCK_N, end, BLOCK_N): keys_pos = start + tl.arange(0, BLOCK_N) key_valid = keys_pos < end blocks = tl.load( @@ -279,6 +288,8 @@ def _paged_prefill_kernel( ) scores = tl.dot(query, tl.trans(keys), input_precision=PRECISION) * SCALE visible = key_valid[None, :] & (keys_pos[None, :] <= positions[:, None]) + if WINDOW: + visible = visible & (keys_pos[None, :] >= positions[:, None] - WINDOW + 1) scores = tl.where(visible, scores, -float("inf")) next_maximum = tl.maximum(maximum, tl.max(scores, axis=1)) # Rows with no visible key in this tile keep -inf; avoid inf - inf. @@ -294,9 +305,13 @@ def _paged_prefill_kernel( mask=key_valid[:, None] & dim_valid[None, :], other=0.0, ) - accumulator = accumulator * correction[:, None] + tl.dot( - weights.to(values.dtype), values, input_precision=PRECISION - ) + rounded = weights.to(values.dtype) + partial = tl.dot(rounded, values, input_precision=PRECISION) + if COMPENSATE: + # Recover the leading bits lost when the FP32 weight is cast for MMA. + residual = (weights - rounded.to(tl.float32)).to(values.dtype) + partial = tl.dot(residual, values, partial, input_precision=PRECISION) + accumulator = accumulator * correction[:, None] + partial normalizer = normalizer * correction + tl.sum(weights, axis=1) maximum = next_maximum out = accumulator / tl.maximum(normalizer, 1.0e-30)[:, None] @@ -311,7 +326,16 @@ def _paged_prefill_kernel( def paged_prefill_attention( - q, key_cache, value_cache, block_tables, context_lengths, cu_seqlens_q, max_seqlen_q + q, + key_cache, + value_cache, + block_tables, + context_lengths, + cu_seqlens_q, + max_seqlen_q, + *, + window=0, + compensate=False, ): """Packed causal prefill over paged K/V, bottom-right aligned per sequence. @@ -324,14 +348,51 @@ def paged_prefill_attention( sequences = block_tables.shape[0] if q.shape[0] == 0 or sequences == 0: return output - block_m, block_n = 64, 64 - q_blocks = triton.cdiv(int(max_seqlen_q), block_m) + if type(window) is not int or window < 0: + raise ValueError("window must be a nonnegative integer") for tensor in (key_cache, value_cache): if tensor.stride(3) != 1: raise ValueError("paged prefill requires a contiguous head dimension") if q.stride(2) != 1: raise ValueError("paged prefill requires contiguous query rows") - _paged_prefill_kernel[(sequences * q_blocks, q.shape[1])]( + _launch_paged_prefill( + q, + key_cache, + value_cache, + block_tables, + context_lengths, + cu_seqlens_q, + output, + max_seqlen_q, + window=window, + compensate=compensate, + ) + return output + + +def _launch_paged_prefill( + q, + key_cache, + value_cache, + block_tables, + context_lengths, + cu_seqlens_q, + output, + max_seqlen_q, + *, + window, + compensate, + compile_only=False, +): + block_m, block_n = _PACKED_WINDOW_TILE if compensate else (64, 64) + q_blocks = triton.cdiv(int(max_seqlen_q), block_m) + grid = (block_tables.shape[0] * q_blocks, q.shape[1]) + launch = ( + partial(_paged_prefill_kernel.warmup, grid=grid) + if compile_only + else _paged_prefill_kernel[grid] + ) + return launch( q, key_cache, value_cache, @@ -359,8 +420,53 @@ def paged_prefill_attention( BLOCK_N=block_n, # FP32 callers (reference checks) need exact products, not TF32. PRECISION="ieee" if q.dtype == torch.float32 else "tf32", + WINDOW=window, + COMPENSATE=compensate, max_q_blocks=q_blocks, num_warps=4, - num_stages=2, + num_stages=1 if compensate else 2, ) - return output + + +def select_packed_window_tile(device, dtype, head_dim, block_size, shape): + if torch.version.hip or dtype != torch.bfloat16: + return None + index = device.index if device.index is not None else torch.cuda.current_device() + try: + return _select_packed_window_tile(index, head_dim, block_size, shape) + except torch.OutOfMemoryError: + # A transient failure must not be cached for other engines. + return None + + +@lru_cache(maxsize=16) +def _select_packed_window_tile(index, head_dim, block_size, shape): + # Warmup only needs tensor metadata. Preserve the KV pool's interleaved layer strides. + q = torch.empty((256, shape.query_heads, head_dim), device="meta", dtype=torch.bfloat16) + kv = q.new_empty((1, shape.kv_layers, block_size, shape.kv_heads, head_dim))[:, 0] + table = q.new_empty((1, 16), dtype=torch.int32) + lengths = q.new_empty(1, dtype=torch.int32) + offsets = q.new_empty(2, dtype=torch.int32) + windows = (0,) if shape.window is None else (0, shape.window) + with torch.cuda.device(index): + limit = driver.active.utils.get_device_properties(index)["max_shared_mem"] + try: + for window in windows: + compiled = _launch_paged_prefill( + q, + kv, + kv, + table, + lengths, + offsets, + q, + len(q), + window=window, + compensate=True, + compile_only=True, + ) + if compiled.metadata.shared > limit: + return None + except TritonError: + return None + return _PACKED_WINDOW_TILE diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index 78f78a78..3766c5b2 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -621,7 +621,7 @@ def _gather(self, requests, workspace, size): hidden[row].copy_(request.hidden_state) return hidden - def _prepare_prefill_coda(self, ids, positions, requests, kv=None): + def _prepare_prefill_coda(self, ids, positions, requests, kv=None, *, packed=None): if not getattr(self.model, "requires_boundary_kv", False): return None, None rows = [ @@ -639,7 +639,9 @@ def _prepare_prefill_coda(self, ids, positions, requests, kv=None): [ids[row] for row in rows], [[0] * len(rows)], [positions[row] for row in rows], - packed=self.cache_manager.attention_capabilities.packed_prefill, + packed=self.cache_manager.attention_capabilities.packed_prefill + if packed is None + else packed, )[0] return ForwardBatch( Operation.CODA, prelude_coda_kv_metadata=kv_metadata, compute_logits=False @@ -716,11 +718,14 @@ def _prepare_shared_loop_inputs(self, batch): return tuple(inputs) def _uses_packed_loop_inputs(self, batch): + capabilities = self.cache_manager.attention_capabilities return ( batch.operation == Operation.PRELUDE and batch.loop_count and self.cache_manager.layout != "shared" - and self.cache_manager.attention_capabilities.packed_prefill + and capabilities.packed_prefill + # Sliding-window batches share staging for packed and per-row metadata. + and not capabilities.packed_sliding_window and self.execution_config.packed_prefill ) @@ -737,7 +742,10 @@ def _prepare_loop_inputs(self, batch, packed_metadata=None): tokens.extend(span.input_tokens) requests.extend([request] * count) outputs.append((len(tokens) - 1, request)) - packed = cache.attention_capabilities.packed_prefill + packed = cache.attention_capabilities.packed_prefill and ( + not cache.attention_capabilities.packed_sliding_window + or all(len(span.request.prompt_token_ids) > 64 for span in batch.spans) + ) size = len(tokens) if packed else self._size(len(tokens)) depth_sets = [[depth + offset for depth in depths] for offset in range(batch.loop_count)] if packed_metadata is not None: @@ -753,7 +761,7 @@ def _prepare_loop_inputs(self, batch, packed_metadata=None): packed=packed, padded_rows=size, ) - coda, coda_rows = self._prepare_prefill_coda(ids, positions, requests) + coda, coda_rows = self._prepare_prefill_coda(ids, positions, requests, packed=packed) forwards = [ PreparedForward( ForwardBatch(Operation.PRELUDE, tokens=tensor, metadata=metadata),