From b17d1f68dc300448e9a10f7269981c5fd6921d2a Mon Sep 17 00:00:00 2001 From: cecilsd <127472276+cecilsd@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:55:04 +0800 Subject: [PATCH 1/6] perf(dory): add opt-in packed Triton prefill --- docs/dory.md | 26 +++++++ tests/test_dory.py | 13 ++++ tests/test_dory_gpu.py | 111 +++++++++++++++++++++++++++ vllm_rlt/attention/backend.py | 15 +++- vllm_rlt/config.py | 2 + vllm_rlt/core/kv_cache_manager.py | 12 ++- vllm_rlt/engine/stage_engine.py | 21 ++++- vllm_rlt/entrypoints/runtime_args.py | 6 ++ vllm_rlt/kernels/paged_attention.py | 26 ++++++- vllm_rlt/kernels/triton_attention.py | 37 +++++++-- vllm_rlt/worker/model_runner.py | 13 +++- 11 files changed, 265 insertions(+), 17 deletions(-) diff --git a/docs/dory.md b/docs/dory.md index d53dfe69..46a38b9e 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -90,6 +90,32 @@ layer id rather than their physical sublayer index. template, resolved without requiring transformers to accept the checkpoint config (its generic validator rejects Dory's `mlp` sublayer entries). +## Opt-in packed Triton prefill + +Set `ExecutionConfig(dory_packed_prefill=True)` in Python or pass +`--dory-packed-prefill` with the Triton attention backend in the CLI. This +path requires Dory BF16, NVIDIA SM 12.0, `per_loop` KV, ordinary prefill, +and synchronous eager single-stage execution. Speculation, preemption, CUDA +graphs and asynchronous scheduling are outside its scope. RTX 5090 is the +only GPU tested. Validate other SM 12.0 GPUs locally. Other GPU +architectures require a separate kernel configuration and validation +before the option can be enabled. + +A batch uses packed attention across PRELUDE, recurrence and the CODA +writes for prompt history when every prompt is longer than 64 tokens. +The CODA that samples the first token uses per-query attention. A batch containing +a shorter prompt or a 64-token prompt uses per-query attention. The +cutoff uses the full prompt length. Decode keeps the per-query path. + +Packed full and sliding-window attention use the same causal ranges as +per-query attention. The matrix products and softmax reductions use a +different order and intermediate rounding. The value product compensates +for the BF16 rounding of softmax weights. Remaining differences can +propagate through Dory's recurrent layers and change a greedy choice. +For reproducible comparisons, use per-query attention and hold the +batch shape fixed. Changes in arrival timing can change the batch shape +even with per-query attention. + ## Rejected today These fail fast rather than degrade silently: diff --git a/tests/test_dory.py b/tests/test_dory.py index 4f47a002..cf6a2274 100644 --- a/tests/test_dory.py +++ b/tests/test_dory.py @@ -670,3 +670,16 @@ def test_exit_written_ignores_boundary_layers(model): # The output stage has not run, so "every layer" is still false. assert not cache.token_written("request", 0, model.config.total_ut_steps - 1) cache.free("request") + + +def test_dory_packed_prefill_requires_a_qualified_backend(model): + with pytest.raises(ValueError, match="packed_sliding_window"): + LLMEngine(model, execution_config=ExecutionConfig(dory_packed_prefill=True)) + + +def test_dory_packed_prefill_rejects_graphs(model): + with pytest.raises(ValueError, match="packed prefill requires synchronous eager"): + LLMEngine( + model, + execution_config=ExecutionConfig(dory_packed_prefill=True, cuda_graphs=True), + ) diff --git a/tests/test_dory_gpu.py b/tests/test_dory_gpu.py index 7941ca00..79160e96 100644 --- a/tests/test_dory_gpu.py +++ b/tests/test_dory_gpu.py @@ -288,3 +288,114 @@ def failing(batch): engine.add_request("a", PROMPTS["a"], params) assert drain(engine)["a"].token_ids == greedy_reference(reference, PROMPTS["a"], 4) assert engine.cache_manager.num_used_blocks == 0 + + +@pytest.mark.parametrize("window", [0, 32]) +def test_dory_packed_attention_matches_per_row_reference(window): + if torch.cuda.get_device_capability() != (12, 0) or torch.version.hip: + pytest.skip("Dory packed prefill requires NVIDIA SM 12.0") + from vllm_rlt.kernels.triton_attention import paged_attention, paged_prefill_attention + + # The first chunk is far enough into its history to skip a full SWA tile. + # The two sequences have distinct pages and different causal offsets. + torch.manual_seed(123) + q = torch.randn((42, 16, 256), device=DEVICE, dtype=torch.bfloat16) + k = torch.randn((10, 16, 4, 256), device=DEVICE, dtype=torch.bfloat16) + v = torch.randn_like(k) + tables = torch.tensor( + [[8, 2, 3, 0, 1, 4, 5, 6, 7], [9, -1, -1, -1, -1, -1, -1, -1, -1]], + device=DEVICE, + dtype=torch.int32, + ) + lengths = torch.tensor([132, 15], device=DEVICE, dtype=torch.int32) + offsets = torch.tensor([0, 35, 42], device=DEVICE, dtype=torch.int32) + row_tables = torch.cat((tables[:1].expand(35, -1), tables[1:].expand(7, -1))) + row_lengths = torch.tensor([*range(98, 133), *range(9, 16)], device=DEVICE, dtype=torch.int32) + expected = paged_attention(q, k, v, row_tables, row_lengths, window=window) + actual = paged_prefill_attention( + q, k, v, tables, lengths, offsets, 35, window=window, compensate=True + ) + torch.testing.assert_close(actual, expected, atol=2e-3, rtol=1e-2) + + +@pytest.mark.parametrize("multi_stream", [True, False]) +def test_dory_packed_prefill_preserves_short_prompt_logits(monkeypatch, multi_stream): + if torch.cuda.get_device_capability() != (12, 0) or torch.version.hip: + pytest.skip("Dory packed prefill requires NVIDIA SM 12.0") + model = make_gpu_model(head_dim=256, max_position_embeddings=256, sliding_window=32).to( + dtype=torch.bfloat16 + ) + prompt = [4, 7, 9, 3, 12, 5, 23] + scheduler = SchedulerConfig( + max_num_seqs=2, + max_num_batched_tokens=128, + prefill_chunk_size=64, + ) + + def run(*, enable): + engine = LLMEngine( + model, + cache_config=CacheConfig(num_blocks=64), + scheduler_config=scheduler, + attention_backend="triton", + execution_config=ExecutionConfig(dory_packed_prefill=enable, multi_stream=multi_stream), + ) + packed_layers = set() + short_logits = [] + sampler = engine.model_runner.sampler + original_sample = sampler.sample + original_attention = engine.cache_manager._attend_prepared + + def sample(logits, params, generator): + if params is short_options: + short_logits.append(logits.detach().cpu()) + return original_sample(logits, params, generator) + + def attention(layer, metadata, q, *, window=None): + if metadata.cu_seqlens_q is not None: + assert all(rid == "long" for rid, _ in metadata.allocations) + packed_layers.add((layer, window is not None)) + return original_attention(layer, metadata, q, window=window) + + short_options = SamplingParams(max_tokens=3, ignore_eos=True) + with monkeypatch.context() as mp: + mp.setattr(sampler, "sample", sample) + mp.setattr(engine.cache_manager, "_attend_prepared", attention) + engine.add_request( + "long", (prompt * 19)[:129], SamplingParams(max_tokens=3, ignore_eos=True) + ) + engine.add_request("short", (prompt * 10)[:64], short_options) + try: + while engine.has_unfinished_requests(): + engine.step() + assert engine.cache_manager.num_used_blocks == 0 + finally: + engine.close() + return torch.stack(short_logits), packed_layers + + baseline, baseline_packed = run(enable=False) + actual, packed = run(enable=True) + assert not baseline_packed + assert {layer for layer, _ in packed} == set(range(model.config.num_kv_layers)) + assert {window for _, window in packed} == {False, True} + assert torch.equal(baseline, actual) + + +def test_dory_packed_prefill_keeps_small_value_residual(): + if torch.cuda.get_device_capability() != (12, 0) or torch.version.hip: + pytest.skip("Dory packed prefill requires NVIDIA SM 12.0") + from vllm_rlt.kernels.triton_attention import paged_attention, paged_prefill_attention + + q = torch.zeros((1, 1, 256), device=DEVICE, dtype=torch.bfloat16) + kv = torch.zeros((1, 16, 1, 256), device=DEVICE, dtype=torch.bfloat16) + k, v = kv.clone(), kv.clone() + q[0, 0, 0], k[0, 1, 0, 0] = 16, -0.5 + # Rounding exp(-0.5) to BF16 before multiplying would zero the numerator. + v[0, 0, 0, 0], v[0, 1, 0, 0] = -0.60546875, 1 + table = torch.tensor([[0]], device=DEVICE, dtype=torch.int32) + length = torch.tensor([2], device=DEVICE, dtype=torch.int32) + offset = torch.tensor([0, 1], device=DEVICE, dtype=torch.int32) + expected = paged_attention(q, k, v, table, length) + actual = paged_prefill_attention(q, k, v, table, length, offset, 1, compensate=True) + assert expected[0, 0, 0] > 0.0006 + torch.testing.assert_close(actual, expected, atol=5e-6, rtol=0) diff --git a/vllm_rlt/attention/backend.py b/vllm_rlt/attention/backend.py index 7891b917..63115223 100644 --- a/vllm_rlt/attention/backend.py +++ b/vllm_rlt/attention/backend.py @@ -25,6 +25,7 @@ class BackendCapabilities: device_context_lengths: bool = False # Mask keys outside a per-layer sliding window (not storage reclamation). sliding_window: bool = False + packed_sliding_window: bool = False @dataclass @@ -124,7 +125,18 @@ def create_backend( for candidate in candidates: try: implementation = _create_explicit(candidate, device, dtype, head_dim, block_size) - if candidate == "triton" and required_capabilities.packed_prefill: + if candidate == "triton" and required_capabilities.packed_sliding_window: + if ( + dtype != torch.bfloat16 + or torch.version.hip + or torch.cuda.get_device_capability(device) != (12, 0) + ): + raise ValueError("packed_sliding_window requires NVIDIA SM 12.0 and BF16") + implementation.kernel = TritonPagedAttention(packed_sliding_window=True) + implementation.capabilities = replace( + implementation.capabilities, packed_prefill=True, packed_sliding_window=True + ) + elif candidate == "triton" and required_capabilities.packed_prefill: # Packed BF16 prefill has different arithmetic from per-row decode. # Opt in for prefill_uva without changing existing split/mixed or # Huginn boundary attention when that option is disabled. @@ -139,6 +151,7 @@ def create_backend( "async_scheduling", "device_context_lengths", "sliding_window", + "packed_sliding_window", ) if getattr(required_capabilities, name) and not getattr(implementation.capabilities, name) diff --git a/vllm_rlt/config.py b/vllm_rlt/config.py index 648781d7..c553a4b6 100644 --- a/vllm_rlt/config.py +++ b/vllm_rlt/config.py @@ -151,10 +151,12 @@ class ExecutionConfig: # Select packed prefill metadata staging (and require packed attention). # The historical name is retained; use_uva controls its transfer mechanism. prefill_uva: bool = False + dory_packed_prefill: bool = False def __post_init__(self): for name in ( "prefill_uva", + "dory_packed_prefill", "use_uva", "async_scheduling", "multi_stream", diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 0d1e8641..ec59c9cc 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -738,8 +738,17 @@ def _attend_prepared( self._require_live_batch(batch) self._validate_tensor(q, len(batch.position_ids), "q", query=True) if window is not None: - return self._attend_windowed(layer, batch, q, window) + if batch.cu_seqlens_q is None: + return self._attend_windowed(layer, batch, q, window) + if not self.attention_capabilities.packed_sliding_window: + raise NotImplementedError("sliding-window attention needs one query row per token") + if type(window) is not int or window <= 0: + raise ValueError("window must be a positive integer") if isinstance(batch, DeviceKVMetadata): + if window is not None: + raise NotImplementedError( + "packed sliding-window attention requires host-planned KV" + ) return self.attention( q, self.key_cache[:, layer], @@ -760,6 +769,7 @@ def _attend_prepared( batch.context_lengths, batch.cu_seqlens_q, batch.max_seqlen_q, + **({} if window is None else {"window": window}), ) return self.attention( q, diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index e16f2e07..d2d62b60 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -17,6 +17,7 @@ from vllm_rlt.engine.connector import StageConnector from vllm_rlt.engine.preemption import PreemptionManager from vllm_rlt.engine.stage_config import StageConfig +from vllm_rlt.models.dory import DoryForCausalLM from vllm_rlt.profiling import Profiler from vllm_rlt.request import FinishReason, Request, RequestOutput, RequestPhase from vllm_rlt.sampling_params import SamplingParams @@ -74,6 +75,22 @@ def __init__( staged_prefill = requires_staged_prefill( scheduler_config, produces_tokens=self.stage_config.produces_tokens ) + if self.execution_config.dory_packed_prefill: + if not isinstance(model, DoryForCausalLM): + raise ValueError("Dory packed prefill requires a Dory model") + if ( + self.execution_config.async_scheduling + or self.execution_config.cuda_graphs + or staged_prefill + or scheduler_config.enable_preemption + or speculative_config is not None + or self.stage_config.role != "prefill_decode" + or cache_config.layout != "last_exited" + ): + raise ValueError( + "Dory packed prefill requires synchronous eager single-stage execution " + "with per_loop KV and ordinary prefill, without speculation or preemption" + ) if getattr(model, "requires_boundary_kv", False): if staged_prefill: raise ValueError( @@ -187,7 +204,9 @@ def __init__( config.head_dim, cache_config.block_size, required_capabilities=BackendCapabilities( - packed_prefill=self.execution_config.prefill_uva, + packed_prefill=self.execution_config.prefill_uva + or self.execution_config.dory_packed_prefill, + packed_sliding_window=self.execution_config.dory_packed_prefill, cuda_graphs=self.execution_config.cuda_graphs, async_scheduling=( parameter.device.type == "cuda" and self.execution_config.async_scheduling diff --git a/vllm_rlt/entrypoints/runtime_args.py b/vllm_rlt/entrypoints/runtime_args.py index 883d884b..f85d2aa0 100644 --- a/vllm_rlt/entrypoints/runtime_args.py +++ b/vllm_rlt/entrypoints/runtime_args.py @@ -29,6 +29,11 @@ def add_runtime_args(parser): action="store_true", help="Use packed prefill metadata staging (requires packed attention)", ) + parser.add_argument( + "--dory-packed-prefill", + action="store_true", + help="Enable packed Triton prefill for Dory BF16 on NVIDIA SM 12.0", + ) parser.add_argument( "--no-uva", dest="use_uva", @@ -119,6 +124,7 @@ def runtime_configs(args): ), execution_config=ExecutionConfig( prefill_uva=args.prefill_uva, + dory_packed_prefill=args.dory_packed_prefill, use_uva=args.use_uva, cuda_graphs=args.cuda_graphs, cuda_graph_max_batch_size=args.cuda_graph_max_batch_size, diff --git a/vllm_rlt/kernels/paged_attention.py b/vllm_rlt/kernels/paged_attention.py index dda4b2ef..c4c669c4 100644 --- a/vllm_rlt/kernels/paged_attention.py +++ b/vllm_rlt/kernels/paged_attention.py @@ -103,19 +103,39 @@ def triton_paged_attention( def triton_paged_prefill( - q, key_cache, value_cache, block_tables, context_lengths, cu_seqlens_q, max_seqlen_q + q, + key_cache, + value_cache, + block_tables, + context_lengths, + cu_seqlens_q, + max_seqlen_q, + *, + window=0, + compensate=False, ): """Packed causal prefill with one block table per sequence (see triton_attention).""" from vllm_rlt.kernels.triton_attention import paged_prefill_attention return paged_prefill_attention( - q, key_cache, value_cache, block_tables, context_lengths, cu_seqlens_q, max_seqlen_q + q, + key_cache, + value_cache, + block_tables, + context_lengths, + cu_seqlens_q, + max_seqlen_q, + window=window, + compensate=compensate, ) class TritonPagedAttention: """Triton single-query decode attention plus packed causal prefill.""" + def __init__(self, *, packed_sliding_window=False): + self.packed_sliding_window = packed_sliding_window + def __call__(self, q, key_cache, value_cache, block_tables, context_lengths): return triton_paged_attention(q, key_cache, value_cache, block_tables, context_lengths) @@ -125,4 +145,4 @@ def windowed(self, q, key_cache, value_cache, block_tables, context_lengths, win ) def prefill(self, *args, **kwargs): - return triton_paged_prefill(*args, **kwargs) + return triton_paged_prefill(*args, **kwargs, compensate=self.packed_sliding_window) diff --git a/vllm_rlt/kernels/triton_attention.py b/vllm_rlt/kernels/triton_attention.py index e8f41288..c3199f57 100644 --- a/vllm_rlt/kernels/triton_attention.py +++ b/vllm_rlt/kernels/triton_attention.py @@ -230,6 +230,8 @@ def _paged_prefill_kernel( BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, PRECISION: tl.constexpr, + WINDOW: tl.constexpr, + COMPENSATE: tl.constexpr, max_q_blocks, ): pid = tl.program_id(0) @@ -261,7 +263,8 @@ def _paged_prefill_kernel( accumulator = tl.zeros((BLOCK_M, BLOCK_D), tl.float32) # Bottom-right causal alignment: the last query row sees every key. end = tl.minimum(kv_len, prefix + (q_block + 1) * BLOCK_M) - for start in range(0, end, BLOCK_N): + first = tl.maximum(0, prefix + q_block * BLOCK_M - WINDOW + 1) if WINDOW else 0 + for start in range(first // BLOCK_N * BLOCK_N, end, BLOCK_N): keys_pos = start + tl.arange(0, BLOCK_N) key_valid = keys_pos < end blocks = tl.load( @@ -279,6 +282,8 @@ def _paged_prefill_kernel( ) scores = tl.dot(query, tl.trans(keys), input_precision=PRECISION) * SCALE visible = key_valid[None, :] & (keys_pos[None, :] <= positions[:, None]) + if WINDOW: + visible = visible & (keys_pos[None, :] >= positions[:, None] - WINDOW + 1) scores = tl.where(visible, scores, -float("inf")) next_maximum = tl.maximum(maximum, tl.max(scores, axis=1)) # Rows with no visible key in this tile keep -inf; avoid inf - inf. @@ -294,9 +299,13 @@ def _paged_prefill_kernel( mask=key_valid[:, None] & dim_valid[None, :], other=0.0, ) - accumulator = accumulator * correction[:, None] + tl.dot( - weights.to(values.dtype), values, input_precision=PRECISION - ) + rounded = weights.to(values.dtype) + partial = tl.dot(rounded, values, input_precision=PRECISION) + if COMPENSATE: + # Recover the leading bits lost when the FP32 weight is cast for MMA. + residual = (weights - rounded.to(tl.float32)).to(values.dtype) + partial = tl.dot(residual, values, partial, input_precision=PRECISION) + accumulator = accumulator * correction[:, None] + partial normalizer = normalizer * correction + tl.sum(weights, axis=1) maximum = next_maximum out = accumulator / tl.maximum(normalizer, 1.0e-30)[:, None] @@ -311,7 +320,16 @@ def _paged_prefill_kernel( def paged_prefill_attention( - q, key_cache, value_cache, block_tables, context_lengths, cu_seqlens_q, max_seqlen_q + q, + key_cache, + value_cache, + block_tables, + context_lengths, + cu_seqlens_q, + max_seqlen_q, + *, + window=0, + compensate=False, ): """Packed causal prefill over paged K/V, bottom-right aligned per sequence. @@ -324,7 +342,10 @@ def paged_prefill_attention( sequences = block_tables.shape[0] if q.shape[0] == 0 or sequences == 0: return output - block_m, block_n = 64, 64 + if type(window) is not int or window < 0: + raise ValueError("window must be a nonnegative integer") + # The compensated head-dim-256 variant must fit SM 12.0's shared memory limit. + block_m, block_n = (32, 64) if compensate else (64, 64) q_blocks = triton.cdiv(int(max_seqlen_q), block_m) for tensor in (key_cache, value_cache): if tensor.stride(3) != 1: @@ -359,8 +380,10 @@ def paged_prefill_attention( BLOCK_N=block_n, # FP32 callers (reference checks) need exact products, not TF32. PRECISION="ieee" if q.dtype == torch.float32 else "tf32", + WINDOW=window, + COMPENSATE=compensate, max_q_blocks=q_blocks, num_warps=4, - num_stages=2, + num_stages=1 if compensate else 2, ) return output diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index 4bdee10b..4373e172 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -619,7 +619,7 @@ def _gather(self, requests, workspace, size): hidden[row].copy_(request.hidden_state) return hidden - def _prepare_prefill_coda(self, ids, positions, requests, kv=None): + def _prepare_prefill_coda(self, ids, positions, requests, kv=None, *, packed=None): if not getattr(self.model, "requires_boundary_kv", False): return None, None rows = [ @@ -637,7 +637,9 @@ def _prepare_prefill_coda(self, ids, positions, requests, kv=None): [ids[row] for row in rows], [[0] * len(rows)], [positions[row] for row in rows], - packed=self.cache_manager.attention_capabilities.packed_prefill, + packed=self.cache_manager.attention_capabilities.packed_prefill + if packed is None + else packed, )[0] return ForwardBatch( Operation.CODA, prelude_coda_kv_metadata=kv_metadata, compute_logits=False @@ -735,7 +737,10 @@ def _prepare_loop_inputs(self, batch, packed_metadata=None): tokens.extend(span.input_tokens) requests.extend([request] * count) outputs.append((len(tokens) - 1, request)) - packed = cache.attention_capabilities.packed_prefill + packed = cache.attention_capabilities.packed_prefill and ( + not self.execution_config.dory_packed_prefill + or all(len(span.request.prompt_token_ids) > 64 for span in batch.spans) + ) size = len(tokens) if packed else self._size(len(tokens)) depth_sets = [[depth + offset for depth in depths] for offset in range(batch.loop_count)] if packed_metadata is not None: @@ -751,7 +756,7 @@ def _prepare_loop_inputs(self, batch, packed_metadata=None): packed=packed, padded_rows=size, ) - coda, coda_rows = self._prepare_prefill_coda(ids, positions, requests) + coda, coda_rows = self._prepare_prefill_coda(ids, positions, requests, packed=packed) forwards = [ PreparedForward( ForwardBatch(Operation.PRELUDE, tokens=tensor, metadata=metadata), From c3e2e7ec1bcfec192ee44012229bc6b55a7d2bcf Mon Sep 17 00:00:00 2001 From: cecilsd <127472276+cecilsd@users.noreply.github.com> Date: Mon, 16 Nov 2026 17:09:05 +0800 Subject: [PATCH 2/6] fix(dory): reject conflicting packed prefill flags --- docs/dory.md | 9 ++++++--- tests/test_tensor_transfer.py | 15 +++++++++++++++ vllm_rlt/entrypoints/runtime_args.py | 5 +++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/docs/dory.md b/docs/dory.md index 8e24f8a6..0cf4978a 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -92,9 +92,12 @@ layer id rather than their physical sublayer index. ## Opt-in packed Triton prefill -Set `ExecutionConfig(dory_packed_prefill=True)` in Python or pass -`--dory-packed-prefill` with the Triton attention backend in the CLI. This -path requires Dory BF16, NVIDIA SM 12.0, `per_loop` KV, ordinary prefill, +In Python, set `attention_backend="triton"` and +`execution_config=ExecutionConfig(dory_packed_prefill=True)`. On the CLI, pass +`--attention-backend triton --dory-packed-prefill`. The `--dory-packed-prefill` +and `--no-packed-prefill` flags are mutually exclusive. + +This path requires Dory BF16, NVIDIA SM 12.0, `per_loop` KV, ordinary prefill, and synchronous eager single-stage execution. Speculation, preemption, CUDA graphs and asynchronous scheduling are outside its scope. RTX 5090 is the only GPU tested. Validate other SM 12.0 GPUs locally. Other GPU diff --git a/tests/test_tensor_transfer.py b/tests/test_tensor_transfer.py index 787dd401..2bb8111b 100644 --- a/tests/test_tensor_transfer.py +++ b/tests/test_tensor_transfer.py @@ -401,3 +401,18 @@ def test_uva_configuration_and_capability(monkeypatch, flags, packed_prefill): assert not use_uva_on("cpu") monkeypatch.setattr(torch.version, "hip", "test") assert not use_uva_on("cuda") + + +@pytest.mark.parametrize( + "flags", + [ + ("--no-packed-prefill", "--dory-packed-prefill"), + ("--dory-packed-prefill", "--no-packed-prefill"), + ], +) +def test_dory_packed_prefill_conflicts_with_explicit_disable(flags): + parser = argparse.ArgumentParser() + add_runtime_args(parser) + with pytest.raises(SystemExit) as error: + parser.parse_args(flags) + assert error.value.code == 2 diff --git a/vllm_rlt/entrypoints/runtime_args.py b/vllm_rlt/entrypoints/runtime_args.py index cd7c4c54..8df18564 100644 --- a/vllm_rlt/entrypoints/runtime_args.py +++ b/vllm_rlt/entrypoints/runtime_args.py @@ -31,13 +31,14 @@ def add_runtime_args(parser): help="Force packed prefill metadata staging (requires packed attention); " "default resolves automatically (on for CUDA with last_exited KV)", ) - parser.add_argument( + packed_prefill_mode = parser.add_mutually_exclusive_group() + packed_prefill_mode.add_argument( "--no-packed-prefill", action="store_false", dest="packed_prefill", help="Keep per-row prefill even when packed attention is available", ) - parser.add_argument( + packed_prefill_mode.add_argument( "--dory-packed-prefill", action="store_true", help="Enable packed Triton prefill for Dory BF16 on NVIDIA SM 12.0", From ffca2ec002f5429688bef77e41f7ec46e4c2bdcf Mon Sep 17 00:00:00 2001 From: cecilsd <127472276+cecilsd@users.noreply.github.com> Date: Tue, 17 Nov 2026 10:12:49 +0800 Subject: [PATCH 3/6] refactor(dory): reuse shared packed prefill option --- docs/dory.md | 8 ++--- tests/test_backend_selection.py | 31 ++++++++++++++++++ tests/test_dory.py | 20 ++++++------ tests/test_dory_gpu.py | 9 ++++-- tests/test_engine.py | 27 +++++++++++----- tests/test_tensor_transfer.py | 6 ++-- vllm_rlt/attention/backend.py | 45 +++++++++++++------------- vllm_rlt/config.py | 4 +-- vllm_rlt/engine/packed_prefill.py | 40 +++++++++++++++++------ vllm_rlt/engine/stage_engine.py | 47 ++++++++++------------------ vllm_rlt/entrypoints/runtime_args.py | 14 +++------ vllm_rlt/worker/model_runner.py | 7 +++-- 12 files changed, 152 insertions(+), 106 deletions(-) diff --git a/docs/dory.md b/docs/dory.md index 0cf4978a..608c531c 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -93,9 +93,9 @@ layer id rather than their physical sublayer index. ## Opt-in packed Triton prefill In Python, set `attention_backend="triton"` and -`execution_config=ExecutionConfig(dory_packed_prefill=True)`. On the CLI, pass -`--attention-backend triton --dory-packed-prefill`. The `--dory-packed-prefill` -and `--no-packed-prefill` flags are mutually exclusive. +`execution_config=ExecutionConfig(packed_prefill=True)`. On the CLI, pass +`--attention-backend triton --packed-prefill`. Dory uses per-query prefill when +the option is omitted or when `--no-packed-prefill` is set. This path requires Dory BF16, NVIDIA SM 12.0, `per_loop` KV, ordinary prefill, and synchronous eager single-stage execution. Speculation, preemption, CUDA @@ -123,7 +123,7 @@ even with per-query attention. These fail fast rather than degrade silently: -- WCPB, mixed or staged prefill; packed prefill metadata (`packed_prefill`); PD. +- WCPB, mixed or staged prefill, and PD. - Prefix caching. - Self-speculation combined with CUDA Graphs, asynchronous scheduling, preemption or `last_loop` KV, and sampled token-pipeline speculation. The diff --git a/tests/test_backend_selection.py b/tests/test_backend_selection.py index 882b7d43..251f8362 100644 --- a/tests/test_backend_selection.py +++ b/tests/test_backend_selection.py @@ -309,6 +309,37 @@ def test_explicit_triton_rejects_unsupported_nvidia_hardware(packages, monkeypat create_backend("triton", torch.device("cuda"), torch.float16, 128, 256) +@pytest.mark.parametrize( + "sm,dtype,hip,expected", + [ + ((12, 0), torch.bfloat16, None, True), + ((12, 1), torch.bfloat16, None, False), + ((8, 9), torch.bfloat16, None, False), + ((12, 0), torch.float32, None, False), + ((12, 0), torch.bfloat16, "test", False), + ], +) +def test_triton_packed_sliding_window_capability(packages, monkeypatch, sm, dtype, hip, expected): + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: sm) + monkeypatch.setattr(torch.version, "hip", hip) + args = (torch.device("cuda"), dtype, 256, 16) + required = BackendCapabilities( + packed_prefill=True, sliding_window=True, packed_sliding_window=True + ) + if not expected: + with pytest.raises( + ValueError, match="packed_sliding_window requires NVIDIA SM 12.0 and BF16" + ): + create_backend("triton", *args, required_capabilities=required) + with pytest.raises(ValueError, match="no compatible attention backend"): + create_backend("auto", *args, required_capabilities=required) + return + backend = create_backend("auto", *args, required_capabilities=required) + assert backend.info["selected_backend"] == "triton" + assert backend.capabilities.packed_prefill and backend.capabilities.packed_sliding_window + assert not create_backend("triton", *args).capabilities.packed_sliding_window + + def test_auto_skips_broken_binary_import(packages, monkeypatch): from vllm_rlt.kernels import flash_attention diff --git a/tests/test_dory.py b/tests/test_dory.py index 9ae89d64..b14a2172 100644 --- a/tests/test_dory.py +++ b/tests/test_dory.py @@ -672,19 +672,21 @@ def test_exit_written_ignores_boundary_layers(model): cache.free("request") -def test_dory_packed_prefill_requires_a_qualified_backend(model): - with pytest.raises(ValueError, match="packed_sliding_window"): - LLMEngine(model, execution_config=ExecutionConfig(dory_packed_prefill=True)) +@pytest.mark.parametrize("packed_prefill", [None, False]) +def test_dory_defaults_to_per_row_prefill(model, packed_prefill): + engine = LLMEngine(model, execution_config=ExecutionConfig(packed_prefill=packed_prefill)) + assert engine.execution_config.packed_prefill is False + assert not engine.cache_manager.attention_capabilities.packed_sliding_window + + +def test_dory_packed_prefill_requires_cuda(model): + with pytest.raises(ValueError, match="packed_prefill requires CUDA"): + LLMEngine(model, execution_config=ExecutionConfig(packed_prefill=True)) def test_dory_packed_prefill_rejects_graphs(model): with pytest.raises(ValueError, match="packed prefill requires synchronous eager"): LLMEngine( model, - execution_config=ExecutionConfig(dory_packed_prefill=True, cuda_graphs=True), + execution_config=ExecutionConfig(packed_prefill=True, cuda_graphs=True), ) - - -def test_generic_packed_prefill_rejects_dory_sliding_windows(model): - with pytest.raises(ValueError, match="packed_prefill does not support sliding-window"): - LLMEngine(model, execution_config=ExecutionConfig(packed_prefill=True)) diff --git a/tests/test_dory_gpu.py b/tests/test_dory_gpu.py index d6cbe7a5..6ddb4dac 100644 --- a/tests/test_dory_gpu.py +++ b/tests/test_dory_gpu.py @@ -341,7 +341,7 @@ def run(*, enable): cache_config=CacheConfig(num_blocks=64), scheduler_config=scheduler, attention_backend="triton", - execution_config=ExecutionConfig(dory_packed_prefill=enable, multi_stream=multi_stream), + execution_config=ExecutionConfig(packed_prefill=enable, multi_stream=multi_stream), ) packed_layers = set() short_logits = [] @@ -365,7 +365,10 @@ def attention(layer, metadata, q, *, window=None): mp.setattr(sampler, "sample", sample) mp.setattr(engine.cache_manager, "_attend_prepared", attention) try: - assert engine.execution_config.packed_prefill is False + assert engine.execution_config.packed_prefill is (enable is True) + assert engine.cache_manager.attention_capabilities.packed_sliding_window is ( + enable is True + ) engine.add_request( "long", (prompt * 19)[:long_tokens], @@ -379,7 +382,7 @@ def attention(layer, metadata, q, *, window=None): engine.close() return torch.stack(short_logits), packed_layers - baseline, baseline_packed = run(enable=False) + baseline, baseline_packed = run(enable=None) actual, packed = run(enable=True) assert not baseline_packed assert {layer for layer, _ in packed} == set(range(model.config.num_kv_layers)) diff --git a/tests/test_engine.py b/tests/test_engine.py index e265904a..bbd15e21 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -423,16 +423,27 @@ def test_packed_prefill_resolves_automatically_when_unset(): common = dict(layout="last_exited", staged_prefill=False) # Unset: on for CUDA, last_exited, ordinary prefill without sliding windows. - assert packed_prefill_eligible(None, device_type="cuda", **common) - assert not packed_prefill_eligible(None, device_type="cpu", **common) - assert not packed_prefill_eligible(None, device_type="cuda", **{**common, "layout": "shared"}) - assert not packed_prefill_eligible(None, device_type="cuda", **common, sliding_window=True) + assert packed_prefill_eligible(ExecutionConfig(), device_type="cuda", **common) + assert not packed_prefill_eligible(ExecutionConfig(), device_type="cpu", **common) assert not packed_prefill_eligible( - None, device_type="cuda", **{**common, "staged_prefill": True} + ExecutionConfig(), device_type="cuda", **{**common, "layout": "shared"} + ) + assert not packed_prefill_eligible( + ExecutionConfig(), device_type="cuda", **common, sliding_window=True + ) + assert not packed_prefill_eligible( + ExecutionConfig(), device_type="cuda", **{**common, "staged_prefill": True} + ) + # Explicit settings override the automatic choice. + assert not packed_prefill_eligible( + ExecutionConfig(packed_prefill=False), device_type="cuda", **common + ) + assert packed_prefill_eligible( + ExecutionConfig(packed_prefill=True), device_type="cuda", **common + ) + assert packed_prefill_eligible( + ExecutionConfig(packed_prefill=True), device_type="cuda", **common, sliding_window=True ) - # Explicit settings win regardless of eligibility. - assert not packed_prefill_eligible(False, device_type="cuda", **common) - assert packed_prefill_eligible(True, device_type="cuda", **common) assert ExecutionConfig().packed_prefill is None with pytest.raises(ValueError, match="packed_prefill"): ExecutionConfig(packed_prefill=1) diff --git a/tests/test_tensor_transfer.py b/tests/test_tensor_transfer.py index 2bb8111b..83900e4f 100644 --- a/tests/test_tensor_transfer.py +++ b/tests/test_tensor_transfer.py @@ -406,11 +406,11 @@ def test_uva_configuration_and_capability(monkeypatch, flags, packed_prefill): @pytest.mark.parametrize( "flags", [ - ("--no-packed-prefill", "--dory-packed-prefill"), - ("--dory-packed-prefill", "--no-packed-prefill"), + ("--no-packed-prefill", "--packed-prefill"), + ("--packed-prefill", "--no-packed-prefill"), ], ) -def test_dory_packed_prefill_conflicts_with_explicit_disable(flags): +def test_packed_prefill_flags_are_mutually_exclusive(flags): parser = argparse.ArgumentParser() add_runtime_args(parser) with pytest.raises(SystemExit) as error: diff --git a/vllm_rlt/attention/backend.py b/vllm_rlt/attention/backend.py index 7fc7e7a3..7013ab1e 100644 --- a/vllm_rlt/attention/backend.py +++ b/vllm_rlt/attention/backend.py @@ -83,17 +83,26 @@ def validate_backend_name(backend: str) -> None: raise ValueError("unknown attention backend") -def _create_explicit(backend, device, dtype, head_dim, block_size): +def _create_explicit(backend, device, dtype, head_dim, block_size, *, packed, packed_window): if backend == "triton": if device.type != "cuda": raise ValueError("the Triton attention backend requires a CUDA or ROCm device") if head_dim > 256: raise ValueError("the Triton attention backend supports head_dim <= 256") - if not torch.version.hip and torch.cuda.get_device_capability(device)[0] < 8: + sm = None if torch.version.hip else torch.cuda.get_device_capability(device) + if sm is not None and sm[0] < 8: raise ValueError("the Triton attention backend requires NVIDIA Compute Capability 8.0+") + if packed_window and (sm != (12, 0) or dtype != torch.bfloat16): + raise ValueError("Triton packed_sliding_window requires NVIDIA SM 12.0 and BF16") # Probe the actual module, including its dependency imports, before selection. importlib.import_module("vllm_rlt.kernels.triton_attention") - return AttentionBackend(TritonPagedAttention(), _TRITON_CAPABILITIES, {"backend": backend}) + capabilities = replace( + _TRITON_CAPABILITIES, + packed_prefill=packed or packed_window, + packed_sliding_window=packed_window, + ) + kernel = TritonPagedAttention(packed_sliding_window=packed_window) + return AttentionBackend(kernel, capabilities, {"backend": backend}) if backend in FLASH_BACKENDS: kernel = FlashPagedAttention(device, dtype, head_dim, block_size, backend) return AttentionBackend(kernel, _FLASH_CAPABILITIES[kernel.generation], dict(kernel.info)) @@ -127,27 +136,15 @@ def create_backend( skipped.append(f"{device.type.upper()} uses Torch reference attention") for candidate in candidates: try: - implementation = _create_explicit(candidate, device, dtype, head_dim, block_size) - if candidate == "triton" and required_capabilities.packed_sliding_window: - if ( - dtype != torch.bfloat16 - or torch.version.hip - or torch.cuda.get_device_capability(device) != (12, 0) - ): - raise ValueError("packed_sliding_window requires NVIDIA SM 12.0 and BF16") - implementation.kernel = TritonPagedAttention(packed_sliding_window=True) - implementation.capabilities = replace( - implementation.capabilities, packed_prefill=True, packed_sliding_window=True - ) - elif candidate == "triton" and ( - required_capabilities.packed_prefill or prefer_packed_prefill - ): - # Packed BF16 prefill has different arithmetic from per-row decode. - # Opt in for packed_prefill without changing existing split/mixed or - # Huginn boundary attention when that option is disabled. - implementation.capabilities = replace( - implementation.capabilities, packed_prefill=True - ) + implementation = _create_explicit( + candidate, + device, + dtype, + head_dim, + block_size, + packed=required_capabilities.packed_prefill or prefer_packed_prefill, + packed_window=required_capabilities.packed_sliding_window, + ) missing = [ name for name in ( diff --git a/vllm_rlt/config.py b/vllm_rlt/config.py index 04f3d3a4..3b03bc30 100644 --- a/vllm_rlt/config.py +++ b/vllm_rlt/config.py @@ -153,15 +153,13 @@ class ExecutionConfig: # None resolves at engine construction: on when the attention backend packs # prefill (Triton opts in, FlashAttention-4 natively) with last_exited KV and # ordinary (non-staged) prefill without sliding-window attention; off otherwise. + # Sliding-window prefill requires explicit opt-in. packed_prefill: bool | None = None - # Dory selects packed sliding-window attention per batch. - dory_packed_prefill: bool = False def __post_init__(self): if self.packed_prefill is not None and type(self.packed_prefill) is not bool: raise ValueError("packed_prefill must be a boolean or None") for name in ( - "dory_packed_prefill", "use_uva", "async_scheduling", "multi_stream", diff --git a/vllm_rlt/engine/packed_prefill.py b/vllm_rlt/engine/packed_prefill.py index 49c49d14..03aa2f16 100644 --- a/vllm_rlt/engine/packed_prefill.py +++ b/vllm_rlt/engine/packed_prefill.py @@ -2,19 +2,39 @@ def packed_prefill_eligible( - packed_prefill, *, device_type, layout, staged_prefill, sliding_window=False + execution, + *, + device_type, + layout, + staged_prefill, + sliding_window=False, + stage_role="prefill_decode", + speculative=False, + preemption=False, ): - """Validate explicit requests and return whether packing may be enabled. + """Validate execution constraints before filtering by backend capability. - Staged prefill and sliding-window attention retain their existing metadata. - Backend support is checked separately by attention selection. + Sliding-window packing is opt-in. Its current implementation requires host-planned + metadata and ordinary synchronous prefill in a single stage. """ - if packed_prefill and sliding_window: - raise ValueError("packed_prefill does not support sliding-window attention") - if packed_prefill and (device_type != "cuda" or layout != "last_exited"): - raise ValueError("packed_prefill requires CUDA packed prefill with last_exited KV") - if packed_prefill is not None: - return bool(packed_prefill) + requested = execution.packed_prefill + if requested: + if sliding_window and ( + execution.async_scheduling + or execution.cuda_graphs + or staged_prefill + or speculative + or preemption + or stage_role != "prefill_decode" + ): + raise ValueError( + "sliding-window packed prefill requires synchronous eager single-stage execution " + "with ordinary prefill, without speculation or preemption" + ) + if device_type != "cuda" or layout != "last_exited": + raise ValueError("packed_prefill requires CUDA packed prefill with last_exited KV") + if requested is not None: + return requested return ( device_type == "cuda" and layout == "last_exited" diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index 91b9ee86..8b5b349a 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -18,7 +18,6 @@ from vllm_rlt.engine.packed_prefill import packed_prefill_eligible from vllm_rlt.engine.preemption import PreemptionManager from vllm_rlt.engine.stage_config import StageConfig -from vllm_rlt.models.dory import DoryForCausalLM from vllm_rlt.profiling import Profiler from vllm_rlt.request import FinishReason, Request, RequestOutput, RequestPhase from vllm_rlt.sampling_params import SamplingParams @@ -76,23 +75,17 @@ def __init__( staged_prefill = requires_staged_prefill( scheduler_config, produces_tokens=self.stage_config.produces_tokens ) - dory_packed = self.execution_config.dory_packed_prefill - if dory_packed: - if not isinstance(model, DoryForCausalLM): - raise ValueError("Dory packed prefill requires a Dory model") - if ( - self.execution_config.async_scheduling - or self.execution_config.cuda_graphs - or staged_prefill - or scheduler_config.enable_preemption - or speculative_config is not None - or self.stage_config.role != "prefill_decode" - or cache_config.layout != "last_exited" - ): - raise ValueError( - "Dory packed prefill requires synchronous eager single-stage execution " - "with per_loop KV and ordinary prefill, without speculation or preemption" - ) + sliding_window = getattr(model, "requires_sliding_window", False) + prefer_packed = packed_prefill_eligible( + self.execution_config, + device_type=parameter.device.type, + layout=cache_config.layout, + staged_prefill=staged_prefill, + sliding_window=sliding_window, + stage_role=self.stage_config.role, + speculative=speculative_config is not None, + preemption=scheduler_config.enable_preemption, + ) if getattr(model, "requires_boundary_kv", False): if staged_prefill: raise ValueError( @@ -192,13 +185,7 @@ def __init__( ) if staged_prefill and cache_config.layout != "last_exited": raise ValueError("staged prefill requires last_exited KV planes") - auto_packed = packed_prefill_eligible( - self.execution_config.packed_prefill, - device_type=parameter.device.type, - layout=cache_config.layout, - staged_prefill=staged_prefill, - sliding_window=getattr(model, "requires_sliding_window", False), - ) + require_packed = self.execution_config.packed_prefill is True attention = create_backend( attention_backend, parameter.device, @@ -206,21 +193,21 @@ def __init__( config.head_dim, cache_config.block_size, required_capabilities=BackendCapabilities( - packed_prefill=bool(self.execution_config.packed_prefill) or dory_packed, - packed_sliding_window=dory_packed, + packed_prefill=require_packed, + packed_sliding_window=require_packed and sliding_window, cuda_graphs=self.execution_config.cuda_graphs, async_scheduling=( parameter.device.type == "cuda" and self.execution_config.async_scheduling ), device_context_lengths=self.async_speculative, - sliding_window=getattr(model, "requires_sliding_window", False), + sliding_window=sliding_window, ), - prefer_packed_prefill=auto_packed, + prefer_packed_prefill=prefer_packed, ) if self.execution_config.packed_prefill is None: self.execution_config = replace( self.execution_config, - packed_prefill=auto_packed and attention.capabilities.packed_prefill, + packed_prefill=prefer_packed and attention.capabilities.packed_prefill, ) num_blocks, self.memory_plan = plan_cache( model, diff --git a/vllm_rlt/entrypoints/runtime_args.py b/vllm_rlt/entrypoints/runtime_args.py index 8df18564..d4a0a976 100644 --- a/vllm_rlt/entrypoints/runtime_args.py +++ b/vllm_rlt/entrypoints/runtime_args.py @@ -24,25 +24,20 @@ def add_runtime_args(parser): parser.add_argument("--kv-watermark", type=float, default=0.0) parser.add_argument("--scheduling-policy", choices=["fcfs", "priority"], default="fcfs") parser.add_argument("--enable-preemption", action="store_true") - parser.add_argument( + packed_prefill_mode = parser.add_mutually_exclusive_group() + packed_prefill_mode.add_argument( "--packed-prefill", action="store_true", default=None, - help="Force packed prefill metadata staging (requires packed attention); " - "default resolves automatically (on for CUDA with last_exited KV)", + help="Enable packed prefill on a compatible backend. " + "Defaults to automatic selection. Sliding-window attention requires explicit opt-in.", ) - packed_prefill_mode = parser.add_mutually_exclusive_group() packed_prefill_mode.add_argument( "--no-packed-prefill", action="store_false", dest="packed_prefill", help="Keep per-row prefill even when packed attention is available", ) - packed_prefill_mode.add_argument( - "--dory-packed-prefill", - action="store_true", - help="Enable packed Triton prefill for Dory BF16 on NVIDIA SM 12.0", - ) parser.add_argument( "--no-uva", dest="use_uva", @@ -133,7 +128,6 @@ def runtime_configs(args): ), execution_config=ExecutionConfig( packed_prefill=args.packed_prefill, - dory_packed_prefill=args.dory_packed_prefill, use_uva=args.use_uva, cuda_graphs=args.cuda_graphs, cuda_graph_max_batch_size=args.cuda_graph_max_batch_size, diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index c048e1c5..3766c5b2 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -718,11 +718,14 @@ def _prepare_shared_loop_inputs(self, batch): return tuple(inputs) def _uses_packed_loop_inputs(self, batch): + capabilities = self.cache_manager.attention_capabilities return ( batch.operation == Operation.PRELUDE and batch.loop_count and self.cache_manager.layout != "shared" - and self.cache_manager.attention_capabilities.packed_prefill + and capabilities.packed_prefill + # Sliding-window batches share staging for packed and per-row metadata. + and not capabilities.packed_sliding_window and self.execution_config.packed_prefill ) @@ -740,7 +743,7 @@ def _prepare_loop_inputs(self, batch, packed_metadata=None): requests.extend([request] * count) outputs.append((len(tokens) - 1, request)) packed = cache.attention_capabilities.packed_prefill and ( - not self.execution_config.dory_packed_prefill + not cache.attention_capabilities.packed_sliding_window or all(len(span.request.prompt_token_ids) > 64 for span in batch.spans) ) size = len(tokens) if packed else self._size(len(tokens)) From a5ea2948929fdc9ec1c3c69045a0fd7380f679d2 Mon Sep 17 00:00:00 2001 From: cecilsd <127472276+cecilsd@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:40:14 +0800 Subject: [PATCH 4/6] perf(dory): use a fixed 32x32 packed prefill tile --- docs/dory.md | 17 ++-- tests/test_backend_selection.py | 77 ++++++++++++------- tests/test_dory_gpu.py | 111 +++++++++++++++++++++++---- vllm_rlt/attention/backend.py | 44 ++++++++--- vllm_rlt/engine/stage_engine.py | 28 +++++-- vllm_rlt/kernels/paged_attention.py | 22 +++++- vllm_rlt/kernels/triton_attention.py | 96 +++++++++++++++++++++-- 7 files changed, 323 insertions(+), 72 deletions(-) diff --git a/docs/dory.md b/docs/dory.md index 608c531c..3defaf94 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -97,12 +97,17 @@ In Python, set `attention_backend="triton"` and `--attention-backend triton --packed-prefill`. Dory uses per-query prefill when the option is omitted or when `--no-packed-prefill` is set. -This path requires Dory BF16, NVIDIA SM 12.0, `per_loop` KV, ordinary prefill, -and synchronous eager single-stage execution. Speculation, preemption, CUDA -graphs and asynchronous scheduling are outside its scope. RTX 5090 is the -only GPU tested. Validate other SM 12.0 GPUs locally. Other GPU -architectures require a separate kernel configuration and validation -before the option can be enabled. +This path uses BF16, `per_loop` KV, ordinary prefill, and synchronous eager +single-stage execution. Speculation, preemption, CUDA graphs and asynchronous +scheduling are outside its scope. + +On NVIDIA CUDA, the packed path uses a 32 × 32 tile. At startup, Triton compiles +the full and sliding-window kernels and checks their shared-memory requirements +against the device limit. The engine uses per-query attention if the check +fails. The result is cached by device and attention geometry within the process. +The check runs before KV memory planning and uses tensor metadata only. +The full BF16 checkpoint and long-context service validation require enough +device memory for the model weights, runtime buffers and KV pool. A batch uses packed attention across PRELUDE, recurrence and the CODA writes for prompt history when every prompt is longer than 64 tokens. diff --git a/tests/test_backend_selection.py b/tests/test_backend_selection.py index 251f8362..8bd134d6 100644 --- a/tests/test_backend_selection.py +++ b/tests/test_backend_selection.py @@ -24,7 +24,7 @@ def import_module(name): if name == "vllm_rlt.kernels.triton_attention": if "triton" not in installed: raise ImportError("triton is not installed") - return SimpleNamespace() + return SimpleNamespace(select_packed_window_tile=lambda *args: None) if name in ("flash_attn", "flash_attn_3.flash_attn_interface", "flash_attn.cute"): if name not in installed: raise ImportError(f"{name} is not installed") @@ -309,35 +309,60 @@ def test_explicit_triton_rejects_unsupported_nvidia_hardware(packages, monkeypat create_backend("triton", torch.device("cuda"), torch.float16, 128, 256) -@pytest.mark.parametrize( - "sm,dtype,hip,expected", - [ - ((12, 0), torch.bfloat16, None, True), - ((12, 1), torch.bfloat16, None, False), - ((8, 9), torch.bfloat16, None, False), - ((12, 0), torch.float32, None, False), - ((12, 0), torch.bfloat16, "test", False), - ], -) -def test_triton_packed_sliding_window_capability(packages, monkeypatch, sm, dtype, hip, expected): - monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: sm) - monkeypatch.setattr(torch.version, "hip", hip) - args = (torch.device("cuda"), dtype, 256, 16) - required = BackendCapabilities( - packed_prefill=True, sliding_window=True, packed_sliding_window=True +def test_triton_advertises_packed_window_only_after_selecting_a_tile(packages, monkeypatch): + from vllm_rlt.attention import backend as selection + from vllm_rlt.kernels.paged_attention import PackedWindowShape + + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (8, 9)) + base_import = selection.importlib.import_module + shape = PackedWindowShape(16, 4, 36, 1024) + calls = [] + + def probe(device, dtype, head_dim, block_size, requested_shape): + calls.append((device, dtype, head_dim, block_size, requested_shape)) + return (32, 32) + + def import_module(name): + if name == "vllm_rlt.kernels.triton_attention": + return SimpleNamespace(select_packed_window_tile=probe) + return base_import(name) + + monkeypatch.setattr(selection.importlib, "import_module", import_module) + args = (torch.device("cuda"), torch.bfloat16, 256, 16) + backend = create_backend( + "auto", + *args, + required_capabilities=BackendCapabilities(sliding_window=True), + packed_window_shape=shape, ) - if not expected: - with pytest.raises( - ValueError, match="packed_sliding_window requires NVIDIA SM 12.0 and BF16" - ): - create_backend("triton", *args, required_capabilities=required) - with pytest.raises(ValueError, match="no compatible attention backend"): - create_backend("auto", *args, required_capabilities=required) - return - backend = create_backend("auto", *args, required_capabilities=required) assert backend.info["selected_backend"] == "triton" + assert backend.info["packed_prefill_tile"] == (32, 32) assert backend.capabilities.packed_prefill and backend.capabilities.packed_sliding_window assert not create_backend("triton", *args).capabilities.packed_sliding_window + assert calls == [(*args, shape)] + + +@pytest.mark.parametrize("backend", ["triton", "auto"]) +def test_packed_window_falls_back_to_per_query_when_probe_has_no_tile( + packages, monkeypatch, backend +): + from vllm_rlt.kernels.paged_attention import PackedWindowShape + + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (8, 9)) + shape = PackedWindowShape(16, 4, 36, 1024) + selected = create_backend( + backend, + torch.device("cuda"), + torch.bfloat16, + 256, + 16, + required_capabilities=BackendCapabilities(sliding_window=True), + packed_window_shape=shape, + ) + assert selected.info["selected_backend"] == "triton" + assert selected.info["packed_prefill_tile"] is None + assert not selected.capabilities.packed_prefill + assert not selected.capabilities.packed_sliding_window def test_auto_skips_broken_binary_import(packages, monkeypatch): diff --git a/tests/test_dory_gpu.py b/tests/test_dory_gpu.py index 6ddb4dac..4f4a5558 100644 --- a/tests/test_dory_gpu.py +++ b/tests/test_dory_gpu.py @@ -290,12 +290,27 @@ def failing(batch): assert engine.cache_manager.num_used_blocks == 0 +def _packed_tile(query_heads, kv_heads, kv_layers=1): + from vllm_rlt.kernels.paged_attention import PackedWindowShape + from vllm_rlt.kernels.triton_attention import select_packed_window_tile + + tile = select_packed_window_tile( + torch.device(DEVICE), + torch.bfloat16, + 256, + 16, + PackedWindowShape(query_heads, kv_heads, kv_layers, 32), + ) + if tile is None: + pytest.skip("No supported packed prefill tile for this device") + return tile + + @pytest.mark.parametrize("window", [0, 32]) def test_dory_packed_attention_matches_per_row_reference(window): - if torch.cuda.get_device_capability() != (12, 0) or torch.version.hip: - pytest.skip("Dory packed prefill requires NVIDIA SM 12.0") from vllm_rlt.kernels.triton_attention import paged_attention, paged_prefill_attention + tile = _packed_tile(16, 4) # The first chunk is far enough into its history to skip a full SWA tile. # The two sequences have distinct pages and different causal offsets. torch.manual_seed(123) @@ -313,7 +328,7 @@ def test_dory_packed_attention_matches_per_row_reference(window): row_lengths = torch.tensor([*range(98, 133), *range(9, 16)], device=DEVICE, dtype=torch.int32) expected = paged_attention(q, k, v, row_tables, row_lengths, window=window) actual = paged_prefill_attention( - q, k, v, tables, lengths, offsets, 35, window=window, compensate=True + q, k, v, tables, lengths, offsets, 35, window=window, compensate=True, tile=tile ) torch.testing.assert_close(actual, expected, atol=2e-3, rtol=1e-2) @@ -323,11 +338,16 @@ def test_dory_packed_attention_matches_per_row_reference(window): def test_dory_packed_prefill_preserves_short_prompt_logits( monkeypatch, multi_stream, long_tokens, max_num_seqs ): - if torch.cuda.get_device_capability() != (12, 0) or torch.version.hip: - pytest.skip("Dory packed prefill requires NVIDIA SM 12.0") + from vllm_rlt.kernels import triton_attention + model = make_gpu_model(head_dim=256, max_position_embeddings=256, sliding_window=32).to( dtype=torch.bfloat16 ) + tile = _packed_tile( + model.config.num_attention_heads, + model.config.num_key_value_heads, + model.config.num_kv_layers, + ) prompt = [4, 7, 9, 3, 12, 5, 23] scheduler = SchedulerConfig( max_num_seqs=max_num_seqs, @@ -335,7 +355,7 @@ def test_dory_packed_prefill_preserves_short_prompt_logits( prefill_chunk_size=64, ) - def run(*, enable): + def run(*, enable, expect_packed): engine = LLMEngine( model, cache_config=CacheConfig(num_blocks=64), @@ -365,9 +385,13 @@ def attention(layer, metadata, q, *, window=None): mp.setattr(sampler, "sample", sample) mp.setattr(engine.cache_manager, "_attend_prepared", attention) try: - assert engine.execution_config.packed_prefill is (enable is True) - assert engine.cache_manager.attention_capabilities.packed_sliding_window is ( - enable is True + assert engine.execution_config.packed_prefill is expect_packed + assert ( + engine.cache_manager.attention_capabilities.packed_sliding_window + is expect_packed + ) + assert engine.cache_manager.attention_info["packed_prefill_tile"] == ( + tile if expect_packed else None ) engine.add_request( "long", @@ -382,19 +406,78 @@ def attention(layer, metadata, q, *, window=None): engine.close() return torch.stack(short_logits), packed_layers - baseline, baseline_packed = run(enable=None) - actual, packed = run(enable=True) + baseline, baseline_packed = run(enable=None, expect_packed=False) + actual, packed = run(enable=True, expect_packed=True) + with monkeypatch.context() as patch: + patch.setattr(triton_attention, "select_packed_window_tile", lambda *args: None) + fallback, fallback_packed = run(enable=True, expect_packed=False) + assert not fallback_packed + assert torch.equal(baseline, fallback) assert not baseline_packed assert {layer for layer, _ in packed} == set(range(model.config.num_kv_layers)) assert {window for _, window in packed} == {False, True} assert torch.equal(baseline, actual) +def test_prefill_tile_probe_rejects_oversized_window(monkeypatch): + from types import SimpleNamespace + + from vllm_rlt.kernels import triton_attention + from vllm_rlt.kernels.paged_attention import PackedWindowShape + + if torch.version.hip: + pytest.skip("NVIDIA shared-memory resource check") + + index = torch.cuda.current_device() + limit = triton_attention.driver.active.utils.get_device_properties(index)["max_shared_mem"] + + def compile_tile(*args, window, **kwargs): + return SimpleNamespace(metadata=SimpleNamespace(shared=limit + bool(window))) + + monkeypatch.setattr(triton_attention, "_launch_paged_prefill", compile_tile) + probe = triton_attention._select_packed_window_tile.__wrapped__ + assert probe(index, 256, 16, PackedWindowShape(16, 4, 36, 1024)) is None + + +def test_prefill_tile_probe_compiles_only_the_default_and_caches_result(monkeypatch): + from types import SimpleNamespace + + from vllm_rlt.kernels import triton_attention as attention + from vllm_rlt.kernels.paged_attention import PackedWindowShape + + if torch.version.hip: + pytest.skip("NVIDIA packed prefill selection") + + device = torch.device(DEVICE) + index = torch.cuda.current_device() + limit = attention.driver.active.utils.get_device_properties(index)["max_shared_mem"] + windows = [] + monkeypatch.setattr(torch.cuda, "get_device_name", lambda device: "Unlisted GPU") + + def compile_tile(*args, window, compile_only, tile, **kwargs): + assert compile_only + assert tile == (32, 32) + assert all(arg.device.type == "meta" for arg in args if isinstance(arg, torch.Tensor)) + windows.append(window) + return SimpleNamespace(metadata=SimpleNamespace(shared=limit)) + + monkeypatch.setattr(attention, "_launch_paged_prefill", compile_tile) + args = (device, torch.bfloat16, 64, 16, PackedWindowShape(4, 2, 2, 128)) + + attention._select_packed_window_tile.cache_clear() + try: + assert attention.select_packed_window_tile(*args) == (32, 32) + assert windows == [0, 128] + assert attention.select_packed_window_tile(*args) == (32, 32) + assert windows == [0, 128] + finally: + attention._select_packed_window_tile.cache_clear() + + def test_dory_packed_prefill_keeps_small_value_residual(): - if torch.cuda.get_device_capability() != (12, 0) or torch.version.hip: - pytest.skip("Dory packed prefill requires NVIDIA SM 12.0") from vllm_rlt.kernels.triton_attention import paged_attention, paged_prefill_attention + tile = _packed_tile(1, 1) q = torch.zeros((1, 1, 256), device=DEVICE, dtype=torch.bfloat16) kv = torch.zeros((1, 16, 1, 256), device=DEVICE, dtype=torch.bfloat16) k, v = kv.clone(), kv.clone() @@ -405,6 +488,6 @@ def test_dory_packed_prefill_keeps_small_value_residual(): length = torch.tensor([2], device=DEVICE, dtype=torch.int32) offset = torch.tensor([0, 1], device=DEVICE, dtype=torch.int32) expected = paged_attention(q, k, v, table, length) - actual = paged_prefill_attention(q, k, v, table, length, offset, 1, compensate=True) + actual = paged_prefill_attention(q, k, v, table, length, offset, 1, compensate=True, tile=tile) assert expected[0, 0, 0] > 0.0006 torch.testing.assert_close(actual, expected, atol=5e-6, rtol=0) diff --git a/vllm_rlt/attention/backend.py b/vllm_rlt/attention/backend.py index 7013ab1e..493f11e9 100644 --- a/vllm_rlt/attention/backend.py +++ b/vllm_rlt/attention/backend.py @@ -8,7 +8,11 @@ import torch from vllm_rlt.kernels.flash_attention import FLASH_BACKENDS, FlashPagedAttention -from vllm_rlt.kernels.paged_attention import TorchPagedAttention, TritonPagedAttention +from vllm_rlt.kernels.paged_attention import ( + PackedWindowShape, + TorchPagedAttention, + TritonPagedAttention, +) @dataclass(frozen=True) @@ -83,7 +87,7 @@ def validate_backend_name(backend: str) -> None: raise ValueError("unknown attention backend") -def _create_explicit(backend, device, dtype, head_dim, block_size, *, packed, packed_window): +def _create_explicit(backend, device, dtype, head_dim, block_size, *, packed, packed_window_shape): if backend == "triton": if device.type != "cuda": raise ValueError("the Triton attention backend requires a CUDA or ROCm device") @@ -92,17 +96,25 @@ def _create_explicit(backend, device, dtype, head_dim, block_size, *, packed, pa sm = None if torch.version.hip else torch.cuda.get_device_capability(device) if sm is not None and sm[0] < 8: raise ValueError("the Triton attention backend requires NVIDIA Compute Capability 8.0+") - if packed_window and (sm != (12, 0) or dtype != torch.bfloat16): - raise ValueError("Triton packed_sliding_window requires NVIDIA SM 12.0 and BF16") - # Probe the actual module, including its dependency imports, before selection. - importlib.import_module("vllm_rlt.kernels.triton_attention") + # Import the implementations before selection and probe opted-in window tiles + # before the engine profiles and allocates its KV pool. + implementation = importlib.import_module("vllm_rlt.kernels.triton_attention") + tile = ( + implementation.select_packed_window_tile( + device, dtype, head_dim, block_size, packed_window_shape + ) + if packed_window_shape is not None + else None + ) capabilities = replace( _TRITON_CAPABILITIES, - packed_prefill=packed or packed_window, - packed_sliding_window=packed_window, + packed_prefill=packed or tile is not None, + packed_sliding_window=tile is not None, + ) + kernel = TritonPagedAttention(packed_window_tile=tile) + return AttentionBackend( + kernel, capabilities, {"backend": backend, "packed_prefill_tile": tile} ) - kernel = TritonPagedAttention(packed_sliding_window=packed_window) - return AttentionBackend(kernel, capabilities, {"backend": backend}) if backend in FLASH_BACKENDS: kernel = FlashPagedAttention(device, dtype, head_dim, block_size, backend) return AttentionBackend(kernel, _FLASH_CAPABILITIES[kernel.generation], dict(kernel.info)) @@ -118,6 +130,7 @@ def create_backend( *, required_capabilities: BackendCapabilities = BackendCapabilities(), prefer_packed_prefill: bool = False, + packed_window_shape: PackedWindowShape | None = None, ) -> AttentionBackend: """Explicit choices fail without fallback; auto tries FA4/3/2, Triton, then Torch. @@ -127,6 +140,7 @@ def create_backend( filter candidates before memory profiling or device-buffer allocation. Preferred packed prefill enables it where available without filtering or reordering candidates; required packed prefill still rejects unsupported ones. + A packed window shape opts into tile selection and keeps row attention available. """ validate_backend_name(backend) candidates = _AUTO_PRIORITY if backend == "auto" else (backend,) @@ -143,7 +157,7 @@ def create_backend( head_dim, block_size, packed=required_capabilities.packed_prefill or prefer_packed_prefill, - packed_window=required_capabilities.packed_sliding_window, + packed_window_shape=packed_window_shape, ) missing = [ name @@ -178,6 +192,14 @@ def create_backend( implementation.info.update( requested_backend=backend, selected_backend=selected, selection_reason=reason ) + if ( + packed_window_shape is not None + and not implementation.capabilities.packed_sliding_window + ): + implementation.info.update( + packed_prefill_tile=None, + packed_prefill_fallback="no supported packed sliding-window tile", + ) logging.getLogger(__name__).info("Attention selection: %s", implementation.info) return implementation raise ValueError("no compatible attention backend; " + "; ".join(skipped)) diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index 8b5b349a..8ab18c76 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -18,6 +18,7 @@ from vllm_rlt.engine.packed_prefill import packed_prefill_eligible from vllm_rlt.engine.preemption import PreemptionManager from vllm_rlt.engine.stage_config import StageConfig +from vllm_rlt.kernels.paged_attention import PackedWindowShape from vllm_rlt.profiling import Profiler from vllm_rlt.request import FinishReason, Request, RequestOutput, RequestPhase from vllm_rlt.sampling_params import SamplingParams @@ -186,6 +187,15 @@ def __init__( if staged_prefill and cache_config.layout != "last_exited": raise ValueError("staged prefill requires last_exited KV planes") require_packed = self.execution_config.packed_prefill is True + kv_layers = getattr(config, "num_kv_layers", config.num_hidden_layers) + window = getattr(config, "sliding_window", None) + packed_window_shape = ( + PackedWindowShape( + config.num_attention_heads, config.num_key_value_heads, kv_layers, window + ) + if require_packed and sliding_window and window + else None + ) attention = create_backend( attention_backend, parameter.device, @@ -193,8 +203,7 @@ def __init__( config.head_dim, cache_config.block_size, required_capabilities=BackendCapabilities( - packed_prefill=require_packed, - packed_sliding_window=require_packed and sliding_window, + packed_prefill=require_packed and not sliding_window, cuda_graphs=self.execution_config.cuda_graphs, async_scheduling=( parameter.device.type == "cuda" and self.execution_config.async_scheduling @@ -202,12 +211,17 @@ def __init__( device_context_lengths=self.async_speculative, sliding_window=sliding_window, ), - prefer_packed_prefill=prefer_packed, + prefer_packed_prefill=prefer_packed and not sliding_window, + packed_window_shape=packed_window_shape, ) - if self.execution_config.packed_prefill is None: + if sliding_window or self.execution_config.packed_prefill is None: + supported = ( + attention.capabilities.packed_sliding_window + if sliding_window + else attention.capabilities.packed_prefill + ) self.execution_config = replace( - self.execution_config, - packed_prefill=prefer_packed and attention.capabilities.packed_prefill, + self.execution_config, packed_prefill=prefer_packed and supported ) num_blocks, self.memory_plan = plan_cache( model, @@ -219,7 +233,7 @@ def __init__( staged_prefill=staged_prefill, ) self.cache_manager = KVCacheManager( - num_layers=getattr(config, "num_kv_layers", config.num_hidden_layers), + num_layers=kv_layers, num_kv_heads=config.num_key_value_heads, head_dim=config.head_dim, max_loops=config.total_ut_steps, diff --git a/vllm_rlt/kernels/paged_attention.py b/vllm_rlt/kernels/paged_attention.py index c4c669c4..4de2af3f 100644 --- a/vllm_rlt/kernels/paged_attention.py +++ b/vllm_rlt/kernels/paged_attention.py @@ -5,6 +5,7 @@ """ import math +from dataclasses import dataclass import torch @@ -102,6 +103,14 @@ def triton_paged_attention( return paged_attention(q, key_cache, value_cache, block_tables, context_lengths, window=window) +@dataclass(frozen=True) +class PackedWindowShape: + query_heads: int + kv_heads: int + kv_layers: int + window: int + + def triton_paged_prefill( q, key_cache, @@ -113,6 +122,7 @@ def triton_paged_prefill( *, window=0, compensate=False, + tile=None, ): """Packed causal prefill with one block table per sequence (see triton_attention).""" from vllm_rlt.kernels.triton_attention import paged_prefill_attention @@ -127,14 +137,15 @@ def triton_paged_prefill( max_seqlen_q, window=window, compensate=compensate, + tile=tile, ) class TritonPagedAttention: """Triton single-query decode attention plus packed causal prefill.""" - def __init__(self, *, packed_sliding_window=False): - self.packed_sliding_window = packed_sliding_window + def __init__(self, *, packed_window_tile=None): + self.packed_window_tile = packed_window_tile def __call__(self, q, key_cache, value_cache, block_tables, context_lengths): return triton_paged_attention(q, key_cache, value_cache, block_tables, context_lengths) @@ -145,4 +156,9 @@ def windowed(self, q, key_cache, value_cache, block_tables, context_lengths, win ) def prefill(self, *args, **kwargs): - return triton_paged_prefill(*args, **kwargs, compensate=self.packed_sliding_window) + return triton_paged_prefill( + *args, + **kwargs, + compensate=self.packed_window_tile is not None, + tile=self.packed_window_tile, + ) diff --git a/vllm_rlt/kernels/triton_attention.py b/vllm_rlt/kernels/triton_attention.py index c3199f57..fd57a489 100644 --- a/vllm_rlt/kernels/triton_attention.py +++ b/vllm_rlt/kernels/triton_attention.py @@ -8,9 +8,15 @@ row serialized its whole context in one program and set the decode step time. """ +from functools import lru_cache, partial + import torch import triton import triton.language as tl +from triton.errors import TritonError +from triton.runtime import driver + +_PACKED_WINDOW_TILE = (32, 32) # Splits per row for single-query attention. A power of two (tl.arange) and a # constant, so the grid is static under CUDA graphs and independent of batch. @@ -330,6 +336,7 @@ def paged_prefill_attention( *, window=0, compensate=False, + tile=None, ): """Packed causal prefill over paged K/V, bottom-right aligned per sequence. @@ -344,15 +351,51 @@ def paged_prefill_attention( return output if type(window) is not int or window < 0: raise ValueError("window must be a nonnegative integer") - # The compensated head-dim-256 variant must fit SM 12.0's shared memory limit. - block_m, block_n = (32, 64) if compensate else (64, 64) - q_blocks = triton.cdiv(int(max_seqlen_q), block_m) for tensor in (key_cache, value_cache): if tensor.stride(3) != 1: raise ValueError("paged prefill requires a contiguous head dimension") if q.stride(2) != 1: raise ValueError("paged prefill requires contiguous query rows") - _paged_prefill_kernel[(sequences * q_blocks, q.shape[1])]( + _launch_paged_prefill( + q, + key_cache, + value_cache, + block_tables, + context_lengths, + cu_seqlens_q, + output, + max_seqlen_q, + window=window, + compensate=compensate, + tile=tile, + ) + return output + + +def _launch_paged_prefill( + q, + key_cache, + value_cache, + block_tables, + context_lengths, + cu_seqlens_q, + output, + max_seqlen_q, + *, + window, + compensate, + tile, + compile_only=False, +): + block_m, block_n = tile or (_PACKED_WINDOW_TILE if compensate else (64, 64)) + q_blocks = triton.cdiv(int(max_seqlen_q), block_m) + grid = (block_tables.shape[0] * q_blocks, q.shape[1]) + launch = ( + partial(_paged_prefill_kernel.warmup, grid=grid) + if compile_only + else _paged_prefill_kernel[grid] + ) + return launch( q, key_cache, value_cache, @@ -386,4 +429,47 @@ def paged_prefill_attention( num_warps=4, num_stages=1 if compensate else 2, ) - return output + + +def select_packed_window_tile(device, dtype, head_dim, block_size, shape): + if torch.version.hip or dtype != torch.bfloat16: + return None + index = device.index if device.index is not None else torch.cuda.current_device() + try: + return _select_packed_window_tile(index, head_dim, block_size, shape) + except torch.OutOfMemoryError: + # A transient failure must not be cached for other engines. + return None + + +@lru_cache(maxsize=16) +def _select_packed_window_tile(index, head_dim, block_size, shape): + # Warmup only needs tensor metadata. Preserve the KV pool's interleaved layer strides. + q = torch.empty((256, shape.query_heads, head_dim), device="meta", dtype=torch.bfloat16) + kv = q.new_empty((1, shape.kv_layers, block_size, shape.kv_heads, head_dim))[:, 0] + table = q.new_empty((1, 16), dtype=torch.int32) + lengths = q.new_empty(1, dtype=torch.int32) + offsets = q.new_empty(2, dtype=torch.int32) + with torch.cuda.device(index): + limit = driver.active.utils.get_device_properties(index)["max_shared_mem"] + try: + for window in (0, shape.window): + compiled = _launch_paged_prefill( + q, + kv, + kv, + table, + lengths, + offsets, + q, + len(q), + window=window, + compensate=True, + tile=_PACKED_WINDOW_TILE, + compile_only=True, + ) + if compiled.metadata.shared > limit: + return None + except TritonError: + return None + return _PACKED_WINDOW_TILE From 6c3da8aac85b38a02db053342fcfd20f2c3e80f3 Mon Sep 17 00:00:00 2001 From: cecilsd <127472276+cecilsd@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:25:23 +0800 Subject: [PATCH 5/6] Fix packed prefill opt-in for full-attention Dory --- docs/dory.md | 7 ++-- tests/test_dory_gpu.py | 48 ++++++++++++++++------------ vllm_rlt/attention/backend.py | 11 ++++--- vllm_rlt/engine/stage_engine.py | 2 +- vllm_rlt/kernels/paged_attention.py | 15 +++------ vllm_rlt/kernels/triton_attention.py | 9 ++---- 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/docs/dory.md b/docs/dory.md index 3defaf94..9ee132bb 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -102,9 +102,10 @@ single-stage execution. Speculation, preemption, CUDA graphs and asynchronous scheduling are outside its scope. On NVIDIA CUDA, the packed path uses a 32 × 32 tile. At startup, Triton compiles -the full and sliding-window kernels and checks their shared-memory requirements -against the device limit. The engine uses per-query attention if the check -fails. The result is cached by device and attention geometry within the process. +the full-attention kernel and, if the model uses a sliding window, the +sliding-window kernel. It checks their shared-memory requirements against the +device limit. The engine uses per-query attention if the check fails. +The result is cached by device and attention geometry within the process. The check runs before KV memory planning and uses tensor metadata only. The full BF16 checkpoint and long-context service validation require enough device memory for the model weights, runtime buffers and KV pool. diff --git a/tests/test_dory_gpu.py b/tests/test_dory_gpu.py index 4f4a5558..e571c0cb 100644 --- a/tests/test_dory_gpu.py +++ b/tests/test_dory_gpu.py @@ -290,7 +290,7 @@ def failing(batch): assert engine.cache_manager.num_used_blocks == 0 -def _packed_tile(query_heads, kv_heads, kv_layers=1): +def _packed_tile(query_heads, kv_heads, kv_layers=1, *, window): from vllm_rlt.kernels.paged_attention import PackedWindowShape from vllm_rlt.kernels.triton_attention import select_packed_window_tile @@ -299,7 +299,7 @@ def _packed_tile(query_heads, kv_heads, kv_layers=1): torch.bfloat16, 256, 16, - PackedWindowShape(query_heads, kv_heads, kv_layers, 32), + PackedWindowShape(query_heads, kv_heads, kv_layers, window), ) if tile is None: pytest.skip("No supported packed prefill tile for this device") @@ -310,7 +310,7 @@ def _packed_tile(query_heads, kv_heads, kv_layers=1): def test_dory_packed_attention_matches_per_row_reference(window): from vllm_rlt.kernels.triton_attention import paged_attention, paged_prefill_attention - tile = _packed_tile(16, 4) + _packed_tile(16, 4, window=window or None) # The first chunk is far enough into its history to skip a full SWA tile. # The two sequences have distinct pages and different causal offsets. torch.manual_seed(123) @@ -328,25 +328,28 @@ def test_dory_packed_attention_matches_per_row_reference(window): row_lengths = torch.tensor([*range(98, 133), *range(9, 16)], device=DEVICE, dtype=torch.int32) expected = paged_attention(q, k, v, row_tables, row_lengths, window=window) actual = paged_prefill_attention( - q, k, v, tables, lengths, offsets, 35, window=window, compensate=True, tile=tile + q, k, v, tables, lengths, offsets, 35, window=window, compensate=True ) torch.testing.assert_close(actual, expected, atol=2e-3, rtol=1e-2) @pytest.mark.parametrize("multi_stream", [True, False]) @pytest.mark.parametrize(("long_tokens", "max_num_seqs"), [(65, 1), (129, 2)]) +@pytest.mark.parametrize("sliding_window", [None, 32], ids=["full", "mixed"]) def test_dory_packed_prefill_preserves_short_prompt_logits( - monkeypatch, multi_stream, long_tokens, max_num_seqs + monkeypatch, multi_stream, long_tokens, max_num_seqs, sliding_window ): from vllm_rlt.kernels import triton_attention - model = make_gpu_model(head_dim=256, max_position_embeddings=256, sliding_window=32).to( - dtype=torch.bfloat16 - ) + layout = {"layer_types": ("full_attention", "mlp") * 4} if sliding_window is None else {} + model = make_gpu_model( + head_dim=256, max_position_embeddings=256, sliding_window=sliding_window, **layout + ).to(dtype=torch.bfloat16) tile = _packed_tile( model.config.num_attention_heads, model.config.num_key_value_heads, model.config.num_kv_layers, + window=sliding_window, ) prompt = [4, 7, 9, 3, 12, 5, 23] scheduler = SchedulerConfig( @@ -415,7 +418,9 @@ def attention(layer, metadata, q, *, window=None): assert torch.equal(baseline, fallback) assert not baseline_packed assert {layer for layer, _ in packed} == set(range(model.config.num_kv_layers)) - assert {window for _, window in packed} == {False, True} + assert {window for _, window in packed} == ( + {False} if sliding_window is None else {False, True} + ) assert torch.equal(baseline, actual) @@ -439,7 +444,10 @@ def compile_tile(*args, window, **kwargs): assert probe(index, 256, 16, PackedWindowShape(16, 4, 36, 1024)) is None -def test_prefill_tile_probe_compiles_only_the_default_and_caches_result(monkeypatch): +@pytest.mark.parametrize(("window", "expected_windows"), [(None, [0]), (128, [0, 128])]) +def test_prefill_tile_probe_compiles_only_the_default_and_caches_result( + monkeypatch, window, expected_windows +): from types import SimpleNamespace from vllm_rlt.kernels import triton_attention as attention @@ -454,22 +462,22 @@ def test_prefill_tile_probe_compiles_only_the_default_and_caches_result(monkeypa windows = [] monkeypatch.setattr(torch.cuda, "get_device_name", lambda device: "Unlisted GPU") - def compile_tile(*args, window, compile_only, tile, **kwargs): - assert compile_only - assert tile == (32, 32) + def compile_tile(*args, **kwargs): + assert (kwargs["BLOCK_M"], kwargs["BLOCK_N"]) == (32, 32) + assert kwargs["COMPENSATE"] assert all(arg.device.type == "meta" for arg in args if isinstance(arg, torch.Tensor)) - windows.append(window) + windows.append(kwargs["WINDOW"]) return SimpleNamespace(metadata=SimpleNamespace(shared=limit)) - monkeypatch.setattr(attention, "_launch_paged_prefill", compile_tile) - args = (device, torch.bfloat16, 64, 16, PackedWindowShape(4, 2, 2, 128)) + monkeypatch.setattr(attention._paged_prefill_kernel, "warmup", compile_tile) + args = (device, torch.bfloat16, 64, 16, PackedWindowShape(4, 2, 2, window)) attention._select_packed_window_tile.cache_clear() try: assert attention.select_packed_window_tile(*args) == (32, 32) - assert windows == [0, 128] + assert windows == expected_windows assert attention.select_packed_window_tile(*args) == (32, 32) - assert windows == [0, 128] + assert windows == expected_windows finally: attention._select_packed_window_tile.cache_clear() @@ -477,7 +485,7 @@ def compile_tile(*args, window, compile_only, tile, **kwargs): def test_dory_packed_prefill_keeps_small_value_residual(): from vllm_rlt.kernels.triton_attention import paged_attention, paged_prefill_attention - tile = _packed_tile(1, 1) + _packed_tile(1, 1, window=None) q = torch.zeros((1, 1, 256), device=DEVICE, dtype=torch.bfloat16) kv = torch.zeros((1, 16, 1, 256), device=DEVICE, dtype=torch.bfloat16) k, v = kv.clone(), kv.clone() @@ -488,6 +496,6 @@ def test_dory_packed_prefill_keeps_small_value_residual(): length = torch.tensor([2], device=DEVICE, dtype=torch.int32) offset = torch.tensor([0, 1], device=DEVICE, dtype=torch.int32) expected = paged_attention(q, k, v, table, length) - actual = paged_prefill_attention(q, k, v, table, length, offset, 1, compensate=True, tile=tile) + actual = paged_prefill_attention(q, k, v, table, length, offset, 1, compensate=True) assert expected[0, 0, 0] > 0.0006 torch.testing.assert_close(actual, expected, atol=5e-6, rtol=0) diff --git a/vllm_rlt/attention/backend.py b/vllm_rlt/attention/backend.py index 493f11e9..fcc58278 100644 --- a/vllm_rlt/attention/backend.py +++ b/vllm_rlt/attention/backend.py @@ -96,7 +96,7 @@ def _create_explicit(backend, device, dtype, head_dim, block_size, *, packed, pa sm = None if torch.version.hip else torch.cuda.get_device_capability(device) if sm is not None and sm[0] < 8: raise ValueError("the Triton attention backend requires NVIDIA Compute Capability 8.0+") - # Import the implementations before selection and probe opted-in window tiles + # Import the implementation before selection and probe the opted-in tile # before the engine profiles and allocates its KV pool. implementation = importlib.import_module("vllm_rlt.kernels.triton_attention") tile = ( @@ -106,12 +106,13 @@ def _create_explicit(backend, device, dtype, head_dim, block_size, *, packed, pa if packed_window_shape is not None else None ) + compensate = tile is not None capabilities = replace( _TRITON_CAPABILITIES, - packed_prefill=packed or tile is not None, - packed_sliding_window=tile is not None, + packed_prefill=packed or compensate, + packed_sliding_window=compensate, ) - kernel = TritonPagedAttention(packed_window_tile=tile) + kernel = TritonPagedAttention(compensate=compensate) return AttentionBackend( kernel, capabilities, {"backend": backend, "packed_prefill_tile": tile} ) @@ -198,7 +199,7 @@ def create_backend( ): implementation.info.update( packed_prefill_tile=None, - packed_prefill_fallback="no supported packed sliding-window tile", + packed_prefill_fallback="no supported packed prefill tile", ) logging.getLogger(__name__).info("Attention selection: %s", implementation.info) return implementation diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index 8ab18c76..b1f98203 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -193,7 +193,7 @@ def __init__( PackedWindowShape( config.num_attention_heads, config.num_key_value_heads, kv_layers, window ) - if require_packed and sliding_window and window + if require_packed and sliding_window else None ) attention = create_backend( diff --git a/vllm_rlt/kernels/paged_attention.py b/vllm_rlt/kernels/paged_attention.py index 4de2af3f..d92d8279 100644 --- a/vllm_rlt/kernels/paged_attention.py +++ b/vllm_rlt/kernels/paged_attention.py @@ -108,7 +108,7 @@ class PackedWindowShape: query_heads: int kv_heads: int kv_layers: int - window: int + window: int | None def triton_paged_prefill( @@ -122,7 +122,6 @@ def triton_paged_prefill( *, window=0, compensate=False, - tile=None, ): """Packed causal prefill with one block table per sequence (see triton_attention).""" from vllm_rlt.kernels.triton_attention import paged_prefill_attention @@ -137,15 +136,14 @@ def triton_paged_prefill( max_seqlen_q, window=window, compensate=compensate, - tile=tile, ) class TritonPagedAttention: """Triton single-query decode attention plus packed causal prefill.""" - def __init__(self, *, packed_window_tile=None): - self.packed_window_tile = packed_window_tile + def __init__(self, *, compensate=False): + self.compensate = compensate def __call__(self, q, key_cache, value_cache, block_tables, context_lengths): return triton_paged_attention(q, key_cache, value_cache, block_tables, context_lengths) @@ -156,9 +154,4 @@ def windowed(self, q, key_cache, value_cache, block_tables, context_lengths, win ) def prefill(self, *args, **kwargs): - return triton_paged_prefill( - *args, - **kwargs, - compensate=self.packed_window_tile is not None, - tile=self.packed_window_tile, - ) + return triton_paged_prefill(*args, **kwargs, compensate=self.compensate) diff --git a/vllm_rlt/kernels/triton_attention.py b/vllm_rlt/kernels/triton_attention.py index fd57a489..ae9a8d90 100644 --- a/vllm_rlt/kernels/triton_attention.py +++ b/vllm_rlt/kernels/triton_attention.py @@ -336,7 +336,6 @@ def paged_prefill_attention( *, window=0, compensate=False, - tile=None, ): """Packed causal prefill over paged K/V, bottom-right aligned per sequence. @@ -367,7 +366,6 @@ def paged_prefill_attention( max_seqlen_q, window=window, compensate=compensate, - tile=tile, ) return output @@ -384,10 +382,9 @@ def _launch_paged_prefill( *, window, compensate, - tile, compile_only=False, ): - block_m, block_n = tile or (_PACKED_WINDOW_TILE if compensate else (64, 64)) + block_m, block_n = _PACKED_WINDOW_TILE if compensate else (64, 64) q_blocks = triton.cdiv(int(max_seqlen_q), block_m) grid = (block_tables.shape[0] * q_blocks, q.shape[1]) launch = ( @@ -450,10 +447,11 @@ def _select_packed_window_tile(index, head_dim, block_size, shape): table = q.new_empty((1, 16), dtype=torch.int32) lengths = q.new_empty(1, dtype=torch.int32) offsets = q.new_empty(2, dtype=torch.int32) + windows = (0,) if shape.window is None else (0, shape.window) with torch.cuda.device(index): limit = driver.active.utils.get_device_properties(index)["max_shared_mem"] try: - for window in (0, shape.window): + for window in windows: compiled = _launch_paged_prefill( q, kv, @@ -465,7 +463,6 @@ def _select_packed_window_tile(index, head_dim, block_size, shape): len(q), window=window, compensate=True, - tile=_PACKED_WINDOW_TILE, compile_only=True, ) if compiled.metadata.shared > limit: From d66082c8e5de62b77cf7a0c0e8e427b43138eff1 Mon Sep 17 00:00:00 2001 From: cecilsd <127472276+cecilsd@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:21:37 +0800 Subject: [PATCH 6/6] fix: preserve packed prefill tri-state selection --- docs/dory.md | 17 ++- docs/runtime.md | 11 +- tests/test_backend_selection.py | 167 +++++++++++++++++++++++++++ tests/test_dory.py | 2 +- tests/test_dory_gpu.py | 9 +- tests/test_engine.py | 38 +++++- vllm_rlt/config.py | 4 +- vllm_rlt/engine/packed_prefill.py | 38 +++--- vllm_rlt/engine/stage_engine.py | 13 +-- vllm_rlt/entrypoints/runtime_args.py | 4 +- 10 files changed, 258 insertions(+), 45 deletions(-) diff --git a/docs/dory.md b/docs/dory.md index 9ee132bb..46cfd8d6 100644 --- a/docs/dory.md +++ b/docs/dory.md @@ -90,12 +90,13 @@ layer id rather than their physical sublayer index. template, resolved without requiring transformers to accept the checkpoint config (its generic validator rejects Dory's `mlp` sublayer entries). -## Opt-in packed Triton prefill +## Packed Triton prefill -In Python, set `attention_backend="triton"` and -`execution_config=ExecutionConfig(packed_prefill=True)`. On the CLI, pass -`--attention-backend triton --packed-prefill`. Dory uses per-query prefill when -the option is omitted or when `--no-packed-prefill` is set. +With `attention_backend="triton"`, the default `packed_prefill=None` automatically +enables packed prefill when execution and backend support it. Set +`ExecutionConfig(packed_prefill=True)` / `--packed-prefill` to require support, +or `packed_prefill=False` / `--no-packed-prefill` to keep per-query prefill. +An unsupported explicit enable request fails at startup. This path uses BF16, `per_loop` KV, ordinary prefill, and synchronous eager single-stage execution. Speculation, preemption, CUDA graphs and asynchronous @@ -104,9 +105,13 @@ scheduling are outside its scope. On NVIDIA CUDA, the packed path uses a 32 × 32 tile. At startup, Triton compiles the full-attention kernel and, if the model uses a sliding window, the sliding-window kernel. It checks their shared-memory requirements against the -device limit. The engine uses per-query attention if the check fails. +device limit. Automatic mode uses per-query attention if the check fails; +explicit enable requests fail through backend capability validation. The result is cached by device and attention geometry within the process. The check runs before KV memory planning and uses tensor metadata only. +The check establishes that the tile fits, not that it is fastest. The fixed +32 × 32 tile was validated on RTX 4060 and RTX 5090; other GPUs and workloads +need measurements to determine their best tile. The full BF16 checkpoint and long-context service validation require enough device memory for the model weights, runtime buffers and KV pool. diff --git a/docs/runtime.md b/docs/runtime.md index 66f515e4..910d3451 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -81,12 +81,17 @@ For a task-assigned GPU, set `CUDA_VISIBLE_DEVICES` and add `--device cuda --att `ExecutionConfig.packed_prefill` controls packed prefill attention and metadata staging. Its default, `None`, selects packed prefill when the selected backend supports it on CUDA with `last_exited` KV and ordinary, non-staged prefill. -Mixed scheduling, wavefront chunked prefill and sliding-window attention keep -their existing paths. These selection rules apply to all models. +Mixed scheduling and wavefront chunked prefill keep their existing paths. +Sliding-window packing additionally requires backend support for both packed +prefill and packed sliding-window attention, with ordinary synchronous eager +single-stage execution, without speculation or preemption. These selection rules +apply to all models. Incompatible automatic configurations keep per-query attention. Set `packed_prefill=True` / `--packed-prefill` to require packed prefill, or `packed_prefill=False` / `--no-packed-prefill` to disable its staging path. -Explicit requests still undergo device, model and backend compatibility checks. +Explicit enable requests fail if execution constraints or required backend +capabilities are unsupported; they do not silently fall back. Explicit disable +does not probe a packed sliding-window tile. `ExecutionConfig.use_uva` independently controls how input buffers are read. `use_uva=False` / `--no-uva` copies them to the GPU and can be combined with diff --git a/tests/test_backend_selection.py b/tests/test_backend_selection.py index 8bd134d6..3778ae71 100644 --- a/tests/test_backend_selection.py +++ b/tests/test_backend_selection.py @@ -382,3 +382,170 @@ def import_module(name): assert "incompatible CUDA extension" in selected.info["selection_reason"] with pytest.raises(OSError, match="incompatible CUDA extension"): create_backend("flash_attn_2", torch.device("cuda"), torch.bfloat16, 128, 256) + + +@pytest.mark.parametrize("requested", [None, True, False]) +@pytest.mark.parametrize("tile", [None, (32, 32)]) +@pytest.mark.parametrize("backend", ["triton", "auto"]) +def test_window_engine_resolves_tristate_before_cache_planning( + packages, monkeypatch, requested, tile, backend +): + from tests.test_dory import tiny_dory_config + from vllm_rlt.attention import backend as selection + from vllm_rlt.engine import stage_engine + + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (8, 9)) + original_import = selection.importlib.import_module + probes = [] + + def import_module(name): + if name == "vllm_rlt.kernels.triton_attention": + + def probe(*args): + probes.append(args[-1]) + return tile + + return SimpleNamespace(select_packed_window_tile=probe) + return original_import(name) + + monkeypatch.setattr(selection.importlib, "import_module", import_module) + config = tiny_dory_config() + model = SimpleNamespace( + config=config, + requires_boundary_kv=True, + requires_sliding_window=True, + parameters=lambda: iter( + [SimpleNamespace(device=torch.device("cuda"), dtype=torch.bfloat16)] + ), + ) + + class Planned(Exception): + pass + + def plan_cache(model, cache, scheduler, execution, selected, **kwargs): + assert execution.packed_prefill is (requested is not False and tile is not None) + assert selected == "triton" + raise Planned + + monkeypatch.setattr(stage_engine, "plan_cache", plan_cache) + error = ValueError if requested is True and tile is None else Planned + with pytest.raises(error): + LLMEngine( + model, + attention_backend=backend, + execution_config=ExecutionConfig(packed_prefill=requested), + ) + assert len(probes) == int(requested is not False) + if probes: + assert probes[0].query_heads == config.num_attention_heads + assert probes[0].kv_layers == config.num_kv_layers + assert probes[0].window == config.sliding_window + + +@pytest.mark.parametrize("missing", ["packed_prefill", "packed_sliding_window"]) +@pytest.mark.parametrize("requested", [None, True]) +def test_window_engine_requires_both_packed_capabilities(monkeypatch, missing, requested): + from dataclasses import replace + + from tests.test_dory import tiny_dory_config + from vllm_rlt.attention import AttentionBackend + from vllm_rlt.attention import backend as selection + from vllm_rlt.engine import stage_engine + + capabilities = replace( + BackendCapabilities(packed_prefill=True, sliding_window=True, packed_sliding_window=True), + **{missing: False}, + ) + monkeypatch.setattr( + selection, + "_create_explicit", + lambda *args, **kwargs: AttentionBackend(None, capabilities, {"backend": "triton"}), + ) + model = SimpleNamespace( + config=tiny_dory_config(), + requires_boundary_kv=True, + requires_sliding_window=True, + parameters=lambda: iter( + [SimpleNamespace(device=torch.device("cuda"), dtype=torch.bfloat16)] + ), + ) + + class Planned(Exception): + pass + + def plan_cache(model, cache, scheduler, execution, selected, **kwargs): + assert requested is None + assert execution.packed_prefill is False + raise Planned + + monkeypatch.setattr(stage_engine, "plan_cache", plan_cache) + error = ValueError if requested is True else Planned + with pytest.raises(error, match=missing if requested else None): + LLMEngine( + model, + attention_backend="triton", + execution_config=ExecutionConfig(packed_prefill=requested), + ) + + +@pytest.mark.parametrize("requested", [None, True]) +@pytest.mark.parametrize( + "asynchronous,graphs,preemption,role", + [ + (True, False, False, "prefill_decode"), + (False, True, False, "prefill_decode"), + (False, False, True, "prefill_decode"), + (False, False, False, "decode"), + ], +) +def test_incompatible_window_engine_does_not_probe_packed_tile( + packages, monkeypatch, requested, asynchronous, graphs, preemption, role +): + from tests.test_dory import tiny_dory_config + from vllm_rlt import SchedulerConfig + from vllm_rlt.attention import backend as selection + from vllm_rlt.engine import stage_engine + from vllm_rlt.engine.stage_config import StageConfig + + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (8, 9)) + original_import = selection.importlib.import_module + + def import_module(name): + if name == "vllm_rlt.kernels.triton_attention": + return SimpleNamespace( + select_packed_window_tile=lambda *args: pytest.fail( + "incompatible execution probed a packed tile" + ) + ) + return original_import(name) + + monkeypatch.setattr(selection.importlib, "import_module", import_module) + model = SimpleNamespace( + config=tiny_dory_config(), + requires_boundary_kv=True, + requires_sliding_window=True, + fixed_loop_count=True, + supports_async_scheduling=True, + parameters=lambda: iter( + [SimpleNamespace(device=torch.device("cuda"), dtype=torch.bfloat16)] + ), + ) + + class Planned(Exception): + pass + + def plan_cache(model, cache, scheduler, execution, selected, **kwargs): + assert requested is None and execution.packed_prefill is False + raise Planned + + monkeypatch.setattr(stage_engine, "plan_cache", plan_cache) + with pytest.raises(ValueError if requested is True else Planned): + LLMEngine( + model, + attention_backend="triton", + stage_config=StageConfig(role=role), + scheduler_config=SchedulerConfig(enable_preemption=preemption), + execution_config=ExecutionConfig( + packed_prefill=requested, async_scheduling=asynchronous, cuda_graphs=graphs + ), + ) diff --git a/tests/test_dory.py b/tests/test_dory.py index b14a2172..45260837 100644 --- a/tests/test_dory.py +++ b/tests/test_dory.py @@ -673,7 +673,7 @@ def test_exit_written_ignores_boundary_layers(model): @pytest.mark.parametrize("packed_prefill", [None, False]) -def test_dory_defaults_to_per_row_prefill(model, packed_prefill): +def test_dory_cpu_keeps_per_row_prefill(model, packed_prefill): engine = LLMEngine(model, execution_config=ExecutionConfig(packed_prefill=packed_prefill)) assert engine.execution_config.packed_prefill is False assert not engine.cache_manager.attention_capabilities.packed_sliding_window diff --git a/tests/test_dory_gpu.py b/tests/test_dory_gpu.py index e571c0cb..1038a0ba 100644 --- a/tests/test_dory_gpu.py +++ b/tests/test_dory_gpu.py @@ -409,14 +409,19 @@ def attention(layer, metadata, q, *, window=None): engine.close() return torch.stack(short_logits), packed_layers - baseline, baseline_packed = run(enable=None, expect_packed=False) + baseline, baseline_packed = run(enable=False, expect_packed=False) + automatic, automatic_packed = run(enable=None, expect_packed=True) actual, packed = run(enable=True, expect_packed=True) with monkeypatch.context() as patch: patch.setattr(triton_attention, "select_packed_window_tile", lambda *args: None) - fallback, fallback_packed = run(enable=True, expect_packed=False) + fallback, fallback_packed = run(enable=None, expect_packed=False) + with pytest.raises(ValueError, match="lacks required capabilities: packed_sliding_window"): + run(enable=True, expect_packed=False) assert not fallback_packed assert torch.equal(baseline, fallback) assert not baseline_packed + assert automatic_packed == packed + assert torch.equal(automatic, actual) assert {layer for layer, _ in packed} == set(range(model.config.num_kv_layers)) assert {window for _, window in packed} == ( {False} if sliding_window is None else {False, True} diff --git a/tests/test_engine.py b/tests/test_engine.py index bbd15e21..e738d8f4 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -422,13 +422,13 @@ def test_packed_prefill_resolves_automatically_when_unset(): from vllm_rlt.engine.packed_prefill import packed_prefill_eligible common = dict(layout="last_exited", staged_prefill=False) - # Unset: on for CUDA, last_exited, ordinary prefill without sliding windows. + # Unset: prefer packing for compatible CUDA execution, including windows. assert packed_prefill_eligible(ExecutionConfig(), device_type="cuda", **common) assert not packed_prefill_eligible(ExecutionConfig(), device_type="cpu", **common) assert not packed_prefill_eligible( ExecutionConfig(), device_type="cuda", **{**common, "layout": "shared"} ) - assert not packed_prefill_eligible( + assert packed_prefill_eligible( ExecutionConfig(), device_type="cuda", **common, sliding_window=True ) assert not packed_prefill_eligible( @@ -454,3 +454,37 @@ def test_packed_prefill_resolves_automatically_when_unset(): LLMEngine(tiny_model(), execution_config=ExecutionConfig(packed_prefill=True)) engine = LLMEngine(tiny_model(), execution_config=ExecutionConfig(packed_prefill=False)) assert engine.execution_config.packed_prefill is False + + +@pytest.mark.parametrize("requested", [None, True, False]) +@pytest.mark.parametrize( + "execution, constraints, error", + [ + ({}, {}, None), + ({}, {"device_type": "cpu"}, "requires CUDA"), + ({}, {"layout": "shared"}, "last_exited"), + ({"async_scheduling": True}, {}, "synchronous eager"), + ({"cuda_graphs": True}, {}, "synchronous eager"), + ({}, {"staged_prefill": True}, "ordinary prefill"), + ({}, {"speculative": True}, "without speculation"), + ({}, {"preemption": True}, "without speculation or preemption"), + ({}, {"stage_role": "prefill"}, "single-stage"), + ({}, {"stage_role": "decode"}, "single-stage"), + ], +) +def test_sliding_window_packed_prefill_tristate(requested, execution, constraints, error): + from vllm_rlt import ExecutionConfig + from vllm_rlt.engine.packed_prefill import packed_prefill_eligible + + options = dict( + device_type="cuda", layout="last_exited", staged_prefill=False, sliding_window=True + ) + options.update(constraints) + config = ExecutionConfig(packed_prefill=requested, **execution) + if requested is True and error is not None: + with pytest.raises(ValueError, match=error): + packed_prefill_eligible(config, **options) + else: + assert packed_prefill_eligible(config, **options) is ( + requested is not False and error is None + ) diff --git a/vllm_rlt/config.py b/vllm_rlt/config.py index 3b03bc30..1619ba80 100644 --- a/vllm_rlt/config.py +++ b/vllm_rlt/config.py @@ -152,8 +152,8 @@ class ExecutionConfig: # use_uva independently controls its transfer mechanism. # None resolves at engine construction: on when the attention backend packs # prefill (Triton opts in, FlashAttention-4 natively) with last_exited KV and - # ordinary (non-staged) prefill without sliding-window attention; off otherwise. - # Sliding-window prefill requires explicit opt-in. + # ordinary (non-staged) prefill. Sliding windows additionally require packed-window + # support and compatible synchronous eager execution; off otherwise. packed_prefill: bool | None = None def __post_init__(self): diff --git a/vllm_rlt/engine/packed_prefill.py b/vllm_rlt/engine/packed_prefill.py index 03aa2f16..82eec246 100644 --- a/vllm_rlt/engine/packed_prefill.py +++ b/vllm_rlt/engine/packed_prefill.py @@ -14,30 +14,28 @@ def packed_prefill_eligible( ): """Validate execution constraints before filtering by backend capability. - Sliding-window packing is opt-in. Its current implementation requires host-planned - metadata and ordinary synchronous prefill in a single stage. + Sliding-window packing requires host-planned metadata and ordinary + synchronous prefill in a single stage. """ requested = execution.packed_prefill - if requested: - if sliding_window and ( - execution.async_scheduling - or execution.cuda_graphs - or staged_prefill - or speculative - or preemption - or stage_role != "prefill_decode" - ): + if requested is False: + return False + if sliding_window and ( + execution.async_scheduling + or execution.cuda_graphs + or staged_prefill + or speculative + or preemption + or stage_role != "prefill_decode" + ): + if requested is True: raise ValueError( "sliding-window packed prefill requires synchronous eager single-stage execution " "with ordinary prefill, without speculation or preemption" ) - if device_type != "cuda" or layout != "last_exited": + return False + if device_type != "cuda" or layout != "last_exited": + if requested is True: raise ValueError("packed_prefill requires CUDA packed prefill with last_exited KV") - if requested is not None: - return requested - return ( - device_type == "cuda" - and layout == "last_exited" - and not staged_prefill - and not sliding_window - ) + return False + return requested is True or not staged_prefill diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index b1f98203..2f9fde18 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -193,7 +193,7 @@ def __init__( PackedWindowShape( config.num_attention_heads, config.num_key_value_heads, kv_layers, window ) - if require_packed and sliding_window + if prefer_packed and sliding_window else None ) attention = create_backend( @@ -203,7 +203,8 @@ def __init__( config.head_dim, cache_config.block_size, required_capabilities=BackendCapabilities( - packed_prefill=require_packed and not sliding_window, + packed_prefill=require_packed, + packed_sliding_window=require_packed and sliding_window, cuda_graphs=self.execution_config.cuda_graphs, async_scheduling=( parameter.device.type == "cuda" and self.execution_config.async_scheduling @@ -214,11 +215,9 @@ def __init__( prefer_packed_prefill=prefer_packed and not sliding_window, packed_window_shape=packed_window_shape, ) - if sliding_window or self.execution_config.packed_prefill is None: - supported = ( - attention.capabilities.packed_sliding_window - if sliding_window - else attention.capabilities.packed_prefill + if self.execution_config.packed_prefill is None: + supported = attention.capabilities.packed_prefill and ( + not sliding_window or attention.capabilities.packed_sliding_window ) self.execution_config = replace( self.execution_config, packed_prefill=prefer_packed and supported diff --git a/vllm_rlt/entrypoints/runtime_args.py b/vllm_rlt/entrypoints/runtime_args.py index d4a0a976..49d2b13f 100644 --- a/vllm_rlt/entrypoints/runtime_args.py +++ b/vllm_rlt/entrypoints/runtime_args.py @@ -29,8 +29,8 @@ def add_runtime_args(parser): "--packed-prefill", action="store_true", default=None, - help="Enable packed prefill on a compatible backend. " - "Defaults to automatic selection. Sliding-window attention requires explicit opt-in.", + help="Require packed prefill on a compatible backend; fail if unsupported. " + "Defaults to automatic selection when execution and backend support it.", ) packed_prefill_mode.add_argument( "--no-packed-prefill",