diff --git a/docs/runtime.md b/docs/runtime.md index 460c5e9c..9c4a0c18 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -63,7 +63,9 @@ if __name__ == "__main__": The default exit mode remains `ouro`, with the original cumulative hazard policy. For random lookahead, sigmoid is a direct score for exiting after one more loop, not a hazard/CDF. `SamplingParams.exit_threshold` compares this score directly. Threshold 1 disables adaptive exit; `max_loops` always caps execution. Adaptive lookahead cannot exit before round 2, although an explicitly configured hard maximum of 1 is honored. All prompt tokens still run the model's full depth. -Ordinary asynchronous execution requires `ouro_delayed`, `random_lookahead` or `trace`; speculative execution has the separate constraints below. It automatically enables reusable static inputs to avoid blocking device metadata transfers. CPU supports the same scheduling state machine for tests, with immediately completed operations; it does not simulate GPU parallel execution. `multi_stream` is an independent CUDA execution feature: `False` uses one stream for core, boundary and copies; `True` separates those streams, with or without async scheduling. +Fixed-loop models accept the request-level `SamplingParams.target_loops` field (offline CLI: `--request-target-loops`). An omitted target defaults to `total_ut_steps`; an explicit target must be between one and the model depth. If `max_loops` is provided, it must match the fixed target. For compatibility with the default, `min_loops=2` is accepted, as is `min_loops` equal to the target; the fixed target determines the exit depth. Targets below the model depth are rejected until per-request prompt prefill is enabled; they are never silently run at full prompt depth. Dynamic-exit models do not accept this field and retain their existing loop-bound and exit-threshold behavior. + +For dynamic-exit models, ordinary asynchronous execution requires `ouro_delayed`, `random_lookahead` or `trace`; fixed-loop models do not need an exit signal. Speculative execution has the separate constraints below. Asynchronous execution automatically enables reusable static inputs to avoid blocking device metadata transfers. CPU supports the same scheduling state machine for tests, with immediately completed operations; it does not simulate GPU parallel execution. `multi_stream` is an independent CUDA execution feature: `False` uses one stream for core, boundary and copies; `True` separates those streams, with or without async scheduling. CLI flags are shared by offline and serving entrypoints: @@ -306,7 +308,7 @@ of that capacity can safely be promised to new work. ## Replay and validation commands -Trace mode expects `depths_by_request`, with one depth per output: full prefill depth first, then each decode input's exit depth. Admission validates coverage and loop bounds. Request IDs must match the trace. The public engine replay fixes depth decisions; token generation still follows the configured sampler. +Trace mode expects `depths_by_request`, with one depth per output: full prefill depth first, then each decode input's exit depth. Trace decisions take precedence over a valid request-level `target_loops` value; the trace controls per-output depths. Trace depths remain bounded by model depth and any supplied `max_loops`. Admission validates coverage and loop bounds. Request IDs must match the trace. The public engine replay fixes depth decisions; token generation still follows the configured sampler. ```python ExitConfig(mode="trace", depths_by_request={"A": [4, 2, 4]}) diff --git a/docs/serving.md b/docs/serving.md index a2e4807c..81cb6999 100644 --- a/docs/serving.md +++ b/docs/serving.md @@ -72,6 +72,7 @@ curl http://127.0.0.1:8000/v1/completions \ | `temperature`, `top_p`, `top_k` | 0, 1, -1; passed to `SamplingParams` | | `seed`, `ignore_eos` | 0, false; request-local RNG and EOS policy | | `min_loops`, `max_loops`, `exit_threshold` | 2, model depth, 1; Ouro extensions | +| `target_loops` | null; fixed-loop models default to model depth; trace replay takes precedence when enabled; otherwise shallower targets are rejected pending prefill support, and `max_loops` must match when supplied | | `stream` | false; true enables SSE | | `stream_options.include_usage` | false; valid with streaming | | `n`, `best_of`, `echo` | only 1, 1, false | @@ -81,7 +82,7 @@ curl http://127.0.0.1:8000/v1/completions \ Unsupported fields/behaviors, invalid values, empty tokenized prompts, impossible KV reservations and context overflow return 400. An unknown model returns 404. Missing or invalid model identifiers return 400. Omit fields to use defaults; -explicit null is accepted only for `max_tokens`, `max_loops`, `stream_options` and the four +explicit null is accepted only for `max_tokens`, `max_loops`, `target_loops`, `stream_options` and the four null-only fields above. Structured errors use the same value for `error.type` and `error.code`; admission messages include diagnostic context/KV limits. The HTTP default uses the model context length minus the number of encoded diff --git a/tests/test_auto_model.py b/tests/test_auto_model.py index 9015e502..9695c701 100644 --- a/tests/test_auto_model.py +++ b/tests/test_auto_model.py @@ -406,6 +406,32 @@ def test_cli_requires_source_for_real_model(monkeypatch): assert error.value.code == 2 +def test_cli_request_and_speculative_targets_are_distinct(monkeypatch): + import vllm_rlt.entrypoints.cli as cli + + client = Mock() + client.engine.model_config.total_ut_steps = 4 + client.generate.return_value = [] + llm = Mock(return_value=client) + monkeypatch.setattr(cli, "LLM", llm) + monkeypatch.setattr( + "sys.argv", + [ + "vllm-rlt", + "--toy", + "--speculative-tokens", + "1", + "--target-loops", + "4", + "--request-target-loops", + "3", + ], + ) + cli.main() + assert llm.call_args.kwargs["speculative_config"].target_loops == 4 + assert client.generate.call_args.args[1].target_loops == 3 + + @pytest.mark.parametrize("answer", ["yes", "no"]) def test_missing_tokenizer_prompts_before_download(monkeypatch, answer): from vllm_rlt.models import load_tokenizer diff --git a/tests/test_dory.py b/tests/test_dory.py index 4f47a002..711cac63 100644 --- a/tests/test_dory.py +++ b/tests/test_dory.py @@ -8,7 +8,7 @@ import torch from tests.reference.dory import dense_dory_reference -from vllm_rlt import CacheConfig, ExecutionConfig, SamplingParams, SchedulerConfig +from vllm_rlt import LLM, CacheConfig, ExecutionConfig, ExitConfig, SamplingParams, SchedulerConfig from vllm_rlt.attention import BackendCapabilities from vllm_rlt.attention.backend import AttentionBackend from vllm_rlt.core.kv_cache_manager import KVCacheManager @@ -589,16 +589,211 @@ def test_engine_serves_fixed_loops_and_matches_greedy_reference(model): assert result.token_ids == tokens[len(prompt) :] +def test_engine_accepts_request_target_loops(model): + target_loops = model.config.total_ut_steps + engine = LLMEngine( + model, + cache_config=CacheConfig(num_blocks=256, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=1, max_num_batched_tokens=4), + ) + params = SamplingParams(max_tokens=2, target_loops=target_loops, ignore_eos=True) + engine.add_request("request", [4, 7, 3], params) + request = engine.scheduler.requests["request"] + assert request.target_loops == target_loops + assert request.loop_limit(model.config.total_ut_steps) == target_loops + finished = {} + while engine.has_unfinished_requests(): + for output in engine.step(): + if output.finished: + finished[output.request_id] = output + assert finished["request"].exit_depths == [target_loops, target_loops] + + +@pytest.mark.parametrize("mode", ["ouro", "ouro_delayed", "random_lookahead"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_fixed_request_ignores_adaptive_exit_threshold(model, mode, asynchronous): + engine = LLMEngine( + model, + cache_config=CacheConfig(num_blocks=256, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=1, max_num_batched_tokens=4), + exit_config=ExitConfig(mode=mode), + execution_config=ExecutionConfig(async_scheduling=asynchronous), + ) + engine.add_request( + "request", + [4, 7, 3], + SamplingParams(max_tokens=2, target_loops=2, exit_threshold=0, ignore_eos=True), + ) + finished = {} + while engine.has_unfinished_requests(): + for output in engine.step(): + if output.finished: + finished[output.request_id] = output + assert finished["request"].exit_depths == [2, 2] + + +@pytest.mark.parametrize("explicit", [False, True]) +def test_single_loop_model_accepts_default_and_explicit_target(explicit): + torch.manual_seed(42) + model = DoryForCausalLM(tiny_dory_config(n_recurrent_loops=1)).eval() + engine = LLMEngine( + model, + cache_config=CacheConfig(num_blocks=64, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=1, max_num_batched_tokens=4), + ) + params = SamplingParams( + max_tokens=2, + target_loops=1 if explicit else None, + max_loops=1 if explicit else None, + ignore_eos=True, + ) + prompt = [4, 7, 3] + engine.add_request("request", prompt, params) + assert engine.scheduler.requests["request"].target_loops == 1 + finished = {} + while engine.has_unfinished_requests(): + for output in engine.step(): + if output.finished: + finished[output.request_id] = output + result = finished["request"] + assert result.exit_depths == [1, 1] + tokens = list(prompt) + for _ in range(2): + logits = dense_dory_reference(model, torch.tensor(tokens))["logits"] + tokens.append(int(logits[-1].argmax())) + assert result.token_ids == tokens[len(prompt) :] + + +def test_default_target_uses_model_depth_above_minimum_default(): + model = DoryForCausalLM(tiny_dory_config(n_recurrent_loops=4)).eval() + engine = LLMEngine( + model, + cache_config=CacheConfig(num_blocks=64, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=1, max_num_batched_tokens=4), + ) + engine.add_request("request", [4, 7, 3], SamplingParams(max_tokens=2, ignore_eos=True)) + assert engine.scheduler.requests["request"].target_loops == 4 + finished = {} + while engine.has_unfinished_requests(): + for output in engine.step(): + if output.finished: + finished[output.request_id] = output + assert finished["request"].exit_depths == [4, 4] + + +def test_llm_passes_request_target_across_stage_process(model): + with LLM(model, cache_config=CacheConfig(num_blocks=64, block_size=2)) as llm: + result = llm.generate( + [[4, 7, 3]], SamplingParams(max_tokens=2, target_loops=2, ignore_eos=True) + ) + assert result[0].exit_depths == [2, 2] + with pytest.raises(ValueError, match="per-request prefill support"): + llm.generate([[4, 7, 3]], SamplingParams(max_tokens=1, target_loops=1)) + + +def test_engine_accepts_legacy_fixed_loop_bounds(model): + engine = LLMEngine( + model, + cache_config=CacheConfig(num_blocks=256, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=1, max_num_batched_tokens=4), + ) + params = SamplingParams(max_tokens=1, min_loops=2, max_loops=2, ignore_eos=True) + engine.add_request("request", [4, 7, 3], params) + assert engine.scheduler.requests["request"].target_loops == model.config.total_ut_steps + engine.abort_request("request") + + +def test_engine_defaults_request_target_loops_to_model_depth(model): + engine = LLMEngine( + model, + cache_config=CacheConfig(num_blocks=256, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=1, max_num_batched_tokens=4), + ) + engine.add_request("request", [4, 7, 3], SamplingParams(max_tokens=1, ignore_eos=True)) + assert engine.scheduler.requests["request"].target_loops == model.config.total_ut_steps + engine.abort_request("request") + + +def test_engine_rejects_invalid_request_target_loops(model): + with pytest.raises(ValueError, match="target_loops"): + LLMEngine( + model, + cache_config=CacheConfig(num_blocks=256, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=1, max_num_batched_tokens=4), + ).add_request( + "request", + [4, 7, 3], + SamplingParams(max_tokens=1, target_loops=model.config.total_ut_steps + 1), + ) + with pytest.raises(ValueError, match="per-request prefill support"): + LLMEngine( + model, + cache_config=CacheConfig(num_blocks=256, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=1, max_num_batched_tokens=4), + ).add_request("request", [4, 7, 3], SamplingParams(max_tokens=1, target_loops=1)) + with pytest.raises(ValueError, match="max_loops"): + LLMEngine( + model, + cache_config=CacheConfig(num_blocks=256, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=1, max_num_batched_tokens=4), + ).add_request( + "request", + [4, 7, 3], + SamplingParams(max_tokens=1, target_loops=1, min_loops=1, max_loops=2), + ) + with pytest.raises(ValueError, match="min_loops"): + LLMEngine( + model, + cache_config=CacheConfig(num_blocks=256, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=1, max_num_batched_tokens=4), + ).add_request( + "request", + [4, 7, 3], + SamplingParams( + max_tokens=1, + target_loops=1, + min_loops=3, + ignore_eos=True, + ), + ) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_trace_overrides_fixed_request_target(model, asynchronous): + engine = LLMEngine( + model, + cache_config=CacheConfig(num_blocks=64, block_size=2), + scheduler_config=SchedulerConfig(max_num_seqs=1, max_num_batched_tokens=4), + exit_config=ExitConfig(mode="trace", depths_by_request={"request": [2, 1]}), + execution_config=ExecutionConfig(async_scheduling=asynchronous), + ) + engine.add_request( + "request", + [4, 7, 3], + SamplingParams(max_tokens=2, target_loops=1, ignore_eos=True), + ) + request = engine.scheduler.requests["request"] + assert request.target_loops is None + finished = {} + while engine.has_unfinished_requests(): + for output in engine.step(): + if output.finished: + finished[output.request_id] = output + assert finished["request"].exit_depths == [2, 1] + + def test_engine_rejects_unqualified_dory_configuration(model): with pytest.raises(ValueError, match="max_loops"): LLMEngine( model, cache_config=CacheConfig(num_blocks=256, block_size=2), scheduler_config=SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=4), - ).add_request("request", [4, 7, 3], SamplingParams(max_tokens=1, min_loops=1, max_loops=1)) - with pytest.raises(ValueError, match="min_loops and max_loops"): - # A fabricated gate of -1e4 becomes 0.0 after sigmoid, so an unchecked - # exit_threshold=0 would let a shallow request "exit" after min_loops. + ).add_request( + "request", + [4, 7, 3], + SamplingParams(max_tokens=1, min_loops=1, max_loops=1), + ) + with pytest.raises(ValueError, match="min_loops"): LLMEngine( model, cache_config=CacheConfig(num_blocks=256, block_size=2), @@ -606,9 +801,7 @@ def test_engine_rejects_unqualified_dory_configuration(model): ).add_request( "request", [4, 7, 3], - SamplingParams( - max_tokens=1, min_loops=1, max_loops=2, exit_threshold=0.0, ignore_eos=True - ), + SamplingParams(max_tokens=1, min_loops=1, max_loops=2, target_loops=2, ignore_eos=True), ) with pytest.raises(ValueError, match="last_exited"): LLMEngine( diff --git a/tests/test_engine.py b/tests/test_engine.py index 1ad00979..30d97978 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -4,7 +4,7 @@ import torch from tests.helpers import tiny_ouro_config -from vllm_rlt import LLM, CacheConfig, SamplingParams, SchedulerConfig +from vllm_rlt import LLM, CacheConfig, ExitConfig, SamplingParams, SchedulerConfig from vllm_rlt.engine.llm_engine import LLMEngine from vllm_rlt.models import OuroForCausalLM from vllm_rlt.request import FinishReason, Request, RequestOutput, RequestPhase @@ -183,6 +183,36 @@ def test_same_id_after_finish_starts_from_a_fresh_generator(): assert second["r"].exit_depths == first["r"].exit_depths +@pytest.mark.parametrize("depth", [1, 2, 4]) +def test_request_fixed_target_is_read_only(depth): + request = Request("request", [2, 4], SamplingParams(), _target_loops=depth) + assert request.target_loops == request.loop_limit(4) == depth + request.loops_done = 1 + assert request.target_loops == depth + with pytest.raises(AttributeError): + request.target_loops = 3 + + +def test_target_loops_are_rejected_for_dynamic_models(): + with pytest.raises(ValueError, match="fixed-loop models"): + LLMEngine(tiny_model(), cache_config=CacheConfig(64, 2)).add_request( + "request", [2, 4], SamplingParams(max_tokens=1, target_loops=1) + ) + + +def test_trace_takes_precedence_over_dynamic_target(): + engine = LLMEngine( + tiny_model(), + cache_config=CacheConfig(64, 2), + exit_config=ExitConfig("trace", depths_by_request={"request": [4, 2]}), + ) + engine.add_request( + "request", [2, 4], SamplingParams(max_tokens=2, target_loops=1, ignore_eos=True) + ) + outputs, _ = drain(engine) + assert outputs["request"].exit_depths == [4, 2] + + def test_abort_drops_the_rng_slot_so_id_reuse_starts_over(): params = SamplingParams(max_tokens=4, temperature=0.8, seed=23, ignore_eos=True) engine = LLMEngine(tiny_model(), cache_config=CacheConfig(64, 2)) @@ -318,6 +348,7 @@ def test_integer_limits_rejected_before_scheduling(invalid): (SamplingParams, "max_tokens"), (SamplingParams, "min_loops"), (SamplingParams, "max_loops"), + (SamplingParams, "target_loops"), (SchedulerConfig, "max_num_seqs"), (SchedulerConfig, "max_num_batched_tokens"), (CacheConfig, "num_blocks"), diff --git a/tests/test_pd.py b/tests/test_pd.py index bb1554c5..4f0d9778 100644 --- a/tests/test_pd.py +++ b/tests/test_pd.py @@ -203,6 +203,8 @@ def ready_client(model, *, stage_config, **options): engine.add_request( "bad_depth", [1], SamplingParams(max_loops=config.total_ut_steps + 1) ) + with pytest.raises(ValueError, match="PD does not support per-request target_loops"): + engine.add_request("target", [1], SamplingParams(target_loops=config.total_ut_steps)) with pytest.raises(ValueError, match="exceeds context capacity"): engine.add_request( "bad_context", [1], SamplingParams(max_tokens=config.max_position_embeddings + 1) diff --git a/tests/test_serving.py b/tests/test_serving.py index d2994a87..42e31637 100644 --- a/tests/test_serving.py +++ b/tests/test_serving.py @@ -123,6 +123,9 @@ def body(**kwargs): {"max_tokens": -1}, {"max_tokens": 1.5}, {"max_tokens": "4"}, + {"target_loops": 0}, + {"target_loops": True}, + {"target_loops": 1.5}, {"temperature": "0"}, {"temperature": float("nan")}, {"top_p": True}, @@ -147,20 +150,26 @@ def test_neutral_client_fields_and_extensions(): logprobs=None, min_loops=2, max_loops=3, + target_loops=3, exit_threshold=0.7, top_k=9, top_p=0.8, ), "test-model", ) - assert spec.include_usage and spec.params.max_loops == 3 and spec.params.top_k == 9 + assert ( + spec.include_usage + and spec.params.max_loops == 3 + and spec.params.target_loops == 3 + and spec.params.top_k == 9 + ) def test_automatic_max_tokens_uses_remaining_context_and_server_cap(): - request = CompletionRequest.parse(body(max_tokens=None, top_k=3), "test-model") + request = CompletionRequest.parse(body(max_tokens=None, top_k=3, target_loops=2), "test-model") assert request.resolve_params(1024, 4096).max_tokens == 3072 capped = request.resolve_params(1024, 4096, max_output_tokens=64) - assert capped.max_tokens == 64 and capped.top_k == 3 + assert capped.max_tokens == 64 and capped.top_k == 3 and capped.target_loops == 2 assert SamplingParams().max_tokens == 16 diff --git a/tests/test_stage_engine.py b/tests/test_stage_engine.py index 6290592a..f8fb5efb 100644 --- a/tests/test_stage_engine.py +++ b/tests/test_stage_engine.py @@ -94,13 +94,14 @@ def test_cancelled_import_cannot_activate_reused_request_id(): @pytest.mark.parametrize("atomic", [False, True]) @pytest.mark.parametrize("asynchronous", [False, True]) -def test_independent_stage_resources_preserve_output_after_import(atomic, asynchronous): +@pytest.mark.parametrize("max_loops", [None, 2]) +def test_independent_stage_resources_preserve_output_after_import(atomic, asynchronous, max_loops): p = make_stage_engine(role="prefill", atomic=atomic, asynchronous=asynchronous) d = make_stage_engine(role="decode", atomic=atomic, asynchronous=asynchronous) local = make_stage_engine(atomic=atomic, asynchronous=asynchronous) p.connector = RecordingConnector() prompt = [1, 2, 3, 4, 5] - params = SamplingParams(max_tokens=3, ignore_eos=True) + params = SamplingParams(max_tokens=3, max_loops=max_loops, ignore_eos=True) source = p.reserve_transfer("r", prompt, params, receiving=False) target = d.reserve_transfer("r", prompt, params, receiving=True) assert source is not target @@ -125,6 +126,7 @@ def test_independent_stage_resources_preserve_output_after_import(atomic, asynch assert d.accept_import(target, source.hidden_state.clone()) assert d.scheduler.requests["r"] is target assert target.prefill_reclaimable_tokens == len(prompt) + assert target.loops_done == d.model.config.total_ut_steps assert p.release_transfer(source) local.add_request("r", prompt, params) assert drain(d) == drain(local) diff --git a/vllm_rlt/core/scheduler.py b/vllm_rlt/core/scheduler.py index 14d95950..9c9bce7f 100644 --- a/vllm_rlt/core/scheduler.py +++ b/vllm_rlt/core/scheduler.py @@ -1232,7 +1232,9 @@ def _apply_prefill_phase_progress( def _should_exit(self, request: Request) -> bool: """Evaluate the policy after recording the completed loop's actual hazard.""" params = request.sampling_params - maximum = params.max_loops or self.total_ut_steps + maximum = request.loop_limit(self.total_ut_steps) + if request.target_loops is not None: + return request.loops_done >= maximum return request.loops_done >= maximum or ( params.exit_threshold < 1.0 and request.loops_done >= params.min_loops @@ -1260,7 +1262,9 @@ def _delayed_signal(self, request: Request, score: float, signal_depth: int) -> def needs_exit_signal(self, request: Request) -> bool: """Whether the next submitted loop needs its predecessor's readback.""" params = request.sampling_params - maximum = params.max_loops or self.total_ut_steps + maximum = request.loop_limit(self.total_ut_steps) + if request.target_loops is not None: + return False # No score can end a request whose minimum depth is its maximum. return ( self.exit_mode != "trace" @@ -1322,13 +1326,15 @@ def update_from_output( request.loops_done += 1 if self.exit_mode == "trace": should_exit = self._trace_exit(request) + elif request.target_loops is not None: + should_exit = self._should_exit(request) else: score = result.gate_scores[row] if self.exit_mode == "ouro": request.remaining_probability *= 1.0 - score should_exit = self._should_exit(request) else: - maximum = request.sampling_params.max_loops or self.total_ut_steps + maximum = request.loop_limit(self.total_ut_steps) should_exit = ( request.loops_done >= maximum or request.pending_exit_depth == request.loops_done @@ -1455,12 +1461,10 @@ def update_after_submission( and self.needs_exit_signal(request) ) request.loops_done += 1 - should_exit = ( - self._trace_exit(request) - if self.exit_mode == "trace" - else request.loops_done - >= (request.sampling_params.max_loops or self.total_ut_steps) - ) + if self.exit_mode == "trace": + should_exit = self._trace_exit(request) + else: + should_exit = request.loops_done >= request.loop_limit(self.total_ut_steps) signal = signals.get(row) if signals else None if consume_signal: if ( @@ -1475,7 +1479,8 @@ def update_after_submission( request.pending_exit_depth = request.loops_done update.exited.append(request) elif ( - self.exit_mode in ("ouro_delayed", "random_lookahead") + request.target_loops is None + and self.exit_mode in ("ouro_delayed", "random_lookahead") and request.sampling_params.exit_threshold < 1 ): update.retain_signal.append(row) diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index e16f2e07..a8100f17 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -19,7 +19,7 @@ from vllm_rlt.engine.stage_config import StageConfig from vllm_rlt.profiling import Profiler from vllm_rlt.request import FinishReason, Request, RequestOutput, RequestPhase -from vllm_rlt.sampling_params import SamplingParams +from vllm_rlt.sampling_params import DEFAULT_MIN_LOOPS, SamplingParams from vllm_rlt.worker.executor import Executor from vllm_rlt.worker.model_runner import ModelRunner from vllm_rlt.worker.output import ExitSignal, ModelRunnerOutput @@ -319,6 +319,44 @@ def _execution_ready(self, request): self._connector_blocked.add(request.request_id) return ready + def _resolve_loop_bounds(self, params): + model_depth = self.model.config.total_ut_steps + fixed_loop_count = getattr(self.model, "fixed_loop_count", False) + trace_mode = self.exit_config.mode == "trace" + + if trace_mode: + if params.target_loops is not None and params.target_loops > model_depth: + raise ValueError("target_loops cannot exceed total_ut_steps") + max_loops = params.max_loops or model_depth + if max_loops > model_depth or (not fixed_loop_count and params.min_loops > max_loops): + raise ValueError("requested loop bounds exceed the model's supported depth") + return (None if fixed_loop_count else model_depth), max_loops + + if params.target_loops is not None and not fixed_loop_count: + raise ValueError("target_loops is only supported for fixed-loop models") + if params.target_loops is not None and params.target_loops > model_depth: + raise ValueError("target_loops cannot exceed total_ut_steps") + + if fixed_loop_count: + target_loops = params.target_loops or model_depth + if params.max_loops is not None and params.max_loops != target_loops: + raise ValueError("max_loops must match the fixed target_loops") + if params.min_loops not in (DEFAULT_MIN_LOOPS, target_loops): + raise ValueError( + "min_loops and max_loops must match the fixed target_loops when specified, " + "except the min_loops default" + ) + if target_loops != model_depth: + raise ValueError( + "target_loops below total_ut_steps requires per-request prefill support" + ) + return target_loops, target_loops + + max_loops = params.max_loops or model_depth + if max_loops > model_depth or params.min_loops > max_loops: + raise ValueError("requested loop bounds exceed the model's supported depth") + return model_depth, max_loops + def add_request( self, request_id: str, @@ -349,16 +387,8 @@ def _add_request(self, request_id, prompt_token_ids, sampling_params, *, trace_i raise ValueError("prompt must contain at least one token") if any(type(t) is not int or not 0 <= t < config.vocab_size for t in prompt_token_ids): raise ValueError("prompt token IDs must be integers within the model vocabulary") - max_loops = params.max_loops or config.total_ut_steps - if max_loops > config.total_ut_steps or params.min_loops > max_loops: - raise ValueError("requested loop bounds exceed the model's supported depth") - if getattr(self.model, "fixed_loop_count", False) and ( - max_loops != config.total_ut_steps or params.min_loops != config.total_ut_steps - ): - raise ValueError( - "this model always runs the configured loop count; min_loops and " - "max_loops must both equal it" - ) + fixed_loop_count = getattr(self.model, "fixed_loop_count", False) + target_loops, max_loops = self._resolve_loop_bounds(params) if self.speculative_config is not None and ( max_loops != self.speculative_config.target_loops or params.exit_threshold != 1.0 ): @@ -378,7 +408,8 @@ def _add_request(self, request_id, prompt_token_ids, sampling_params, *, trace_i len(trace) < params.max_tokens or trace[0] != config.total_ut_steps or any( - type(d) is not int or not params.min_loops <= d <= max_loops + type(d) is not int + or not (1 if fixed_loop_count else params.min_loops) <= d <= max_loops for d in trace[1 : params.max_tokens] ) ): @@ -406,6 +437,9 @@ def _add_request(self, request_id, prompt_token_ids, sampling_params, *, trace_i prefill_depth_ranges=Request.prefill_boundaries( len(prompt_token_ids), config.total_ut_steps ), + _target_loops=( + target_loops if fixed_loop_count and self.exit_config.mode != "trace" else None + ), ) ) @@ -661,7 +695,7 @@ def accept_import(self, request, hidden, *, cached_tokens=0): request.num_prefilled_tokens = request.num_preluded_tokens = count request.prefill_window_origin = count request.prefill_reclaimable_tokens = count - request.loops_done = self.model.config.total_ut_steps + request.loops_done = request.target_loops or self.model.config.total_ut_steps request.hidden_state = hidden # Keep RECEIVING until the scheduler grants a compute slot. The lease # protects the imported allocation while it waits alongside local work. diff --git a/vllm_rlt/entrypoints/cli.py b/vllm_rlt/entrypoints/cli.py index a9b8025c..e0d6ecef 100644 --- a/vllm_rlt/entrypoints/cli.py +++ b/vllm_rlt/entrypoints/cli.py @@ -80,6 +80,7 @@ def main(): parser.add_argument("--mode", choices=["refill", "no_refill", "mixed"], default="refill") parser.add_argument("--max-tokens", type=int, default=16) parser.add_argument("--max-loops", type=int, default=None) + parser.add_argument("--request-target-loops", type=int, default=None) parser.add_argument("--min-loops", type=int, default=None) parser.add_argument("--exit-threshold", type=float, default=1.0) parser.add_argument("--temperature", type=float, default=0.0) @@ -138,6 +139,7 @@ def main(): params = SamplingParams( max_tokens=args.max_tokens, max_loops=args.max_loops, + target_loops=args.request_target_loops, min_loops=min_loops, exit_threshold=args.exit_threshold, temperature=args.temperature, diff --git a/vllm_rlt/models/dory.py b/vllm_rlt/models/dory.py index 8e6a388e..5f60482b 100644 --- a/vllm_rlt/models/dory.py +++ b/vllm_rlt/models/dory.py @@ -480,8 +480,7 @@ class DoryForCausalLM(nn.Module): """Inference-only Dory with eager KV boundaries and a graphable recurrent core.""" requires_boundary_kv = True - # No Ouro-style learned exit gate: the reviewed implementation always runs - # the configured loop count, so requesting a shallower depth is unsupported. + # No Ouro-style learned exit gate: a request uses one fixed target depth. fixed_loop_count = True # Fixed loops need no exit signal, so ordinary asynchronous scheduling can # submit a request's next loop without reading the previous one back. diff --git a/vllm_rlt/pd/engine.py b/vllm_rlt/pd/engine.py index cebf8015..cae52f06 100644 --- a/vllm_rlt/pd/engine.py +++ b/vllm_rlt/pd/engine.py @@ -280,6 +280,8 @@ def add_request(self, request_id, prompt_token_ids, sampling_params=None, *, tra raise ValueError("PD pending-request limit reached") params = sampling_params or SamplingParams() cfg = self.model_config + if params.target_loops is not None: + raise ValueError("PD does not support per-request target_loops") if self.speculative_config is not None: if ( params.exit_threshold != 1.0 diff --git a/vllm_rlt/request.py b/vllm_rlt/request.py index ef784437..66322808 100644 --- a/vllm_rlt/request.py +++ b/vllm_rlt/request.py @@ -67,6 +67,21 @@ class Request: compute_released: bool = False finish_reason: FinishReason | None = None exit_trace: tuple[int, ...] = field(default=(), repr=False) + _target_loops: int | None = field(default=None, repr=False) + + def __post_init__(self): + if self._target_loops is not None and ( + type(self._target_loops) is not int or self._target_loops < 1 + ): + raise ValueError("target_loops must be a positive integer") + + @property + def target_loops(self) -> int | None: + return self._target_loops + + def loop_limit(self, default: int) -> int: + """Return the request's fixed target or its legacy loop upper bound.""" + return self.target_loops or self.sampling_params.max_loops or default @staticmethod def prefill_boundaries( diff --git a/vllm_rlt/sampling_params.py b/vllm_rlt/sampling_params.py index 30009bde..640a8011 100644 --- a/vllm_rlt/sampling_params.py +++ b/vllm_rlt/sampling_params.py @@ -1,6 +1,8 @@ import math from dataclasses import dataclass +DEFAULT_MIN_LOOPS = 2 + @dataclass(frozen=True) class SamplingParams: @@ -9,22 +11,27 @@ class SamplingParams: top_p: float = 1.0 top_k: int = -1 seed: int = 0 - min_loops: int = 2 + min_loops: int = DEFAULT_MIN_LOOPS max_loops: int | None = None exit_threshold: float = 1.0 ignore_eos: bool = False priority: int = 0 + target_loops: int | None = None def __post_init__(self): if type(self.priority) is not int: raise ValueError("priority must be an integer") - for name in ("max_tokens", "min_loops", "max_loops"): + for name in ("max_tokens", "min_loops", "max_loops", "target_loops"): value = getattr(self, name) - if name == "max_loops" and value is None: + if name in ("max_loops", "target_loops") and value is None: continue if type(value) is not int or value < 1: raise ValueError(f"{name} must be a positive integer") - if self.max_loops is not None and self.max_loops < self.min_loops: + if ( + self.max_loops is not None + and self.max_loops < self.min_loops + and not (self.target_loops == self.max_loops and self.min_loops == DEFAULT_MIN_LOOPS) + ): raise ValueError("max_loops must be at least min_loops") if not math.isfinite(self.temperature) or self.temperature < 0: raise ValueError("temperature must be finite and nonnegative") diff --git a/vllm_rlt/worker/model_runner.py b/vllm_rlt/worker/model_runner.py index 4bdee10b..ae3766a6 100644 --- a/vllm_rlt/worker/model_runner.py +++ b/vllm_rlt/worker/model_runner.py @@ -1226,13 +1226,11 @@ def submit(self, scheduled_batch: SchedulerOutput | None = None, *, batch: Execu and batch.operation == Operation.RECURRENT and not any( span.emit_gate + and span.request.target_loops is None and self.exit_config.mode in ("ouro_delayed", "random_lookahead") and span.request.sampling_params.exit_threshold < 1 and span.depth + batch.loop_count - < ( - span.request.sampling_params.max_loops - or self.model.config.total_ut_steps - ) + < span.request.loop_limit(self.model.config.total_ut_steps) for span in batch.spans ) ):