diff --git a/examples/rl_algorithms/decoupled-ppo-c2-agentic.json b/examples/rl_algorithms/decoupled-ppo-c2-agentic.json new file mode 100644 index 000000000..ed0b982b3 --- /dev/null +++ b/examples/rl_algorithms/decoupled-ppo-c2-agentic.json @@ -0,0 +1,53 @@ +{ + "advantage": { + "estimator": "grpo", + "reward_binary": false, + "reward_postprocess": null, + "rewards_normalization": true, + "std_normalization": true, + "whiten": false + }, + "correction": { + "batch_normalize": false, + "function": null, + "level": "token", + "lower": 0.0, + "method": "tis", + "mismatch_metrics": false, + "opsm_delta": null, + "ratio_mask": "none", + "upper": 2.0, + "use_rollout_logprobs": false, + "veto": null + }, + "entropy_coef": 0.0, + "execution": { + "group_required": true, + "max_policy_staleness": 4, + "needs_critic": false, + "needs_rollout_logprobs": false + }, + "kl": { + "coef": null, + "estimator": null, + "placement": "none", + "unbiased": false + }, + "loss": { + "aggregation": "default", + "custom_loss": null, + "eps_clip": null, + "eps_clip_c": null, + "eps_clip_high": null, + "reducer": null, + "variant": "policy_loss" + }, + "plugins": [], + "sampling": { + "filter": null, + "max_replacements": null, + "over_sampling_batch_size": null, + "overlong_filter": false + }, + "schema": "yeto-rl-algorithm-spec-v2" +} \ No newline at end of file diff --git a/examples/rl_algorithms/decoupled-ppo-c2.json b/examples/rl_algorithms/decoupled-ppo-c2.json new file mode 100644 index 000000000..b874e8155 --- /dev/null +++ b/examples/rl_algorithms/decoupled-ppo-c2.json @@ -0,0 +1,53 @@ +{ + "advantage": { + "estimator": "grpo", + "reward_binary": false, + "reward_postprocess": null, + "rewards_normalization": true, + "std_normalization": true, + "whiten": false + }, + "correction": { + "batch_normalize": false, + "function": null, + "level": "token", + "lower": 0.0, + "method": "tis", + "mismatch_metrics": false, + "opsm_delta": null, + "ratio_mask": "none", + "upper": 2.0, + "use_rollout_logprobs": false, + "veto": null + }, + "entropy_coef": 0.0, + "execution": { + "group_required": true, + "max_policy_staleness": 2, + "needs_critic": false, + "needs_rollout_logprobs": false + }, + "kl": { + "coef": null, + "estimator": null, + "placement": "none", + "unbiased": false + }, + "loss": { + "aggregation": "default", + "custom_loss": null, + "eps_clip": null, + "eps_clip_c": null, + "eps_clip_high": null, + "reducer": null, + "variant": "policy_loss" + }, + "plugins": [], + "sampling": { + "filter": null, + "max_replacements": null, + "over_sampling_batch_size": null, + "overlong_filter": false + }, + "schema": "yeto-rl-algorithm-spec-v2" +} \ No newline at end of file diff --git a/openspec/changes/rl-offpolicy-correction-passthrough/design.md b/openspec/changes/rl-offpolicy-correction-passthrough/design.md new file mode 100644 index 000000000..2ae50e619 --- /dev/null +++ b/openspec/changes/rl-offpolicy-correction-passthrough/design.md @@ -0,0 +1,120 @@ +# Design + +## Context + +Backend facts are from `infra-drafts/SESSION23/sampler-trainer-startup-explore.md` §9.2–§9.4 and from `infra-drafts/SESSION24/DISCUSS.md` D2 (code reading on verl fork `acad9875` and Miles fork `s17-infra-fork-m`; no GPU run). User decisions are from explore §10 and DISCUSS.md D2 (2026-10-14). + +Terms: +- Start version of a trajectory: the policy version that generated its first response token. +- Version gap of a trajectory: current trainer version minus its start version. +- `k`: the admission limit. A new trajectory is not started when its start version would be more than `k` behind. +- `A_hard`: the consumption limit. It is the existing `max_policy_age`. `A_hard ≥ k`. +- `π_behav`: the per-token logprob that the rollout engine returned (behavior policy). +- `π_prox`: the proximal policy. The trainer recomputes it as the "old logprob" before the update. +- `C`: the upper truncation bound of the importance weight. + +Defaults: `A_hard` keeps its current default, so the default behavior does not change. Recommended values go into example configs only. + +## Decisions + +### D1. Neutral fields, written in full +`CorrectionSpec` gets `method`, `level`, `lower`, `upper`, `veto`, `batch_normalize`, `ratio_mask`. `lower` and `upper` are required when `method` is not `none`. `upper` is `C`. The reason: the backends have different defaults (explore §9.4), so a missing field gives different math on two islands. + +`C` has no agreed default. Miles and verl both default to 2.0 (DISCUSS.md D2). The AReaL example may use 5.0 (unverified). Example configs use 2.0 until a GPU run gives evidence. + +### D2. Decoupled PPO is the training objective +All response tokens enter the loss. For each token: +- importance weight `w = min(π_prox / π_behav, C)`; +- PPO clip ratio `π_θ / π_prox`, clipped as usual. + +This matches decoupled PPO in AReaL. Code reading says both backends can express it without source changes (DISCUSS.md D2, unverified on GPU). + +Miles mapping: +- `--use-tis --tis-clip C` (plus `--tis-clip-low` from `lower`). +- `--use-rollout-logprobs` is refused: it makes `π_rollout` the clip anchor, so `π_prox` is lost. +- `--skip-actor-forward-only` is refused: it skips the forward pass that computes `π_prox`. + +verl mapping: +- `algorithm.rollout_correction.bypass_mode=False`, `rollout_is=token`, `rollout_is_threshold=C`. +- The `fully_async` yaml sets `bypass_mode: True` (`verl/experimental/fully_async_policy/config/fully_async_ppo_trainer.yaml:98`, `fully_async_ppo_megatron_trainer.yaml:97`, checked) and `rollout_is` defaults to `sequence` (DISCUSS.md D2). The adapter writes all three values explicitly. + +`π_prox` differs between backends. Miles recomputes it on the weights before each update. verl takes it from the weights of the first local step of the current cycle (DISCUSS.md D2). The adapter records the anchor policy (`prox_anchor=per_step` or `first_local_step`) in the manifest and the tape. A run that mixes the two across islands gets `correction_config_warning`. + +L6.6 result (code reading, verl fork `acad9875`, checked): no Python code in the fork reads `actor.use_rollout_log_probs`; only the config dataclasses declare it. The `fully_async` yaml value `True` has no effect. With `bypass_mode=False`, `FullyAsyncTrainer._compute_old_log_prob` recomputes `old_log_probs` on the weights of local step 1 of the sync cycle. A CPU test (`tests/test_rl_verl_offpolicy_correction.py`) fails if a reader appears. Not run on GPU. + +### D3. Backend mapping table (ratio masks and other methods) + +| Neutral | Miles | verl | +|---|---|---| +| `tis`, token (default for decoupled PPO) | `--use-tis --tis-clip=upper --tis-clip-low=lower` | `rollout_is=token`, `rollout_is_threshold=upper`; `lower>0` cannot map → warning | +| `ratio_mask=icepop`, token | `--custom-tis-function-path=…icepop_function`, `--tis-clip*` = range | `rollout_is=token`, `rollout_is_threshold="lower_upper"` | +| `ratio_mask=mis`, token or sequence | MIS plugin, `tis_mode=mask`, bounds | `rollout_rs=token_k1` or `seq_mean_k1` (geometric) with bounds | +| `opsm` | `--use-opsm --opsm-delta` | no mapping → warning | +| `veto` | MIS plugin `rs_veto_threshold` | no direct mapping → warning | + +`ratio_mask` defaults to `none`. Rows that say "warning" do not refuse in phase 1. The adapter emits `correction_config_warning` with the field, the backend, and the reason. The event records the setting that actually ran. + +Loss denominator after a ratio mask: Miles keeps the original denominator. The neutral meaning is "denominator unchanged". L6.6 result (CPU, verl and Miles functions loaded from source, checked): verl IcePop (`rollout_is_threshold="lower_upper"`) sets the weight to 0 and keeps `response_mask`, so the denominator is unchanged and the weights equal Miles `icepop_function`. verl rejection sampling (`rollout_rs`, the MIS row) removes tokens from `response_mask`; `batch_num_tokens` is the sum of that mask, so the denominator shrinks. No verl setting keeps it, so the MIS row on verl emits a `ratio_mask` warning. TIS with `lower=0` gives equal weights on both backends; with `lower>0` verl does not raise small weights (warning row in D3). + +### D4. Version gap from the start version +The gap of a trajectory is the current trainer version minus its start version. Miles and yeto compute the gap from the oldest token version in the group. With continued generation the oldest token is the first token, so the two rules give the same value (inferred; not checked case by case). The spec requires a value equal to the start-version rule. L6.12 checks the equality on recorded tapes. If it fails, the adapter computes the start version from `TokenProvenance`. + +L6.12 result (2026-10-15, L6.10 GPU runs, `s1-runs/s24-l6.10/`, `A_hard=1`): the equality holds on all recorded data, but the continued case was not exercised. +- Miles (two runs, 2,560 samples, about 1.2 M response tokens, `--rl-debug-dump`): every response token has a version span. In every sample the first-token version equals the oldest version, and versions never decrease along a sample. No sample spans more than one version (no carry-over happened in Miles `fully_async`), so a continued trajectory across two weight updates is still checked only by the CPU test. +- verl reports only the first and last version per trajectory (`min/max global_steps`). The start version is the first one by construction. In the verl run with thinking on, 54 of 320 groups spanned two versions. No per-token check is possible on verl. +No change to the adapter is needed now. Recheck on a Miles run that carries groups across updates. + +### D5. Three limits +1. Admission (`k`), approximate: + - Miles: buffer capacity plus `--max-weight-staleness` at consumption (`miles/rollout/fully_async_data_buffer.py:197`, checked). + - verl: `async_training.staleness_threshold` sample quota (yaml default `0.1`, `fully_async_ppo_trainer.yaml:12`, checked). + - Neither backend refuses a new trajectory by its exact start version. Exact admission is deferred; the decision depends on the drop ratio in L6.10. +2. In flight: generation continues across weight updates. Miles uses partial rollout; verl uses `partial_rollout=True` (yaml default, `fully_async_ppo_trainer.yaml:23`, checked). Miles `mask_offpolicy_in_partial_rollout` SHALL be off: when on, Miles masks the old-version tokens of a continued sample (`miles/rollout/sglang_rollout.py:291`, checked), which is the version mask this design rejects. The adapter refuses it. +3. Consumption (`A_hard`): a group whose gap exceeds `A_hard` is dropped whole. Miles does this through `--max-weight-staleness`. verl has no drop by version at consumption; the yeto verl adapter adds it at round collection. + +`SUPPORT.max_policy_age` is 1 on both adapters today (`yeto/rl/adapters/miles/policy_age.py:36`, `yeto/rl/adapters/verl/policy_age.py:22`, on `s23-g3-fp`, checked). L6.9 raises the cap so that `A_hard ≥ 2` is allowed. The new cap value is set by L6.9 together with the backend limits. + +### D6. Monitoring +Per round, emit `offpolicy_gate` with: +- counts per reason (`kept`, `group_dropped`, and backend admission drops where the backend reports them); +- a response length histogram of dropped trajectories, with bucket edges fixed in code; +- the ratio of truncated tokens (`w` hit `C`) by turn position. Miles gives token-level data. verl gives only trajectory-level min and max version, so on verl the adapter reports per-trajectory values and marks the field `granularity=trajectory`. + +The dashboard reads this event. + +### D7. `cross_version_is` is observe-only +It does not change the loss today (call site `yeto/rl/adapters/miles/carry_over.py:326` per explore). It keeps the per-version-segment statistics that `PolicyAgeGovernor` needs. Its clip bounds come from `CorrectionSpec.lower/upper`. `StalenessPolicy.correction` derives from the spec instead of the fixed `"tis"`. + +### D8. Phase 1 contract: manifest and tape, not hash +The correction group is removed from the canonical JSON that produces `algorithm_sha256`. A test locks that a change in correction fields does not change the hash in phase 1. The full group, the backend-mapped values, and `prox_anchor` go into the run manifest and each island tape. The launcher compares the groups of all islands at start. A difference gives a warning and an event. + +The MIS plugin path is a PluginRef. Its source hash already enters the spec (`algorithm.py:435` per explore). Phase 1 keeps that. + +### D9. Phase 2 (later) +After phase 1 has one passing GPU run, the correction group enters `algorithm_sha256`, and unmappable settings and cross-island differences are refused at launch (same refusal path as `yeto/rl/adapters/verl/entry.py:34`). This needs a hash migration under `hash-migration.md`. L6b SHALL land at the same version boundary as the semantic hash replacement of change `rl-identity-tiering`, so that the hash changes once. + +## Alternatives considered + +### A1. Two thresholds with version masks (rejected 2026-10-14) +The earlier design had `A_soft ≤ A_hard`. A token gap `≤ A_soft` got TIS. A gap between `A_soft` and `A_hard` got a mask: whole turns for agentic tasks, a token prefix for other tasks. A gap over `A_hard` dropped the group. + +Reasons to reject: +- Position bias. The oldest tokens are always the first tokens or turns. The opening turns get no gradient, often, and reward credit moves to later turns. +- A mask does not fix the real error. Token-level IS cannot correct the state distribution shift. The error term of the policy improvement bound depends on the oldest version used. So the main control is a version limit at rollout, and IS is a supplement. +- Cost: the mask needs per-turn version records and a loss denominator aligned across backends. +- Industry practice (AReaL, Solar Open 2, verl) is continued generation, all tokens in the loss, and IS correction (DISCUSS.md D2). + +`A_soft` is removed. + +### A2. Ratio masks (IcePop, MIS) as the main tool (rejected as default) +A ratio mask looks at how far the policies differ, not how old the sample is. Both backends have it. It stays available through `ratio_mask`, off by default, because it drops signal like a version mask and it does not give an interpretable hard limit. + +### A3. Exact admission now (deferred) +Exact admission needs a Miles source change. The user chose the approximation first and a decision after L6.10. + +## Risks + +- Approximate admission can admit trajectories older than `k`. Consumption drops them at `A_hard`, which wastes rollout compute. The drop ratio in `offpolicy_gate` is the control. +- `π_prox` differs between Miles and verl (D2). Cross-backend metrics are not exactly comparable. +- verl loss denominator after a ratio mask and verl `use_rollout_log_probs` with `bypass_mode=False`: checked on CPU by L6.6 (see D2, D3). Not run on GPU. +- Whether sglang returns `weight_versions` per token is partly verified. In L6.10 the Miles island's SGLang returned version spans for all 2,560 samples and they covered every response token. No version changed inside one generate call, so a call with more than one span was not observed. The truncation-by-turn metric and D4 depend on it. diff --git a/openspec/changes/rl-offpolicy-correction-passthrough/proposal.md b/openspec/changes/rl-offpolicy-correction-passthrough/proposal.md new file mode 100644 index 000000000..e708286d0 --- /dev/null +++ b/openspec/changes/rl-offpolicy-correction-passthrough/proposal.md @@ -0,0 +1,48 @@ +# Proposal + +## Why + +A trainer that starts and stops, and a sampler that keeps generating while the trainer trains, produce samples that are 0 to A versions old. Long trajectories also cross several versions. The trainer must correct these samples, not only filter them. + +Today (source: `infra-drafts/SESSION23/sampler-trainer-startup-explore.md` §5 and §9): +- yeto has one hard filter: a group is dropped when its oldest token is older than the limit (`split_by_age`, `yeto/rl/engine/policy_age.py:115`). +- yeto caps `max_policy_age` at 1 on both backends (`SUPPORT`, `yeto/rl/adapters/miles/policy_age.py:36` and `yeto/rl/adapters/verl/policy_age.py:22` on `s23-g3-fp`). A limit of 2 or more is refused today. +- The neutral `CorrectionSpec.method` has only `none | tis | opsm | custom` (`yeto/rl/engine/algorithm.py:783-792` on `d6c58fb8`). The verl adapter accepts only `none, tis` (`yeto/rl/adapters/verl/config.py:28`). +- Both backends already implement truncated importance sampling (TIS), IcePop, and masked importance sampling (MIS). Their defaults differ (explore §9.4). + +User decisions (2026-10-14; source: `infra-drafts/SESSION24/DISCUSS.md` D2, which replaces the two-threshold mask decision Q3 of explore §9.1): +- Use decoupled PPO, close to AReaL. Do not mask tokens by version. The `A_soft` threshold is removed. +- The version gap of a trajectory is the current trainer version minus the trajectory start version. +- Three limits: admission (do not start a trajectory that is already more than `k` versions behind), in flight (continue generation across weight updates; do not abort), consumption (drop the whole group when the gap is over `A_hard`). +- All response tokens enter the loss. The importance weight is `π_prox / π_behav`, truncated at `C`. The PPO clip anchor is `π_prox`. +- Admission uses existing backend mechanisms. Miles source is not changed. Exact admission is deferred until GPU runs show the drop ratio. +- verl and Miles are both in scope. Each backend uses its own async mode: Miles `fully_async`, verl `verl/experimental/fully_async_policy`. +- Ratio masks (IcePop, MIS) stay in the backends. yeto passes their settings through. They are optional and off by default. +- Phase 1 (explore §10): correction settings go into the run manifest and the tape. A mismatch gives a warning and an event. It does not refuse the launch and it does not enter the algorithm hash. + +## What Changes + +- Extend the neutral `CorrectionSpec` with explicit fields: `method` (`none | tis | icepop | mis_mask | opsm | custom`), `level` (`token | sequence | geometric`), `lower`, `upper`, `veto`, `batch_normalize`, `ratio_mask` (default off). yeto writes every field. No field takes a backend default. +- Map decoupled PPO to both backends. Miles: `--use-tis --tis-clip C`; `--use-rollout-logprobs` and `--skip-actor-forward-only` are refused. verl: `bypass_mode=False`, `rollout_is=token`, `rollout_is_threshold=C`, set explicitly over the `fully_async` yaml defaults. +- Version gap from the trajectory start version. The existing oldest-token rule is accepted only where it equals the start version. +- Admission by approximation: Miles buffer capacity plus `--max-weight-staleness`; verl `async_training.staleness_threshold`. +- In flight: Miles partial rollout with `mask_offpolicy_in_partial_rollout` off; verl `partial_rollout=True`. +- Consumption drop over `A_hard` (`max_policy_age`). Miles has it; yeto adds it in the verl adapter. Raise the `SUPPORT.max_policy_age` cap so that `A_hard ≥ 2` is allowed. +- Monitoring: `offpolicy_gate` with counts per reason, length histogram of dropped trajectories, truncation ratio by turn position. +- Phase 1 warn-only contract. Phase 2 (L6b): the correction group enters `algorithm_sha256`. + +## Capabilities + +### New Capabilities +- `rl-offpolicy-correction`: neutral correction settings passed to the backend, decoupled PPO on Miles and verl async modes, the three version limits, drop and truncation monitoring, and the phase 1 warn-only contract. + +### Modified Capabilities + + +## Impact + +- `yeto/rl/engine/algorithm.py` (`CorrectionSpec`, hash exclusion), `yeto/rl/algos/mismatch_correction.py`. +- `yeto/rl/adapters/miles/algorithm_flags.py`, `yeto/rl/adapters/miles/policy_age.py`, `yeto/rl/adapters/verl/config.py`, `yeto/rl/adapters/verl/policy_age.py`, verl round collection (`fully_async_round.py`). +- `yeto/rl/engine/policy_age.py`, `yeto/rl/engine/island_ledger.py` (`StalenessPolicy.correction` default). +- Run manifest and tape writers; launcher cross-island compare. +- Depends on P0-S per-trajectory version evidence (schema 4) and the S23 integration branch. These are not on `d6c58fb8`; see GOAL.md prerequisites. diff --git a/openspec/changes/rl-offpolicy-correction-passthrough/specs/rl-offpolicy-correction/spec.md b/openspec/changes/rl-offpolicy-correction-passthrough/specs/rl-offpolicy-correction/spec.md new file mode 100644 index 000000000..4f9337934 --- /dev/null +++ b/openspec/changes/rl-offpolicy-correction-passthrough/specs/rl-offpolicy-correction/spec.md @@ -0,0 +1,99 @@ +# Spec Delta + +## Purpose + +Correct off-policy samples with decoupled PPO on the backends, limit sample age at admission and consumption, and record every drop. Phase 1 warns on mismatches and does not refuse. + +## ADDED Requirements + +### Requirement: Correction settings are explicit and passed through +The neutral `CorrectionSpec` SHALL carry `method`, `level`, `lower`, `upper`, `veto`, `batch_normalize`, and `ratio_mask`. When `method` is not `none`, `lower` and `upper` SHALL be given. `ratio_mask` SHALL default to `none`. The adapters SHALL map these fields to backend settings and SHALL NOT compute importance weights in yeto. + +#### Scenario: IcePop on Miles +- **WHEN** the spec has `ratio_mask=icepop`, `level=token`, `lower=0.5`, `upper=5.0` +- **THEN** the Miles flags select the IcePop function with the range 0.5 to 5.0 + +#### Scenario: IcePop on verl +- **WHEN** the same spec runs on verl +- **THEN** the verl config has `rollout_is=token` and `rollout_is_threshold="0.5_5.0"` + +### Requirement: Decoupled PPO objective +Every response token SHALL enter the loss. The importance weight SHALL be `π_prox / π_behav`, truncated at `upper`, where `π_behav` is the per-token rollout logprob. The PPO clip anchor SHALL be `π_prox`, recomputed by the trainer. On Miles the adapter SHALL set `--use-tis --tis-clip ` and SHALL refuse `--use-rollout-logprobs` and `--skip-actor-forward-only`. On verl the adapter SHALL write `bypass_mode=False`, `rollout_is=token`, and `rollout_is_threshold=` explicitly. The manifest and tape SHALL record which weights the backend uses for `π_prox`. + +#### Scenario: verl yaml defaults overridden +- **WHEN** a verl island starts from the `fully_async` yaml that sets `bypass_mode: True` +- **THEN** the effective config has `bypass_mode=False`, `rollout_is=token`, and `rollout_is_threshold` equal to `upper` + +#### Scenario: Forbidden Miles flag +- **WHEN** a Miles island config contains `--use-rollout-logprobs` or `--skip-actor-forward-only` +- **THEN** the adapter refuses the config and names the flag + +#### Scenario: Proximal anchor differs +- **WHEN** a Miles island and a verl island run in one job +- **THEN** each tape records its `prox_anchor`, and the launcher emits one `correction_config_warning` that lists both values + +### Requirement: Version gap from the start version +The version gap of a trajectory SHALL equal the current trainer version minus the trajectory start version. An implementation that uses the oldest token version SHALL give the same value. + +#### Scenario: Continued trajectory +- **WHEN** a trajectory starts at version 3, continues across updates to versions 4 and 5, and the trainer is at version 6 +- **THEN** its gap is 3 + +### Requirement: Three version limits +Admission SHALL use existing backend mechanisms: Miles buffer capacity with `--max-weight-staleness`, and verl `async_training.staleness_threshold`. In-flight generation SHALL continue across weight updates: Miles partial rollout with `mask_offpolicy_in_partial_rollout` off, and verl `partial_rollout=True`. At consumption, a group with gap over `A_hard` SHALL be dropped whole on both backends. `A_hard` SHALL be the existing `max_policy_age`. The adapters SHALL accept `A_hard ≥ 2`. No token SHALL be masked because of its version. + +#### Scenario: Miles partial-rollout mask refused +- **WHEN** a Miles island config enables `mask_offpolicy_in_partial_rollout` +- **THEN** the adapter refuses the config + +#### Scenario: verl drop at consumption +- **WHEN** a verl round collects a group with gap `A_hard + 1` +- **THEN** the yeto verl adapter drops the whole group with reason `group_dropped` + +#### Scenario: A_hard of 2 accepted +- **WHEN** a run sets `max_policy_age=2` +- **THEN** both adapters accept it and map it to their backend limits + +#### Scenario: Defaults keep behavior +- **WHEN** a run does not set new fields +- **THEN** `max_policy_age` keeps its current default and meaning + +### Requirement: Phase 1 warns and does not refuse +In phase 1, a correction setting that a backend cannot map, or a correction group that differs between islands, SHALL produce a `correction_config_warning` event and a log warning. It SHALL NOT refuse the launch. The correction group SHALL NOT change `algorithm_sha256`. Refusals of forbidden flags in other requirements are not affected. + +#### Scenario: TIS lower bound on verl +- **WHEN** a verl island gets `method=tis` with `lower=0.1` +- **THEN** the launch continues and the event names the field `lower`, the backend verl, and the value it ran with + +#### Scenario: Islands differ +- **WHEN** two islands have different `upper` values +- **THEN** each island starts, and the launcher emits one warning that lists both values + +#### Scenario: Hash unchanged +- **WHEN** only the correction group changes between two specs +- **THEN** both specs have the same `algorithm_sha256` + +### Requirement: Correction settings are recorded +The run manifest and each island tape SHALL contain the full correction group and the backend values it mapped to. + +#### Scenario: After a run +- **WHEN** a run ends +- **THEN** the manifest and every island tape contain the correction group, and a reader can compare them without the launch logs + +### Requirement: Drop and truncation monitoring +Each training round SHALL emit `offpolicy_gate` with counts per reason, a fixed-bucket response length histogram of dropped trajectories, and the ratio of truncated importance weights by turn position. When the backend gives only trajectory-level versions, the event SHALL mark the field `granularity=trajectory`. + +#### Scenario: Round with drops +- **WHEN** a round drops two groups of lengths 300 and 5000 tokens +- **THEN** the `group_dropped` histogram has one count in each bucket that holds these lengths + +#### Scenario: verl granularity +- **WHEN** a verl island emits `offpolicy_gate` +- **THEN** the truncation-by-turn field has `granularity=trajectory` + +### Requirement: Both backends use their own async mode +The correction settings and version limits SHALL run on Miles `fully_async` and on verl `fully_async_policy`. yeto SHALL NOT add its own async loop for either backend. + +#### Scenario: verl island with A_hard 2 +- **WHEN** a verl island runs with `max_policy_age=2` and decoupled PPO +- **THEN** it runs through verl `FullyAsyncRollouter` and `FullyAsyncTrainer`, and emits `offpolicy_gate` with the same reasons as a Miles island diff --git a/openspec/changes/rl-offpolicy-correction-passthrough/tasks.md b/openspec/changes/rl-offpolicy-correction-passthrough/tasks.md new file mode 100644 index 000000000..3621e4bda --- /dev/null +++ b/openspec/changes/rl-offpolicy-correction-passthrough/tasks.md @@ -0,0 +1,30 @@ +# Tasks + +Codes follow the phase codes of `infra-drafts/SESSION23/sampler-trainer-startup-explore.md` §8. This change holds L6 (phase 1) and L6b (phase 2). Effort is in sub-agent person-days and is an estimate. The 2026-10-14 revision (DISCUSS.md D2) removed the old L6.7 version-mask task. The new D2 work (L6.7, L6.9, L6.12, L6.13, and the verl drop in L6.11) is about 3–4 d without exact admission (estimate from DISCUSS.md material). + +## L6. Phase 1: pass-through, decoupled PPO, version limits, monitoring + +- [x] L6.1 Extend `CorrectionSpec` (design D1) and `CORRECTION_METHODS`, including `ratio_mask` (default `none`); validation of required bounds. Acceptance: CPU unit tests for valid and invalid specs. GPU: no. Estimate: 0.5 d +- [x] L6.2 Exclude the correction group from `algorithm_sha256`; lock it with a test. Write the group, mapped values, and `prox_anchor` to the run manifest and island tape (design D8). Acceptance: CPU tests (hash unchanged; manifest and tape contain the group). GPU: no. Estimate: 1 d +- [x] L6.3 Miles mapping of design D3 rows (IcePop through the custom function path, MIS through the plugin config). Acceptance: CPU tests on generated flags for each row. GPU: no. Estimate: 1 d +- [x] L6.4 verl mapping of design D3 rows: extend `SUPPORTED_CORRECTIONS`; warnings for unmappable fields; denominator warning when `ratio_mask` is on. Acceptance: CPU tests on the generated config and on `correction_config_warning`. GPU: no. Estimate: 1 d +- [x] L6.5 Launcher cross-island compare with warning event, including `prox_anchor`. Acceptance: CPU test with two islands that differ. GPU: no. Estimate: 0.5 d +- [x] L6.6 CPU parity test: same logprobs through the Miles and verl TIS and IcePop functions; compare weight tensors; record the verl mask denominator; check verl `use_rollout_log_probs` with `bypass_mode=False`. Acceptance: test passes or the difference is documented in design.md. GPU: no. Estimate: 1 d (result: TIS with lower bound > 0 differs between backends and gives a warning; IcePop agrees; verl rejection sampling shrinks the denominator, so MIS on verl gives a warning) +- [x] L6.7 Decoupled PPO mapping (design D2). Miles: `--use-tis --tis-clip C`; refuse `--use-rollout-logprobs`, `--skip-actor-forward-only`, `mask_offpolicy_in_partial_rollout`. verl: write `bypass_mode=False`, `rollout_is=token`, `rollout_is_threshold=C` over the `fully_async` yaml defaults; `partial_rollout=True`. Example configs with `C=2.0`. Acceptance: CPU tests on generated flags and overrides; each refused flag gives a clear error. GPU: no. Estimate: 1 d +- [x] L6.8 `offpolicy_gate` event (design D6): counts per reason, dropped-length histogram, truncation ratio by turn position (token level on Miles, trajectory level on verl). Dashboard reads it. Acceptance: CPU test on the event for both backends; dashboard view reviewed by the user by hand. GPU: no. Estimate: 1 d +- [x] L6.9 Raise the `SUPPORT.max_policy_age` cap on Miles and verl adapters so that `A_hard ≥ 2` is allowed; set the new cap from backend limits; map admission (`k`) to Miles buffer capacity plus `--max-weight-staleness` and verl `staleness_threshold`. Acceptance: CPU tests that `A_hard=2` passes `SUPPORT.check` and maps to both backends; existing policy-age tests pass. GPU: no. Estimate: 0.5 d +- [x] L6.10 GPU check across backends: small model, one Miles island (Miles `fully_async`) and one verl island (verl `fully_async_policy`), decoupled PPO, `A_hard=2`, `C=2.0`. Depends on L6.11. Acceptance: manifest and tapes of both islands show the group and `prox_anchor`; `offpolicy_gate` events from both, with the drop ratio recorded; backend metrics (`rollout_corr/*`, `tis*`) present; record whether sglang returns `weight_versions` per token; training runs to the planned round count. The drop ratio feeds the exact-admission decision. GPU: yes, review before launch. Estimate: 1 d + about $11 (2×1 H100 for about 1 h at $5.3/card-hour; to be fixed at review) (GPU verified 2026-10-15, evidence: infra-drafts/SESSION24/gpu/L6.10-review.md and infra-drafts/s1-runs/s24-l6.10/; ran as two single-island launches with A_hard=1; verl drop branch triggered, 1 group in round 4, 0.31%; not verified: Miles drop branch, not triggered) +- [x] L6.11 verl async path: run the correction mapping on verl `verl/experimental/fully_async_policy` (`FullyAsyncRollouter`, `MessageQueue`, `FullyAsyncTrainer`; verl fork `acad9875`). Reuse the yeto adapter (`yeto/rl/adapters/verl/island_entry.py` `fully_async_plan`, `fully_async_round.py`, `fully_async_runner.py`). Add the consumption drop over `A_hard` at round collection (design D5). Do not use `one_step_off_policy`. Close the gaps listed in S23 P0-S-Q1..Q4. Acceptance: CPU tests on overrides and on the drop at round collection; one verl-only GPU smoke run before L6.10. GPU: yes, review before launch. Estimate: 2.5–3 d + about 1 card-hour (about $5) (GPU verified 2026-10-14, evidence: infra-drafts/SESSION24/gpu/L6.11-review.md; verl single island, 5 rounds, backend=verl, bypass=False, rollout_is=token, C=2.0; not verified: A_hard drop branch, drop=0) +- [x] L6.12 Check that the oldest-token gap equals the start-version gap (design D4) on recorded Miles and verl tapes; if not, compute the start version from `TokenProvenance`. Acceptance: CPU test on a continued trajectory across two weight updates; result written to design.md. GPU: no. Estimate: 0.5 d +- [x] L6.13 `cross_version_is` observe-only with bounds from the spec; `StalenessPolicy.correction` derived (design D7). Acceptance: existing governor tests pass; new test that bounds follow the spec. GPU: no. Estimate: 0.5 d +- [x] L6.14 Count the Miles buffer stale drops in `offpolicy_gate` with the reason code `backend_stale_dropped`, separate from the yeto gate drops. Acceptance: CPU test on the event. GPU: no. Estimate: 0.5 d +- [x] L6.15 The verl island writes the correction record (group, `prox_anchor`, warnings) to its tape; `RolloutBatchHandle` carries a `backend` field so that verl events are not labelled as Miles. Acceptance: CPU tests; default spec writes nothing and golden samples stay the same. GPU: no. Estimate: 0.5 d + +Total L6 (estimate): about 12–12.5 d plus GPU cost of about $16. Exact admission is not included. + +## L6b. Phase 2: tighten (after L6.10 passes) + +L6b lands at the same version boundary as the semantic hash replacement of change `rl-identity-tiering`. + +- [x] L6b.1 Put the correction group into `algorithm_sha256`; hash migration and golden samples per `hash-migration.md`. Acceptance: CPU tests; golden samples regenerated. GPU: no. Estimate: 1 d +- [x] L6b.2 Refuse unmappable settings and cross-island differences at launch, and also at island start. Acceptance: CPU tests for each design D3 "warning" row. GPU: no. Estimate: 0.5 d diff --git a/openspec/changes/yeto-framework-decoupling/hash-migration.md b/openspec/changes/yeto-framework-decoupling/hash-migration.md index faa9f730a..afaf366dd 100644 --- a/openspec/changes/yeto-framework-decoupling/hash-migration.md +++ b/openspec/changes/yeto-framework-decoupling/hash-migration.md @@ -273,6 +273,12 @@ design D6：去耦合后 `AlgorithmSpec.sha256()` 按中立名与新源码重新 - Only change: the source hash of `yeto/rl/adapters/miles/rollout_meta_hook.py` `1188451c…` → `a4fa982b…` (the three plugin entries in `plugins`, all 8 samples). Cause: fully_async rounds now report `execution_mode` on the carry-over event (ff6105fc). `algorithm_sha256`, contract hashes, `ports_runtime_fingerprint` and the Miles argv are unchanged. - Regenerated with `python tests/decoupling_golden.py --write`. +## S24 L6.2 correction group out of the hash (rl-offpolicy-correction-passthrough D8, branch s24-l6-core, 2026-10-14) + +- Cause: in phase 1 the correction group is recorded in the run manifest and island tapes, not hashed. `AlgorithmSpec.sha256()` hashes a fixed default correction group (`identity_dict()`); `canonical_json()` still carries the full group, now with the neutral field names (`level`, `lower`, `upper`, `veto`, `batch_normalize`, `ratio_mask`; `tis_clip`/`tis_clip_low` are accepted input aliases). A custom correction function's source hash stays in the identity as a plugin reference. +- Affected golden samples (3 of 8): `grpo_tis.json` — `algorithm_sha256` `5a8a5a9f…` → `27df1133…` (the default-spec hash), with the derived launcher/learner/execution-profile fields; `drgrpo.json` and `seq_adv_maxrl.json` — `algorithm_spec` text only (new field names), hashes unchanged. The other 5 samples and `fake_engine_tapes.json` are unchanged. +- Hash boundary: runs with a TIS (or other non-default) correction group recorded before this change carry the old hash; runs after it carry the hash of the same spec without its correction group. Compare across the boundary by the recorded `algorithm_spec`, not by hash. Phase 2 (L6b.1) puts the group back into the hash at the same version boundary as the `rl-identity-tiering` semantic hash. +- Accepted by the main agent (SESSION24 GOAL §6). Regenerated with `python tests/decoupling_golden.py --write`; the diff touches only these 3 files. ## S24 cold-start image pin (branch s24-image, 2026-10-14) - `MILES_NEXT_COMMIT` 386ae3b07 → 17aeb5cb5 (michaellchung/miles `s24-miles-import`: light Ray actors stay off `megatron_utils`, rl-trainer-cold-start-cost L2.1; master port held until use, S23 defect 9). No loss path change; appended to `MILES_FORK_LINEAGE`. `MILES_NEXT_IMAGE` `@sha256:bcd5ffd3…` → `@sha256:b304f857…` (tag 17aeb5c-2fa8801; the layer also carries peft 0.20.0 and the lazy-modelopt Megatron-Bridge `__init__.py`, L2.2–L2.4; build record `openspec/changes/rl-infra-spec/evidence/ports-image/2026-10-14-17aeb5c-2fa8801/`). diff --git a/tests/golden/decoupling/drgrpo.json b/tests/golden/decoupling/drgrpo.json index 303ed917d..e41802e86 100644 --- a/tests/golden/decoupling/drgrpo.json +++ b/tests/golden/decoupling/drgrpo.json @@ -12,13 +12,17 @@ "whiten": false }, "correction": { + "batch_normalize": false, "function": null, + "level": "token", + "lower": null, "method": "none", "mismatch_metrics": false, "opsm_delta": null, - "tis_clip": null, - "tis_clip_low": null, - "use_rollout_logprobs": false + "ratio_mask": "none", + "upper": null, + "use_rollout_logprobs": false, + "veto": null }, "entropy_coef": 0.0, "execution": { diff --git a/tests/golden/decoupling/grpo_tis.json b/tests/golden/decoupling/grpo_tis.json index 4a4a8bd28..b6dbc2e3b 100644 --- a/tests/golden/decoupling/grpo_tis.json +++ b/tests/golden/decoupling/grpo_tis.json @@ -1,7 +1,7 @@ { - "algorithm_sha256": "5a8a5a9ff5abb317fc8159ab56c07293dc3bc99e084542880b6d32cc5a02b441", - "algorithm_sha256_launcher_expected": "5a8a5a9ff5abb317fc8159ab56c07293dc3bc99e084542880b6d32cc5a02b441", - "algorithm_sha256_learner_ports_spec": "5a8a5a9ff5abb317fc8159ab56c07293dc3bc99e084542880b6d32cc5a02b441", + "algorithm_sha256": "6a33a8e904537257d7d58517557368c7c1663beb0c042c1f714c06b0a231ce77", + "algorithm_sha256_launcher_expected": "6a33a8e904537257d7d58517557368c7c1663beb0c042c1f714c06b0a231ce77", + "algorithm_sha256_learner_ports_spec": "6a33a8e904537257d7d58517557368c7c1663beb0c042c1f714c06b0a231ce77", "algorithm_spec": { "advantage": { "estimator": "grpo", @@ -12,13 +12,17 @@ "whiten": false }, "correction": { + "batch_normalize": false, "function": null, + "level": "token", + "lower": 0.0, "method": "tis", "mismatch_metrics": false, "opsm_delta": null, - "tis_clip": 2.0, - "tis_clip_low": 0.0, - "use_rollout_logprobs": false + "ratio_mask": "none", + "upper": 2.0, + "use_rollout_logprobs": false, + "veto": null }, "entropy_coef": 0.0, "execution": { @@ -61,10 +65,10 @@ "sha256": "aeee22abefe0a1e756fe8e23e5f2af3931a3eee946b355bc0044cace2a365ea6" }, "execution_profile": { - "contract_hash": "sha256:ff3574071f62f2b71fb1eb71f5e32b71e272d1345b5cb280100429ab8a8c10f8", + "contract_hash": "sha256:a0805cacf3cfc5d515caa8418aa031c5f1ec1d2c3bd6ef56f6b67e71c95ce740", "profile": { "algorithm_contract": "on-policy", - "algorithm_spec_sha256": "5a8a5a9ff5abb317fc8159ab56c07293dc3bc99e084542880b6d32cc5a02b441", + "algorithm_spec_sha256": "6a33a8e904537257d7d58517557368c7c1663beb0c042c1f714c06b0a231ce77", "allowed_overlap": [], "execution_mode": "partitioned-serial", "groups_per_batch": 4, @@ -80,7 +84,7 @@ "schema": "yeto-rl-execution-profile-v1" } }, - "launch_algorithm_sha256": "5a8a5a9ff5abb317fc8159ab56c07293dc3bc99e084542880b6d32cc5a02b441", + "launch_algorithm_sha256": "6a33a8e904537257d7d58517557368c7c1663beb0c042c1f714c06b0a231ce77", "launch_cli": [ "--rl-placement", "fixed-partition", diff --git a/tests/golden/decoupling/seq_adv_maxrl.json b/tests/golden/decoupling/seq_adv_maxrl.json index b63f918b4..af43aa566 100644 --- a/tests/golden/decoupling/seq_adv_maxrl.json +++ b/tests/golden/decoupling/seq_adv_maxrl.json @@ -16,13 +16,17 @@ "whiten": false }, "correction": { + "batch_normalize": false, "function": null, + "level": "token", + "lower": null, "method": "none", "mismatch_metrics": false, "opsm_delta": null, - "tis_clip": null, - "tis_clip_low": null, - "use_rollout_logprobs": false + "ratio_mask": "none", + "upper": null, + "use_rollout_logprobs": false, + "veto": null }, "entropy_coef": 0.0, "execution": { diff --git a/tests/test_rl_algorithm_flags.py b/tests/test_rl_algorithm_flags.py index 7537beab9..917864ae2 100644 --- a/tests/test_rl_algorithm_flags.py +++ b/tests/test_rl_algorithm_flags.py @@ -218,7 +218,6 @@ def test_default_and_v1_argv_unchanged(): tis_clip_low=0.5, mismatch_metrics=True)), ["--use-tis", "--tis-clip", "5.0", "--tis-clip-low", "0.5", "--custom-tis-function-path", REF.path, "--get-mismatch-metrics"]), - (dict(correction=CorrectionSpec(use_rollout_logprobs=True)), ["--use-rollout-logprobs"]), (dict(correction=CorrectionSpec(method="opsm", opsm_delta=1e-4)), ["--use-opsm", "--opsm-delta", "0.0001"]), ] diff --git a/tests/test_rl_algorithm_provenance.py b/tests/test_rl_algorithm_provenance.py index c636bf82f..1bc2670ac 100644 --- a/tests/test_rl_algorithm_provenance.py +++ b/tests/test_rl_algorithm_provenance.py @@ -287,11 +287,11 @@ def test_export_cli_reads_spec_file(tmp_path, monkeypatch): def test_selection_event_records_absorbed_flags_and_allowance(): - launch = _launch(AlgorithmSpec(), ("--eps-clip-high", "0.28", "--use-rollout-logprobs")) + launch = _launch(AlgorithmSpec(), ("--eps-clip-high", "0.28", "--normalize-advantages")) event = selection_event(launch=launch, algorithm=launch.algorithm, miles_commit="x", unverified_mechanisms=("features:clip_higher",)) assert event["rl/algorithm_absorbed_flags"] == {"--eps-clip-high": "0.28", - "--use-rollout-logprobs": True} + "--normalize-advantages": True} assert event["rl/unverified_mechanisms"] == ["features:clip_higher"] assert event["rl/algorithm_spec_sha256"] == launch.algorithm.sha256() plain = selection_event(launch=_launch(AlgorithmSpec()), algorithm=AlgorithmSpec(), diff --git a/tests/test_rl_algorithm_spec_v2.py b/tests/test_rl_algorithm_spec_v2.py index e4d89957f..482db1dec 100644 --- a/tests/test_rl_algorithm_spec_v2.py +++ b/tests/test_rl_algorithm_spec_v2.py @@ -69,10 +69,10 @@ (lambda: KlSpec(unbiased="yes"), "kl.unbiased"), (lambda: AdvantageSpec(estimator="dapo"), "advantage.estimator"), (lambda: AdvantageSpec(whiten=1), "advantage.whiten"), - (lambda: CorrectionSpec(method="tis"), "correction.tis_clip"), + (lambda: CorrectionSpec(method="tis"), "correction.upper"), (lambda: CorrectionSpec(method="opsm"), "correction.opsm_delta"), (lambda: CorrectionSpec(method="bogus"), "correction.method"), - (lambda: CorrectionSpec(method="tis", tis_clip=-1, tis_clip_low=0), "correction.tis_clip"), + (lambda: CorrectionSpec(method="tis", tis_clip=-1, tis_clip_low=0), "correction.upper"), (lambda: SamplingSpec(max_replacements=-1, filter=BOUNDED_NONZERO_STD_FILTER), "sampling.max_replacements"), (lambda: SamplingSpec(over_sampling_batch_size=0), "sampling.over_sampling_batch_size"), @@ -155,7 +155,6 @@ def test_r0_golden_hashes_unchanged(): dict(advantage=AdvantageSpec(std_normalization=False)), dict(advantage=AdvantageSpec(estimator="gspo")), dict(kl=KlSpec(placement="loss", coef=0.01, estimator="k1")), - dict(correction=CorrectionSpec(method="opsm", opsm_delta=1e-4)), dict(sampling=SamplingSpec(over_sampling_batch_size=8)), dict(execution=ExecutionSpec(needs_rollout_logprobs=True)), dict(entropy_coef=0.01), @@ -169,6 +168,13 @@ def test_new_field_switches_to_v2_and_changes_hash(change): assert spec.sha256() != BOUNDED.sha256() +def test_correction_group_switches_to_v2_and_is_hashed(): + # rl-offpolicy-correction-passthrough L6b.1: the group is hashed. + spec = AlgorithmSpec(correction=CorrectionSpec(method="opsm", opsm_delta=1e-4)) + assert json.loads(spec.canonical_json())["schema"] == ALGORITHM_SPEC_SCHEMA_V2 + assert spec.sha256() != AlgorithmSpec().sha256() + + def test_int_float_and_order_equivalence(): a = AlgorithmSpec.from_dict({"schema": ALGORITHM_SPEC_SCHEMA_V2, "loss": {"eps_clip": 0.2, "eps_clip_high": 0.28}, diff --git a/tests/test_rl_miles_carry_over.py b/tests/test_rl_miles_carry_over.py index 1e11921f0..e94f3d267 100644 --- a/tests/test_rl_miles_carry_over.py +++ b/tests/test_rl_miles_carry_over.py @@ -239,11 +239,12 @@ def test_governor_fallback_lowers_the_rollout_limit_through_the_sink(sink, tmp_p def test_miles_stage_two_support_and_single_turn_check(): from yeto.rl.adapters.miles.policy_age import SUPPORT, check_task, policy_age_argv - # agentic-rollout-utilization 5 (stage 3) raised the declared stage; limit still <= 1 - assert (SUPPORT.stage, SUPPORT.max_policy_age) == (3, 1) + # agentic-rollout-utilization 5 (stage 3); L6.9 raised the cap to 4 (A_hard >= 2) + assert (SUPPORT.stage, SUPPORT.max_policy_age) == (3, 4) SUPPORT.check(1) - with pytest.raises(PolicyAgeError, match="at most 1"): - SUPPORT.check(2) + SUPPORT.check(2) + with pytest.raises(PolicyAgeError, match="at most 4"): + SUPPORT.check(5) assert policy_age_argv(0) == () and policy_age_argv(1) == ("--partial-rollout",) check_task(1) check_task(0, custom_agent="x.y", custom_generate="x.z") diff --git a/tests/test_rl_mismatch_correction.py b/tests/test_rl_mismatch_correction.py index cce300a96..19f6c457d 100644 --- a/tests/test_rl_mismatch_correction.py +++ b/tests/test_rl_mismatch_correction.py @@ -130,7 +130,7 @@ def test_observe_translation_and_identity(): [ (lambda: spec(method="custom", function=mc.observe_ref().to_dict()), "mismatch_metrics"), (lambda: spec(method="custom", function=mc.observe_ref().to_dict(), - mismatch_metrics=True, tis_clip=2.0), "tis_clip"), + mismatch_metrics=True, tis_clip=2.0), "correction.upper"), (lambda: spec(method="custom", function=mc.observe_ref().to_dict(), mismatch_metrics=True, use_rollout_logprobs=True), "tis_with_rollout_logprobs"), @@ -193,15 +193,15 @@ def test_icepop_translation_and_hash(): "--custom-tis-function-path", mc.ICEPOP_PATH, "--get-mismatch-metrics", ] assert s.correction.function.sha256 == mc.ICEPOP_SOURCE_SHA256 - assert icepop(low=0.4).sha256() != s.sha256() + assert icepop(low=0.4).sha256() != s.sha256() # L6b.1: the bounds are hashed @pytest.mark.parametrize( "build, needle", [ (lambda: spec(method="custom", function=mc.icepop_ref().to_dict(), tis_clip=5.0), - "tis_clip_low"), - (lambda: spec(method="custom", function=mc.icepop_ref().to_dict()), "tis_clip"), + "correction.lower"), + (lambda: spec(method="custom", function=mc.icepop_ref().to_dict()), "correction.upper"), (lambda: icepop(low=5.0, high=0.5), "5.0"), (lambda: icepop(low=5.0, high=5.0), "smaller"), ], @@ -310,7 +310,7 @@ def test_opsm_source_identity(): trainer, rollout = opsm(), opsm(source="rollout") assert trainer.correction.opsm_old_logprob_source == "trainer" assert opsm(source="trainer").sha256() == trainer.sha256() - assert trainer.sha256() != rollout.sha256() + assert trainer.sha256() != rollout.sha256() # L6b.1: the group is hashed assert json.loads(rollout.canonical_json())["correction"]["opsm_old_logprob_source"] == "rollout" # Not selecting OPSM: the field (at its default) does not affect the hash. assert spec(opsm_old_logprob_source="trainer").sha256() == AlgorithmSpec().sha256() @@ -358,6 +358,7 @@ def test_mis_translation(): "tis_lower_bound": 0.999, "tis_mode": "mask", "tis_upper_bound": 1.001, "use_rs": False, } assert s.correction.function.sha256 == PluginRef.from_path(mc.MIS_PATH).sha256 + # L6b.1: MIS fields are hashed assert mis(mode="clip", low=0.5).sha256() != mis(mode="mask", low=0.5).sha256() @@ -376,7 +377,7 @@ def test_runtime_attrs_unchanged_without_mis(): (lambda: mis(mode="truncate", low=0.5), "unused"), (lambda: mis(mode="clip", low=3.0, high=2.0), "smaller"), (lambda: mis(level="geometric", mis_batch_normalize=True), "token|sequence"), - (lambda: mis(tis_clip=2.0), "tis_clip"), + (lambda: mis(tis_clip=2.0), "correction.lower/upper"), (lambda: spec(mis_mode="mask"), "require the MIS function"), (lambda: tis(mis_upper_bound=2.0), "require the MIS function"), ], diff --git a/tests/test_rl_offpolicy_correction.py b/tests/test_rl_offpolicy_correction.py new file mode 100644 index 000000000..ac3555bb8 --- /dev/null +++ b/tests/test_rl_offpolicy_correction.py @@ -0,0 +1,471 @@ +"""rl-offpolicy-correction-passthrough (CPU): neutral correction fields, +hash (L6b.1), Miles mapping, decoupled PPO refusals, refusals (L6b.2), +offpolicy_gate, A_hard >= 2, start-version gap and observe-only bounds.""" + +from __future__ import annotations + +import json +from types import SimpleNamespace + +import pytest + +from yeto.rl.adapters.miles import carry_over +from yeto.rl.adapters.miles import offpolicy_correction as mo +from yeto.rl.adapters.miles.algorithm_flags import algorithm_argv +from yeto.rl.adapters.miles.config import MilesConfigError, check_extra_argv +from yeto.rl.algos import mismatch_correction as mc +from yeto.rl.engine import offpolicy_correction as oc +from yeto.rl.engine.algorithm import AlgorithmSpec, AlgorithmSpecError, CorrectionSpec +from yeto.rl.engine.island_ledger import StalenessPolicy +from yeto.rl.engine.policy_age import PolicyAgeError +from yeto.rl.engine.version_segments import TokenProvenance + + +def decoupled(upper: float = 2.0, **kw) -> AlgorithmSpec: + """Decoupled PPO with C = upper (the example-config value is 2.0).""" + return AlgorithmSpec(correction=CorrectionSpec(method="tis", lower=0.0, upper=upper, **kw)) + + +# -- L6.1 neutral fields ------------------------------------------------------ + + +def test_fields_written_in_full_with_defaults(): + group = CorrectionSpec(method="tis", lower=0.0, upper=2.0).to_dict() + for name in ("method", "level", "lower", "upper", "veto", "batch_normalize", "ratio_mask"): + assert name in group + assert (group["level"], group["ratio_mask"], group["veto"], group["batch_normalize"]) == ( + "token", "none", None, False) + + +def test_old_bound_names_are_aliases(): + old = CorrectionSpec(method="tis", tis_clip=2.0, tis_clip_low=0.1) + assert old == CorrectionSpec(method="tis", upper=2.0, lower=0.1) + assert (old.tis_clip, old.tis_clip_low) == (2.0, 0.1) + assert CorrectionSpec.from_dict({"method": "tis", "tis_clip": 2, "tis_clip_low": 0}) == \ + CorrectionSpec(method="tis", upper=2.0, lower=0.0) + with pytest.raises(AlgorithmSpecError, match="old name of correction.upper"): + CorrectionSpec(method="tis", tis_clip=2.0, upper=3.0, lower=0.0) + + +@pytest.mark.parametrize("kw, needle", [ + (dict(method="tis", upper=2.0), "correction.upper and correction.lower"), + (dict(method="tis", lower=0.0), "correction.upper and correction.lower"), + (dict(ratio_mask="icepop"), "requires correction.method='tis'"), + (dict(method="tis", lower=0.0, upper=2.0, ratio_mask="bogus"), "correction.ratio_mask"), + (dict(method="tis", lower=0.0, upper=2.0, level="turn"), "correction.level"), + (dict(method="tis", lower=0.0, upper=2.0, level="geometric"), "ratio_mask='mis'"), + (dict(veto=1e-4), "correction.veto"), + (dict(batch_normalize=True), "correction.batch_normalize"), + (dict(level="sequence"), "correction.level"), + (dict(method="tis", lower=0.0, upper=2.0, veto=0.0), "correction.veto"), + (dict(method="tis", lower=0.0, upper=2.0, batch_normalize=1), "correction.batch_normalize"), +]) +def test_invalid_specs(kw, needle): + with pytest.raises(AlgorithmSpecError, match=needle.replace(".", r"\.").replace("(", r"\(")): + CorrectionSpec(**kw) + + +def test_valid_specs(): + CorrectionSpec(method="tis", lower=0.5, upper=5.0, ratio_mask="icepop") + CorrectionSpec(method="tis", lower=0.5, upper=2.0, ratio_mask="mis", level="geometric", + veto=1e-4, batch_normalize=True) + assert not decoupled().rejections() + assert any("smaller" in p for p in + AlgorithmSpec(correction=CorrectionSpec(method="tis", lower=3.0, upper=2.0)) + .rejections()) + + +# -- L6.2 / L6b.1 hash and records ------------------------------------------- + + +@pytest.mark.parametrize("correction", [ + CorrectionSpec(method="tis", lower=0.0, upper=2.0), + CorrectionSpec(method="tis", lower=0.0, upper=5.0), + CorrectionSpec(method="tis", lower=0.5, upper=5.0, ratio_mask="icepop"), + CorrectionSpec(method="tis", lower=0.5, upper=2.0, ratio_mask="mis", veto=1e-4), + CorrectionSpec(method="opsm", opsm_delta=1e-4), +]) +def test_correction_group_changes_the_hash(correction): + # L6b.1: the correction group is in the hash + base = AlgorithmSpec(entropy_coef=0.01) + spec = base.replace(correction=correction) + assert spec.sha256() != base.sha256() + assert AlgorithmSpec.from_dict(json.loads(spec.canonical_json())) == spec + + +def test_each_correction_field_changes_the_hash(): + tis = AlgorithmSpec(correction=CorrectionSpec(method="tis", lower=0.0, upper=2.0)) + variants = [tis.replace(correction=CorrectionSpec(method="tis", lower=0.0, upper=5.0)), + tis.replace(correction=CorrectionSpec(method="tis", lower=0.5, upper=2.0)), + tis.replace(correction=CorrectionSpec(method="tis", lower=0.0, upper=2.0, + level="sequence"))] + assert len({tis.sha256(), *(v.sha256() for v in variants)}) == 4 + + +def test_default_hashes_are_unchanged(): + # values from before the change (base c4ae38a6) + assert AlgorithmSpec().sha256() == \ + "27df1133c924e7a337e90246c7e1e30699017f461e9f35b1dd165a0263e65bea" + assert AlgorithmSpec(loss={"variant": "policy_loss", "eps_clip": 0.3}).sha256() == \ + "64f4876cb7a99d2650731844dfc7da29dbbe4b638dc3837f5916cf4e98c4f2f4" + assert AlgorithmSpec(entropy_coef=0.1).sha256() == \ + "e5a86b389f80e46d9e36bf5942687fc9d9a14106aac34159c680952724251e82" + + +def test_custom_function_source_hash_stays_in_the_identity(): + ref = mc.icepop_ref() + spec = AlgorithmSpec(correction={"method": "custom", "function": ref.to_dict(), + "tis_clip": 5.0, "tis_clip_low": 0.5}) + other = AlgorithmSpec(correction={"method": "custom", "tis_clip": 5.0, "tis_clip_low": 0.5, + "function": {"path": ref.path, "sha256": "1" * 64}}) + assert spec.sha256() != other.sha256() != AlgorithmSpec().sha256() + + +def test_manifest_section_records_group_mapping_and_anchor(): + spec = decoupled() + section = oc.run_correction_manifest(spec, {"0": "miles", "1": "miles"}) + record = section["islands"]["0"] + assert record["group"] == spec.correction.to_dict() + assert record["mapped"] == {"--use-tis": True, "--tis-clip": "2.0", "--tis-clip-low": "0.0"} + assert record["prox_anchor"] == "per_step" + assert section["warnings"] == [] + + +def test_backend_without_mapping_is_recorded_and_warned(): + from yeto.rl.engine import backends + + backends.register(backends.BackendEntry("fake", "x", {})) + try: + section = oc.run_correction_manifest(decoupled(), {"0": "fake"}) + assert section["islands"]["0"]["mapped"] is None + assert [w["backend"] for w in section["warnings"]] == ["fake"] + # a default group needs no mapping: no warning + assert oc.run_correction_manifest(AlgorithmSpec(), {"0": "fake"})["warnings"] == [] + finally: + backends.unregister("fake") + + +def test_island_tape_gets_record_and_warnings_then_refuses(tmp_path): + # L6b.2: the tape keeps the record and the warning; the island start is refused + from yeto.rl.adapters.miles import island_entry + + events = [] + island_entry_append = island_entry._append_ports_event + island_entry._append_ports_event = lambda _a, _m, e: events.append(e) + try: + island_entry._append_correction_record(SimpleNamespace(), SimpleNamespace(), + AlgorithmSpec()) + assert events == [] # default tapes unchanged + spec = decoupled(level="sequence") + with pytest.raises(oc.CorrectionConfigError, match="level on backend 'miles'"): + island_entry._append_correction_record(SimpleNamespace(), SimpleNamespace(), spec) + finally: + island_entry._append_ports_event = island_entry_append + assert events[0]["event"] == oc.CORRECTION_RECORD_EVENT + assert events[0]["group"]["level"] == "sequence" and events[0]["prox_anchor"] == "per_step" + assert [(e["event"], e["field"], e["backend"]) for e in events[1:]] == [ + (oc.CORRECTION_CONFIG_WARNING, "level", "miles")] + + +# -- L6.3 Miles mapping ------------------------------------------------------- + + +def test_miles_tis_row(): + assert algorithm_argv(decoupled()) == ["--use-tis", "--tis-clip", "2.0", "--tis-clip-low", "0.0"] + + +def test_miles_icepop_row(): + spec = AlgorithmSpec(correction=CorrectionSpec(method="tis", lower=0.5, upper=5.0, + ratio_mask="icepop")) + assert algorithm_argv(spec) == ["--use-tis", "--tis-clip", "5.0", "--tis-clip-low", "0.5", + "--custom-tis-function-path", mc.ICEPOP_PATH] + assert mc.selected_corrections(spec) == ("icepop",) + assert spec.to_legacy_runtime_attrs() == AlgorithmSpec().to_legacy_runtime_attrs() + + +def test_miles_mis_row(): + spec = AlgorithmSpec(correction=CorrectionSpec(method="tis", lower=0.5, upper=2.0, + ratio_mask="mis", level="sequence", + veto=1e-4, batch_normalize=True)) + assert algorithm_argv(spec) == ["--use-tis", "--tis-clip", "2.0", "--tis-clip-low", "0.5", + "--custom-tis-function-path", mc.MIS_PATH] + attrs = {k: v for k, v in spec.to_legacy_runtime_attrs().items() if not k.startswith("yeto_")} + assert attrs == { + "rs_level": "sequence", "rs_lower_bound": 0.5, "rs_upper_bound": 2.0, + "rs_veto_threshold": 1e-4, "tis_batch_normalize": True, "tis_level": "sequence", + "tis_lower_bound": 0.5, "tis_mode": "mask", "tis_upper_bound": 2.0, "use_rs": True} + assert mc.selected_corrections(spec) == ("mis_mask",) + assert mo.correction_mapping(spec).warnings == () + + +def test_miles_opsm_row(): + spec = AlgorithmSpec(correction=CorrectionSpec(method="opsm", opsm_delta=1e-4)) + assert algorithm_argv(spec) == ["--use-opsm", "--opsm-delta", "0.0001"] + + +@pytest.mark.parametrize("kw, field", [ + (dict(level="sequence"), "level"), + (dict(veto=1e-4), "veto"), + (dict(batch_normalize=True), "batch_normalize"), +]) +def test_miles_unmappable_fields_warn(kw, field): + mapping = mo.correction_mapping(decoupled(**kw)) + assert [w.field for w in mapping.warnings] == [field] + assert mapping.warnings[0].backend == "miles" + event = mapping.warnings[0].event() + assert event["event"] == "correction_config_warning" and "ran_with" in event + + +# -- L6.7 decoupled PPO: refused Miles flags ----------------------------------- + + +@pytest.mark.parametrize("flag", ["--use-rollout-logprobs", "--skip-actor-forward-only", + "--mask-offpolicy-in-partial-rollout"]) +def test_forbidden_miles_flag_is_refused_by_name(flag): + with pytest.raises(MilesConfigError, match=flag): + check_extra_argv([flag], decoupled()) + with pytest.raises(oc.CorrectionConfigError, match=flag): + mo.refuse_forbidden_argv(["--use-tis", flag]) + + +def test_rollout_logprobs_from_the_spec_is_refused_in_the_final_argv(): + spec = AlgorithmSpec(correction=CorrectionSpec(use_rollout_logprobs=True)) + assert "--use-rollout-logprobs" in mo.correction_mapping(spec).argv + with pytest.raises(oc.CorrectionConfigError, match="--use-rollout-logprobs"): + mo.refuse_forbidden_argv(algorithm_argv(spec)) + + +def test_decoupled_ppo_records_per_step_anchor(): + mapping = mo.correction_mapping(decoupled()) + assert mapping.prox_anchor == oc.PROX_PER_STEP + assert mapping.argv == ("--use-tis", "--tis-clip", "2.0", "--tis-clip-low", "0.0") + + +# -- L6.5 cross-island comparison --------------------------------------------- + + +def _record(upper: float, anchor: str) -> dict: + mapping = oc.CorrectionMapping(backend="x", prox_anchor=anchor) + return oc.correction_record(decoupled(upper).correction, mapping, backend="x") + + +def test_islands_with_different_upper_give_one_warning(): + event = oc.compare_island_records({"0": _record(2.0, "per_step"), "1": _record(5.0, "per_step")}) + assert event["event"] == "correction_config_warning" + assert event["differences"] == {"upper": {"0": 2.0, "1": 5.0}} + + +def test_different_prox_anchor_is_listed(): + event = oc.compare_island_records({"miles": _record(2.0, "per_step"), + "verl": _record(2.0, "first_local_step")}) + assert event["differences"] == {"prox_anchor": {"miles": "per_step", + "verl": "first_local_step"}} + assert oc.compare_island_records({"0": _record(2.0, "per_step"), + "1": _record(2.0, "per_step")}) is None + + +def test_launcher_manifest_section(): + from tests.test_rl_launcher import _args + from yeto.launch.rl_ports import _offpolicy_correction_manifest + + args = _args([]) + section = _offpolicy_correction_manifest(args, decoupled()) + assert "0" in section["islands"] and section["warnings"] == [] + # L6b.2: an unmappable field refuses the launch + with pytest.raises(ValueError, match="level on backend 'miles'"): + _offpolicy_correction_manifest(args, decoupled(level="sequence")) + + +def test_refuse_names_every_warning(): + oc.refuse_correction_warnings([]) + cross = oc.compare_island_records({"0": _record(2.0, "per_step"), + "1": _record(5.0, "per_step")}) + unmappable = oc.CorrectionWarning("veto", "verl", "no mapping").event() + with pytest.raises(oc.CorrectionConfigError) as exc: + oc.refuse_correction_warnings([unmappable, cross]) + assert "veto on backend 'verl': no mapping" in str(exc.value) + assert "islands differ in ['upper']" in str(exc.value) + + +# -- L6.8 offpolicy_gate -------------------------------------------------------- + + +def test_dropped_length_histogram_fixed_buckets(): + hist = oc.length_histogram([300, 5000]) + assert hist == {"0-255": 0, "256-1023": 1, "1024-4095": 0, "4096-16383": 1, "16384+": 0} + + +def test_gate_event_granularity(): + event = oc.offpolicy_gate_event(backend="verl", counts={"kept": 3}, dropped_lengths={}, + truncation_by_turn={0: (1, 2)}, granularity="trajectory") + assert event["granularity"] == "trajectory" + assert event["truncation_by_turn"] == {"0": {"truncated": 1, "total": 2, "ratio": 0.5}} + with pytest.raises(ValueError): + oc.offpolicy_gate_event(backend="x", counts={}, dropped_lengths={}, + truncation_by_turn=None, granularity="turn") + + +class _Span: + def __init__(self, version, start, end): + self.version, self.abs_start, self.abs_end = str(version), start, end + + +class _Sample: + def __init__(self, group, spans, length, prompt=3, **kw): + self.group_index, self.response_length = group, length + self.tokens = [0] * (prompt + length) + self.weight_versions = [SimpleNamespace(spans=[_Span(v, prompt + a, prompt + b) + for v, a, b in spans])] + self.__dict__.update(kw) + + +def test_miles_buffer_filter_drops_feed_the_gate(): + args = SimpleNamespace() + old_a = [_Sample(0, [(1, 0, 300)], 300)] + old_b = [_Sample(1, [(1, 0, 5000)], 5000)] + fresh = [_Sample(2, [(4, 0, 10)], 10)] + kept = carry_over.carry_over_buffer_filter(args, [old_a, old_b, fresh], 4, + current_version=4, limit=2) + assert kept == [fresh] + fields = carry_over.take_carry_stats(args, 4) + gate = oc.gate_from_carry_over(fields, kept=1, backend="miles") + assert oc.GATE_DROPPED_LENGTHS not in fields + assert gate["counts"] == {"group_dropped": 2, "kept": 1, "unknown_version_dropped": 0} + hist = gate["dropped_length_histogram"]["group_dropped"] + assert (hist["256-1023"], hist["4096-16383"]) == (1, 1) + assert gate["granularity"] == "token" + + +def test_miles_truncation_by_turn_is_token_level(): + # two model turns (loss_mask runs) of 2 tokens, separated by one tool token + sample = _Sample(0, [(3, 0, 5)], 5, loss_mask=[1, 1, 0, 1, 1], + rollout_log_probs=[-0.5] * 5) + current = [0.5, -0.5, -0.5, 0.4, -0.5] # ratios e^1, 1, (tool), e^0.9, 1 + args = SimpleNamespace(tis_clip=2.0, tis_clip_low=0.0) + out = carry_over.estimate_cross_version_truncation(args, [sample], 4, lambda _s: current) + assert out[carry_over.TRUNCATED_BY_TURN] == {0: (1, 2), 1: (1, 2)} + + +def test_driver_tapes_offpolicy_gate_as_its_own_event(): + from yeto.rl.engine.driver import IslandDriver + + emitted = [] + fake = SimpleNamespace(emit=lambda name, **kw: emitted.append((name, kw)), + clock=lambda: 0.0, _labels=lambda: {}) + carry = {"max_policy_age": 2, "over_age_discarded_groups": 1, + oc.GATE_DROPPED_LENGTHS: {"group_dropped": [300]}, + oc.TRUNCATED_BY_TURN: {0: (1, 2)}} + batch = SimpleNamespace(carry_over=carry, policy_version=3, groups=(object(),)) + IslandDriver._emit_carry_over(fake, 3, batch) + assert [name for name, _ in emitted] == [oc.OFFPOLICY_GATE_EVENT, "rl_rollout_carry_over"] + assert emitted[0][1]["counts"]["kept"] == 1 + assert emitted[0][1]["counts"]["group_dropped"] == 1 + assert emitted[0][1]["granularity"] == "token" + assert oc.GATE_DROPPED_LENGTHS not in emitted[1][1] + assert oc.TRUNCATED_BY_TURN not in emitted[1][1] + + + +@pytest.mark.parametrize("backend, label", [("verl", "verl"), ("miles", "miles"), + (None, "unknown")]) +def test_driver_labels_offpolicy_gate_with_the_batch_backend(backend, label): + from yeto.rl.engine.driver import IslandDriver + from yeto.rl.engine.ports import RolloutBatchHandle + + emitted = [] + fake = SimpleNamespace(emit=lambda name, **kw: emitted.append((name, kw)), + clock=lambda: 0.0, _labels=lambda: {}) + batch = RolloutBatchHandle( + rollout_id=1, policy_version=1, policy_hash="h", groups=(), completed=0, aborted=0, + carry_over={oc.GATE_DROPPED_LENGTHS: {}, oc.GATE_GRANULARITY: oc.GRANULARITY_TRAJECTORY}, + backend=backend) + IslandDriver._emit_carry_over(fake, 1, batch) + assert emitted[0][0] == oc.OFFPOLICY_GATE_EVENT + assert emitted[0][1]["backend"] == label + assert emitted[0][1]["granularity"] == oc.GRANULARITY_TRAJECTORY + +# -- L6.9 A_hard >= 2 ---------------------------------------------------------- + + +def test_a_hard_two_passes_support_and_maps_to_both_backends(): + from yeto.rl.adapters.miles.policy_age import SUPPORT as MILES, policy_age_argv + from yeto.rl.adapters.verl.policy_age import SUPPORT as VERL, policy_age_overrides + + MILES.check(2) + VERL.check(2) + assert policy_age_argv(2, overlap=True) == ("--fully-async", "--max-weight-staleness", "2") + assert "async_training.staleness_threshold=2.0" in policy_age_overrides(2, groups_per_round=4) + with pytest.raises(PolicyAgeError): + MILES.check(MILES.max_policy_age + 1) + + +def test_miles_admission_mapping(): + assert mo.admission_argv(0) == () + assert mo.admission_argv(2) == ("--max-weight-staleness", "2") + assert mo.admission_argv(2, admission_limit=1) == ( + "--max-weight-staleness", "2", "--async-data-buffer-capacity-factor", "1") + with pytest.raises(ValueError, match="k=3"): + mo.admission_argv(2, admission_limit=3) + + +# -- L6.12 start version -------------------------------------------------------- + + +def test_continued_trajectory_gap_from_start_version(): + provenance = TokenProvenance((3, 3), (-0.1, -0.1)).extend(4, [-0.2]).extend(5, [-0.3, -0.3]) + assert oc.start_version_gap(6, provenance.versions) == 3 + assert oc.oldest_is_start(provenance.versions) is True + assert 6 - provenance.oldest == oc.start_version_gap(6, provenance.versions) + + +def test_tape_segments_check(): + records = [{"policy_versions": [[3, 0, 2], [4, 2, 3], [5, 3, 5]]}, + {"policy_versions": [[5, 0, 2], [4, 2, 3]]}, # not a continuation order + {"reward": 1.0}] + assert oc.segments_start_check(records) == { + "checked": 2, "oldest_is_start": 1, "oldest_before_start": 1} + + +# -- L6.13 cross_version_is: observe-only, from the spec --------------------------- + + +def test_cross_version_bounds_follow_the_spec(): + assert oc.cross_version_bounds(decoupled(5.0).correction) == (0.0, 5.0) + icepop = CorrectionSpec(method="tis", lower=0.5, upper=5.0, ratio_mask="icepop") + assert oc.cross_version_bounds(icepop) == (0.5, 5.0) + assert oc.cross_version_bounds(CorrectionSpec()) == oc.OBSERVE_DEFAULT_BOUNDS + + +def test_staleness_policy_correction_is_derived(): + assert StalenessPolicy.for_algorithm(decoupled()).correction == "tis" + icepop = AlgorithmSpec(correction=CorrectionSpec(method="tis", lower=0.5, upper=5.0, + ratio_mask="icepop")) + assert StalenessPolicy.for_algorithm(icepop).correction == "icepop" + assert StalenessPolicy.for_algorithm(AlgorithmSpec()).correction == "tis" + + +def test_miles_rollout_handle_keeps_the_gate_inputs(): + # L6.10 GPU: the Miles handle dropped the gate keys, so the driver taped no offpolicy_gate + from yeto.rl.adapters.miles.rollout import _carry_over_fields + + payload = {"max_policy_age": 1, "over_age_discarded_groups": 1, + oc.GATE_DROPPED_LENGTHS: {"group_dropped": [300], "unknown_version_dropped": []}, + oc.TRUNCATED_BY_TURN: {"0": [1, 2]}} + fields = _carry_over_fields(payload) + gate = oc.gate_from_carry_over(fields, kept=3, backend="miles") + assert gate["counts"] == {"group_dropped": 1, "kept": 3, "unknown_version_dropped": 0} + assert gate["truncation_by_turn"]["0"]["truncated"] == 1 + assert oc.GATE_DROPPED_LENGTHS not in fields + + +@pytest.mark.parametrize("stale, expected", [(2, 2), (0, 0), (None, None)]) +def test_miles_buffer_stale_drops_count_apart_from_the_gate(stale, expected): + # GOAL §6: Miles' own --fully-async buffer drops get their own reason code + from yeto.rl.adapters.miles.rollout import _carry_over_fields + + payload = {"max_policy_age": 1, "over_age_discarded_groups": 1, "stale_dropped_groups": stale, + oc.GATE_DROPPED_LENGTHS: {"group_dropped": [300], "unknown_version_dropped": []}} + gate = oc.gate_from_carry_over(_carry_over_fields(payload), kept=3, backend="miles") + assert gate["counts"]["group_dropped"] == 1 + assert gate["counts"].get(oc.GATE_BACKEND_STALE_DROPPED) == expected + assert oc.GATE_BACKEND_STALE_DROPPED not in gate["dropped_length_histogram"] diff --git a/tests/test_rl_policy_age.py b/tests/test_rl_policy_age.py index 9e1b3f780..3f82926f6 100644 --- a/tests/test_rl_policy_age.py +++ b/tests/test_rl_policy_age.py @@ -57,7 +57,7 @@ def test_elastic_join_carries_the_bound_identity(monkeypatch): assert a != b -@pytest.mark.parametrize("backend,limit,stage", [("miles", 2, 3), ("verl", 2, 2)]) +@pytest.mark.parametrize("backend,limit,stage", [("miles", 5, 3), ("verl", 5, 2)]) def test_launcher_refuses_an_unsupported_limit_before_launch(backend, limit, stage): from tests.test_rl_launcher import _args from yeto.launcher import _prepare_rl_args diff --git a/tests/test_rl_verl_fully_async_64b.py b/tests/test_rl_verl_fully_async_64b.py index 986f64eed..145cd8a2f 100644 --- a/tests/test_rl_verl_fully_async_64b.py +++ b/tests/test_rl_verl_fully_async_64b.py @@ -63,17 +63,19 @@ def test_limit_drives_staleness_threshold_and_partial_rollout(): assert not any("staleness" in o for o in vconf.build_overrides(run)) -def test_verl_declares_stage_two_up_to_limit_one(): +def test_verl_declares_stage_two_up_to_limit_four(): from yeto.rl.adapters.verl.entry import verl_capabilities from yeto.rl.adapters.verl.policy_age import SUPPORT, policy_age_overrides - assert (SUPPORT.stage, SUPPORT.max_policy_age) == (2, 1) + assert (SUPPORT.stage, SUPPORT.max_policy_age) == (2, 4) assert policy_age_overrides(0) == () assert policy_age_overrides(1, groups_per_round=8) == ( "async_training.staleness_threshold=1.0", "async_training.partial_rollout=True", "async_training.trigger_parameter_sync_step=1", "async_training.require_batches=1") with pytest.raises(PolicyAgeError, match="supports up to stage 2"): - policy_age_overrides(2, groups_per_round=8) + policy_age_overrides(5, groups_per_round=8) + # L6.9: A_hard = 2 maps to staleness_threshold 2 (decoupled PPO needs >= 2) + assert "async_training.staleness_threshold=2.0" in policy_age_overrides(2, groups_per_round=8) sync_caps = verl_capabilities("sha256:" + "0" * 64) assert sync_caps.execution.max_policy_staleness == 0 and sync_caps.placements == {"colocated"} fa = verl_capabilities("sha256:" + "0" * 64, fully_async_limit=1) @@ -325,6 +327,9 @@ def test_driver_runs_fully_async_rounds_with_carry_over_and_discard(tmp_path): assert carry[2]["over_age_discarded_groups"] == 1 and carry[2]["over_age_discarded_tokens"] == 10 cutoff = [e for e in events if e["event"] == "rl_rollout_cutoff"] assert [e["rollout_id"] for e in cutoff] == [2] + gates = [e for e in events if e["event"] == "offpolicy_gate"] + assert [e["rollout_id"] for e in gates] == [0, 1, 2] + assert {(e["backend"], e["granularity"]) for e in gates} == {("verl", "trajectory")} trained = [e for e in events if e["event"] == "rl_round_trained"] assert [e["aborted_in_flight_groups"] for e in trained] == [0, 0, 1] # never null (async10) checks = [e for e in events if e["event"] == "rl_publication_check"] diff --git a/tests/test_rl_verl_fully_async_translate.py b/tests/test_rl_verl_fully_async_translate.py index 65d64557b..5b5e9699b 100644 --- a/tests/test_rl_verl_fully_async_translate.py +++ b/tests/test_rl_verl_fully_async_translate.py @@ -77,4 +77,4 @@ def test_resumed_calls_give_per_token_versions_and_logprobs(): def test_verl_declares_stage_two_after_6_4b(): from yeto.rl.adapters.verl.policy_age import SUPPORT - assert (SUPPORT.stage, SUPPORT.max_policy_age) == (2, 1) + assert (SUPPORT.stage, SUPPORT.max_policy_age) == (2, 4) diff --git a/tests/test_rl_verl_offpolicy_correction.py b/tests/test_rl_verl_offpolicy_correction.py new file mode 100644 index 000000000..d51b4705a --- /dev/null +++ b/tests/test_rl_verl_offpolicy_correction.py @@ -0,0 +1,346 @@ +"""rl-offpolicy-correction-passthrough, verl side (L6.4, L6.6, L6.11 CPU part). + +CPU only: no verl install, no Ray. The parity tests (L6.6) load the correction +functions straight from the verl fork and the Miles checkout on this machine +and skip when either is absent. +""" + +from __future__ import annotations + +import importlib.util +import os +import sys +import types +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch + +from yeto.rl.adapters.verl import fully_async_round as far +from yeto.rl.adapters.verl import offpolicy_correction as voc +from yeto.rl.adapters.verl.fully_async_translate import VersionMap +from yeto.rl.engine import offpolicy_correction as oc +from yeto.rl.engine.algorithm import AlgorithmSpec, CorrectionSpec, ExecutionSpec + +K = voc.PREFIX +DECOUPLED = AlgorithmSpec(execution=ExecutionSpec(max_policy_staleness=2), + correction=CorrectionSpec(method="tis", lower=0.0, upper=2.0)) + + +def _spec(**correction) -> AlgorithmSpec: + return AlgorithmSpec(correction=CorrectionSpec(**correction)) + + +def _fields(mapping) -> set[str]: + return {w.field for w in mapping.warnings} + + +# ------------------------------------------------------------------ L6.4 / L6.7 mapping + + +def test_decoupled_ppo_writes_all_three_values_over_the_fully_async_defaults(): + m = voc.correction_mapping(DECOUPLED) + assert m.backend == "verl" and m.prox_anchor == oc.PROX_FIRST_LOCAL_STEP + assert m.values[K + "bypass_mode"] is False + assert m.values[K + "rollout_is"] == "token" + assert m.values[K + "rollout_is_threshold"] == 2.0 + assert {K + "bypass_mode=False", K + "rollout_is=token", + K + "rollout_is_threshold=2.0", K + "rollout_rs=null"} <= set(m.overrides) + assert m.warnings == () + + +def test_tis_lower_warns_and_runs_without_a_lower_bound(): + m = voc.correction_mapping(_spec(method="tis", lower=0.5, upper=2.0)) + (w,) = m.warnings + assert (w.field, w.backend, w.ran_with) == ("lower", "verl", 0.0) + assert m.values[K + "rollout_is_threshold"] == 2.0 + + +def test_icepop_maps_to_a_lower_upper_string_without_denominator_warning(): + m = voc.correction_mapping(_spec(method="tis", lower=0.5, upper=5.0, ratio_mask="icepop")) + assert m.values[K + "rollout_is_threshold"] == "0.5_5.0" + assert K + "rollout_is_threshold='0.5_5.0'" in m.overrides + assert m.warnings == () + zero = voc.correction_mapping(_spec(method="tis", lower=0.0, upper=5.0, ratio_mask="icepop")) + assert _fields(zero) == {"lower"} and zero.warnings[0].ran_with == voc.ICEPOP_MIN_LOWER + + +def test_mis_maps_to_rejection_sampling_and_warns_on_the_denominator(): + m = voc.correction_mapping(_spec(method="tis", lower=0.5, upper=2.0, ratio_mask="mis")) + assert m.values[K + "rollout_rs"] == "token_k1" + assert m.values[K + "rollout_rs_threshold"] == "0.5_2.0" + assert m.values[K + "rollout_is"] is None + assert _fields(m) == {"ratio_mask"} + geo = voc.correction_mapping(_spec(method="tis", lower=0.5, upper=2.0, ratio_mask="mis", + level="geometric")) + assert geo.values[K + "rollout_rs"] == "seq_mean_k1" and _fields(geo) == {"ratio_mask"} + seq = voc.correction_mapping(_spec(method="tis", lower=0.5, upper=2.0, ratio_mask="mis", + level="sequence")) + assert _fields(seq) == {"ratio_mask", "level"} + + +def test_unmappable_fields_warn(): + assert _fields(voc.correction_mapping(_spec(method="tis", lower=0.0, upper=2.0, veto=1e-4))) == {"veto"} + assert _fields(voc.correction_mapping(_spec(method="opsm", opsm_delta=0.1))) == {"method"} + norm = voc.correction_mapping(_spec(method="tis", lower=0.0, upper=2.0, batch_normalize=True)) + assert norm.values[K + "rollout_is_batch_normalize"] is True and norm.warnings == () + seq = voc.correction_mapping(_spec(method="tis", lower=0.0, upper=2.0, level="sequence")) + assert seq.values[K + "rollout_is"] == "sequence" and seq.warnings == () + + +def test_bypass_mode_true_is_refused_with_the_setting_named(): + with pytest.raises(oc.CorrectionConfigError, match="bypass_mode=True"): + voc.refuse_forbidden_overrides([K + "bypass_mode=true"]) + m = voc.correction_mapping(_spec(method="tis", lower=0.0, upper=2.0, use_rollout_logprobs=True)) + with pytest.raises(oc.CorrectionConfigError, match="pi_prox"): + voc.apply_to_overrides([], m) + voc.refuse_forbidden_overrides([K + "bypass_mode=False"]) + + +def test_apply_replaces_every_rollout_correction_entry_once(): + out = voc.apply_to_overrides(["a=1", K + "bypass_mode=True", K + "rollout_is=sequence", + "+" + K + "rollout_is_batch_normalize=True"], + voc.correction_mapping(DECOUPLED)) + keys = [o.split("=", 1)[0] for o in out] + assert len(keys) == len(set(keys)) and "a=1" in out + assert K + "bypass_mode=False" in out and K + "rollout_is=token" in out + + +def test_backend_role_feeds_the_run_manifest_and_the_cross_island_warning(): + assert oc.backend_mapping(DECOUPLED, "verl").prox_anchor == oc.PROX_FIRST_LOCAL_STEP + manifest = oc.run_correction_manifest(DECOUPLED, {"i0": "miles", "i1": "verl"}) + assert manifest["islands"]["i1"]["mapped"][K + "bypass_mode"] is False + (cross,) = [w for w in manifest["warnings"] if w.get("scope") == "cross_island"] + assert cross["differences"]["prox_anchor"] == {"i0": oc.PROX_PER_STEP, + "i1": oc.PROX_FIRST_LOCAL_STEP} + + +# ------------------------------------------------------------------ L6.11 fully_async path + + +@pytest.mark.parametrize("limit", [2, 4]) +def test_fully_async_plan_runs_the_correction_mapping(limit, tmp_path): + import json + + from yeto.rl.adapters.verl.island_entry import append_correction_record, fully_async_plan + + import yeto.rl.adapters.verl.config as vconf + + run = vconf.VerlRunConfig(model_path="/m", train_file="/t", val_file="/v", out_dir="/o", + groups_per_round=8, samples_per_group=4) + spec = AlgorithmSpec(execution=ExecutionSpec(max_policy_staleness=limit), + correction=CorrectionSpec(method="tis", lower=0.5, upper=2.0)) + tape = tmp_path / "tape.jsonl" + args = SimpleNamespace(groups_per_round=8, samples_per_group=4, global_rounds=3, + event_tape=str(tape), learner_id=0) + out, keys = fully_async_plan(limit, vconf.build_overrides(run), spec.to_dict(), args, "none") + # L6b.2: lower > 0 cannot map on verl; the tape keeps the record, the start is refused + with pytest.raises(oc.CorrectionConfigError, match="lower on backend 'verl'"): + append_correction_record(args, spec.to_dict()) + append_correction_record(args, AlgorithmSpec().to_dict()) # default: tape unchanged + # D8: the island tape records the group, the verl values and prox_anchor + events = [json.loads(line) for line in tape.read_text().splitlines()] + assert events[0]["event"] == oc.CORRECTION_RECORD_EVENT and events[0]["backend"] == "verl" + assert events[0]["prox_anchor"] == oc.PROX_FIRST_LOCAL_STEP + assert events[0]["group"]["upper"] == 2.0 + assert [(e["event"], e["field"]) for e in events[1:]] == [(oc.CORRECTION_CONFIG_WARNING, "lower")] + assert f"async_training.staleness_threshold={float(limit)}" in out + assert "async_training.partial_rollout=True" in out + assert [o for o in out if o.startswith(K + "bypass_mode=")] == [K + "bypass_mode=False"] + assert K + "rollout_is=token" in out and K + "rollout_is_threshold=2.0" in out + assert K + "bypass_mode" in keys + + +def _vmap(n: int) -> VersionMap: + vm = VersionMap() + for v in range(n): + vm.record(v, v) + return vm + + +def _meta(uid, lo, hi, lengths): + n = len(lengths) + return far.sample_meta({"uid": [uid] * n, "min_global_steps": [lo] * n, + "max_global_steps": [hi] * n}, lengths) + + +def test_round_collection_drops_whole_groups_over_a_hard_and_feeds_the_gate(): + rc = far.RoundCollector(_vmap(6), current_outer=5, limit=2, required=2) + assert rc.offer(_meta("fresh", 5, 5, [10, 20])) + assert not rc.offer(_meta("old", 2, 5, [300, 5000])) # oldest v2: gap 3 > A_hard 2 + assert not rc.offer(_meta("unk", None, None, [7, 7])) + assert rc.offer(_meta("edge", 3, 4, [1, 1])) # gap 2 == A_hard: kept + fields = rc.carry_over_fields() + assert fields["over_age_discarded_groups"] == 1 + # the driver builds offpolicy_gate from these keys and removes them + gate = oc.gate_from_carry_over(fields, kept=len(rc.kept), backend="verl") + assert oc.GATE_DROPPED_LENGTHS not in fields and oc.GATE_GRANULARITY not in fields + assert gate["backend"] == "verl" and gate["granularity"] == oc.GRANULARITY_TRAJECTORY + assert gate["counts"] == {oc.GATE_KEPT: 2, oc.GATE_GROUP_DROPPED: 1, + oc.GATE_UNKNOWN_VERSION_DROPPED: 1} + assert gate["dropped_length_histogram"][oc.GATE_GROUP_DROPPED] == \ + oc.length_histogram([300, 5000]) + assert gate["truncation_by_turn"] is None + + +# ------------------------------------------------------------------ L6.6 CPU parity + + +VERL_SRC = Path(os.environ.get("YETO_VERL_SRC", "/home/michael/work/verl-fork-acad9875")) +MILES_SRC = Path(os.environ.get("YETO_MILES_SRC", "/home/michael/work/miles-next")) +VERL_HELPER = VERL_SRC / "verl/trainer/ppo/rollout_corr_helper.py" +MILES_CORRECTIONS = MILES_SRC / "miles/backends/training_utils/loss_hub/corrections.py" + + +def _load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def engines(): + if not (VERL_HELPER.is_file() and MILES_CORRECTIONS.is_file()): + pytest.skip("verl fork or Miles checkout not on this machine") + # rollout_corr_helper imports verl_F (masked_sum / masked_mean: copied below + # from verl/utils/torch_functional.py at acad9875), DataProto and two config + # classes (type hints only). Stubs replace the heavy package imports. + stub = types.ModuleType("verl.utils.torch_functional") + + def masked_sum(values, mask, axis=None): + return (torch.where(mask.bool(), values, 0.0) * mask).sum(axis=axis) + + def masked_mean(values, mask, axis=None): + return masked_sum(values, mask, axis) / (mask.sum(axis=axis) + 1e-8) + + stub.masked_sum, stub.masked_mean = masked_sum, masked_mean + + class DataProto: + @classmethod + def from_dict(cls, tensors): + obj = cls() + obj.batch = tensors + return obj + + names = {"verl": types.ModuleType("verl"), "verl.utils": types.ModuleType("verl.utils"), + "verl.utils.torch_functional": stub, + "verl.protocol": types.SimpleNamespace(DataProto=DataProto), + "verl.trainer.config.algorithm": types.SimpleNamespace(RolloutCorrectionConfig=object), + "verl.workers.config.actor": types.SimpleNamespace(PolicyLossConfig=object)} + names["verl"].utils = names["verl.utils"] + names["verl.utils"].torch_functional = stub + saved = {k: sys.modules.get(k) for k in names} + sys.modules.update(names) + try: + verl = _load("_yeto_parity_verl_rollout_corr", VERL_HELPER) + finally: + for k, v in saved.items(): + if v is None: + sys.modules.pop(k, None) + else: + sys.modules[k] = v + miles = _load("_yeto_parity_miles_corrections", MILES_CORRECTIONS) + return verl, miles + + +def _logprobs(seed: int = 3): + g = torch.Generator().manual_seed(seed) + rollout = -torch.rand(4, 16, generator=g, dtype=torch.float64) * 3 + train = rollout + torch.randn(4, 16, generator=g, dtype=torch.float64) * 0.8 + mask = torch.ones(4, 16, dtype=torch.float64) + mask[1, 12:] = 0 + mask[3, 5:] = 0 + return train, rollout, mask + + +def _miles_weights(miles, fn, train, rollout, mask, lower, upper): + rows = [m.bool() for m in mask] + flat_t = [t[r] for t, r in zip(train, rows)] + flat_r = [t[r] for t, r in zip(rollout, rows)] + n = sum(int(r.sum()) for r in rows) + args = SimpleNamespace(tis_clip=upper, tis_clip_low=lower) + pg, masks, _ = getattr(miles, fn)(args, pg_loss=torch.ones(n, dtype=torch.float64), + train_log_probs=flat_t, rollout_log_probs=flat_r, + loss_masks=[r.double() for r in rows]) + return pg, masks + + +def _verl(verl, train, rollout, mask, **kw): + weights, new_mask, _ = verl.compute_rollout_correction_and_rejection_mask( + old_log_prob=train, rollout_log_prob=rollout, response_mask=mask, **kw) + return (None if weights is None else weights.batch["rollout_is_weights"][mask.bool()]), new_mask + + +def test_parity_tis_with_lower_zero_matches(engines): + verl, miles = engines + train, rollout, mask = _logprobs() + miles_w, _ = _miles_weights(miles, "vanilla_tis_function", train, rollout, mask, 0.0, 2.0) + verl_w, new_mask = _verl(verl, train, rollout, mask, rollout_is="token", rollout_is_threshold=2.0) + torch.testing.assert_close(verl_w, miles_w) + assert torch.equal(new_mask, mask) + + +def test_parity_tis_lower_differs_as_warned(engines): + verl, miles = engines + train, rollout, mask = _logprobs() + miles_w, _ = _miles_weights(miles, "vanilla_tis_function", train, rollout, mask, 0.5, 2.0) + verl_w, _ = _verl(verl, train, rollout, mask, rollout_is="token", rollout_is_threshold=2.0) + low = miles_w > verl_w + assert low.any() and torch.all(miles_w[low] == 0.5) + torch.testing.assert_close(verl_w[~low], miles_w[~low]) + + +def test_parity_icepop_matches_and_keeps_the_denominator(engines): + verl, miles = engines + train, rollout, mask = _logprobs() + miles_w, miles_masks = _miles_weights(miles, "icepop_function", train, rollout, mask, 0.5, 2.0) + verl_w, new_mask = _verl(verl, train, rollout, mask, rollout_is="token", + rollout_is_threshold="0.5_2.0") + torch.testing.assert_close(verl_w, miles_w) + assert (verl_w == 0).any() + assert torch.equal(new_mask, mask) # verl IcePop zeroes weights; loss_mask unchanged + assert sum(int(m.sum()) for m in miles_masks) == int(mask.sum()) + + +def test_verl_rejection_sampling_shrinks_the_denominator(engines): + verl, _ = engines + train, rollout, mask = _logprobs() + _, new_mask = _verl(verl, train, rollout, mask, rollout_rs="token_k1", + rollout_rs_threshold="0.5_2.0") + assert 0 < new_mask.sum() < mask.sum() + ratio = torch.exp(train - rollout) + kept = mask.bool() & (ratio >= 0.5) & (ratio <= 2.0) + assert torch.equal(new_mask.bool(), kept) + + +def test_verl_use_rollout_log_probs_is_not_read_by_any_code(): + """L6.6: ``actor.use_rollout_log_probs`` (fully_async yaml: True) has no + reader in the fork, so with ``bypass_mode=False`` pi_prox is recomputed.""" + if not VERL_SRC.is_dir(): + pytest.skip("verl fork not on this machine") + readers = [p for p in (VERL_SRC / "verl").rglob("*.py") + if "use_rollout_log_probs" in p.read_text(errors="ignore") + and p.name != "actor.py"] + assert readers == [] + + +# ------------------------------------------------------------------ example configs + + +@pytest.mark.parametrize("name,a_hard", [("decoupled-ppo-c2", 2), ("decoupled-ppo-c2-agentic", 4)]) +def test_decoupled_ppo_examples_map_on_both_backends(name, a_hard): + from yeto.rl.adapters.miles.offpolicy_correction import admission_argv + from yeto.rl.adapters.miles.offpolicy_correction import correction_mapping as miles_mapping + from yeto.rl.adapters.verl.policy_age import policy_age_overrides + + examples = Path(__file__).resolve().parents[1] / "examples" / "rl_algorithms" + spec = AlgorithmSpec.from_json_file(str(examples / f"{name}.json")) + assert spec.rejections() == [] and spec.execution.max_policy_staleness == a_hard + assert miles_mapping(spec).argv == ("--use-tis", "--tis-clip", "2.0", "--tis-clip-low", "0.0") + assert admission_argv(a_hard) == ("--max-weight-staleness", str(a_hard)) + verl = voc.correction_mapping(spec) + assert verl.values[K + "rollout_is_threshold"] == 2.0 and verl.warnings == () + assert f"async_training.staleness_threshold={float(a_hard)}" in \ + policy_age_overrides(a_hard, groups_per_round=8) diff --git a/tests/test_rl_verl_policy_age.py b/tests/test_rl_verl_policy_age.py index 055c66913..9f02ad767 100644 --- a/tests/test_rl_verl_policy_age.py +++ b/tests/test_rl_verl_policy_age.py @@ -59,11 +59,11 @@ def test_verl_declares_stage_two_and_limit_zero_overrides_are_empty(): from yeto.rl.adapters.verl.entry import verl_capabilities from yeto.rl.adapters.verl.policy_age import SUPPORT, policy_age_overrides - assert (SUPPORT.stage, SUPPORT.max_policy_age) == (2, 1) # 6.4b: fully_async path + assert (SUPPORT.stage, SUPPORT.max_policy_age) == (2, 4) # 6.4b; L6.9 raised the cap assert verl_capabilities("sha256:" + "0" * 64).execution.max_policy_staleness == 0 assert policy_age_overrides(0) == () with pytest.raises(PolicyAgeError, match="backend 'verl' supports up to stage 2"): - policy_age_overrides(2, groups_per_round=4) + policy_age_overrides(5, groups_per_round=4) def test_launcher_accepts_verl_limit_one(): diff --git a/yeto/launch/manifest.py b/yeto/launch/manifest.py index fc4d62650..7dba9a903 100644 --- a/yeto/launch/manifest.py +++ b/yeto/launch/manifest.py @@ -49,6 +49,9 @@ def _write_run_manifest(args) -> dict | None: # launch-preflight-guards: preflight readings, negative_test, island_overrides **(getattr(args, "_launch_preflight_manifest", None) or {}), "island_placements": rl_island_placement_records(args), + # rl-offpolicy-correction-passthrough D8: correction group per island + **({"offpolicy_correction": args._offpolicy_correction_manifest} + if getattr(args, "_offpolicy_correction_manifest", None) else {}), "written_unix": time.time(), } print(f"[launcher] RL image {manifest['rl_image']}, " diff --git a/yeto/launch/rl_ports.py b/yeto/launch/rl_ports.py index 811dea937..c66b32bcd 100644 --- a/yeto/launch/rl_ports.py +++ b/yeto/launch/rl_ports.py @@ -115,6 +115,23 @@ def _prepare_ports_algorithm(args) -> None: _check_critic_warmup_options(args, spec) args.rl_algorithm_spec_json = spec.canonical_json() args.rl_expected_algorithm_sha256 = spec.sha256() + args._offpolicy_correction_manifest = _offpolicy_correction_manifest(args, spec) + + +def _offpolicy_correction_manifest(args, spec) -> dict: + """rl-offpolicy-correction-passthrough D8/L6.5: every island's correction + record. A warning (unmappable field, island difference) refuses the launch + (phase 2, L6b.2).""" + + from ..rl.engine.offpolicy_correction import (refuse_correction_warnings, + run_correction_manifest) + from .topology import rl_island_args + + backends = {str(m): getattr(rl_island_args(args, m), "rl_backend", None) or "miles" + for m in range(rl_island_count(args))} + section = run_correction_manifest(spec, backends) + refuse_correction_warnings(section["warnings"]) + return section def _check_critic_warmup_options(args, spec) -> None: diff --git a/yeto/rl/adapters/miles/algorithm_flags.py b/yeto/rl/adapters/miles/algorithm_flags.py index d4f7ab1c1..5e534a04f 100644 --- a/yeto/rl/adapters/miles/algorithm_flags.py +++ b/yeto/rl/adapters/miles/algorithm_flags.py @@ -140,17 +140,9 @@ def use_kl_loss(spec): def _correction_rows() -> list[FlagMapping]: def use_tis(spec): - c = spec.correction - if c.method not in ("tis", "custom"): - return [] - argv = ["--use-tis"] - if c.tis_clip is not None: - argv += ["--tis-clip", _num(c.tis_clip)] - if c.tis_clip_low is not None: - argv += ["--tis-clip-low", _num(c.tis_clip_low)] - if c.function is not None: - argv += ["--custom-tis-function-path", c.function.path] - return argv + from .offpolicy_correction import tis_argv # neutral fields -> Miles (D3) + + return tis_argv(spec) def use_opsm(spec): c = spec.correction @@ -161,10 +153,10 @@ def use_opsm(spec): return [ FlagMapping("--use-tis", "correction.method", True, lambda raw: True, lambda _: [("correction.method", "tis")], use_tis), - FlagMapping("--tis-clip", "correction.tis_clip", False, _float, - lambda v: [("correction.tis_clip", v)], lambda spec: []), - FlagMapping("--tis-clip-low", "correction.tis_clip_low", False, _float, - lambda v: [("correction.tis_clip_low", v)], lambda spec: []), + FlagMapping("--tis-clip", "correction.upper", False, _float, + lambda v: [("correction.upper", v)], lambda spec: []), + FlagMapping("--tis-clip-low", "correction.lower", False, _float, + lambda v: [("correction.lower", v)], lambda spec: []), FlagMapping("--custom-tis-function-path", "correction.function", False, _plugin, lambda v: [("correction.method", "custom"), ("correction.function", v)], lambda spec: []), diff --git a/yeto/rl/adapters/miles/carry_over.py b/yeto/rl/adapters/miles/carry_over.py index 215829f32..4e4eca4a0 100644 --- a/yeto/rl/adapters/miles/carry_over.py +++ b/yeto/rl/adapters/miles/carry_over.py @@ -39,6 +39,12 @@ from collections.abc import Callable, Sequence from typing import Any +from yeto.rl.engine.offpolicy_correction import ( + GATE_GROUP_DROPPED, + GATE_DROPPED_LENGTHS, + GATE_UNKNOWN_VERSION_DROPPED, + TRUNCATED_BY_TURN, +) from yeto.rl.engine.policy_versions import engine_version_number from yeto.rl.engine.version_segments import ( ProvenanceError, @@ -48,6 +54,7 @@ ) CARRY_STATS_ATTR = "_yeto_carry_over_stats" +DROPPED_LENGTHS = "dropped_lengths" CARRY_FIELDS = ( "carried_in_groups", "carried_in_trajectories", @@ -129,14 +136,19 @@ def _trajectories(group: Sequence[Any]) -> int: return sum(1 for s in _flat(group) if int(getattr(s, "response_length", 0) or 0) > 0) -def _stats_for(args: Any, current_version: int) -> dict[str, int]: +def _stats_for(args: Any, current_version: int) -> dict[str, Any]: stats = getattr(args, CARRY_STATS_ATTR, None) if not isinstance(stats, dict) or stats.get("version") != current_version: - stats = {"version": current_version, **{k: 0 for k in CARRY_FIELDS}} + stats = {"version": current_version, **{k: 0 for k in CARRY_FIELDS}, + DROPPED_LENGTHS: {GATE_GROUP_DROPPED: [], GATE_UNKNOWN_VERSION_DROPPED: []}} setattr(args, CARRY_STATS_ATTR, stats) return stats +def _trajectory_lengths(group: Sequence[Any]) -> list[int]: + return [n for n in (int(getattr(s, "response_length", 0) or 0) for s in _flat(group)) if n] + + def carry_over_buffer_filter(args: Any, buffer: list, num_samples: int, *, current_version: int, limit: int) -> list: """Select up to ``num_samples`` buffered groups to continue at ``current_version``. @@ -150,8 +162,10 @@ def carry_over_buffer_filter(args: Any, buffer: list, num_samples: int, *, known, unknown = group_versions(group) if unknown: stats["unknown_version_discarded_groups"] += 1 + stats[DROPPED_LENGTHS][GATE_UNKNOWN_VERSION_DROPPED] += _trajectory_lengths(group) continue if known and current_version - min(known) > limit: + stats[DROPPED_LENGTHS][GATE_GROUP_DROPPED] += _trajectory_lengths(group) stats["over_age_discarded_groups"] += 1 stats["over_age_discarded_trajectories"] += _trajectories(group) stats["over_age_discarded_tokens"] += _response_tokens(group) @@ -167,14 +181,30 @@ def carry_over_buffer_filter(args: Any, buffer: list, num_samples: int, *, return selected -def take_carry_stats(args: Any, current_version: int | None) -> dict[str, int]: - """This round's carry-over counts (zeros when nothing was buffered); consumed.""" +def take_carry_round(args: Any, current_version: int | None + ) -> tuple[dict[str, int], dict[str, list[int]]]: + """This round's carry-over counts and the response lengths of the + trajectories discarded per ``offpolicy_gate`` reason; consumed.""" stats = getattr(args, CARRY_STATS_ATTR, None) if hasattr(args, CARRY_STATS_ATTR): setattr(args, CARRY_STATS_ATTR, None) if not isinstance(stats, dict) or stats.get("version") != current_version: - return {k: 0 for k in CARRY_FIELDS} - return {k: int(stats[k]) for k in CARRY_FIELDS} + return ({k: 0 for k in CARRY_FIELDS}, + {GATE_GROUP_DROPPED: [], GATE_UNKNOWN_VERSION_DROPPED: []}) + lengths = stats.get(DROPPED_LENGTHS) or {} + return ({k: int(stats[k]) for k in CARRY_FIELDS}, + {reason: list(lengths.get(reason, ())) + for reason in (GATE_GROUP_DROPPED, GATE_UNKNOWN_VERSION_DROPPED)}) + + +def take_carry_stats(args: Any, current_version: int | None) -> dict[str, Any]: + """This round's carry-over counts (zeros when nothing was buffered); consumed. + + Also carries the dropped response lengths under :data:`GATE_DROPPED_LENGTHS` + (rl-offpolicy-correction-passthrough D6); the driver turns them into + ``offpolicy_gate`` and removes the key from ``rl_rollout_carry_over``.""" + counts, lengths = take_carry_round(args, current_version) + return {**counts, GATE_DROPPED_LENGTHS: lengths} def group_versions_record(group: Sequence[Any]) -> dict[str, Any]: @@ -293,6 +323,7 @@ def estimate_cross_version_truncation(args: Any, samples: Sequence[Any], current unknown_tokens = 0 near_zero = 0 near_zero_examples: list[dict[str, Any]] = [] + by_turn: dict[int, list[int]] = {} # turn index -> [truncated, cross-version tokens] def fail(why: str) -> None: nonlocal failed @@ -325,11 +356,16 @@ def fail(why: str) -> None: tuple(min(float(p), 0.0) for p in generated)) corrections.append(cross_version_is(provenance, current, current_version, clip_low=clip_low, clip_high=clip_high)) + turns = turn_indices(sample, len(versions)) for i, (v, g, c) in enumerate(zip(provenance.versions, provenance.logprobs, current, strict=True)): if v == current_version: continue r = math.exp(float(c) - g) ratios.append(r) + if turns[i] is not None: + counts = by_turn.setdefault(turns[i], [0, 0]) + counts[0] += not clip_low <= r <= clip_high + counts[1] += 1 if r < NEAR_ZERO_RATIO: # 5.5 follow-up: where do zero ratios come from near_zero += 1 if len(near_zero_examples) < NEAR_ZERO_EXAMPLES: @@ -342,6 +378,8 @@ def fail(why: str) -> None: extra["cross_version_unscored_reasons"] = dict(sorted(reasons.items())[:8]) if unknown_tokens: extra["cross_version_unknown_version_tokens"] = unknown_tokens + if by_turn: # rl-offpolicy-correction D6 (absent when none: old key set kept) + extra[TRUNCATED_BY_TURN] = {turn: tuple(c) for turn, c in sorted(by_turn.items())} if near_zero: # absent when none (old key set kept) extra["cross_version_ratio_near_zero"] = near_zero extra["cross_version_ratio_near_zero_examples"] = near_zero_examples @@ -355,6 +393,22 @@ def fail(why: str) -> None: } +def turn_indices(sample: Any, n: int) -> list[int | None]: + """Per response token: its model turn (0, 1, ...; runs of ``loss_mask`` 1), + None for a token outside the loss (tool output). No mask: one turn.""" + mask = list(getattr(sample, "loss_mask", None) or ()) + if len(mask) != n: + return [0] * n + out: list[int | None] = [] + turn, inside = -1, False + for m in mask: + if m and not inside: + turn += 1 + inside = bool(m) + out.append(turn if inside else None) + return out + + NEAR_ZERO_RATIO = 1e-6 NEAR_ZERO_EXAMPLES = 8 diff --git a/yeto/rl/adapters/miles/config.py b/yeto/rl/adapters/miles/config.py index 556ec57d5..d0c966786 100644 --- a/yeto/rl/adapters/miles/config.py +++ b/yeto/rl/adapters/miles/config.py @@ -596,6 +596,18 @@ def reject_fault_tolerance_flags(tokens: Sequence[str]) -> None: ) +def _refuse_decoupled_ppo_breakers(tokens: Sequence[str]) -> None: + """rl-offpolicy-correction-passthrough D2: flags that remove pi_prox or mask + tokens by version are refused, with the flag named.""" + from .offpolicy_correction import refuse_forbidden_argv + from yeto.rl.engine.offpolicy_correction import CorrectionConfigError + + try: + refuse_forbidden_argv(tokens) + except CorrectionConfigError as exc: + raise MilesConfigError(str(exc)) from exc + + def check_extra_argv(extra_argv: Sequence[str], algorithm: AlgorithmSpec | None = None): """Refuse FT/adapter-owned flags; absorb mapped algorithm flags. @@ -606,6 +618,7 @@ def check_extra_argv(extra_argv: Sequence[str], algorithm: AlgorithmSpec | None """ reject_fault_tolerance_flags(extra_argv) + _refuse_decoupled_ppo_breakers(extra_argv) mapped = mapped_flags() unmapped_objective = objective_flags() - mapped for flag in _flags(extra_argv): @@ -1085,6 +1098,7 @@ def translate_run_config( values.extend((flag, engine)) values.extend(extra_argv) reject_fault_tolerance_flags(values[1:]) # generated argv never carries FT flags + _refuse_decoupled_ppo_breakers(values) # also when the spec emitted the flag # IR-1: the final argv never pairs the session server with partial rollout # (also re-checked on the parsed namespace by validate_parsed_args). check_session_server_partial_rollout(agent.use_session_server, "--partial-rollout" in values) diff --git a/yeto/rl/adapters/miles/island_entry.py b/yeto/rl/adapters/miles/island_entry.py index 2532d2e1f..98288d6c1 100644 --- a/yeto/rl/adapters/miles/island_entry.py +++ b/yeto/rl/adapters/miles/island_entry.py @@ -444,6 +444,7 @@ def verify_ports_algorithm(args, miles_args, launch) -> None: f"island {args.learner_id} algorithm hash {actual} differs from the launcher's " f"expected {expected.lower()}; refusing to join outer sync" ) + _append_correction_record(args, miles_args, algorithm) miles_args.yeto_rl_algorithm_absorbed_flags = dict(launch.absorbed_flags) miles_args.yeto_rl_outer_sync = not getattr(args, "rl_single_island_no_sync", False) # INFRA compares it with the runtime AlgorithmSpec before connect_island_ray @@ -456,6 +457,29 @@ def verify_ports_algorithm(args, miles_args, launch) -> None: ) +def _append_correction_record(args, miles_args, algorithm) -> None: + """rl-offpolicy-correction-passthrough D8: the island tape records the full + correction group, the Miles values and ``prox_anchor``. A field Miles + cannot map refuses the start (phase 2, L6b.2), after the record and its + ``correction_config_warning`` events are on the tape.""" + from yeto.rl.engine.offpolicy_correction import (CORRECTION_RECORD_EVENT, correction_record, + refuse_correction_warnings) + + from .offpolicy_correction import BACKEND, correction_mapping + + if algorithm.correction.is_default(): + return # default tapes unchanged; the run manifest still records the group + mapping = correction_mapping(algorithm) + _append_ports_event(args, miles_args, { + "event": CORRECTION_RECORD_EVENT, + **correction_record(algorithm.correction, mapping, backend=BACKEND), + }) + warnings = [warning.event() for warning in mapping.warnings] + for warning in warnings: + _append_ports_event(args, miles_args, warning) + refuse_correction_warnings(warnings) + + def _require_ports_supported(args, extra_argv: Sequence[str] = ()) -> None: """Reject combinations outside the R0 ports matrix before any startup.""" diff --git a/yeto/rl/adapters/miles/offpolicy_correction.py b/yeto/rl/adapters/miles/offpolicy_correction.py new file mode 100644 index 000000000..1dd8a1295 --- /dev/null +++ b/yeto/rl/adapters/miles/offpolicy_correction.py @@ -0,0 +1,173 @@ +"""Miles mapping of the neutral correction group +(change ``rl-offpolicy-correction-passthrough`` L6.3, L6.7, L6.9). + +:func:`correction_mapping` turns ``AlgorithmSpec.correction`` into Miles +argv (design D2/D3); :func:`algorithm_flags.algorithm_argv` emits the same +argv through its ``--use-tis`` row. Mapping table (design D3): + +==================== ========================================================== +neutral Miles +==================== ========================================================== +tis, token ``--use-tis --tis-clip upper --tis-clip-low lower`` +ratio_mask=icepop the same plus ``--custom-tis-function-path`` IcePop +ratio_mask=mis the same with the MIS function; mask mode, ``level``, + bounds, ``batch_normalize`` and ``veto`` as namespace + attributes (``mismatch_correction.neutral_mis_config``) +opsm ``--use-opsm --opsm-delta`` +==================== ========================================================== + +Rows Miles cannot express give a :class:`CorrectionWarning`; phase 2 (L6b.2) +refuses the start on them: ``level`` other than token without the MIS mask, and ``veto`` / +``batch_normalize`` without the MIS mask. + +Decoupled PPO (design D2) needs pi_prox, which Miles recomputes on the +weights before each update (``prox_anchor=per_step``). :data:`FORBIDDEN` +lists the flags that remove it or mask by version; :func:`refuse_forbidden_argv` +refuses them with the flag named. + +Admission (design D5, approximate): :func:`admission_argv` maps the +consumption limit ``A_hard`` and an optional admission limit ``k`` to +``--max-weight-staleness`` and the fully_async buffer capacity. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from yeto.rl.algos.mismatch_correction import ICEPOP_PATH, MIS_PATH, mis_config +from yeto.rl.engine.algorithm import AlgorithmSpec +from yeto.rl.engine.offpolicy_correction import ( + PROX_PER_STEP, + CorrectionMapping, + CorrectionWarning, + ForbiddenSetting, + refuse_forbidden, +) +from yeto.rl.engine.policy_age import validate_limit + +BACKEND = "miles" +PROX_ANCHOR = PROX_PER_STEP + +FORBIDDEN = ( + ForbiddenSetting(BACKEND, "--use-rollout-logprobs", + "it makes the rollout log-probabilities the PPO clip anchor, so pi_prox " + "is lost"), + ForbiddenSetting(BACKEND, "--skip-actor-forward-only", + "it skips the actor forward pass that computes pi_prox"), + ForbiddenSetting(BACKEND, "--mask-offpolicy-in-partial-rollout", + "it masks the older-version tokens of a continued sample; every response " + "token must enter the loss"), +) +FORBIDDEN_FLAGS = tuple(f.setting for f in FORBIDDEN) + +# Miles fully_async data buffer: capacity = factor * rollout_batch_size +# (miles/rollout/fully_async_data_buffer.py:136, Miles fork read 2026-10-14). +BUFFER_CAPACITY_FLAG = "--async-data-buffer-capacity-factor" + + +def _num(value: float) -> str: + return repr(float(value)) + + +def _function_path(spec: AlgorithmSpec) -> str | None: + c = spec.correction + if c.function is not None: + return c.function.path + return {"icepop": ICEPOP_PATH, "mis": MIS_PATH}.get(c.ratio_mask) + + +def tis_argv(spec: AlgorithmSpec) -> list[str]: + """``--use-tis`` and its bounds / function for a tis or custom correction.""" + + c = spec.correction + if c.method not in ("tis", "custom"): + return [] + argv = ["--use-tis"] + if c.upper is not None: + argv += ["--tis-clip", _num(c.upper)] + if c.lower is not None: + argv += ["--tis-clip-low", _num(c.lower)] + function = _function_path(spec) + if function is not None: + argv += ["--custom-tis-function-path", function] + return argv + + +def _warnings(spec: AlgorithmSpec) -> list[CorrectionWarning]: + c = spec.correction + if c.method != "tis" or c.ratio_mask == "mis": + return [] + out = [] + if c.level != "token": + out.append(CorrectionWarning( + "level", BACKEND, f"Miles TIS{' / IcePop' if c.ratio_mask == 'icepop' else ''} " + "weights tokens only; level other than token needs ratio_mask='mis'", "token")) + if c.veto is not None: + out.append(CorrectionWarning( + "veto", BACKEND, "Miles has a veto only in the MIS function (ratio_mask='mis')", None)) + if c.batch_normalize: + out.append(CorrectionWarning( + "batch_normalize", BACKEND, + "Miles normalizes weights only in the MIS function (ratio_mask='mis')", False)) + return out + + +def _values(argv: Sequence[str]) -> dict[str, Any]: + values: dict[str, Any] = {} + i = 0 + while i < len(argv): + flag = argv[i] + if i + 1 < len(argv) and not argv[i + 1].startswith("--"): + values[flag] = argv[i + 1] + i += 2 + else: + values[flag] = True + i += 1 + return values + + +def correction_mapping(spec: AlgorithmSpec) -> CorrectionMapping: + """The Miles settings of ``spec.correction`` with warnings and ``prox_anchor``.""" + + c = spec.correction + argv = tis_argv(spec) + if c.opsm_delta is not None: + argv += ["--use-opsm", "--opsm-delta", _num(c.opsm_delta)] + if c.use_rollout_logprobs: + argv.append("--use-rollout-logprobs") # refused by refuse_forbidden_argv + if c.mismatch_metrics: + argv.append("--get-mismatch-metrics") + attrs = mis_config(spec) + values = {**_values(argv), **{f"namespace.{k}": v for k, v in sorted(attrs.items())}} + return CorrectionMapping(backend=BACKEND, prox_anchor=PROX_ANCHOR, values=values, + warnings=tuple(_warnings(spec)), argv=tuple(argv), + runtime_attrs=attrs) + + +def refuse_forbidden_argv(argv: Sequence[str]) -> None: + """Refuse a Miles argv that holds a :data:`FORBIDDEN` flag (names the flag).""" + + refuse_forbidden((token.split("=", 1)[0] for token in argv), FORBIDDEN) + + +def admission_argv(max_policy_age: int, *, admission_limit: int | None = None) -> tuple[str, ...]: + """Approximate admission and consumption limits on Miles fully_async (D5). + + Consumption: ``--max-weight-staleness A_hard`` drops a group whose oldest + token is more than ``A_hard`` versions behind (whole group). Admission: + Miles has no limit by start version; a buffer of ``k`` rounds of groups + (``--async-data-buffer-capacity-factor k``) bounds how far generation can + run ahead. ``k`` None keeps Miles' buffer default. ``k <= A_hard``. + """ + + limit = validate_limit(max_policy_age) + if limit == 0: + return () + argv = ["--max-weight-staleness", str(limit)] + if admission_limit is not None: + k = validate_limit(admission_limit) + if not 1 <= k <= limit: + raise ValueError(f"admission limit k={k} must be in [1, A_hard={limit}]") + argv += [BUFFER_CAPACITY_FLAG, str(k)] + return tuple(argv) diff --git a/yeto/rl/adapters/miles/policy_age.py b/yeto/rl/adapters/miles/policy_age.py index a93785417..d52886feb 100644 --- a/yeto/rl/adapters/miles/policy_age.py +++ b/yeto/rl/adapters/miles/policy_age.py @@ -33,7 +33,12 @@ from yeto.rl.engine.policy_age import STAGE_NAMES, PolicyAgeError, PolicyAgeSupport, validate_limit -SUPPORT = PolicyAgeSupport(backend="miles", stage=3, max_policy_age=1) +# rl-offpolicy-correction-passthrough L6.9: the cap was 1 (the only limit +# measured on GPU). Decoupled PPO needs A_hard >= 2. Miles itself has no upper +# bound (--max-weight-staleness and --agentic-suspend-max-rounds take any int); +# 4 bounds the rollout compute a dropped group can waste. Sub-agent decision, +# 2026-10-14; limits above 1 are not yet run on GPU (L6.10). +SUPPORT = PolicyAgeSupport(backend="miles", stage=3, max_policy_age=4) PARTIAL_ROLLOUT_FLAGS = ("--partial-rollout",) AGENTIC_GENERATE = "miles.rollout.generate_hub.agentic_tool_call.generate" diff --git a/yeto/rl/adapters/miles/rollout.py b/yeto/rl/adapters/miles/rollout.py index 9b49dd193..4a61ce1db 100644 --- a/yeto/rl/adapters/miles/rollout.py +++ b/yeto/rl/adapters/miles/rollout.py @@ -30,6 +30,11 @@ from collections.abc import Mapping from types import SimpleNamespace +from yeto.rl.engine.offpolicy_correction import ( + GATE_DROPPED_LENGTHS, + GATE_GRANULARITY, + TRUNCATED_BY_TURN, +) from yeto.rl.engine.policy_versions import PublishedPolicies from yeto.rl.engine.ports import GroupMetadata, RolloutBatchHandle, over_sampling_fields from .traits import MILES_ABORT_MECHANISM @@ -460,6 +465,7 @@ def handle_from_metadata( aborted_in_flight_tokens=payload.get("aborted_in_flight_tokens"), aborted_in_flight_unknown_groups=payload.get("aborted_in_flight_unknown_groups"), abort_mechanism=MILES_ABORT_MECHANISM, + backend="miles", tool_wait_seconds=( float(payload["tool_wait_seconds"]) if "tool_wait_seconds" in payload else None ), @@ -507,6 +513,8 @@ def handle_from_metadata( "resumed_groups", "resumed_done_groups", "suspend_expired_trajectories", "policy_age_exceeded_trajectories", "suspended_trajectories", "suspended_env_seconds", "suspend_retried_turns", + # rl-offpolicy-correction-passthrough D5/D6: the driver pops these into ``offpolicy_gate`` + GATE_DROPPED_LENGTHS, GATE_GRANULARITY, TRUNCATED_BY_TURN, ) diff --git a/yeto/rl/adapters/verl/fully_async_ports.py b/yeto/rl/adapters/verl/fully_async_ports.py index b339348a2..0f59ff8b2 100644 --- a/yeto/rl/adapters/verl/fully_async_ports.py +++ b/yeto/rl/adapters/verl/fully_async_ports.py @@ -136,6 +136,7 @@ def generate(self, rollout_id: int, *, expected_policy_version: str | None = Non completed=len(groups), aborted=0, payload=None, submitted_groups=pulled, abort_mechanism=DISCARD_MECHANISM if collector.discarded else None, carry_over=carry, + backend="verl", # Same field and meaning as Miles: groups aborted in flight and NOT # trained. verl's partial rollout continues interrupted requests # (counted as resumed_trajectories in carry_over, like Miles' diff --git a/yeto/rl/adapters/verl/fully_async_round.py b/yeto/rl/adapters/verl/fully_async_round.py index 7791b3a3e..d428bd0f2 100644 --- a/yeto/rl/adapters/verl/fully_async_round.py +++ b/yeto/rl/adapters/verl/fully_async_round.py @@ -29,6 +29,13 @@ from dataclasses import dataclass, field from typing import Any +from yeto.rl.engine.offpolicy_correction import ( + GATE_DROPPED_LENGTHS, + GATE_GRANULARITY, + GATE_GROUP_DROPPED, + GATE_UNKNOWN_VERSION_DROPPED, + GRANULARITY_TRAJECTORY, +) from yeto.rl.engine.overlap import ROLLOUT_OVERLAP_MODE from .fully_async_translate import ( @@ -285,9 +292,17 @@ def discard_tally(self) -> dict[str, int]: "unknown_groups": sum(1 for v in self.discarded if v.unknown)} def carry_over_fields(self) -> dict[str, Any]: - """The round's ``rl_rollout_carry_over`` fields (Miles names, carry_over.CARRY_FIELDS).""" + """The round's ``rl_rollout_carry_over`` fields (Miles names, carry_over.CARRY_FIELDS). + + Also the ``offpolicy_gate`` inputs the driver turns into its own event + (rl-offpolicy-correction-passthrough D5/D6): verl has no drop by version + at consumption, so this collector is it (a group more than ``limit`` + versions behind is dropped whole). verl gives versions per trajectory, + so the granularity is ``trajectory``; truncation by turn is not reported. + """ carried = [v for v in self.kept if v.versions and min(v.versions) < self.current_outer] over_age = [v for v in self.discarded if not v.unknown] + unknown = [v for v in self.discarded if v.unknown] return { "execution_mode": ROLLOUT_OVERLAP_MODE, "max_policy_age": self.limit, @@ -297,15 +312,18 @@ def carry_over_fields(self) -> dict[str, Any]: "over_age_discarded_groups": len(over_age), "over_age_discarded_trajectories": sum(v.meta.n for v in over_age), "over_age_discarded_tokens": sum(sum(v.meta.lengths) for v in over_age), - "unknown_version_discarded_groups": sum(1 for v in self.discarded if v.unknown), + "unknown_version_discarded_groups": len(unknown), "cross_version_tokens": sum(v.older_tokens for v in self.kept), "trained_response_tokens": sum(sum(v.meta.lengths) for v in self.kept), "resumed_trajectories": sum(v.resumed for v in self.kept), "segment_token_mismatches": sum(v.segment_mismatches for v in self.kept), "discard_reasons": sorted({v.reason for v in self.discarded if v.reason})[:5], + GATE_DROPPED_LENGTHS: { + GATE_GROUP_DROPPED: [n for v in over_age for n in v.meta.lengths], + GATE_UNKNOWN_VERSION_DROPPED: [n for v in unknown for n in v.meta.lengths]}, + GATE_GRANULARITY: GRANULARITY_TRAJECTORY, } - def group_metadata(collector: RoundCollector, rollout_id: int, published: Mapping[int, str]): """Kept groups as :class:`GroupMetadata`; the policy token names the oldest version (the driver checks it against the hashes it published within the window).""" diff --git a/yeto/rl/adapters/verl/island_entry.py b/yeto/rl/adapters/verl/island_entry.py index 03ce5a310..24ba30b1e 100644 --- a/yeto/rl/adapters/verl/island_entry.py +++ b/yeto/rl/adapters/verl/island_entry.py @@ -25,6 +25,7 @@ import sys import time from pathlib import Path +from types import SimpleNamespace from . import config as vconf from .pins import VERL_COMMIT, VERL_ROOT, expected_versions @@ -287,13 +288,16 @@ def main(argv=None) -> int: algo = algorithm_options(spec_dict) chat = json.loads(args.apply_chat_template_kwargs) if args.apply_chat_template_kwargs else {} max_prompt = 512 + limit = int(args.rl_max_policy_age or 0) run = vconf.VerlRunConfig( model_path=model_path, train_file=train_file, val_file=val_file, out_dir=str(work / "out"), lora_rank=args.lora_r, lora_alpha=args.lora_r, lr=args.inner_lr, groups_per_round=args.groups_per_round, samples_per_group=args.samples_per_group, max_prompt_len=max_prompt, max_response_len=args.rollout_max_response_len, seed=args.seed, total_rounds=args.global_rounds, correction=algo["correction"], - tis_lower=algo.get("tis_lower", 0.0), tis_upper=algo["tis_upper"], + # limit > 0: the correction is mapped by offpolicy_correction in + # fully_async_plan (lower > 0 warns there, D3); the sync path keeps its check. + tis_lower=0.0 if limit else algo.get("tis_lower", 0.0), tis_upper=algo["tis_upper"], gpu_memory_utilization=float(os.environ.get("YETO_VERL_GPU_MEM", "0.5")), reward_path=str(Path(__file__).with_name("reward_fn.py")), reward_name=function_for(args.reward_function), chat_template_kwargs=chat, @@ -310,10 +314,10 @@ def main(argv=None) -> int: sync = "strict" if sync != "none" and not args.syncer: raise SystemExit("--syncer is required unless --rl-single-island-no-sync") - limit = int(args.rl_max_policy_age or 0) asserted_keys = vconf.ASSERTED_KEYS if limit: # agentic-rollout-utilization 6.4b: fully_async path overrides, asserted_keys = fully_async_plan(limit, overrides, spec_dict, args, sync) + append_correction_record(args, spec_dict) test = json.loads(TEST_FILE.read_text()) if TEST_FILE.is_file() else {} exit_after = (test.get("exit_after_version") or {}).get(str(lid)) env_exit = os.environ.get("YETO_VERL_TEST_EXIT_AFTER", "") # TEST ONLY "ISLAND:VERSION[,...]" @@ -369,12 +373,18 @@ def main(argv=None) -> int: def fully_async_plan(limit: int, overrides: list[str], spec_dict: dict | None, args, sync: str): """Limit > 0: fully_async overrides + asserted keys; the algorithm contract is checked here (bounded staleness needs ``execution.max_policy_staleness >= limit`` - and a TIS correction) so a wrong spec fails before verl starts.""" + and a TIS correction) so a wrong spec fails before verl starts. + + The ``algorithm.rollout_correction`` overrides come from the correction + mapping (rl-offpolicy-correction-passthrough D2/D3): decoupled PPO with + ``bypass_mode=False`` written over the fully_async yaml default; a + forbidden setting is refused here.""" from yeto.rl.engine.algorithm import AlgorithmSpec from yeto.rl.engine.execution_profile import check_algorithm_contract from .fully_async_round import (FULLY_ASYNC_ASSERTED_KEYS, execution_profile_for, fully_async_run_overrides) + from .offpolicy_correction import apply_to_overrides, correction_mapping from .policy_age import SUPPORT SUPPORT.check(limit) @@ -384,6 +394,7 @@ def fully_async_plan(limit: int, overrides: list[str], spec_dict: dict | None, a check_algorithm_contract(profile, spec) out = fully_async_run_overrides(overrides, limit, groups_per_round=args.groups_per_round, rounds=args.global_rounds) + out = apply_to_overrides(out, correction_mapping(spec)) from yeto.hw.catalog import device_family if device_family() == "ascend": @@ -394,6 +405,32 @@ def fully_async_plan(limit: int, overrides: list[str], spec_dict: dict | None, a return out, tuple(vconf.ASSERTED_KEYS) + FULLY_ASYNC_ASSERTED_KEYS +def append_correction_record(args, spec_dict: dict | None) -> None: + """rl-offpolicy-correction-passthrough D8 (as Miles' island entry): the island + tape records the full correction group, the verl values and ``prox_anchor``. + A field verl cannot map refuses the start (phase 2, L6b.2), after the record + and its ``correction_config_warning`` events are on the tape.""" + from yeto.rl.engine.algorithm import AlgorithmSpec + from yeto.rl.engine.events import append_tape_event + from yeto.rl.engine.offpolicy_correction import (CORRECTION_RECORD_EVENT, correction_record, + refuse_correction_warnings) + + from .offpolicy_correction import BACKEND, correction_mapping + + spec = AlgorithmSpec.from_dict(spec_dict) if spec_dict else AlgorithmSpec() + if spec.correction.is_default(): + return # default tapes unchanged + mapping = correction_mapping(spec) + tape = SimpleNamespace(yeto_rl_event_tape=args.event_tape, + yeto_rl_learner_id=args.learner_id) + append_tape_event(tape, {"event": CORRECTION_RECORD_EVENT, + **correction_record(spec.correction, mapping, backend=BACKEND)}) + warnings = [warning.event() for warning in mapping.warnings] + for warning in warnings: + append_tape_event(tape, warning) + refuse_correction_warnings(warnings) + + def _override_value(overrides, key): for item in overrides: if item.startswith(key + "="): diff --git a/yeto/rl/adapters/verl/offpolicy_correction.py b/yeto/rl/adapters/verl/offpolicy_correction.py new file mode 100644 index 000000000..ccebdad95 --- /dev/null +++ b/yeto/rl/adapters/verl/offpolicy_correction.py @@ -0,0 +1,176 @@ +"""verl mapping of the neutral correction group +(change ``rl-offpolicy-correction-passthrough`` L6.4, L6.6, L6.7, L6.11). + +:func:`correction_mapping` turns ``AlgorithmSpec.correction`` into Hydra +overrides under ``algorithm.rollout_correction`` (verl fork ``acad9875``, +``verl/trainer/ppo/rollout_corr_helper.py``). Mapping table (design D3): + +========================== ===================================================== +neutral verl +========================== ===================================================== +tis, token / sequence ``rollout_is=token|sequence``, ``rollout_is_threshold=upper`` +ratio_mask=icepop ``rollout_is_threshold='lower_upper'`` (weights outside + the range set to 0; ``loss_mask`` unchanged) +ratio_mask=mis ``rollout_rs=token_k1|seq_mean_k1``, + ``rollout_rs_threshold='lower_upper'`` (rejected tokens + leave ``response_mask``: the loss denominator shrinks) +batch_normalize ``rollout_is_batch_normalize=True`` +========================== ===================================================== + +Rows verl cannot express give a :class:`CorrectionWarning`; phase 2 (L6b.2) refuses +the start on them: +``lower > 0`` on plain TIS (verl clamps the upper side only), level +``sequence`` with the MIS mask (verl has the geometric mean only), ``veto``, +``opsm``, a custom function, and the MIS mask (denominator). + +Decoupled PPO (design D2): ``bypass_mode=False`` makes verl recompute +``old_log_probs`` as pi_prox; the fully_async trainer computes it on the +weights of the first local step of the current sync cycle +(``fully_async_trainer.py`` ``_compute_old_log_prob``), hence +``prox_anchor=first_local_step``. ``fully_async_ppo_trainer.yaml`` sets +``bypass_mode: True``, so every mapping writes ``bypass_mode`` explicitly and +:data:`FORBIDDEN` refuses ``bypass_mode=True``. + +``actor.use_rollout_log_probs`` (yaml: True) is not forbidden: no Python code +in the fork reads it (``grep`` over ``verl/`` at ``acad9875``: only the config +dataclasses declare it), so it does not change pi_prox (L6.6). +""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +from yeto.rl.engine.algorithm import AlgorithmSpec +from yeto.rl.engine.offpolicy_correction import ( + PROX_FIRST_LOCAL_STEP, + CorrectionMapping, + CorrectionWarning, + ForbiddenSetting, + refuse_forbidden, +) + +BACKEND = "verl" +PROX_ANCHOR = PROX_FIRST_LOCAL_STEP +PREFIX = "algorithm.rollout_correction." +BYPASS_KEY = PREFIX + "bypass_mode" + +# verl's own default (rollout_correction.yaml) when the spec gives no upper bound. +DEFAULT_UPPER = 2.0 +# verl IcePop needs lower > 0 (_parse_rollout_is_threshold); a missing or zero +# lower runs with this bound, which keeps every positive weight in practice. +ICEPOP_MIN_LOWER = 1e-8 + +FORBIDDEN = ( + ForbiddenSetting(BACKEND, f"{BYPASS_KEY}=True", + "it sets old_log_probs = rollout_log_probs, so pi_prox is lost"), +) + + +def _num(value: float) -> str: + return repr(float(value)) + + +def _range(lower: float, upper: float) -> str: + return f"{_num(lower)}_{_num(upper)}" + + +def _override_value(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return str(value) + if isinstance(value, float): + return _num(value) + if isinstance(value, str) and "_" in value: + return f"'{value}'" # "lower_upper" stays a string for Hydra + return str(value) + + +def _mapped(spec: AlgorithmSpec) -> tuple[dict[str, Any], list[CorrectionWarning]]: + c = spec.correction + values: dict[str, Any] = {"bypass_mode": bool(c.use_rollout_logprobs), + "rollout_is": None, "rollout_rs": None} + warnings: list[CorrectionWarning] = [] + + def warn(field: str, reason: str, ran_with: Any = None) -> None: + warnings.append(CorrectionWarning(field, BACKEND, reason, ran_with)) + + if c.method == "opsm": + warn("method", "verl has no OPSM; the run has no correction", "none") + if c.method == "custom": + warn("function", "verl takes no custom correction function; the run has no correction", + "none") + if c.method != "tis": + return values, warnings + + upper = DEFAULT_UPPER if c.upper is None else float(c.upper) + lower = 0.0 if c.lower is None else float(c.lower) + if c.veto is not None: + warn("veto", "verl has no per-sequence veto", None) + + if c.ratio_mask == "mis": + if lower <= 0.0: + warn("lower", "verl rejection sampling needs lower > 0; the k1 rule uses 1/upper", + 1.0 / upper) + lower = 1.0 / upper + values["rollout_rs"] = "token_k1" if c.level == "token" else "seq_mean_k1" + values["rollout_rs_threshold"] = _range(lower, upper) + if c.level == "sequence": + warn("level", "verl rejection sampling has no product-of-ratios rule; the " + "geometric mean (seq_mean_k1) runs", "geometric") + warn("ratio_mask", "verl removes rejected tokens from response_mask, so the loss " + "denominator shrinks; the neutral meaning is 'denominator unchanged'", + "denominator reduced") + if c.batch_normalize: + warn("batch_normalize", "verl normalizes IS weights only; the MIS mask maps to " + "rejection sampling without weights", False) + return values, warnings + + values["rollout_is"] = c.level # token | sequence (geometric needs the MIS mask) + if c.ratio_mask == "icepop": + if lower <= 0.0: + warn("lower", "verl IcePop needs lower > 0", ICEPOP_MIN_LOWER) + lower = ICEPOP_MIN_LOWER + values["rollout_is_threshold"] = _range(lower, upper) + else: + values["rollout_is_threshold"] = upper + if lower > 0.0: + warn("lower", "verl TIS clamps the upper side only (weights below lower pass " + "through)", 0.0) + values["rollout_is_batch_normalize"] = bool(c.batch_normalize) + return values, warnings + + +def correction_mapping(spec: AlgorithmSpec) -> CorrectionMapping: + """The verl settings of ``spec.correction`` with warnings and ``prox_anchor``.""" + + values, warnings = _mapped(spec) + keyed = {PREFIX + k: v for k, v in values.items()} + overrides = tuple(f"{k}={_override_value(v)}" for k, v in keyed.items()) + return CorrectionMapping(backend=BACKEND, prox_anchor=PROX_ANCHOR, values=keyed, + warnings=tuple(warnings), overrides=overrides) + + +def _normalized(override: str) -> str: + key, sep, value = override.lstrip("+~").partition("=") + if sep and value.strip().strip("'\"").lower() in ("true", "false"): + value = value.strip().strip("'\"").capitalize() + return f"{key}{sep}{value}" + + +def refuse_forbidden_overrides(overrides: Iterable[str]) -> None: + """Refuse Hydra overrides that hold a :data:`FORBIDDEN` assignment (names it).""" + + refuse_forbidden((_normalized(o) for o in overrides), FORBIDDEN) + + +def apply_to_overrides(overrides: Iterable[str], mapping: CorrectionMapping) -> list[str]: + """``overrides`` with every ``algorithm.rollout_correction.*`` entry replaced by + the mapping's (one value per key), then checked against :data:`FORBIDDEN`.""" + + out = [o for o in overrides if not o.lstrip("+~").startswith(PREFIX)] + out += list(mapping.overrides) + refuse_forbidden_overrides(out) + return out + diff --git a/yeto/rl/adapters/verl/policy_age.py b/yeto/rl/adapters/verl/policy_age.py index d5824c364..755c6ba68 100644 --- a/yeto/rl/adapters/verl/policy_age.py +++ b/yeto/rl/adapters/verl/policy_age.py @@ -17,9 +17,11 @@ from yeto.rl.engine.policy_age import PolicyAgeSupport, validate_limit # 6.4b (10-09): stage 2 through the fully_async path (rollouter + trainer on -# separate GPUs). The largest limit is 1 -- the only one the pre-launch review -# (rl-verl-backend 6.6) tests; sub-agent decision, raise after a GPU pass. -SUPPORT = PolicyAgeSupport(backend="verl", stage=2, max_policy_age=1) +# separate GPUs). rl-offpolicy-correction-passthrough L6.9: the cap was 1; +# decoupled PPO needs A_hard >= 2. verl has no upper bound of its own +# (staleness_threshold = N, fully_async_translate.staleness_threshold_for); +# 4 matches the Miles cap. Limits above 1 are not yet run on GPU (L6.10). +SUPPORT = PolicyAgeSupport(backend="verl", stage=2, max_policy_age=4) def policy_age_overrides(limit: int, *, groups_per_round: int | None = None) -> tuple[str, ...]: diff --git a/yeto/rl/adapters/verl/ports_impl.py b/yeto/rl/adapters/verl/ports_impl.py index bed469f12..dad0f2801 100644 --- a/yeto/rl/adapters/verl/ports_impl.py +++ b/yeto/rl/adapters/verl/ports_impl.py @@ -253,6 +253,7 @@ def generate(self, rollout_id: int, *, expected_policy_version: str | None = Non rollout_id=rollout_id, policy_version=version, policy_hash=digest, groups=tuple(metas), completed=len(metas), aborted=0, payload=batch, batch_summary=summary, data_cursor=self.data_cursor(), + backend="verl", ) def abort(self) -> None: diff --git a/yeto/rl/algos/mismatch_correction.py b/yeto/rl/algos/mismatch_correction.py index 768eb8da8..2f53962dc 100644 --- a/yeto/rl/algos/mismatch_correction.py +++ b/yeto/rl/algos/mismatch_correction.py @@ -11,7 +11,7 @@ mechanism spec masks ================== =============================================== ========= mismatch_observe method=custom, function=:data:`OBSERVE_PATH` no - tis (P0 built-in) method=tis, tis_clip, tis_clip_low no + tis (P0 built-in) method=tis, upper, lower no icepop method=custom, function=:data:`ICEPOP_PATH` yes opsm_trainer OPSM, opsm_old_logprob_source=trainer yes opsm_rollout OPSM, opsm_old_logprob_source=rollout yes @@ -105,14 +105,16 @@ def selected_corrections(spec: AlgorithmSpec) -> tuple[str, ...]: out = [] c = spec.correction fn = _function(spec) - if c.method == "tis": + if c.method == "tis" and c.ratio_mask == "none": out.append("tis") if fn == OBSERVE_PATH: out.append("mismatch_observe") - if fn == ICEPOP_PATH: + if fn == ICEPOP_PATH or c.ratio_mask == "icepop": out.append("icepop") if fn == MIS_PATH: out.append("mis_mask" if c.mis_mode == "mask" else "mis") + if c.ratio_mask == "mis": # neutral MIS ratio mask (rl-offpolicy-correction D3) + out.append("mis_mask") if _uses_opsm(spec): out.append(f"opsm_{c.opsm_old_logprob_source}") return tuple(out) @@ -205,18 +207,18 @@ def _positive(path: str, value: Any): def _reject_bounds(spec: AlgorithmSpec) -> str | None: c = spec.correction fn = _function(spec) - if fn == ICEPOP_PATH and (c.tis_clip is None or c.tis_clip_low is None): - missing = [n for n in ("tis_clip", "tis_clip_low") if getattr(c, n) is None] + if fn == ICEPOP_PATH and (c.upper is None or c.lower is None): + missing = [n for n in ("upper", "lower") if getattr(c, n) is None] return ( f"IcePop needs explicit interval bounds: missing " f"{', '.join('correction.' + m for m in missing)} (no engine default is inherited)" ) - if (c.method == "tis" or fn == ICEPOP_PATH) and c.tis_clip is not None \ - and c.tis_clip_low is not None and not c.tis_clip_low < c.tis_clip: + if (c.method == "tis" or fn == ICEPOP_PATH) and c.upper is not None \ + and c.lower is not None and not c.lower < c.upper: what = "IcePop interval" if fn == ICEPOP_PATH else "TIS clip" return ( - f"{what}: correction.tis_clip_low={c.tis_clip_low} must be smaller than " - f"correction.tis_clip={c.tis_clip}" + f"{what}: correction.lower={c.lower} must be smaller than " + f"correction.upper={c.upper}" ) return None @@ -226,8 +228,8 @@ def _reject_observe(spec: AlgorithmSpec) -> str | None: if _function(spec) != OBSERVE_PATH: return None problems = [] - if c.tis_clip is not None or c.tis_clip_low is not None: - problems.append("correction.tis_clip/tis_clip_low (observe-only weights are 1)") + if c.upper is not None or c.lower is not None: + problems.append("correction.upper/lower (observe-only weights are 1)") if _uses_opsm(spec): problems.append("OPSM (observe-only cannot be combined with any correction)") if not c.mismatch_metrics: @@ -295,8 +297,8 @@ def _reject_mis(spec: AlgorithmSpec) -> str | None: ) if c.mis_batch_normalize and c.mis_level == "geometric": problems.append("correction.mis_batch_normalize supports mis_level token|sequence only") - if c.tis_clip is not None or c.tis_clip_low is not None: - problems.append("MIS uses mis_lower_bound/mis_upper_bound, not correction.tis_clip*") + if c.upper is not None or c.lower is not None: + problems.append("MIS uses mis_lower_bound/mis_upper_bound, not correction.lower/upper") if problems: return "MIS: " + "; ".join(problems) return None @@ -322,9 +324,11 @@ def mis_config(spec: AlgorithmSpec) -> dict[str, Any]: Empty unless the MIS function is selected. """ + c = spec.correction + if c.ratio_mask == "mis": + return neutral_mis_config(c) if _function(spec) != MIS_PATH: return {} - c = spec.correction return { "rs_level": c.mis_level, "rs_lower_bound": None, @@ -339,6 +343,29 @@ def mis_config(spec: AlgorithmSpec) -> dict[str, Any]: } +def neutral_mis_config(c: Any) -> dict[str, Any]: + """MIS attributes for the neutral ``ratio_mask='mis'`` (rl-offpolicy-correction D3). + + Mask mode over ``[lower, upper]`` at ``level``. ``veto`` needs Miles' + rejection-sampling branch (``use_rs``); its RS range is the same interval, + so RS adds only the veto (vendor/miles_mis.py, read 2026-10-14, not run on GPU). + """ + + veto = c.veto is not None + return { + "rs_level": c.level, + "rs_lower_bound": c.lower if veto else None, + "rs_upper_bound": c.upper if veto else None, + "rs_veto_threshold": c.veto, + "tis_batch_normalize": bool(c.batch_normalize), + "tis_level": c.level, + "tis_lower_bound": c.lower, + "tis_mode": "mask", + "tis_upper_bound": c.upper, + "use_rs": veto, + } + + register_runtime_attrs("mismatch_mis", mis_config) diff --git a/yeto/rl/engine/algorithm.py b/yeto/rl/engine/algorithm.py index 4ef3d3cd8..2ce1f4f7f 100644 --- a/yeto/rl/engine/algorithm.py +++ b/yeto/rl/engine/algorithm.py @@ -116,6 +116,18 @@ def normalize_filter_name(name: str | None) -> str | None: KL_PLACEMENTS = ("none", "reward", "loss") KL_ESTIMATORS = ("k1", "k2", "k3", "low_var_kl") CORRECTION_METHODS = ["none", "tis", "opsm", "custom"] +# rl-offpolicy-correction-passthrough D1/D3: neutral correction fields. +CORRECTION_LEVELS = ("token", "sequence", "geometric") +# Optional ratio masks on top of method='tis' (default none, design A2). +RATIO_MASKS = ("none", "icepop", "mis") +# The correction group as every pre-D1 hash saw a default spec. sha256() hashes +# a default group as this constant, so default specs keep their hash (D8, L6b.1). +DEFAULT_CORRECTION_IDENTITY = ( + ("function", None), ("method", "none"), ("mismatch_metrics", False), ("opsm_delta", None), + ("tis_clip", None), ("tis_clip_low", None), ("use_rollout_logprobs", False), +) +# Pre-D1 names of the bounds, still accepted on input (constructor and JSON). +CORRECTION_FIELD_ALIASES = {"tis_clip": "upper", "tis_clip_low": "lower"} DYNAMIC_SAMPLING_FILTERS = { BOUNDED_NONZERO_STD_FILTER, STOCK_NONZERO_STD_FILTER, SECRLENV_GROUP_CHECK_FILTER, } @@ -787,10 +799,28 @@ def __post_init__(self) -> None: @dataclass(frozen=True) class CorrectionSpec(_Group): + """Off-policy / mismatch correction (rl-offpolicy-correction-passthrough D1). + + The neutral fields are written in full because the backends have + different defaults: ``level`` (token | sequence | geometric), ``lower`` and + ``upper`` (the importance-weight bounds; ``upper`` is the truncation bound + ``C``), ``veto``, ``batch_normalize`` and ``ratio_mask`` (none | icepop | + mis, on top of ``method='tis'``). Adapters map them to backend settings; + yeto never computes the weights. ``tis_clip`` / ``tis_clip_low`` are the + pre-D1 names of ``upper`` / ``lower``: accepted on input, read-only after. + + Phase 1 (design D8): the group does not enter ``AlgorithmSpec.sha256()``; + the run manifest and the island tapes record it instead. + """ + GROUP = "correction" method: str = "none" # none | tis | opsm | custom (custom tis function) - tis_clip: float | None = None - tis_clip_low: float | None = None + level: str = "token" + lower: float | None = None + upper: float | None = None + veto: float | None = None + batch_normalize: bool = False + ratio_mask: str = "none" opsm_delta: float | None = None function: PluginRef | None = None # --custom-tis-function-path use_rollout_logprobs: bool = False @@ -800,22 +830,43 @@ class CorrectionSpec(_Group): def __post_init__(self) -> None: s = object.__setattr__ s(self, "method", _choice("correction.method", self.method, CORRECTION_METHODS)) - s(self, "tis_clip", _number("correction.tis_clip", self.tis_clip, low=0.0, low_open=True)) - s(self, "tis_clip_low", _number("correction.tis_clip_low", self.tis_clip_low, low=0.0)) + s(self, "level", _choice("correction.level", self.level, CORRECTION_LEVELS)) + s(self, "ratio_mask", _choice("correction.ratio_mask", self.ratio_mask, RATIO_MASKS)) + s(self, "upper", _number("correction.upper", self.upper, low=0.0, low_open=True)) + s(self, "lower", _number("correction.lower", self.lower, low=0.0)) + s(self, "veto", _number("correction.veto", self.veto, low=0.0, low_open=True)) s(self, "opsm_delta", _number("correction.opsm_delta", self.opsm_delta, low=0.0, low_open=True)) s(self, "function", _plugin("correction.function", self.function)) + _boolean("correction.batch_normalize", self.batch_normalize) _boolean("correction.use_rollout_logprobs", self.use_rollout_logprobs) _boolean("correction.mismatch_metrics", self.mismatch_metrics) if self.method == "custom" and self.function is None: raise AlgorithmSpecError("correction.function is required for correction.method='custom'") if self.method != "custom" and self.function is not None: raise AlgorithmSpecError("correction.function requires correction.method='custom'") - if self.method not in ("tis", "custom") and ( - self.tis_clip is not None or self.tis_clip_low is not None - ): + weighting = self.method in ("tis", "custom") + if not weighting and (self.upper is not None or self.lower is not None): raise AlgorithmSpecError( - "correction.tis_clip/tis_clip_low require correction.method tis or custom" + "correction.upper/lower (tis_clip/tis_clip_low) require correction.method " + "tis or custom" + ) + if not weighting: + given = [name for name, default in (("level", "token"), ("veto", None), + ("batch_normalize", False)) + if getattr(self, name) != default] + if given: + raise AlgorithmSpecError( + f"{', '.join('correction.' + n for n in given)} require correction.method " + "tis or custom" + ) + if self.ratio_mask != "none" and self.method != "tis": + raise AlgorithmSpecError( + f"correction.ratio_mask={self.ratio_mask!r} requires correction.method='tis'" + ) + if self.level == "geometric" and self.ratio_mask != "mis" and self.method == "tis": + raise AlgorithmSpecError( + "correction.level='geometric' needs correction.ratio_mask='mis'" ) if self.method == "none" and self.opsm_delta is not None: raise AlgorithmSpecError( @@ -824,17 +875,54 @@ def __post_init__(self) -> None: ) if self.method == "opsm" and self.opsm_delta is None: raise AlgorithmSpecError("correction.opsm_delta is required for correction.method='opsm'") - if self.method == "tis" and (self.tis_clip is None or self.tis_clip_low is None): + if self.method == "tis" and (self.upper is None or self.lower is None): raise AlgorithmSpecError( - "correction.tis_clip and correction.tis_clip_low are required for " - "correction.method='tis' (no engine default is inherited)" + "correction.upper and correction.lower (tis_clip, tis_clip_low) are required " + "for correction.method='tis' (no engine default is inherited)" ) - if self.mismatch_metrics and self.method not in ("tis", "custom"): + if self.mismatch_metrics and not weighting: raise AlgorithmSpecError( "correction.mismatch_metrics needs a tis/custom correction function" ) self._normalize_ext() + @property + def tis_clip(self) -> float | None: + return self.upper + + @property + def tis_clip_low(self) -> float | None: + return self.lower + + @classmethod + def from_dict(cls, payload: Any): + if isinstance(payload, Mapping): + payload = _rename_correction_aliases(dict(payload)) + return super().from_dict(payload) + + +def _rename_correction_aliases(values: dict[str, Any]) -> dict[str, Any]: + for alias, name in CORRECTION_FIELD_ALIASES.items(): + if alias not in values: + continue + value = values.pop(alias) + if name in values and values[name] != value: + raise AlgorithmSpecError( + f"correction.{alias} is the old name of correction.{name}; give one of them") + values[name] = value + return values + + +_correction_init = CorrectionSpec.__init__ + + +def _correction_init_with_aliases(self, *args: Any, **kwargs: Any) -> None: + _correction_init(self, *args, **_rename_correction_aliases(kwargs)) + + +# The pre-D1 keyword names stay accepted; dataclasses.replace() sees only fields. +CorrectionSpec.__init__ = _correction_init_with_aliases # type: ignore[method-assign] + @dataclass(frozen=True) class SamplingSpec(_Group): @@ -1267,12 +1355,29 @@ def effective_default_at(self, path: str) -> Any: return self.default_at(path) def canonical_json(self) -> str: + """The full spec (correction group included): manifests, tapes, islands.""" return json.dumps( self.to_dict(), sort_keys=True, separators=(",", ":"), allow_nan=False ) + def identity_dict(self) -> dict[str, Any]: + """The payload of :meth:`sha256` (rl-offpolicy-correction-passthrough D8, L6b.1). + + The correction group is hashed as written. A default group is hashed + as the pre-D1 default group (:data:`DEFAULT_CORRECTION_IDENTITY`), so + every spec whose correction group is the default keeps its hash. + """ + + out = self.to_dict() + if "correction" in out and self.correction.is_default(): + out["correction"] = dict(DEFAULT_CORRECTION_IDENTITY) + return out + def sha256(self) -> str: - return hashlib.sha256(self.canonical_json().encode("utf-8")).hexdigest() + payload = json.dumps( + self.identity_dict(), sort_keys=True, separators=(",", ":"), allow_nan=False + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() @property def schema(self) -> str: diff --git a/yeto/rl/engine/backends.py b/yeto/rl/engine/backends.py index 59b23f6d3..243e07c98 100644 --- a/yeto/rl/engine/backends.py +++ b/yeto/rl/engine/backends.py @@ -56,6 +56,8 @@ class BackendEntry: # identity BackendIdentity of this adapter (phase 5) # image Modal-built engine image recipe (verl: ``verl-build:``) # policy_age SUPPORT: PolicyAgeSupport (agentic-rollout-utilization 2.1/6.3) + # offpolicy_correction correction_mapping(spec) + FORBIDDEN (rl-offpolicy-correction + # -passthrough; neutral side: yeto.rl.engine.offpolicy_correction) # overlay source patch applied to the engine checkout (launcher setup, run manifest) roles: Mapping[str, str] = field(default_factory=dict) @@ -79,6 +81,7 @@ class BackendEntry: "binding": f"{_MILES}.binding", "identity": f"{_MILES}.identity", "policy_age": f"{_MILES}.policy_age", + "offpolicy_correction": f"{_MILES}.offpolicy_correction", "overlay": f"{_MILES}.overlay", }), # rl-verl-backend: first step FSDP2 + vLLM, one GPU per island (roles the @@ -94,6 +97,7 @@ class BackendEntry: "identity": f"{_VERL}.identity", "image": f"{_VERL}.image", "policy_age": f"{_VERL}.policy_age", + "offpolicy_correction": f"{_VERL}.offpolicy_correction", }), } diff --git a/yeto/rl/engine/driver.py b/yeto/rl/engine/driver.py index 4ab523892..dacbdb77c 100644 --- a/yeto/rl/engine/driver.py +++ b/yeto/rl/engine/driver.py @@ -86,6 +86,7 @@ train_blockers, ) from .leave_reason import leave_reason_for +from .offpolicy_correction import OFFPOLICY_GATE_EVENT, gate_from_carry_over from .overlap import EvalOverlap, EvalStarter, overlap_refusal from .ports import ( Placement, @@ -1259,7 +1260,14 @@ def _emit_carry_over(self, rollout_id: int, batch: Any) -> None: fields = getattr(batch, "carry_over", None) if not fields: return + fields = dict(fields) groups = getattr(batch, "groups", ()) or () + gate = gate_from_carry_over(fields, kept=len(groups), + backend=getattr(batch, "backend", None) or "unknown") + if gate: # rl-offpolicy-correction-passthrough D6: its own event per round + self.emit(OFFPOLICY_GATE_EVENT, rollout_id=rollout_id, + policy_version=int(getattr(batch, "policy_version", rollout_id)), + t=self.clock(), **self._labels(), **gate) self.emit("rl_rollout_carry_over", rollout_id=rollout_id, policy_version=int(getattr(batch, "policy_version", rollout_id)), t=self.clock(), **{**self._labels(), **dict(fields)}, diff --git a/yeto/rl/engine/island_ledger.py b/yeto/rl/engine/island_ledger.py index 8b6f2dee5..58d97277e 100644 --- a/yeto/rl/engine/island_ledger.py +++ b/yeto/rl/engine/island_ledger.py @@ -108,6 +108,14 @@ class StalenessPolicy: # Algorithm families whose value/critic needs on-policy data (design Q3). on_policy_only: bool = False + @classmethod + def for_algorithm(cls, spec: Any, **kwargs: Any) -> "StalenessPolicy": + """``correction`` derived from the algorithm spec, not the fixed ``"tis"`` + (rl-offpolicy-correction-passthrough D7).""" + from .offpolicy_correction import staleness_correction + + return cls(correction=staleness_correction(spec), **kwargs) + def __post_init__(self) -> None: if self.correction not in IS_CORRECTIONS: raise LedgerError(f"correction {self.correction!r} is not one of {IS_CORRECTIONS}") diff --git a/yeto/rl/engine/offpolicy_correction.py b/yeto/rl/engine/offpolicy_correction.py new file mode 100644 index 000000000..759b63142 --- /dev/null +++ b/yeto/rl/engine/offpolicy_correction.py @@ -0,0 +1,418 @@ +"""Off-policy correction pass-through: the backend-neutral part +(change ``rl-offpolicy-correction-passthrough``). + +yeto does not compute importance weights. A backend adapter maps the neutral +``CorrectionSpec`` (design D1) to its own settings and returns a +:class:`CorrectionMapping`. This module holds what every adapter shares: + +* :class:`CorrectionMapping` / :class:`CorrectionWarning` -- the mapped + values, the proximal-policy anchor (``prox_anchor``, design D2) and the + fields the backend could not map (``correction_config_warning``, D3); +* :func:`refuse_correction_warnings` -- phase 2 (L6b.2): an unmappable field + or an island difference refuses the launch; +* :class:`ForbiddenSetting` / :func:`refuse_forbidden` -- backend settings + that break decoupled PPO (Miles ``--use-rollout-logprobs``, verl + ``bypass_mode=True``, ...). They are refused at launch with the setting + named, also in phase 1; +* :func:`correction_record` / :func:`compare_island_records` -- the manifest + and tape record (D8) and the cross-island comparison (D2, D8); +* :func:`start_version` / :func:`start_version_gap` -- the version gap from + the start version (D4); +* :func:`offpolicy_gate_event` -- the per-round drop and truncation event (D6); +* :func:`cross_version_bounds` / :func:`staleness_correction` -- the + observe-only cross-version IS takes its bounds and name from the spec (D7). + +Phase 1 warned on unmappable settings and on island differences. Phase 2 +(L6b.2) refuses them: the launcher refuses the run manifest's warnings and each +island refuses the fields its backend cannot map. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from yeto.rl.engine.algorithm import AlgorithmSpec, CorrectionSpec + +CORRECTION_RECORD_SCHEMA = "yeto-offpolicy-correction-v1" +# Tape / manifest event names (design D2, D3, D6, D8). +CORRECTION_RECORD_EVENT = "rl_offpolicy_correction" +CORRECTION_CONFIG_WARNING = "correction_config_warning" +OFFPOLICY_GATE_EVENT = "offpolicy_gate" + +# Which weights the backend uses for pi_prox (design D2). +PROX_PER_STEP = "per_step" # recomputed on the weights before each update (Miles) +PROX_FIRST_LOCAL_STEP = "first_local_step" # first local step of the cycle (verl) +PROX_ANCHORS = (PROX_PER_STEP, PROX_FIRST_LOCAL_STEP) + +# offpolicy_gate reasons (design D6). Backends may add their own admission drops. +GATE_KEPT = "kept" +GATE_GROUP_DROPPED = "group_dropped" # gap over A_hard at consumption, whole group +GATE_UNKNOWN_VERSION_DROPPED = "unknown_version_dropped" +GATE_REASONS = (GATE_KEPT, GATE_GROUP_DROPPED, GATE_UNKNOWN_VERSION_DROPPED) +# Groups the backend dropped as stale inside its own buffer (Miles --fully-async +# ``stale_dropped_groups``), counted apart from the yeto gate drops. Present only +# when the backend reported the count; no lengths (the backend does not give them). +GATE_BACKEND_STALE_DROPPED = "backend_stale_dropped" +# Response-length bucket edges (tokens) of the dropped-trajectory histogram. +# Fixed in code so that rounds and runs stay comparable. +DROPPED_LENGTH_EDGES = (256, 1024, 4096, 16384) +GRANULARITY_TOKEN = "token" +GRANULARITY_TRAJECTORY = "trajectory" +GRANULARITIES = (GRANULARITY_TOKEN, GRANULARITY_TRAJECTORY) +# Round-metadata keys an adapter fills for the driver (removed from +# rl_rollout_carry_over): dropped lengths per reason, truncation per turn, +# and the granularity of the turn counts. +GATE_DROPPED_LENGTHS = "offpolicy_gate_dropped_lengths" +TRUNCATED_BY_TURN = "cross_version_truncated_by_turn" +GATE_GRANULARITY = "offpolicy_gate_granularity" + + +class CorrectionConfigError(ValueError): + """A backend setting that breaks the decoupled PPO objective; launch refused.""" + + +# -------------------------------------------------------------------------- +# mapping, warnings, forbidden settings +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class CorrectionWarning: + """One field a backend cannot map as written (design D3 "warning" rows).""" + + field: str # neutral field, e.g. "lower" + backend: str + reason: str + ran_with: Any = None # the value the backend actually runs with + + def to_dict(self) -> dict[str, Any]: + return {"field": self.field, "backend": self.backend, "reason": self.reason, + "ran_with": self.ran_with} + + def event(self) -> dict[str, Any]: + return {"event": CORRECTION_CONFIG_WARNING, "level": "warning", **self.to_dict()} + + +@dataclass(frozen=True) +class CorrectionMapping: + """What an adapter made of the correction group. + + ``values``: backend setting -> value (Miles: flag -> argv value or True; + verl: Hydra key -> value). ``argv`` / ``runtime_attrs`` / ``overrides`` + are the forms the adapter passes on (empty when unused by the backend). + """ + + backend: str + prox_anchor: str + values: Mapping[str, Any] = field(default_factory=dict) + warnings: tuple[CorrectionWarning, ...] = () + argv: tuple[str, ...] = () + runtime_attrs: Mapping[str, Any] = field(default_factory=dict) + overrides: tuple[str, ...] = () + + def __post_init__(self) -> None: + if self.prox_anchor not in PROX_ANCHORS: + raise ValueError(f"prox_anchor must be one of {PROX_ANCHORS}, got {self.prox_anchor!r}") + + +@dataclass(frozen=True) +class ForbiddenSetting: + """A backend setting refused at launch because it breaks decoupled PPO.""" + + backend: str + setting: str # Miles flag, verl Hydra assignment ("key=value") + reason: str + + def message(self) -> str: + return (f"{self.setting} is refused on backend {self.backend!r}: {self.reason} " + "(rl-offpolicy-correction-passthrough D2)") + + +def refuse_forbidden(present: Iterable[str], forbidden: Iterable[ForbiddenSetting]) -> None: + """Raise :class:`CorrectionConfigError` naming the first forbidden setting in ``present``.""" + + table = {f.setting: f for f in forbidden} + for setting in present: + if setting in table: + raise CorrectionConfigError(table[setting].message()) + + +def refuse_correction_warnings(warnings: Sequence[Mapping[str, Any]]) -> None: + """Phase 2 (L6b.2): raise :class:`CorrectionConfigError` naming every + ``correction_config_warning`` (unmappable field, backend without a + mapping, difference between islands). No warnings: no-op.""" + + if not warnings: + return + lines = [] + for warning in warnings: + if warning.get("scope") == "cross_island": + lines.append(f"islands differ in {sorted(warning.get('differences') or {})}: " + f"{warning.get('reason')}") + else: + lines.append(f"{warning.get('field')} on backend {warning.get('backend')!r}: " + f"{warning.get('reason')}") + raise CorrectionConfigError( + "correction settings refused (rl-offpolicy-correction-passthrough L6b.2): " + + "; ".join(lines)) + + +# -------------------------------------------------------------------------- +# manifest / tape record and the cross-island comparison (D2, D8) +# -------------------------------------------------------------------------- + + +def correction_record(correction: CorrectionSpec, mapping: CorrectionMapping | None, + *, backend: str) -> dict[str, Any]: + """The full group, the backend values and ``prox_anchor`` (manifest and tape). + + ``mapping`` None: the backend has no correction mapping yet; the record + says so (``mapped`` None) and the caller warns. + """ + + return { + "schema": CORRECTION_RECORD_SCHEMA, + "backend": backend, + "group": correction.to_dict(), + "mapped": None if mapping is None else dict(mapping.values), + "prox_anchor": None if mapping is None else mapping.prox_anchor, + "warnings": [] if mapping is None else [w.to_dict() for w in mapping.warnings], + } + + +def compare_island_records(records: Mapping[str, Mapping[str, Any]]) -> dict[str, Any] | None: + """One ``correction_config_warning`` listing every field that differs + between islands (group fields and ``prox_anchor``), or None. + + ``records``: island name -> :func:`correction_record`. + """ + + if len(records) < 2: + return None + islands = sorted(records) + keys = sorted({k for r in records.values() for k in (r.get("group") or {})}) + differences: dict[str, dict[str, Any]] = {} + for key in keys: + values = {i: (records[i].get("group") or {}).get(key) for i in islands} + if len({_comparable(v) for v in values.values()}) > 1: + differences[key] = values + anchors = {i: records[i].get("prox_anchor") for i in islands} + if len(set(anchors.values())) > 1: + differences["prox_anchor"] = anchors + if not differences: + return None + return {"event": CORRECTION_CONFIG_WARNING, "level": "warning", "scope": "cross_island", + "reason": "islands run different correction settings", "differences": differences} + + +def backend_mapping(spec: AlgorithmSpec, backend: str) -> CorrectionMapping | None: + """The ``offpolicy_correction`` role of ``backend`` applied to ``spec``. + + Adapter interface (role ``offpolicy_correction`` in + :mod:`yeto.rl.engine.backends`): a module with ``correction_mapping(spec) + -> CorrectionMapping`` and ``FORBIDDEN`` (a tuple of + :class:`ForbiddenSetting`). None when the backend has no such role yet. + """ + + from . import backends + + try: + module = backends.module("offpolicy_correction", backend) + except backends.UnknownBackend: + return None + return module.correction_mapping(spec) + + +def run_correction_manifest(spec: AlgorithmSpec, + island_backends: Mapping[str, str]) -> dict[str, Any]: + """The run manifest's ``offpolicy_correction`` section (design D8, L6.5). + + ``island_backends``: island name -> backend. Returns the record of every + island and the ``correction_config_warning`` events: one per field a + backend cannot map, one per backend without a mapping, and at most one + for differences between islands. The caller refuses them + (:func:`refuse_correction_warnings`). + """ + + records: dict[str, dict[str, Any]] = {} + warnings: list[dict[str, Any]] = [] + mappings: dict[str, CorrectionMapping | None] = {} + for island, backend in sorted(island_backends.items()): + if backend not in mappings: + mapping = mappings[backend] = backend_mapping(spec, backend) + if mapping is None and not spec.correction.is_default(): + warnings.append(CorrectionWarning( + "*", backend, "the backend adapter has no correction mapping yet; the " + "group is recorded but not passed on").event()) + elif mapping is not None: + warnings.extend(w.event() for w in mapping.warnings) + records[island] = correction_record(spec.correction, mappings[backend], backend=backend) + cross = compare_island_records(records) + if cross is not None: + warnings.append(cross) + return {"islands": records, "warnings": warnings} + + +def _comparable(value: Any) -> Any: + if isinstance(value, Mapping): + return tuple(sorted((k, _comparable(v)) for k, v in value.items())) + if isinstance(value, list): + return tuple(_comparable(v) for v in value) + return value + + +# -------------------------------------------------------------------------- +# version gap from the start version (D4) +# -------------------------------------------------------------------------- + + +def start_version(token_versions: Sequence[int | None]) -> int | None: + """The version of the first response token whose version is known.""" + + return next((v for v in token_versions if v is not None), None) + + +def start_version_gap(trainer_version: int, token_versions: Sequence[int | None]) -> int | None: + """Current trainer version minus the trajectory start version (design D4).""" + + start = start_version(token_versions) + return None if start is None else trainer_version - start + + +def oldest_is_start(token_versions: Sequence[int | None]) -> bool | None: + """L6.12: whether the oldest token version equals the start version. + + True for a continued trajectory (versions never decrease along the + response). None when no version is known. + """ + + known = [v for v in token_versions if v is not None] + return None if not known else min(known) == known[0] + + +def segments_start_check(records: Iterable[Mapping[str, Any]], + key: str = "policy_versions") -> dict[str, int]: + """L6.12 on a tape: counts of records whose oldest segment version is (not) + the first one. ``key`` holds ``[[version, start, end), ...]`` segments + (``rl_trajectory_reward.policy_versions``).""" + + out = {"checked": 0, "oldest_is_start": 0, "oldest_before_start": 0} + for record in records: + segments = record.get(key) + if not segments or not isinstance(segments[0], (list, tuple)): + continue + versions = [int(s[0]) for s in sorted(segments, key=lambda s: int(s[1]))] + out["checked"] += 1 + out["oldest_is_start" if oldest_is_start(versions) else "oldest_before_start"] += 1 + return out + + +# -------------------------------------------------------------------------- +# offpolicy_gate (D6) +# -------------------------------------------------------------------------- + + +def length_bucket(length: int) -> str: + low = 0 + for edge in DROPPED_LENGTH_EDGES: + if length < edge: + return f"{low}-{edge - 1}" + low = edge + return f"{low}+" + + +def length_histogram(lengths: Iterable[int]) -> dict[str, int]: + """Fixed-bucket histogram; every bucket is present (zeros included).""" + + edges = (0, *DROPPED_LENGTH_EDGES) + out = {f"{lo}-{hi - 1}": 0 for lo, hi in zip(edges, edges[1:])} + out[f"{DROPPED_LENGTH_EDGES[-1]}+"] = 0 + for length in lengths: + out[length_bucket(int(length))] += 1 + return out + + +def offpolicy_gate_event(*, backend: str, counts: Mapping[str, int], + dropped_lengths: Mapping[str, Sequence[int]], + truncation_by_turn: Mapping[int, tuple[int, int]] | None, + granularity: str, **extra: Any) -> dict[str, Any]: + """The ``offpolicy_gate`` fields of one round (design D6). + + ``counts``: reason -> groups. ``dropped_lengths``: reason -> response + lengths of the dropped trajectories. ``truncation_by_turn``: turn index + -> (truncated, total); None when the backend reported nothing. + ``granularity`` says whether the turn counts are tokens or trajectories. + """ + + if granularity not in GRANULARITIES: + raise ValueError(f"granularity must be one of {GRANULARITIES}, got {granularity!r}") + turns = None + if truncation_by_turn is not None: + turns = {str(turn): {"truncated": int(t), "total": int(n), + "ratio": (t / n) if n else None} + for turn, (t, n) in sorted(truncation_by_turn.items())} + return { + "backend": backend, + "counts": {k: int(v) for k, v in sorted(counts.items())}, + "dropped_length_edges": list(DROPPED_LENGTH_EDGES), + "dropped_length_histogram": {reason: length_histogram(lengths) + for reason, lengths in sorted(dropped_lengths.items())}, + "truncation_by_turn": turns, + "granularity": granularity, + **extra, + } + + +def gate_from_carry_over(fields: dict[str, Any], *, kept: int, backend: str + ) -> dict[str, Any] | None: + """Pop the gate keys from a round's carry-over fields and build the + ``offpolicy_gate`` event fields; None when the adapter gave none.""" + + lengths = fields.pop(GATE_DROPPED_LENGTHS, None) + turns = fields.pop(TRUNCATED_BY_TURN, None) + granularity = fields.pop(GATE_GRANULARITY, GRANULARITY_TOKEN) + if lengths is None: + return None + counts = {GATE_KEPT: kept, + GATE_GROUP_DROPPED: int(fields.get("over_age_discarded_groups", 0) or 0), + GATE_UNKNOWN_VERSION_DROPPED: + int(fields.get("unknown_version_discarded_groups", 0) or 0)} + backend_stale = fields.get("stale_dropped_groups") + if isinstance(backend_stale, int): # None = the backend did not expose it + counts[GATE_BACKEND_STALE_DROPPED] = backend_stale + if turns is not None: + turns = {int(k): tuple(v) for k, v in turns.items()} + return offpolicy_gate_event(backend=backend, counts=counts, dropped_lengths=lengths, + truncation_by_turn=turns, granularity=granularity) + + +# -------------------------------------------------------------------------- +# cross_version_is: observe-only, bounds and name from the spec (D7) +# -------------------------------------------------------------------------- + +# Bounds the observe-only estimate used before D7; kept when the spec has none. +OBSERVE_DEFAULT_BOUNDS = (0.0, 2.0) + + +def cross_version_bounds(correction: CorrectionSpec) -> tuple[float, float]: + """``(clip_low, clip_high)`` for :func:`version_segments.cross_version_is`. + + The spec's ``lower`` / ``upper``; :data:`OBSERVE_DEFAULT_BOUNDS` for a + missing bound. ``cross_version_is`` needs ``low <= 1 <= high``, so the + bounds are clamped to that and the estimate stays observe-only. + """ + + low = OBSERVE_DEFAULT_BOUNDS[0] if correction.lower is None else float(correction.lower) + high = OBSERVE_DEFAULT_BOUNDS[1] if correction.upper is None else float(correction.upper) + return min(low, 1.0), max(high, 1.0) + + +def staleness_correction(spec: AlgorithmSpec, default: str = "tis") -> str: + """The ``StalenessPolicy.correction`` name derived from the spec (D7).""" + + from yeto.rl.algos.mismatch_correction import selected_corrections + + named = [m for m in selected_corrections(spec) if m != "mismatch_observe"] + return named[0] if named else default diff --git a/yeto/rl/engine/ports.py b/yeto/rl/engine/ports.py index 402830a70..9399d2bcf 100644 --- a/yeto/rl/engine/ports.py +++ b/yeto/rl/engine/ports.py @@ -158,6 +158,10 @@ class RolloutBatchHandle: # (:data:`OVER_SAMPLING_KEYS`: submit batch sizes, refill calls, filtered / # kept / surplus groups, groups in flight at the end). None = not reported. over_sampling: Mapping[str, int] | None = field(default=None, compare=False) + # rl-offpolicy-correction-passthrough L6.11: the engine backend that built + # this batch ("miles", "verl", ...), set by the adapter; it labels the + # driver's ``offpolicy_gate`` event. None = not reported ("unknown"). + backend: str | None = field(default=None, compare=False) def mismatched_groups(self, expected_token: str) -> tuple[GroupMetadata, ...]: return tuple(g for g in self.groups if g.policy_token != expected_token)