Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions .github/workflows/provider-limit-park-policy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
name: provider-limit-park-policy

# Runs ONLY on this branch: focused gate for the limit-handling policy
# (visible exactly-once park/resume notices, delegate-child fail-fast with a
# structured error, cap expiry fails the run, structured 9router limit fields).
# The repo's own full-repo CI (tests.yml) also fires on the PR.
on:
push:
branches: [feat/provider-limit-park-policy]
permissions:
contents: read
concurrency:
group: provider-limit-park-policy
cancel-in-progress: true

jobs:
gate:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

- uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
with:
enable-cache: true
cache-dependency-glob: |
pyproject.toml
uv.lock

- name: Python 3.14 (.python-version)
run: uv python install 3.14

- name: "Private PM environment (canonical command; setup inputs modified: 4 UV_CONCURRENT caps = 1)"
env:
UV_CONCURRENT_DOWNLOADS: '1'
UV_CONCURRENT_BUILDS: '1'
UV_CONCURRENT_INSTALLS: '1'
UV_CONCURRENT_CACHE_READS: '1'
run: |
PY="$(uv python find 3.14)"
"$PY" -m pm.build_env --source . --out ci --group dev --group test --python "$PY"

- name: Focused limit-policy regressions (must PASS)
env:
TZ: UTC
LANG: C.UTF-8
HERMES_HOME: ${{ runner.temp }}/hermes-home
run: |
mkdir -p "$HERMES_HOME"
HERMES_PYTHON="$PWD/ci/bin/python" bash scripts/run_tests.sh \
tests/agent/test_provider_reset_wait.py \
tests/agent/test_provider_reset_cooldown.py \
tests/agent/test_turn_api_error_stream_parse.py \
tests/agent/test_kanban_turn_recovery.py \
--file-retries 0 --file-timeout 240 -j 2 -q 2>&1 | tee limit-policy.log

- name: Gate on the focused result
run: |
grep -E "passed" limit-policy.log
! grep -qE "[1-9][0-9]* failed" limit-policy.log

- name: Upload evidence
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: limit-policy-evidence
path: limit-policy.log
if-no-files-found: warn
31 changes: 30 additions & 1 deletion acp_adapter/gateway_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,34 @@ async def _teardown(self):
self._failure = None
self._changed.notify_all()

_t3_kind_cache: bool | None = None

async def _t3_delegated(self) -> bool:
"""Whether this ACP session is a T3 Code delegate_task child: the thread-bound MCP
credential reads its own thread, and ``relationshipToParent == 'subagent'`` marks it.
False outside T3 and on any failure — fail-open to the visible top-level park."""
if self._t3_kind_cache is not None:
return self._t3_kind_cache
self._t3_kind_cache = False
env = self._environ
node = env.get("T3_ACP_MCP_NODE")
if not (node and env.get("T3_ACP_MCP_ENDPOINT") and env.get("T3_ACP_MCP_AUTHORIZATION")):
return False
import json
entry = env.get("T3_ACP_MCP_ENTRYPOINT")
argv = [node, *([entry] if entry else []), "acp-mcp-call", "t3_thread_read",
json.dumps({"limit": 1, "maxCharsPerItem": 200})]
try:
proc = await asyncio.create_subprocess_exec(
*argv, env={**os.environ, **env, "ELECTRON_RUN_AS_NODE": "1"},
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL)
out, _ = await asyncio.wait_for(proc.communicate(), 20)
thread = json.loads(out or b"{}").get("thread") or {}
self._t3_kind_cache = thread.get("relationshipToParent") == "subagent"
except Exception:
logger.debug("T3 delegate detection failed", exc_info=True)
return self._t3_kind_cache

@gateway_errors
async def new_session(self, cwd, mcp_servers=None, **kwargs):
client = await self._client()
Expand All @@ -373,10 +401,11 @@ async def new_session(self, cwd, mcp_servers=None, **kwargs):
flags = env_launch_flags(self._environ)
if flags.get("safe_mode") or flags.get("ignore_user_config"):
flags.update(_bypass_route(self._environ))
create_kwargs = {"t3_delegated": True} if await self._t3_delegated() else {}
snapshot = await client.rpc("session.create", request_id=uuid.uuid4().hex,
source="acp", cwd=_translate_acp_cwd(cwd),
editor={"mcp_servers": self._mcp_specs(mcp_servers),
"edit_approval_policy": "ask"}, **flags)
"edit_approval_policy": "ask"}, **flags, **create_kwargs)
session_id = snapshot["session_id"]
await self._adopt(session_id, snapshot)
if mcp_servers:
Expand Down
5 changes: 5 additions & 0 deletions agent/conversation_loop.py
Original file line number Diff line number Diff line change
Expand Up @@ -1745,6 +1745,11 @@ def _run_conversation_turn(
if early_result is not None:
return early_result

# The call succeeded: a provider-limit park episode opened earlier in this
# iteration closes here, with its one resume notice.
from agent.turn_provider_reset import close_provider_limit_episode
close_provider_limit_episode(agent, s._retry)

_rs = _run_phase(apply_retry_restarts, agent, s)
if _rs.action == "break":
break
Expand Down
24 changes: 21 additions & 3 deletions agent/error_classifier.py
Original file line number Diff line number Diff line change
Expand Up @@ -1000,8 +1000,9 @@ def _by_status(c: _Ctx) -> Optional[Verdict]:
# evidence wins over the outer transport status; a bare 5xx stays transport.
if 500 <= status < 600 and ('[429]' in c.msg or 'rate_limit_error' in c.msg):
reset = _rate_limit_reset_seconds(c.msg, c.body, c.headers)
return _v(_R.rate_limit, **_ROTATE_FALLBACK,
error_context={'reset_at': time.time() + reset} if reset else {})
ctx = {'reset_at': time.time() + reset} if reset else {}
ctx.update(_structured_limit_context(c.body))
return _v(_R.rate_limit, **_ROTATE_FALLBACK, error_context=ctx)
default = _V_FORMAT_ERROR if 400 <= status < 500 else _V_SERVER_ERROR if 500 <= status < 600 else None
return _STATUS_HANDLERS[status](c) if status in _STATUS_HANDLERS else default

Expand Down Expand Up @@ -1127,7 +1128,9 @@ def _status_429(c: _Ctx) -> Verdict:
# Carry the reset window so the terminal copy can name it instead of "wait a minute" (#89401).
reset = _rate_limit_reset_seconds(c.msg, c.body, c.headers)
if reset:
return _v(_R.rate_limit, **_ROTATE_FALLBACK, error_context={"reset_at": time.time() + reset})
return _v(_R.rate_limit, **_ROTATE_FALLBACK,
error_context={'reset_at': time.time() + reset,
**_structured_limit_context(c.body)})
return _V_RATE_LIMIT


Expand Down Expand Up @@ -1330,6 +1333,21 @@ def _has_usage_limit_transient_signal(error_msg: str, body: dict, response_heade
return False


def _structured_limit_context(body) -> dict:
"""Structured provider-limit fields for the park/child errors: 9router ships
``code`` + ``reset_at`` + ``scope`` (soonest reset for combos); pass what is present."""
payload = body if isinstance(body, dict) else {}
err = payload.get('error') if isinstance(payload.get('error'), dict) else {}
out: dict = {}
for src in (payload, err):
for key in ('limit_kind', 'limit_scope', 'scope'):
if src.get(key) and key not in out:
out[key] = src[key]
if src.get('code') and 'limit_code' not in out:
out['limit_code'] = src['code']
return out


def _rate_limit_reset_seconds(error_msg: str, body: dict, response_headers) -> Optional[float]:
"""Seconds until a 429's window reopens, from the body's reset fields, ``Retry-After`` or the
message grammar (``retry after Ns`` / ``resets in 4hr``); None when the response names none."""
Expand Down
79 changes: 74 additions & 5 deletions agent/turn_provider_reset.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,56 @@
PROVIDER_RESET_PROBE_INTERVAL_S = 5 * 60


def _hhmm(epoch: float) -> str:
return time.strftime('%H:%M', time.localtime(epoch))


def _delegated_session(agent) -> bool:
"""Delegate children never park on a limit: fail the run so T3 surfaces the limit
to the parent thread, whose agent decides retry later / switch model / take over.
True for in-process delegate_task children (and descendants carrying the env
marker) and for T3 ACP sessions the gateway stamped ``t3_delegated`` at launch."""
try:
from agent.delegation_context import is_delegated_child_process_context
if is_delegated_child_process_context():
return True
except Exception:
pass
return getattr(agent, '_t3_delegated', False) is True


def _open_episode(agent, retry, reset_at: float) -> None:
"""Open the park episode with its ONE park notice — never re-announced while the
episode stays open, no matter how many 5-minute probe parks follow."""
if getattr(retry, 'provider_limit_episode', None):
return
provider = getattr(agent, 'provider', '') or 'unknown'
model = getattr(agent, 'model', '') or 'unknown'
retry.provider_limit_episode = {'started': time.time(), 'provider': provider, 'model': model}
agent._emit_warning(
f"Stuck on provider limit: {provider}/{model}, resets ~{_hhmm(reset_at)} WIB; "
"auto-retrying every 5 min")


def close_provider_limit_episode(agent, retry) -> None:
"""Its ONE resume notice, when a parked episode's next API call succeeds; called from
the turn loop's success path. No-op without an open episode and after the first."""
episode = getattr(retry, 'provider_limit_episode', None)
if not episode or episode.get('resumed'):
return
episode['resumed'] = True
minutes = max(1, round((time.time() - episode['started']) / 60))
agent._emit_warning(f"Resumed at {_hhmm(time.time())} after {minutes} min parked")


def _terminal_failure(messages, api_call_count, reason: str, detail: str):
return ('return', {
'final_response': detail, 'messages': messages, 'api_calls': api_call_count,
'completed': False, 'failed': True, 'error': detail,
'failure_reason': reason, 'failure_retryable': False,
})


def wait_for_provider_reset(agent, error, classified, context, retry, *,
messages, conversation_history, api_call_count):
if not classified.retryable:
Expand All @@ -25,16 +75,35 @@ def wait_for_provider_reset(agent, error, classified, context, retry, *,
reset = _parse_absolute_timestamp(context.get('reset_at'))
seconds = None if reset is None else reset - time.time()
if seconds is None or not math.isfinite(seconds) or seconds <= 0:
return None
return None # no declared reset: an ordinary error — no park, no special casing
if _delegated_session(agent):
# Structured, immediate failure: provider, model, limit kind, reset_at (+ any
# structured scope/code the provider body carried) for the parent to act on.
reset_at = time.time() + seconds
extra = ' '.join(f"{k}={context[k]}" for k in
('limit_code', 'limit_kind', 'limit_scope', 'scope') if k in context)
reason = getattr(classified, 'reason', None)
detail = (f"provider_limit: provider={getattr(agent, 'provider', '') or '?'} "
f"model={getattr(agent, 'model', '') or '?'} "
f"kind={getattr(reason, 'value', reason) or '?'} "
f"reset_at={time.strftime('%Y-%m-%dT%H:%M:%S%z', time.localtime(reset_at))}"
+ (f" {extra}" if extra else ""))
return _terminal_failure(messages, api_call_count, 'provider_limit', detail)
now = time.monotonic()
if retry.provider_reset_deadline is None:
retry.provider_reset_deadline = now + PROVIDER_RESET_WAIT_CAP_S
_open_episode(agent, retry, time.time() + seconds)
wait = min(seconds + random.uniform(1, 3),
PROVIDER_RESET_PROBE_INTERVAL_S + random.uniform(0, 30))
PROVIDER_RESET_PROBE_INTERVAL_S + random.uniform(0, 30))
if wait > retry.provider_reset_deadline - now:
# Do not retry BEFORE a declared reset beyond our total attended bound.
# Let the existing bounded retry/terminal policy handle that response.
return None
# Cap expired: FAIL the run with a clear limit error instead of looping forever.
return _terminal_failure(
messages, api_call_count, 'provider_limit_wait_cap',
f"Provider limit did not clear within {PROVIDER_RESET_WAIT_CAP_S // 3600}h: "
f"provider={getattr(agent, 'provider', '') or '?'} "
f"model={getattr(agent, 'model', '') or '?'} "
f"next reset in ~{seconds / 60:.0f} min. Giving up per limit policy; "
"resend later or switch provider/model.")
agent._emit_diagnostic_status(f'⏳ Provider cooldown: waiting {wait:.0f}s until reset; retry budget unchanged.')
agent._emit_diagnostic_wait(f'⏳ Waiting for provider reset in {wait:.0f}s (retry budget unchanged)')
result = interruptible_backoff_sleep(agent, wait, retry, messages=messages,
Expand Down
3 changes: 3 additions & 0 deletions agent/turn_retry_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ class TurnRetryState:

# Transport / rate-limit recovery
provider_reset_deadline: float | None = None
# Open provider-limit park episode: keys started/provider/model, then resumed=True.
# Drives the exactly-once park and resume notices (turn_provider_reset).
provider_limit_episode: dict | None = None
primary_recovery_attempted: bool = False
has_retried_429: bool = False
# Credit-limited 402 ("can only afford N tokens") already retried with a lowered output cap.
Expand Down
5 changes: 5 additions & 0 deletions gateway/session_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,4 +231,9 @@ def _resolve_turn_agent(self, turn_route, platform_key, combined_ephemeral, max_
cache[ctx.session_key] = (agent, sig, msg_count, ctx.session_id)
runner._enforce_agent_cache_cap()
logger.debug("Created new agent for session %s (sig=%s)", ctx.session_key, sig)
# A T3 delegate_task child fails fast on provider limits instead of parking; the
# flag rides the session's frozen launch config and is re-read every turn so a
# cached agent built before the mark landed still picks it up.
from gateway.session_policy import config_flag, policy_for_source, T3_DELEGATED
agent._t3_delegated = config_flag(policy_for_source(runner, ctx.source), T3_DELEGATED)
return agent, found.reused
3 changes: 3 additions & 0 deletions gateway/session_local.py
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,9 @@ def stage_local_session(authority, actor, params, *, trusted_policy=None, truste
{**json.loads(policy.request_json), 'model_requested': raw_model, 'route_derived': derived},
sort_keys=True))
marks = {LOCAL_DESKTOP_CHAT: True} if desktop_local and policy.source == 'gui' else {}
# A T3 delegate child never parks on a provider limit (fails fast to the parent).
if params.pop('t3_delegated', False) is True:
marks[T3_DELEGATED] = True
if policy.model is None:
from hermes_cli.anon_auth import GUEST_MODEL
policy = replace(policy, model=_resolve_gateway_model(policy.config()) or None)
Expand Down
14 changes: 13 additions & 1 deletion gateway/session_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
'provider', 'base_url', 'reasoning', 'max_turns', 'ignore_rules', 'api_key', 'editor',
'yolo', 'safe_mode', 'ignore_user_config',
'skills', 'checkpoints', 'accept_hooks', 'pass_session_id', 'follow_profile_config',
'service_tier', 'cwd_explicit', 'run_budget', 'verbose', 'row_cwd', 'client_env'})
'service_tier', 'cwd_explicit', 'run_budget', 'verbose', 'row_cwd', 'client_env',
't3_delegated'})
# Sections a ``follow_profile_config`` session re-reads from the owning profile's CURRENT config each
# turn (main's Bot Chat row), credentials included: a frozen key never crosses a profile route change.
FOLLOW_SECTIONS = ('model', 'providers', 'custom_providers', 'fallback_providers', 'fallback_model', 'model_aliases',
Expand All @@ -32,13 +33,24 @@
# an exact create retry from a socket whose declaration has not landed carries the same creation request,
# and ``launch_request`` never echoes it.
LOCAL_DESKTOP_CHAT = 'desktop_local'
# A T3 Code delegate_task child session (the ACP relay's own thread is a subagent): its runs
# fail fast on provider limits instead of parking — the parent thread decides what to do.
T3_DELEGATED = 't3_delegated'
# A launch field's state as a committed session edit left it, in the policy's own config: ``True`` the
# session chose it (``/reasoning <level>``, a model pick carrying one), ``False`` a reset gave it back to
# the profile. Either wins over the creation request (the launch identity, never rewritten) and the older
# pick receipts.
SESSION_PINS = 'session_pins'


def config_flag(policy, key: str) -> bool:
"""True when the session's frozen launch config carries a True ``key`` mark."""
try:
return json.loads(policy.config_json).get(key) is True
except Exception:
return False


def session_pinned(config, field, launched):
"""Whether *field* is the session's own choice: its last committed session edit, else *launched*."""
pins = config.get(SESSION_PINS)
Expand Down
Loading
Loading