diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index 275dda78..97d880e7 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -147,9 +147,25 @@ # The arms ``run --arm`` takes. "pe" is arm 2 as run on 2026-10-14 and # 2026-10-15. "web" swaps the PolicyEngine tool for the provider's own web -# search, and "web+pe" offers both. -ARMS = ("pe", "web", "web+pe") -ARM_SUFFIX = {"pe": "+pe-tool", "web": "+web", "web+pe": "+web+pe-tool"} +# search, and "web+pe" offers both. "none" offers neither: its first request is +# the board's own no-tools request (the forced answer function, or the JSON reply +# with no tools), so a prompt version other than the board's has its own +# no-tools baseline. Repairs, retries and truncation then follow this pilot's +# loop, not the board's; they matter only when a first answer does not parse. +ARMS = ("none", "pe", "web", "web+pe") +ARM_SUFFIX = { + "none": "+no-tools", + "pe": "+pe-tool", + "web": "+web", + "web+pe": "+web+pe-tool", +} +# A web arm's request timeout. A search-heavy turn can outlast the board's +# (300 s for Gemini 4 Argon, the harness default for its card; its web-only arm +# never finished scenario_005 within it in six attempts on 2026-10-21), so the +# web arms allow this much. A timed-out web request is not retried: three +# 900 s attempts would hold a worker for 45 minutes, so the household ends as +# provider_error and a resume pass reruns it. +WEB_REQUEST_TIMEOUT_SECONDS = 900 _CALCULATE_OPENING = "You also have a `calculate_household` function." WEB_SENTENCE = "You also have web search, which searches the public web." @@ -275,6 +291,60 @@ "{who}. Where a credit depends on income tax liability, compute that " "liability from the section 1 tax rate schedules, not the Tax Table", ), + ( + # policyengine-us 2.41.0's state_income_tax_before_refundable_credits + # list adds id_pbf for Idaho ("Form 40 permanent building fund tax, + # added before refundable credits"); no other state adds such a tax. + # GPT-6 Sol's tool answers for scenario_007 were $10 short without it. + re.compile( + r"(state individual income tax after nonrefundable credits and before " + r"refundable credits, excluding local income and payroll taxes)" + ), + "{who}; include Idaho's $10 permanent building fund tax where it applies", + ), + ( + # The reference rule (reference sidecar, quoted in each + # reference_audit/2026-09-28/fixes/latest_c_* module): "a scored + # reference follows from the stated facts and from law published before + # the 2026-07-03 reference freeze". Where no 2026 amount was published by + # then, the reference holds the last one published before it: California's + # 2025 indexed amounts, Idaho's zero-rate thresholds, the IRS 2025 + # optional sales tax tables, and some Maryland, Michigan and Minnesota + # amounts. SNAP holds the FY2026 schedule (allotments, deductions, + # utility allowances, asset limits) for October to December; "Not held: + # the poverty guideline", so income limits move to the 2026 guideline in + # October. Web arms had applied SNAP's FY2027 schedule, published in + # August 2026 (GPT-6 Sol, scenarios 008 and 012). + re.compile( + r"(Do not infer unlisted income, expenses, assets, benefit receipt, " + r"rent, or health coverage\.)" + ), + "{who} Apply the law and program parameters published before July 3, " + "2026. Where a 2026 amount had not been published by then, use the latest " + "amount published before that date (for example, a state's 2025 " + "inflation-adjusted amounts, or the IRS's 2025 optional sales tax " + "tables). For SNAP, keep the fiscal year 2026 allotments, deductions, " + "utility allowances and asset limits through December 2026; the income " + "limits use the 2026 HHS poverty guidelines from October 2026.", + ), + ( + # policyengine-us documents employer_sponsored_insurance_premiums as + # "Annual employer-paid health insurance premiums". Three web arms read + # the bare label as premiums the household paid (scenario_000). + re.compile(r"employer sponsored insurance premiums: "), + "employer-paid health insurance premiums (paid by the employer, not by " + "the household): ", + ), + ( + # spm_unit_energy_subsidy, the SPM's energy subsidy (home energy + # assistance). No scored output reads it. policyengine-us never reads the + # amount for SNAP: it deems a utility allowance from the state's + # always-SUA flag, and in 2026 only for elderly or disabled households + # (heat_and_eat_requires_elderly_disabled, P.L. 119-21 section 10103). + # The engine label hid what the amount is. + re.compile(r"spm unit energy subsidy: "), + "home energy assistance received: ", + ), ) # What a JSON-contract web arm hears once its search-only and calculation turns @@ -1055,6 +1125,8 @@ class ModelProfile: @property def notice(self) -> str: tool = self.contract == "tool" + if self.arm == "none": + return "" if self.arm == "web": notice = WEB_NOTICE if tool else JSON_WEB_NOTICE elif self.arm == "web+pe": @@ -1065,11 +1137,11 @@ def notice(self) -> str: @property def uses_calculator(self) -> bool: - return self.arm != "web" + return self.arm in ("pe", "web+pe") @property def uses_web(self) -> bool: - return self.arm != "pe" + return self.arm in ("web", "web+pe") @property def provider(self) -> str: @@ -1116,7 +1188,7 @@ def model_profile( f"unknown prompt {prompt_version!r}; the prompts are " f"{', '.join(PROMPT_VERSIONS)}" ) - if benchmark_search_rule and arm == "pe": + if benchmark_search_rule and arm not in ("web", "web+pe"): raise SystemExit("the benchmark-search rule needs a web arm") model_id = MODELS[name] return ModelProfile( @@ -1338,7 +1410,11 @@ def request_kwargs(self, base: dict, tools: list[dict], tool_choice) -> dict: # then rebuilds the message in another block order. if self.cache_breakpoint and messages[-1].get("role") != "assistant": messages = [*messages[:-1], _with_cache_breakpoint(messages[-1])] - kwargs.update({"messages": messages, "tools": tools}) + kwargs["messages"] = messages + # A request with no tools sends no tools key at all, as the board's + # JSON-contract request does. + if tools: + kwargs["tools"] = tools if tool_choice is not None: kwargs["tool_choice"] = tool_choice if self.budget_key not in kwargs and "max_tokens" in kwargs: @@ -1674,8 +1750,13 @@ class BudgetStop(RuntimeError): pass +def is_timeout(error: Exception) -> bool: + """A provider or wall-clock timeout (litellm.Timeout and its kin).""" + return any("Timeout" in cls.__name__ for cls in type(error).__mro__) + + def _request_with_retries( - completion_fn, kwargs: dict + completion_fn, kwargs: dict, retry_timeouts: bool = True ) -> tuple[Any, float, float, float]: from policybench.eval_no_tools import _run_request_with_wall_timeout, _should_retry @@ -1686,7 +1767,11 @@ def _request_with_retries( response = _run_request_with_wall_timeout(completion_fn, kwargs) return response, started_at, time.time(), time.perf_counter() - started except Exception as error: - if attempt == MAX_PROVIDER_ATTEMPTS - 1 or not _should_retry(error): + if ( + attempt == MAX_PROVIDER_ATTEMPTS - 1 + or not _should_retry(error) + or (not retry_timeouts and is_timeout(error)) + ): raise time.sleep(2 * (2**attempt)) raise RuntimeError("unreachable") @@ -1735,6 +1820,10 @@ def run_household( offered = ([web_search_tool(profile)] if profile.uses_web else []) + ( [calculate_tool] if profile.uses_calculator else [] ) + if profile.uses_web: + base["timeout"] = max( + float(base.get("timeout") or 0), WEB_REQUEST_TIMEOUT_SECONDS + ) if profile.uses_web and profile.provider == "gemini": base["include_server_side_tool_invocations"] = True if profile.uses_web and profile.provider == "openai": @@ -1746,7 +1835,11 @@ def run_household( ResponsesConversation(prompt) if profile.api == "responses" else ChatConversation( - prompt, cache_breakpoint=profile.model_id.startswith("claude-") + # The board's own request marks no prompt-cache breakpoint, so the + # no-tools arm, its stand-in, marks none either. + prompt, + cache_breakpoint=profile.model_id.startswith("claude-") + and profile.arm != "none", ) ) json_contract = profile.contract == "json" @@ -1779,7 +1872,9 @@ def run_household( def tools_and_choice(kind: str): if json_contract: - return list(offered), "auto" + # The board's JSON contract sends no tools, so the no-tools arm + # sends none and no tool_choice either. + return list(offered), ("auto" if offered else None) if kind == "repair": return ( [ @@ -1788,7 +1883,9 @@ def tools_and_choice(kind: str): ], forced_answer_choice(profile), ) - if kind == "forced_answer": + if kind == "forced_answer" or not offered: + # With no tools offered, the board forces its answer function from + # the first turn; so does the no-tools arm. return [*offered, answer_tool], forced_answer_choice(profile) return [*offered, answer_tool], "required" @@ -1913,7 +2010,7 @@ def run_calculations(turn_record: dict, calls: list[Call]) -> bool: if repair_rounds: kind = "repair" elif not json_contract and ( - forced or tool_turns + search_only_turns >= turn_cap + forced or tool_turns + search_only_turns >= turn_cap or not offered ): kind, forced = "forced_answer", True else: @@ -1930,7 +2027,7 @@ def run_calculations(turn_record: dict, calls: list[Call]) -> bool: escalated_from = None try: response, started_at, completed_at, latency = _request_with_retries( - completion_fn, kwargs + completion_fn, kwargs, retry_timeouts=not profile.uses_web ) turn = conversation.read(response, kwargs.get(budget_key)) if ( @@ -1953,7 +2050,7 @@ def run_calculations(turn_record: dict, calls: list[Call]) -> bool: escalated_from = kwargs.get(budget_key) kwargs[budget_key] = ceiling response, started_at, completed_at, latency = _request_with_retries( - completion_fn, kwargs + completion_fn, kwargs, retry_timeouts=not profile.uses_web ) turn = conversation.read(response, kwargs.get(budget_key)) except Exception as error: @@ -2540,6 +2637,13 @@ def command_run(args) -> None: "tool_notice": profile.notice, "calculate_tool": calculate_tool_schema() if profile.uses_calculator else None, "web_search_tool": web_search_tool(profile) if profile.uses_web else None, + # The timeout actually sent: the board's for the model, or the web arms'. + "request_timeout_seconds": build_initial_request( + board["scenarios"][ids[0]], board["variables"][ids[0]], profile + )[1].get("timeout") + if ids and not profile.uses_web + else (WEB_REQUEST_TIMEOUT_SECONDS if profile.uses_web else None), + "web_timeouts_retried": not profile.uses_web, "search_usd_per_1k": ( SEARCH_USD_PER_1K[profile.provider] if profile.uses_web else None ), @@ -2557,13 +2661,18 @@ def command_run(args) -> None: print(f"{len(todo)} households to run; ${spent_before:.4f} already spent") if not todo: return - pool = ToolPool(size=args.workers) + # Arms without the calculator start no engine workers. + pool = ToolPool(size=args.workers) if profile.uses_calculator else None + + def no_calculator(arguments) -> tuple[dict, float]: + return {"error": "not_run", "message": "no calculator in this arm"}, 0.0 + try: def one(sid: str) -> dict: if spend.exhausted(args.reserve_usd): return {"scenario_id": sid, "status": "budget_stop"} - if pool.counters()["lost"] >= args.workers: + if pool is not None and pool.counters()["lost"] >= args.workers: # Every worker is gone: start nothing more. The household keeps # no record, so a resume runs it. return { @@ -2575,7 +2684,7 @@ def one(sid: str) -> dict: record = run_household( board["scenarios"][sid], board["variables"][sid], - pool.call, + pool.call if pool is not None else no_calculator, completion_fn, spend, profile=profile, @@ -2584,6 +2693,12 @@ def one(sid: str) -> dict: record = { "scenario_id": sid, "model": profile.name, + # The setup keys a resume and score check, so a folder with + # this record still resumes and scores. + "arm": profile.arm, + "prompt_version": profile.prompt_version, + "prompt_wording_sha256": prompt_wording_sha256(profile), + "benchmark_search_rule": profile.benchmark_search_rule, "variables": board["variables"][sid], "status": "harness_error", "error": f"{type(error).__name__}: {error}"[:MAX_MESSAGE_CHARS], @@ -2614,8 +2729,9 @@ def one(sid: str) -> dict: ) as executor: results = list(executor.map(one, todo)) finally: - counters = pool.counters() - pool.close() + counters = pool.counters() if pool is not None else {} + if pool is not None: + pool.close() stopped = [r["scenario_id"] for r in results if r.get("status") == "budget_stop"] rerunnable = sorted( r["scenario_id"] for r in results if r.get("status") in RERUNNABLE_STATUSES @@ -4040,7 +4156,7 @@ def main(argv: list[str] | None = None) -> None: "--arm", choices=ARMS, default="pe", - help="the tools: PolicyEngine (pe), web search (web), or both", + help="the tools: none, PolicyEngine (pe), web search (web), or both", ) p.add_argument( "--prompt", diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index 80d455c7..a5d531b0 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -2289,6 +2289,7 @@ def test_web_notices_name_search_and_keep_the_answer_instruction(): assert both.split("function. ", 1)[1] == pe.split("function. ", 1)[1] labels = {arm: pilot.model_profile(SOL, arm).arm2_label for arm in pilot.ARMS} assert labels == { + "none": "gpt-6-sol+no-tools", "pe": "gpt-6-sol+pe-tool", "web": "gpt-6-sol+web", "web+pe": "gpt-6-sol+web+pe-tool", @@ -3078,7 +3079,10 @@ def test_the_v2_pilot_prompt_names_policyengine_nowhere_in_any_household(): old, new = _board_part(old, v1), _board_part(new, v2) assert "policyengine" not in new.lower() assert "policyengine" not in json.dumps(kwargs["_answer_tool"]).lower() - # Only the four definition lines differ. + # Only the rewritten lines differ: the four PolicyEngine-naming + # definitions, the two federal lines and the state tax line (rate + # schedule, Idaho's fund tax), the intro (law date), and the two + # relabeled facts. old_lines, new_lines = old.splitlines(), new.splitlines() assert len(old_lines) == len(new_lines) changed = [(a, b) for a, b in zip(old_lines, new_lines) if a != b] @@ -3086,10 +3090,22 @@ def test_the_v2_pilot_prompt_names_policyengine_nowhere_in_any_household(): "- federal_income_tax_before_refundable_credits:", "- federal_refundable_credits:", ) - assert all("PolicyEngine" in a or a.startswith(federal) for a, _ in changed) + other = ( + "- state_income_tax_before_refundable_credits:", + "Estimate the requested tax and benefit outputs", + "- employer sponsored insurance premiums: ", + "- spm unit energy subsidy: ", + ) + assert all( + "PolicyEngine" in a or a.startswith(federal + other) for a, _ in changed + ) assert len(changed) == old.count("PolicyEngine") + sum( - line in old for line in federal + 1 for line in old_lines if line.startswith(federal + other) ) + # The law date, the payer and the plain energy label are stated. + assert "published before July 3, 2026" in new + assert "employer sponsored insurance premiums" not in new + assert "spm unit energy subsidy" not in new # d1263: the v2 prompt states the rate-schedule convention on both # federal lines whose values the schedule choice can move. for line in federal: @@ -3435,3 +3451,148 @@ def test_a_run_record_carries_its_wording_hash(): make_scenario(), VARIABLES, fake_tool, model, pilot.Spend(50.0), profile=profile ) assert record["prompt_wording_sha256"] == pilot.prompt_wording_sha256(profile) + + +# --------------------------------------------------------------------------- +# The no-tools arm: the board's own request, for prompt versions the board +# has no rows for. +# --------------------------------------------------------------------------- + + +def _first_request(name: str, scenario, variables, prompt_version="v1"): + profile = pilot.model_profile(name, "none", prompt_version) + values = {v: 0 for v in variables} + if profile.api == "responses": + model = FakeResponses([responses_response([sol_submit(values)], "r")]) + elif profile.contract == "json": + model = FakeModel([response([], content=json_answer(values))]) + else: + model = FakeModel([response([submit(values)])]) + record = pilot.run_household( + scenario, variables, fake_tool, model, pilot.Spend(50.0), profile=profile + ) + return profile, record, model.requests[0] + + +def test_the_no_tools_arm_sends_the_boards_own_request(): + """Differential: for every household and contract, the arm's first request + is the request policybench.eval_no_tools builds for the board, key for key + (budget, timeout, tools and their schemas, tool choice, response format), + except the board's disk-cache flag, which a live run does not use.""" + from policybench.eval_no_tools import ( + _chat_completion_request_kwargs, + _responses_request_kwargs, + ) + + board = pilot.load_board() + for sid in sorted(board["scenarios"]): + scenario, variables = board["scenarios"][sid], board["variables"][sid] + for name in (pilot.MODEL_NAME, SOL, OPUS): + profile, record, sent = _first_request(name, scenario, variables) + assert record["status"] == "complete" and profile.notice == "" + if profile.api == "responses": + _, ref = _responses_request_kwargs( + scenario, variables, profile.model_id + ) + else: + messages, ref = _chat_completion_request_kwargs( + scenario, variables, profile.model_id + ) + ref = {**ref, "messages": messages} + ref = {k: v for k, v in ref.items() if k != "caching"} + sent = json.loads(json.dumps(sent, default=str)) + ref = json.loads(json.dumps(ref, default=str)) + assert sent == ref, (sid, name, sorted(set(sent) ^ set(ref))) + + +def test_the_no_tools_arm_on_v2_pilot_sends_the_rewritten_prompt(): + board = pilot.load_board() + sid = "scenario_000" + profile, record, sent = _first_request( + SOL, board["scenarios"][sid], board["variables"][sid], "v2-pilot" + ) + assert "published before July 3, 2026" in sent["input"] + assert "policyengine" not in sent["input"].lower() + assert record["arm"] == "none" and profile.arm2_label == ( + "gpt-6-sol+no-tools@v2-pilot" + ) + with pytest.raises(SystemExit): + pilot.model_profile(SOL, "none", "v2-pilot", True) + + +def test_only_web_arms_get_the_longer_timeout(): + board = pilot.load_board() + scenario, variables = ( + board["scenarios"]["scenario_000"], + board["variables"]["scenario_000"], + ) + values = {v: 0 for v in variables} + timeouts = {} + for arm in pilot.ARMS: + model = FakeModel([response([submit(values)])]) + pilot.run_household( + scenario, + variables, + fake_tool, + model, + pilot.Spend(50.0), + profile=pilot.model_profile(pilot.MODEL_NAME, arm), + ) + timeouts[arm] = model.requests[0]["timeout"] + assert timeouts["web"] == timeouts["web+pe"] == pilot.WEB_REQUEST_TIMEOUT_SECONDS + assert timeouts["none"] == timeouts["pe"] < pilot.WEB_REQUEST_TIMEOUT_SECONDS + + +def test_a_web_arm_does_not_retry_a_timed_out_request(monkeypatch): + """Three 900 s attempts would hold a worker for 45 minutes: a web arm's + timed-out request ends the household as provider_error, which a resume + reruns. Other arms keep the board's retries.""" + import litellm + + monkeypatch.setattr(pilot.time, "sleep", lambda seconds: None) + for arm, attempts in (("web", 1), ("pe", pilot.MAX_PROVIDER_ATTEMPTS)): + calls = [] + + def timing_out(**kwargs): + calls.append(kwargs) + raise litellm.Timeout("timed out", model="m", llm_provider="openai") + + record = pilot.run_household( + make_scenario(), + VARIABLES, + fake_tool, + timing_out, + pilot.Spend(50.0), + profile=pilot.model_profile(pilot.MODEL_NAME, arm), + ) + assert record["status"] == "provider_error" + assert len(calls) == attempts, arm + assert pilot.is_timeout(litellm.Timeout("t", model="m", llm_provider="openai")) + assert not pilot.is_timeout(ValueError("x")) + + +def test_the_no_tools_arm_marks_no_prompt_cache_breakpoint(): + model = FakeModel([response([], content=json_answer({"snap": 1, VARIABLES[1]: 2}))]) + pilot.run_household( + make_scenario(), + VARIABLES, + fake_tool, + model, + pilot.Spend(50.0), + profile=pilot.model_profile(OPUS, "none"), + ) + assert "cache_control" not in json.dumps(model.requests[0]) + + +def test_the_law_date_sentence_states_the_fallback_and_the_snap_split(): + board = pilot.load_board() + prompt, _ = pilot.build_initial_request( + board["scenarios"]["scenario_000"], + board["variables"]["scenario_000"], + pilot.model_profile(SOL, "pe", "v2-pilot"), + ) + assert "use the latest amount published before that date" in prompt + assert "2025 optional sales tax tables" in prompt + assert "asset limits through December 2026" in prompt + assert "2026 HHS poverty guidelines from October 2026" in prompt + assert "fiscal year 2026 values for all of 2026" not in prompt