From 01b077f14ec61fef8d24e0e4c2758ba432421187 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 21 Oct 2026 23:57:45 -0400 Subject: [PATCH 1/2] Ready the tool pilot's v2-pilot run: prompt fixes, a no-tools arm, web timeout Max (2026-10-21, d1800): run all three models in all four arms on 100 households, on the v2-pilot wording. v2-pilot wording, from the pilot's misses, each checked against policyengine-us 2.41.0 or the reference rule: - Law date: "Apply the law and program parameters published before July 3, 2026; for SNAP, use the fiscal year 2026 values for all of 2026". That is the reference rule; the web arms had applied SNAP's FY2027 values. - "employer sponsored insurance premiums" now reads "employer-paid health insurance premiums (paid by the employer, not by the household)", as the engine documents the variable. Web arms had deducted them. - "spm unit energy subsidy" now reads "home energy assistance received". - The state income tax definition names Idaho's $10 permanent building fund tax, the one extra in the engine's state tax list. Run options: - --arm none sends the board's own no-tools request: the forced answer function, or Opus's JSON reply with no tools key. A differential test checks it against eval_no_tools' builders for all three contracts. - Web arms use a 900 s request timeout. Argon's web-only arm never finished scenario_005 within the board's 300 s. - Arms without the calculator start no engine workers. Tests: 144 passed. Five planted bugs were each caught. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 107 ++++++++++++++++++++++++----- tests/test_tool_pilot_argon.py | 120 ++++++++++++++++++++++++++++++++- 2 files changed, 209 insertions(+), 18 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index 275dda78..5395bade 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -147,9 +147,21 @@ # The arms ``run --arm`` takes. "pe" is arm 2 as run on 2026-10-14 and # 2026-10-15. "web" swaps the PolicyEngine tool for the provider's own web -# search, and "web+pe" offers both. -ARMS = ("pe", "web", "web+pe") -ARM_SUFFIX = {"pe": "+pe-tool", "web": "+web", "web+pe": "+web+pe-tool"} +# search, and "web+pe" offers both. "none" offers neither: the board's own +# no-tools request (the forced answer function, or the JSON reply with no tools), +# so a prompt version other than the board's has its own no-tools baseline. +ARMS = ("none", "pe", "web", "web+pe") +ARM_SUFFIX = { + "none": "+no-tools", + "pe": "+pe-tool", + "web": "+web", + "web+pe": "+web+pe-tool", +} +# A web arm's request timeout. A search-heavy turn can outlast the board's +# (Gemini 4 Argon's model card gives 300 s, and its web-only arm never finished +# scenario_005 within it in six attempts on 2026-10-21), so the web arms allow +# this much and the manifest records it. +WEB_REQUEST_TIMEOUT_SECONDS = 900 _CALCULATE_OPENING = "You also have a `calculate_household` function." WEB_SENTENCE = "You also have web search, which searches the public web." @@ -275,6 +287,48 @@ "{who}. Where a credit depends on income tax liability, compute that " "liability from the section 1 tax rate schedules, not the Tax Table", ), + ( + # policyengine-us 2.41.0's state_income_tax_before_refundable_credits + # list adds id_pbf for Idaho ("Form 40 permanent building fund tax, + # added before refundable credits"); no other state adds such a tax. + # GPT-6 Sol's tool answers for scenario_007 were $10 short without it. + re.compile( + r"(state individual income tax after nonrefundable credits and before " + r"refundable credits, excluding local income and payroll taxes)" + ), + "{who}; include Idaho's $10 permanent building fund tax where it applies", + ), + ( + # The reference rule (reference_audit sidecar, quoted in + # fixes/latest_c_snap_hold_fy2026.py): "A scored reference follows from + # the stated facts and from law published before the 2026-07-03 + # reference freeze", and SNAP holds the FY2026 schedule for October to + # December 2026. Web arms otherwise apply FY2027 values, published in + # August 2026 (GPT-6 Sol, scenarios 008 and 012). + re.compile( + r"(Do not infer unlisted income, expenses, assets, benefit receipt, " + r"rent, or health coverage\.)" + ), + "{who} Apply the law and program parameters published before July 3, " + "2026; for SNAP, use the fiscal year 2026 values for all of 2026.", + ), + ( + # policyengine-us documents employer_sponsored_insurance_premiums as + # "Annual employer-paid health insurance premiums". Three web arms read + # the bare label as premiums the household paid (scenario_000). + re.compile(r"employer sponsored insurance premiums: "), + "employer-paid health insurance premiums (paid by the employer, not by " + "the household): ", + ), + ( + # spm_unit_energy_subsidy, the SPM's home energy assistance. In 2026 + # policyengine-us deems a SNAP utility allowance from energy assistance + # only for elderly or disabled households + # (heat_and_eat_requires_elderly_disabled, P.L. 119-21 section 10103). + # The engine label hid what the amount is. + re.compile(r"spm unit energy subsidy: "), + "home energy assistance received: ", + ), ) # What a JSON-contract web arm hears once its search-only and calculation turns @@ -1055,6 +1109,8 @@ class ModelProfile: @property def notice(self) -> str: tool = self.contract == "tool" + if self.arm == "none": + return "" if self.arm == "web": notice = WEB_NOTICE if tool else JSON_WEB_NOTICE elif self.arm == "web+pe": @@ -1065,11 +1121,11 @@ def notice(self) -> str: @property def uses_calculator(self) -> bool: - return self.arm != "web" + return self.arm in ("pe", "web+pe") @property def uses_web(self) -> bool: - return self.arm != "pe" + return self.arm in ("web", "web+pe") @property def provider(self) -> str: @@ -1116,7 +1172,7 @@ def model_profile( f"unknown prompt {prompt_version!r}; the prompts are " f"{', '.join(PROMPT_VERSIONS)}" ) - if benchmark_search_rule and arm == "pe": + if benchmark_search_rule and arm not in ("web", "web+pe"): raise SystemExit("the benchmark-search rule needs a web arm") model_id = MODELS[name] return ModelProfile( @@ -1338,7 +1394,11 @@ def request_kwargs(self, base: dict, tools: list[dict], tool_choice) -> dict: # then rebuilds the message in another block order. if self.cache_breakpoint and messages[-1].get("role") != "assistant": messages = [*messages[:-1], _with_cache_breakpoint(messages[-1])] - kwargs.update({"messages": messages, "tools": tools}) + kwargs["messages"] = messages + # A request with no tools sends no tools key at all, as the board's + # JSON-contract request does. + if tools: + kwargs["tools"] = tools if tool_choice is not None: kwargs["tool_choice"] = tool_choice if self.budget_key not in kwargs and "max_tokens" in kwargs: @@ -1735,6 +1795,10 @@ def run_household( offered = ([web_search_tool(profile)] if profile.uses_web else []) + ( [calculate_tool] if profile.uses_calculator else [] ) + if profile.uses_web: + base["timeout"] = max( + float(base.get("timeout") or 0), WEB_REQUEST_TIMEOUT_SECONDS + ) if profile.uses_web and profile.provider == "gemini": base["include_server_side_tool_invocations"] = True if profile.uses_web and profile.provider == "openai": @@ -1779,7 +1843,9 @@ def run_household( def tools_and_choice(kind: str): if json_contract: - return list(offered), "auto" + # The board's JSON contract sends no tools, so the no-tools arm + # sends none and no tool_choice either. + return list(offered), ("auto" if offered else None) if kind == "repair": return ( [ @@ -1788,7 +1854,9 @@ def tools_and_choice(kind: str): ], forced_answer_choice(profile), ) - if kind == "forced_answer": + if kind == "forced_answer" or not offered: + # With no tools offered, the board forces its answer function from + # the first turn; so does the no-tools arm. return [*offered, answer_tool], forced_answer_choice(profile) return [*offered, answer_tool], "required" @@ -2540,6 +2608,9 @@ def command_run(args) -> None: "tool_notice": profile.notice, "calculate_tool": calculate_tool_schema() if profile.uses_calculator else None, "web_search_tool": web_search_tool(profile) if profile.uses_web else None, + "request_timeout_seconds": WEB_REQUEST_TIMEOUT_SECONDS + if profile.uses_web + else None, "search_usd_per_1k": ( SEARCH_USD_PER_1K[profile.provider] if profile.uses_web else None ), @@ -2557,13 +2628,18 @@ def command_run(args) -> None: print(f"{len(todo)} households to run; ${spent_before:.4f} already spent") if not todo: return - pool = ToolPool(size=args.workers) + # Arms without the calculator start no engine workers. + pool = ToolPool(size=args.workers) if profile.uses_calculator else None + + def no_calculator(arguments) -> tuple[dict, float]: + return {"error": "not_run", "message": "no calculator in this arm"}, 0.0 + try: def one(sid: str) -> dict: if spend.exhausted(args.reserve_usd): return {"scenario_id": sid, "status": "budget_stop"} - if pool.counters()["lost"] >= args.workers: + if pool is not None and pool.counters()["lost"] >= args.workers: # Every worker is gone: start nothing more. The household keeps # no record, so a resume runs it. return { @@ -2575,7 +2651,7 @@ def one(sid: str) -> dict: record = run_household( board["scenarios"][sid], board["variables"][sid], - pool.call, + pool.call if pool is not None else no_calculator, completion_fn, spend, profile=profile, @@ -2614,8 +2690,9 @@ def one(sid: str) -> dict: ) as executor: results = list(executor.map(one, todo)) finally: - counters = pool.counters() - pool.close() + counters = pool.counters() if pool is not None else {} + if pool is not None: + pool.close() stopped = [r["scenario_id"] for r in results if r.get("status") == "budget_stop"] rerunnable = sorted( r["scenario_id"] for r in results if r.get("status") in RERUNNABLE_STATUSES @@ -4040,7 +4117,7 @@ def main(argv: list[str] | None = None) -> None: "--arm", choices=ARMS, default="pe", - help="the tools: PolicyEngine (pe), web search (web), or both", + help="the tools: none, PolicyEngine (pe), web search (web), or both", ) p.add_argument( "--prompt", diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index 80d455c7..abc9d2db 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -2289,6 +2289,7 @@ def test_web_notices_name_search_and_keep_the_answer_instruction(): assert both.split("function. ", 1)[1] == pe.split("function. ", 1)[1] labels = {arm: pilot.model_profile(SOL, arm).arm2_label for arm in pilot.ARMS} assert labels == { + "none": "gpt-6-sol+no-tools", "pe": "gpt-6-sol+pe-tool", "web": "gpt-6-sol+web", "web+pe": "gpt-6-sol+web+pe-tool", @@ -3078,7 +3079,10 @@ def test_the_v2_pilot_prompt_names_policyengine_nowhere_in_any_household(): old, new = _board_part(old, v1), _board_part(new, v2) assert "policyengine" not in new.lower() assert "policyengine" not in json.dumps(kwargs["_answer_tool"]).lower() - # Only the four definition lines differ. + # Only the rewritten lines differ: the four PolicyEngine-naming + # definitions, the two federal lines and the state tax line (rate + # schedule, Idaho's fund tax), the intro (law date), and the two + # relabeled facts. old_lines, new_lines = old.splitlines(), new.splitlines() assert len(old_lines) == len(new_lines) changed = [(a, b) for a, b in zip(old_lines, new_lines) if a != b] @@ -3086,10 +3090,22 @@ def test_the_v2_pilot_prompt_names_policyengine_nowhere_in_any_household(): "- federal_income_tax_before_refundable_credits:", "- federal_refundable_credits:", ) - assert all("PolicyEngine" in a or a.startswith(federal) for a, _ in changed) + other = ( + "- state_income_tax_before_refundable_credits:", + "Estimate the requested tax and benefit outputs", + "- employer sponsored insurance premiums: ", + "- spm unit energy subsidy: ", + ) + assert all( + "PolicyEngine" in a or a.startswith(federal + other) for a, _ in changed + ) assert len(changed) == old.count("PolicyEngine") + sum( - line in old for line in federal + 1 for line in old_lines if line.startswith(federal + other) ) + # The law date, the payer and the plain energy label are stated. + assert "published before July 3, 2026" in new + assert "employer sponsored insurance premiums" not in new + assert "spm unit energy subsidy" not in new # d1263: the v2 prompt states the rate-schedule convention on both # federal lines whose values the schedule choice can move. for line in federal: @@ -3435,3 +3451,101 @@ def test_a_run_record_carries_its_wording_hash(): make_scenario(), VARIABLES, fake_tool, model, pilot.Spend(50.0), profile=profile ) assert record["prompt_wording_sha256"] == pilot.prompt_wording_sha256(profile) + + +# --------------------------------------------------------------------------- +# The no-tools arm: the board's own request, for prompt versions the board +# has no rows for. +# --------------------------------------------------------------------------- + + +def _first_request(name: str, scenario, variables, prompt_version="v1"): + profile = pilot.model_profile(name, "none", prompt_version) + values = {v: 0 for v in variables} + if profile.api == "responses": + model = FakeResponses([responses_response([sol_submit(values)], "r")]) + elif profile.contract == "json": + model = FakeModel([response([], content=json_answer(values))]) + else: + model = FakeModel([response([submit(values)])]) + record = pilot.run_household( + scenario, variables, fake_tool, model, pilot.Spend(50.0), profile=profile + ) + return profile, record, model.requests[0] + + +def test_the_no_tools_arm_sends_the_boards_own_request(): + """Differential: for each contract, the arm's first request matches what + policybench.eval_no_tools builds for the board, in prompt, tools, tool + choice and response format.""" + from policybench.eval_no_tools import ( + _chat_completion_request_kwargs, + _responses_request_kwargs, + ) + + board = pilot.load_board() + for sid in ("scenario_000", "scenario_012"): + scenario, variables = board["scenarios"][sid], board["variables"][sid] + for name in (pilot.MODEL_NAME, SOL, OPUS): + profile, record, sent = _first_request(name, scenario, variables) + assert record["status"] == "complete" and profile.notice == "" + if profile.api == "responses": + _, ref = _responses_request_kwargs( + scenario, variables, profile.model_id + ) + assert sent["input"] == ref["input"] + assert [t["name"] for t in sent["tools"]] == [ + t["name"] for t in ref["tools"] + ] + else: + messages, ref = _chat_completion_request_kwargs( + scenario, variables, profile.model_id + ) + text = sent["messages"][0]["content"] + text = text if isinstance(text, str) else text[0]["text"] + assert text == messages[0]["content"] + assert [t["function"]["name"] for t in sent.get("tools") or []] == [ + t["function"]["name"] for t in ref.get("tools") or [] + ] + assert sent.get("response_format") == ref.get("response_format") + assert sent.get("tool_choice") == ref.get("tool_choice") + if profile.contract == "json": + assert "tools" not in sent and "tool_choice" not in sent + + +def test_the_no_tools_arm_on_v2_pilot_sends_the_rewritten_prompt(): + board = pilot.load_board() + sid = "scenario_000" + profile, record, sent = _first_request( + SOL, board["scenarios"][sid], board["variables"][sid], "v2-pilot" + ) + assert "published before July 3, 2026" in sent["input"] + assert "policyengine" not in sent["input"].lower() + assert record["arm"] == "none" and profile.arm2_label == ( + "gpt-6-sol+no-tools@v2-pilot" + ) + with pytest.raises(SystemExit): + pilot.model_profile(SOL, "none", "v2-pilot", True) + + +def test_only_web_arms_get_the_longer_timeout(): + board = pilot.load_board() + scenario, variables = ( + board["scenarios"]["scenario_000"], + board["variables"]["scenario_000"], + ) + values = {v: 0 for v in variables} + timeouts = {} + for arm in pilot.ARMS: + model = FakeModel([response([submit(values)])]) + pilot.run_household( + scenario, + variables, + fake_tool, + model, + pilot.Spend(50.0), + profile=pilot.model_profile(pilot.MODEL_NAME, arm), + ) + timeouts[arm] = model.requests[0]["timeout"] + assert timeouts["web"] == timeouts["web+pe"] == pilot.WEB_REQUEST_TIMEOUT_SECONDS + assert timeouts["none"] == timeouts["pe"] < pilot.WEB_REQUEST_TIMEOUT_SECONDS From 274527b9d0e37ca3d5d62fe099e7415fae6f87db Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 22 Oct 2026 07:15:38 -0400 Subject: [PATCH 2/2] Address review r1 of the v2-pilot run prep - Law date: the sentence now gives the reference's fallback. Where no 2026 amount was published before July 3, 2026, the latest earlier one applies (California, Idaho, Maryland, Michigan and Minnesota amounts, and the IRS 2025 sales tax tables, per each fix module's rule). It also states SNAP's hold precisely: FY2026 allotments, deductions, utility allowances and asset limits through December, with income limits on the 2026 poverty guidelines from October ("Not held: the poverty guideline"). - Web arms don't retry a timed-out request. Three 900 s attempts would hold a worker for 45 minutes; the household ends as provider_error, which a resume reruns. - The no-tools arm marks no prompt-cache breakpoint, as the board's request marks none, and records its first turn as forced. The differential test now compares the whole first request with the board's, key for key, across all 100 households and three contracts. - The energy relabel's comment now states the engine's mechanism: it deems the utility allowance from the state flag and never reads the amount. - The manifest records the timeout actually sent. Error records carry the arm and prompt keys that resume and score check. Tests: 147 passed. Three more planted bugs were each caught. Co-Authored-By: Claude Opus 5.5 --- scripts/tool_pilot_argon.py | 87 ++++++++++++++++++++++++---------- tests/test_tool_pilot_argon.py | 83 +++++++++++++++++++++++++------- 2 files changed, 128 insertions(+), 42 deletions(-) diff --git a/scripts/tool_pilot_argon.py b/scripts/tool_pilot_argon.py index 5395bade..97d880e7 100644 --- a/scripts/tool_pilot_argon.py +++ b/scripts/tool_pilot_argon.py @@ -147,9 +147,11 @@ # The arms ``run --arm`` takes. "pe" is arm 2 as run on 2026-10-14 and # 2026-10-15. "web" swaps the PolicyEngine tool for the provider's own web -# search, and "web+pe" offers both. "none" offers neither: the board's own -# no-tools request (the forced answer function, or the JSON reply with no tools), -# so a prompt version other than the board's has its own no-tools baseline. +# search, and "web+pe" offers both. "none" offers neither: its first request is +# the board's own no-tools request (the forced answer function, or the JSON reply +# with no tools), so a prompt version other than the board's has its own +# no-tools baseline. Repairs, retries and truncation then follow this pilot's +# loop, not the board's; they matter only when a first answer does not parse. ARMS = ("none", "pe", "web", "web+pe") ARM_SUFFIX = { "none": "+no-tools", @@ -158,9 +160,11 @@ "web+pe": "+web+pe-tool", } # A web arm's request timeout. A search-heavy turn can outlast the board's -# (Gemini 4 Argon's model card gives 300 s, and its web-only arm never finished -# scenario_005 within it in six attempts on 2026-10-21), so the web arms allow -# this much and the manifest records it. +# (300 s for Gemini 4 Argon, the harness default for its card; its web-only arm +# never finished scenario_005 within it in six attempts on 2026-10-21), so the +# web arms allow this much. A timed-out web request is not retried: three +# 900 s attempts would hold a worker for 45 minutes, so the household ends as +# provider_error and a resume pass reruns it. WEB_REQUEST_TIMEOUT_SECONDS = 900 _CALCULATE_OPENING = "You also have a `calculate_household` function." @@ -299,18 +303,29 @@ "{who}; include Idaho's $10 permanent building fund tax where it applies", ), ( - # The reference rule (reference_audit sidecar, quoted in - # fixes/latest_c_snap_hold_fy2026.py): "A scored reference follows from - # the stated facts and from law published before the 2026-07-03 - # reference freeze", and SNAP holds the FY2026 schedule for October to - # December 2026. Web arms otherwise apply FY2027 values, published in + # The reference rule (reference sidecar, quoted in each + # reference_audit/2026-09-28/fixes/latest_c_* module): "a scored + # reference follows from the stated facts and from law published before + # the 2026-07-03 reference freeze". Where no 2026 amount was published by + # then, the reference holds the last one published before it: California's + # 2025 indexed amounts, Idaho's zero-rate thresholds, the IRS 2025 + # optional sales tax tables, and some Maryland, Michigan and Minnesota + # amounts. SNAP holds the FY2026 schedule (allotments, deductions, + # utility allowances, asset limits) for October to December; "Not held: + # the poverty guideline", so income limits move to the 2026 guideline in + # October. Web arms had applied SNAP's FY2027 schedule, published in # August 2026 (GPT-6 Sol, scenarios 008 and 012). re.compile( r"(Do not infer unlisted income, expenses, assets, benefit receipt, " r"rent, or health coverage\.)" ), "{who} Apply the law and program parameters published before July 3, " - "2026; for SNAP, use the fiscal year 2026 values for all of 2026.", + "2026. Where a 2026 amount had not been published by then, use the latest " + "amount published before that date (for example, a state's 2025 " + "inflation-adjusted amounts, or the IRS's 2025 optional sales tax " + "tables). For SNAP, keep the fiscal year 2026 allotments, deductions, " + "utility allowances and asset limits through December 2026; the income " + "limits use the 2026 HHS poverty guidelines from October 2026.", ), ( # policyengine-us documents employer_sponsored_insurance_premiums as @@ -321,9 +336,10 @@ "the household): ", ), ( - # spm_unit_energy_subsidy, the SPM's home energy assistance. In 2026 - # policyengine-us deems a SNAP utility allowance from energy assistance - # only for elderly or disabled households + # spm_unit_energy_subsidy, the SPM's energy subsidy (home energy + # assistance). No scored output reads it. policyengine-us never reads the + # amount for SNAP: it deems a utility allowance from the state's + # always-SUA flag, and in 2026 only for elderly or disabled households # (heat_and_eat_requires_elderly_disabled, P.L. 119-21 section 10103). # The engine label hid what the amount is. re.compile(r"spm unit energy subsidy: "), @@ -1734,8 +1750,13 @@ class BudgetStop(RuntimeError): pass +def is_timeout(error: Exception) -> bool: + """A provider or wall-clock timeout (litellm.Timeout and its kin).""" + return any("Timeout" in cls.__name__ for cls in type(error).__mro__) + + def _request_with_retries( - completion_fn, kwargs: dict + completion_fn, kwargs: dict, retry_timeouts: bool = True ) -> tuple[Any, float, float, float]: from policybench.eval_no_tools import _run_request_with_wall_timeout, _should_retry @@ -1746,7 +1767,11 @@ def _request_with_retries( response = _run_request_with_wall_timeout(completion_fn, kwargs) return response, started_at, time.time(), time.perf_counter() - started except Exception as error: - if attempt == MAX_PROVIDER_ATTEMPTS - 1 or not _should_retry(error): + if ( + attempt == MAX_PROVIDER_ATTEMPTS - 1 + or not _should_retry(error) + or (not retry_timeouts and is_timeout(error)) + ): raise time.sleep(2 * (2**attempt)) raise RuntimeError("unreachable") @@ -1810,7 +1835,11 @@ def run_household( ResponsesConversation(prompt) if profile.api == "responses" else ChatConversation( - prompt, cache_breakpoint=profile.model_id.startswith("claude-") + # The board's own request marks no prompt-cache breakpoint, so the + # no-tools arm, its stand-in, marks none either. + prompt, + cache_breakpoint=profile.model_id.startswith("claude-") + and profile.arm != "none", ) ) json_contract = profile.contract == "json" @@ -1981,7 +2010,7 @@ def run_calculations(turn_record: dict, calls: list[Call]) -> bool: if repair_rounds: kind = "repair" elif not json_contract and ( - forced or tool_turns + search_only_turns >= turn_cap + forced or tool_turns + search_only_turns >= turn_cap or not offered ): kind, forced = "forced_answer", True else: @@ -1998,7 +2027,7 @@ def run_calculations(turn_record: dict, calls: list[Call]) -> bool: escalated_from = None try: response, started_at, completed_at, latency = _request_with_retries( - completion_fn, kwargs + completion_fn, kwargs, retry_timeouts=not profile.uses_web ) turn = conversation.read(response, kwargs.get(budget_key)) if ( @@ -2021,7 +2050,7 @@ def run_calculations(turn_record: dict, calls: list[Call]) -> bool: escalated_from = kwargs.get(budget_key) kwargs[budget_key] = ceiling response, started_at, completed_at, latency = _request_with_retries( - completion_fn, kwargs + completion_fn, kwargs, retry_timeouts=not profile.uses_web ) turn = conversation.read(response, kwargs.get(budget_key)) except Exception as error: @@ -2608,9 +2637,13 @@ def command_run(args) -> None: "tool_notice": profile.notice, "calculate_tool": calculate_tool_schema() if profile.uses_calculator else None, "web_search_tool": web_search_tool(profile) if profile.uses_web else None, - "request_timeout_seconds": WEB_REQUEST_TIMEOUT_SECONDS - if profile.uses_web - else None, + # The timeout actually sent: the board's for the model, or the web arms'. + "request_timeout_seconds": build_initial_request( + board["scenarios"][ids[0]], board["variables"][ids[0]], profile + )[1].get("timeout") + if ids and not profile.uses_web + else (WEB_REQUEST_TIMEOUT_SECONDS if profile.uses_web else None), + "web_timeouts_retried": not profile.uses_web, "search_usd_per_1k": ( SEARCH_USD_PER_1K[profile.provider] if profile.uses_web else None ), @@ -2660,6 +2693,12 @@ def one(sid: str) -> dict: record = { "scenario_id": sid, "model": profile.name, + # The setup keys a resume and score check, so a folder with + # this record still resumes and scores. + "arm": profile.arm, + "prompt_version": profile.prompt_version, + "prompt_wording_sha256": prompt_wording_sha256(profile), + "benchmark_search_rule": profile.benchmark_search_rule, "variables": board["variables"][sid], "status": "harness_error", "error": f"{type(error).__name__}: {error}"[:MAX_MESSAGE_CHARS], diff --git a/tests/test_tool_pilot_argon.py b/tests/test_tool_pilot_argon.py index abc9d2db..a5d531b0 100644 --- a/tests/test_tool_pilot_argon.py +++ b/tests/test_tool_pilot_argon.py @@ -3475,16 +3475,17 @@ def _first_request(name: str, scenario, variables, prompt_version="v1"): def test_the_no_tools_arm_sends_the_boards_own_request(): - """Differential: for each contract, the arm's first request matches what - policybench.eval_no_tools builds for the board, in prompt, tools, tool - choice and response format.""" + """Differential: for every household and contract, the arm's first request + is the request policybench.eval_no_tools builds for the board, key for key + (budget, timeout, tools and their schemas, tool choice, response format), + except the board's disk-cache flag, which a live run does not use.""" from policybench.eval_no_tools import ( _chat_completion_request_kwargs, _responses_request_kwargs, ) board = pilot.load_board() - for sid in ("scenario_000", "scenario_012"): + for sid in sorted(board["scenarios"]): scenario, variables = board["scenarios"][sid], board["variables"][sid] for name in (pilot.MODEL_NAME, SOL, OPUS): profile, record, sent = _first_request(name, scenario, variables) @@ -3493,24 +3494,15 @@ def test_the_no_tools_arm_sends_the_boards_own_request(): _, ref = _responses_request_kwargs( scenario, variables, profile.model_id ) - assert sent["input"] == ref["input"] - assert [t["name"] for t in sent["tools"]] == [ - t["name"] for t in ref["tools"] - ] else: messages, ref = _chat_completion_request_kwargs( scenario, variables, profile.model_id ) - text = sent["messages"][0]["content"] - text = text if isinstance(text, str) else text[0]["text"] - assert text == messages[0]["content"] - assert [t["function"]["name"] for t in sent.get("tools") or []] == [ - t["function"]["name"] for t in ref.get("tools") or [] - ] - assert sent.get("response_format") == ref.get("response_format") - assert sent.get("tool_choice") == ref.get("tool_choice") - if profile.contract == "json": - assert "tools" not in sent and "tool_choice" not in sent + ref = {**ref, "messages": messages} + ref = {k: v for k, v in ref.items() if k != "caching"} + sent = json.loads(json.dumps(sent, default=str)) + ref = json.loads(json.dumps(ref, default=str)) + assert sent == ref, (sid, name, sorted(set(sent) ^ set(ref))) def test_the_no_tools_arm_on_v2_pilot_sends_the_rewritten_prompt(): @@ -3549,3 +3541,58 @@ def test_only_web_arms_get_the_longer_timeout(): timeouts[arm] = model.requests[0]["timeout"] assert timeouts["web"] == timeouts["web+pe"] == pilot.WEB_REQUEST_TIMEOUT_SECONDS assert timeouts["none"] == timeouts["pe"] < pilot.WEB_REQUEST_TIMEOUT_SECONDS + + +def test_a_web_arm_does_not_retry_a_timed_out_request(monkeypatch): + """Three 900 s attempts would hold a worker for 45 minutes: a web arm's + timed-out request ends the household as provider_error, which a resume + reruns. Other arms keep the board's retries.""" + import litellm + + monkeypatch.setattr(pilot.time, "sleep", lambda seconds: None) + for arm, attempts in (("web", 1), ("pe", pilot.MAX_PROVIDER_ATTEMPTS)): + calls = [] + + def timing_out(**kwargs): + calls.append(kwargs) + raise litellm.Timeout("timed out", model="m", llm_provider="openai") + + record = pilot.run_household( + make_scenario(), + VARIABLES, + fake_tool, + timing_out, + pilot.Spend(50.0), + profile=pilot.model_profile(pilot.MODEL_NAME, arm), + ) + assert record["status"] == "provider_error" + assert len(calls) == attempts, arm + assert pilot.is_timeout(litellm.Timeout("t", model="m", llm_provider="openai")) + assert not pilot.is_timeout(ValueError("x")) + + +def test_the_no_tools_arm_marks_no_prompt_cache_breakpoint(): + model = FakeModel([response([], content=json_answer({"snap": 1, VARIABLES[1]: 2}))]) + pilot.run_household( + make_scenario(), + VARIABLES, + fake_tool, + model, + pilot.Spend(50.0), + profile=pilot.model_profile(OPUS, "none"), + ) + assert "cache_control" not in json.dumps(model.requests[0]) + + +def test_the_law_date_sentence_states_the_fallback_and_the_snap_split(): + board = pilot.load_board() + prompt, _ = pilot.build_initial_request( + board["scenarios"]["scenario_000"], + board["variables"]["scenario_000"], + pilot.model_profile(SOL, "pe", "v2-pilot"), + ) + assert "use the latest amount published before that date" in prompt + assert "2025 optional sales tax tables" in prompt + assert "asset limits through December 2026" in prompt + assert "2026 HHS poverty guidelines from October 2026" in prompt + assert "fiscal year 2026 values for all of 2026" not in prompt