Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/runbook.md
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,7 @@ Google deprecated `gemini-3.5-flash` and `gemini-3.7-flash` on 2026-10-08
([changelog](https://ai.google.dev/gemini-api/docs/changelog)). Requests under
those names now route to `gemini-3.6-flash` and `gemini-3.8-flash`, which are
rows of their own, so neither row can be re-run under its name. A synchronous
re-run or row repair would not show the re-route: litellm 1.88.1 sets the
re-run or row repair would not show the re-route: litellm 1.88.x sets the
response model to the requested name, which PolicyBench records as
`provider_resolved_model`, and ignores Gemini's `modelVersion`. The Gemini
batch path, which calls google-genai directly, does record `modelVersion`.
Expand Down
2 changes: 1 addition & 1 deletion policybench/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@
# answers were requested on 2026-08-18, before Gemini 3.8 Flash was
# released (2026-09-02), so they are Gemini 3.7 Flash, and this name can
# no longer re-run them. A synchronous run, which row repairs also use,
# would not show either re-route: litellm 1.88.1 sets the response model
# would not show either re-route: litellm 1.88.x sets the response model
# to the requested name and never reads Gemini's modelVersion, so
# provider_resolved_model records gemini-3.7-flash or gemini-3.5-flash.
# The Gemini batch path (google-genai) does record modelVersion. GET
Expand Down
23 changes: 12 additions & 11 deletions scripts/tool_pilot_argon.py
Original file line number Diff line number Diff line change
Expand Up @@ -430,9 +430,10 @@ def refuse_policyengine(label: str, value) -> None:
# Times a JSON-contract reply that searched but gave no answer is sent back
# as it is, Anthropic's continuation for a paused server-tool turn.
MAX_PAUSE_CONTINUATIONS = 3
# The litellm whose Gemini search-tool handling allow_gemini_search_with_functions
# was written against.
GEMINI_SEARCH_PATCH_LITELLM = "1.88.1"
# The litellm releases whose Gemini search-tool handling
# allow_gemini_search_with_functions was checked against. It was written against
# 1.88.1; 1.88.6's litellm/llms/vertex_ai/gemini/ is byte-identical to 1.88.1's.
GEMINI_SEARCH_PATCH_LITELLM = frozenset({"1.88.1", "1.88.6"})

CALCULATE_DESCRIPTION = (
"Run PolicyEngine US (policyengine-us 2.41.0) on one household and return "
Expand Down Expand Up @@ -1223,20 +1224,20 @@ def web_search_tool(profile: ModelProfile) -> dict:
def allow_gemini_search_with_functions() -> None:
"""Keep Gemini's search tool beside function tools in litellm.

litellm 1.88.1 drops googleSearch from any request that also declares
functions unless ``include_server_side_tool_invocations`` is set. It reads
that flag only among the mapped OpenAI parameters, and a ``completion``
keyword reaches it later, as a provider extra, after the search tool is
gone. Gemini 3 and 4 models take both together with the flag
litellm 1.88.1 and 1.88.6 drop googleSearch from any request that also
declares functions unless ``include_server_side_tool_invocations`` is set.
They read that flag only among the mapped OpenAI parameters, and a
``completion`` keyword reaches it later, as a provider extra, after the
search tool is gone. Gemini 3 and 4 models take both together with the flag
(ai.google.dev/gemini-api/docs/tool-combination), and the web arms always
send it, so both of litellm's drops are skipped. Refuses any other litellm,
whose code this was not checked against.
"""
version = importlib.metadata.version("litellm")
if version != GEMINI_SEARCH_PATCH_LITELLM:
if version not in GEMINI_SEARCH_PATCH_LITELLM:
raise SystemExit(
f"litellm {version} is installed; the Gemini search patch was "
f"checked against {GEMINI_SEARCH_PATCH_LITELLM} only"
f"checked against {', '.join(sorted(GEMINI_SEARCH_PATCH_LITELLM))} only"
)
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
Expand Down Expand Up @@ -1466,7 +1467,7 @@ def _attach_deferred_results(self, message: dict) -> None:

When Claude calls a server tool beside a client tool, the API runs the
server tool on the next request, and that response begins with its
result, without the call. litellm 1.88.1 re-sends a server result only
result, without the call. litellm 1.88.x re-sends a server result only
beside its call in the same message, so left in place the result drops
out of later requests, and the API refuses the call that lacks it (a
live probe got that 400 on the third request). The result joins its
Expand Down
15 changes: 13 additions & 2 deletions tests/test_tool_pilot_argon.py
Original file line number Diff line number Diff line change
Expand Up @@ -2762,14 +2762,25 @@ def tool_keys():
)
return sorted(k for t in params.get("tools", []) for k in t)

if importlib.metadata.version("litellm") != pilot.GEMINI_SEARCH_PATCH_LITELLM:
pytest.skip("the patch refuses other litellm versions")
# Installed from the lock, litellm is a release the patch was checked
# against (the lock test below), so this runs rather than skips.
assert importlib.metadata.version("litellm") in pilot.GEMINI_SEARCH_PATCH_LITELLM
assert tool_keys() == ["function_declarations"] # litellm drops search
pilot.allow_gemini_search_with_functions()
assert tool_keys() == ["function_declarations", "googleSearch"]
assert calculate_tool["function"]["name"] == pilot.TOOL_NAME


def test_the_locked_litellm_is_one_the_gemini_patch_was_checked_against():
"""A lock bump past the checked releases fails here, before a live web arm
refuses to start."""
import tomllib

lock = tomllib.loads((Path(__file__).resolve().parents[1] / "uv.lock").read_text())
(version,) = {p["version"] for p in lock["package"] if p["name"] == "litellm"}
assert version in pilot.GEMINI_SEARCH_PATCH_LITELLM


def test_the_gemini_patch_refuses_another_litellm(monkeypatch):
_keep_litellm_gemini_config(monkeypatch)
monkeypatch.setattr(pilot.importlib.metadata, "version", lambda name: "9.9.9")
Expand Down
Loading
Loading