diff --git a/.env.example b/.env.example
index 802d7a9..1f67bb3 100644
--- a/.env.example
+++ b/.env.example
@@ -9,3 +9,10 @@ MERIDIAN_DATABASE_URL=postgresql://meridian:meridian@localhost:5433/meridian
# Proxy mode: record | replay | passthrough
# `passthrough` is rejected by config validation in gate mode.
MERIDIAN_PROXY_MODE=replay
+
+# OpenAI-compatible providers (Groq, OpenAI, vLLM, Ollama): the proxy translates
+# the agent's Anthropic-format calls, so the agent and cassettes do not change.
+# GROQ_API_KEY=gsk-replace-me
+# MERIDIAN_UPSTREAM_API=openai
+# MERIDIAN_UPSTREAM_BASE_URL=https://api.groq.com/openai/v1
+# MERIDIAN_UPSTREAM_MODEL=llama-3.3-70b-versatile
diff --git a/.github/workflows/real-model-benchmark.yml b/.github/workflows/real-model-benchmark.yml
new file mode 100644
index 0000000..aa97188
--- /dev/null
+++ b/.github/workflows/real-model-benchmark.yml
@@ -0,0 +1,87 @@
+name: real-model-benchmark
+
+# Records and scores the checkout-agent suite against a REAL model, through the
+# Meridian proxy, and publishes pass^k, the minimum detectable effect and the
+# verdicts as an artifact. Unlike self-eval, this one spends money: it needs the
+# GROQ_API_KEY or ANTHROPIC_API_KEY repository secret and is manual-only, with a hard budget cap
+# enforced by the proxy.
+
+on:
+ workflow_dispatch:
+ inputs:
+ provider:
+ description: Which provider to measure (needs its repository secret)
+ type: choice
+ options: [groq, anthropic]
+ default: groq
+ model:
+ description: Model id at the provider (e.g. llama-3.3-70b-versatile or claude-haiku-4-5-20251001)
+ default: llama-3.3-70b-versatile
+ trials:
+ description: Trials per task (n)
+ default: "10"
+ k:
+ description: The k in pass^k
+ default: "3"
+ budget_cents:
+ description: Hard run budget in cents (the proxy stops spending at this)
+ default: "200"
+
+permissions:
+ contents: read
+
+jobs:
+ benchmark:
+ runs-on: ubuntu-latest
+ timeout-minutes: 120
+ env:
+ # Only the chosen provider's key is exported. The proxy takes the first
+ # credential it finds, so exporting both could send one provider's key to
+ # the other.
+ GROQ_API_KEY: ${{ inputs.provider == 'groq' && secrets.GROQ_API_KEY || '' }}
+ ANTHROPIC_API_KEY: ${{ inputs.provider == 'anthropic' && secrets.ANTHROPIC_API_KEY || '' }}
+ MERIDIAN_UPSTREAM_API: ${{ inputs.provider == 'groq' && 'openai' || 'anthropic' }}
+ MERIDIAN_UPSTREAM_BASE_URL: ${{ inputs.provider == 'groq' && 'https://api.groq.com/openai/v1' || 'https://api.anthropic.com' }}
+ MERIDIAN_UPSTREAM_MODEL: ${{ inputs.model }}
+ steps:
+ - uses: actions/checkout@v4
+ - uses: astral-sh/setup-uv@v5
+ with:
+ enable-cache: true
+ - run: uv sync --frozen
+ - name: Check the secret exists
+ run: test -n "$GROQ_API_KEY$ANTHROPIC_API_KEY" || { echo "no API key secret for provider ${{ inputs.provider }}"; exit 1; }
+ - name: Build images and repin the suite
+ run: |
+ uv run meridian snapshot build ./envs/checkout --tag checkout:dev \
+ --write-ref --update-suite ./suites/checkout-agent
+ uv run meridian snapshot build ./envs/proxy --tag meridian-proxy:dev --context .
+ - name: Record and score against the real model
+ run: |
+ mkdir -p out cassettes
+ uv run meridian run \
+ --suite ./suites/checkout-agent \
+ --n "${{ inputs.trials }}" --k "${{ inputs.k }}" \
+ --proxy-mode record --live-provider \
+ --cassettes ./cassettes --budget "${{ inputs.budget_cents }}" \
+ --out out/result.json --html out/report.html
+ - name: Summarise
+ if: always()
+ run: |
+ {
+ echo "## Real-model benchmark (${{ inputs.provider }} / ${{ inputs.model }})"
+ echo
+ echo "n=${{ inputs.trials }} k=${{ inputs.k }} budget=${{ inputs.budget_cents }}c"
+ echo
+ if [ -f out/result.json ]; then echo '```json'; cat out/result.json; echo '```'; fi
+ } >> "$GITHUB_STEP_SUMMARY"
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: real-model-benchmark
+ path: |
+ out/
+ cassettes/
+ - name: Sweep
+ if: always()
+ run: uv run meridian sweep
diff --git a/README.md b/README.md
index af607ed..46b714e 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,18 @@ So here is Meridian measured by Meridian, regenerated by CI on every push to
| **Replay fidelity** | **5 / 5** | archived runs re-materialized from their manifests alone and reproduced exactly. |
| **Contamination probe** | **pass / fail** | passes with isolation on, **fails** with `--unsafe-shared-env`. A probe that cannot fail proves nothing. |
-`n=5` · `k=3` · tolerance `0.03` · baseline suite pass^3 `0.486` · mean run 32.4s / 51¢ · Meridian `0.1.0` · [`benchmarks/self-eval.json`](benchmarks/self-eval.json), regenerate with `make self-eval` (~30 min)
+> **Read this before the numbers.** The self-eval above runs against a
+> deterministic **stub provider** ([`stub_provider.py`](src/meridian/runtime/proxy/stub_provider.py)),
+> not a real LLM. The stub reproduces the Anthropic wire format and "slips" one
+> trial in five on a schedule, so the *pipeline* (record → replay → gate →
+> report) is exercised for free and reproducibly. These numbers show the gate's
+> **statistical machinery behaves as designed on a controlled source of
+> variance**. They are **not** evidence about how any real model or agent
+> behaves. A real-model benchmark is tracked in the roadmap and will be
+> published separately, with its own pass^k, minimum detectable effect, and
+> inconclusive verdicts.
+
+`n=5` · `k=3` · tolerance `0.03` · baseline suite pass^3 `0.486` · mean run 32.4s / 51¢ notional (token usage × default price table, not real spend) · Meridian `0.1.0` · [`benchmarks/self-eval.json`](benchmarks/self-eval.json), regenerate with `make self-eval` (~30 min)
**The miss, stated plainly.** An arithmetic bug seeded into the tool that writes
the invoice dropped suite pass^3 by 0.114 — a real regression, correctly measured
diff --git a/src/meridian/runtime/orchestrator.py b/src/meridian/runtime/orchestrator.py
index deeb2d2..fb7d08a 100644
--- a/src/meridian/runtime/orchestrator.py
+++ b/src/meridian/runtime/orchestrator.py
@@ -8,6 +8,7 @@
from __future__ import annotations
+import os
import time
import uuid
from collections.abc import Callable, Sequence
@@ -104,6 +105,12 @@ async def execute(client: DockerClient, request: RunRequest) -> RunOutcome:
run_budget_cents=config.execution.budget_cents,
use_stub_provider=request.use_stub_provider,
stub_slip_every=request.stub_slip_every,
+ # A live run reaches its provider only through the proxy. Set
+ # MERIDIAN_UPSTREAM_API=openai and MERIDIAN_UPSTREAM_BASE_URL to use an
+ # OpenAI-compatible provider (e.g. https://api.groq.com/openai/v1).
+ upstream_base_url=os.environ.get("MERIDIAN_UPSTREAM_BASE_URL") or None,
+ upstream_api=os.environ.get("MERIDIAN_UPSTREAM_API") or None,
+ upstream_model=os.environ.get("MERIDIAN_UPSTREAM_MODEL") or None,
)
progress(f"proxy up in {config.execution.proxy_mode} mode on {handle.network_name}")
diff --git a/src/meridian/runtime/proxy/openai_compat.py b/src/meridian/runtime/proxy/openai_compat.py
new file mode 100644
index 0000000..c8db324
--- /dev/null
+++ b/src/meridian/runtime/proxy/openai_compat.py
@@ -0,0 +1,191 @@
+"""Translate Anthropic messages <-> OpenAI chat completions.
+
+The agent and the cassettes speak the Anthropic messages shape. Providers that
+only offer an OpenAI-compatible endpoint (Groq, OpenAI, Together, vLLM, Ollama)
+are reached by translating at the proxy, so the agent, the cassette format and
+the isolation model do not change, and the credential still lives only in the
+proxy container.
+
+Pure functions: no I/O, no HTTP. Unit tested without a daemon.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+# OpenAI finish_reason -> Anthropic stop_reason.
+_STOP_REASONS = {
+ "stop": "end_turn",
+ "length": "max_tokens",
+ "tool_calls": "tool_use",
+ "function_call": "tool_use",
+ "content_filter": "end_turn",
+}
+
+
+def _flatten_text(content: Any) -> str:
+ """Anthropic allows a string or a list of text blocks for system/user text."""
+ if isinstance(content, str):
+ return content
+ parts = [str(b.get("text", "")) for b in content or [] if b.get("type") == "text"]
+ return "\n".join(parts)
+
+
+def _tool_result_text(content: Any) -> str:
+ if isinstance(content, str):
+ return content
+ if isinstance(content, list):
+ return _flatten_text(content)
+ return "" if content is None else json.dumps(content, sort_keys=True)
+
+
+def _translate_message(message: dict[str, Any]) -> list[dict[str, Any]]:
+ role = message["role"]
+ content = message.get("content")
+ if isinstance(content, str):
+ return [{"role": role, "content": content}]
+
+ blocks = content or []
+ if role == "assistant":
+ text = _flatten_text(blocks)
+ tool_calls = [
+ {
+ "id": block["id"],
+ "type": "function",
+ "function": {
+ "name": block["name"],
+ "arguments": json.dumps(block.get("input", {}), sort_keys=True),
+ },
+ }
+ for block in blocks
+ if block.get("type") == "tool_use"
+ ]
+ out: dict[str, Any] = {"role": "assistant", "content": text or None}
+ if tool_calls:
+ out["tool_calls"] = tool_calls
+ return [out]
+
+ # user turn: tool_result blocks become their own `tool` messages, which OpenAI
+ # requires to directly follow the assistant tool_calls message.
+ translated: list[dict[str, Any]] = []
+ for block in blocks:
+ if block.get("type") == "tool_result":
+ translated.append(
+ {
+ "role": "tool",
+ "tool_call_id": block["tool_use_id"],
+ "content": _tool_result_text(block.get("content")),
+ }
+ )
+ text = _flatten_text([b for b in blocks if b.get("type") == "text"])
+ if text:
+ translated.append({"role": "user", "content": text})
+ return translated
+
+
+def _translate_tool_choice(choice: Any) -> Any:
+ if not isinstance(choice, dict):
+ return None
+ kind = choice.get("type")
+ if kind == "auto":
+ return "auto"
+ if kind == "any":
+ return "required"
+ if kind == "none":
+ return "none"
+ if kind == "tool":
+ return {"type": "function", "function": {"name": choice["name"]}}
+ return None
+
+
+def to_openai_request(body: dict[str, Any], *, model: str | None = None) -> dict[str, Any]:
+ """Anthropic /v1/messages request -> OpenAI /chat/completions request."""
+ messages: list[dict[str, Any]] = []
+ system = body.get("system")
+ if system:
+ messages.append({"role": "system", "content": _flatten_text(system)})
+ for message in body.get("messages", []):
+ messages.extend(_translate_message(message))
+
+ request: dict[str, Any] = {
+ "model": model or body.get("model", ""),
+ "messages": messages,
+ }
+ if "max_tokens" in body:
+ request["max_tokens"] = body["max_tokens"]
+ for field in ("temperature", "top_p"):
+ if field in body:
+ request[field] = body[field]
+ if body.get("stop_sequences"):
+ request["stop"] = body["stop_sequences"]
+ tools = body.get("tools")
+ if tools:
+ request["tools"] = [
+ {
+ "type": "function",
+ "function": {
+ "name": tool["name"],
+ "description": tool.get("description", ""),
+ "parameters": tool.get("input_schema", {"type": "object", "properties": {}}),
+ },
+ }
+ for tool in tools
+ ]
+ choice = _translate_tool_choice(body.get("tool_choice"))
+ if choice is not None:
+ request["tool_choice"] = choice
+ return request
+
+
+def _parse_arguments(raw: Any) -> dict[str, Any]:
+ if isinstance(raw, dict):
+ return raw
+ if not raw:
+ return {}
+ try:
+ parsed = json.loads(raw)
+ except json.JSONDecodeError:
+ # Models occasionally emit malformed tool JSON. Surface it rather than
+ # dropping the call: the agent's handling of it is part of what is measured.
+ return {"_raw_arguments": str(raw)}
+ return parsed if isinstance(parsed, dict) else {"_value": parsed}
+
+
+def from_openai_response(response: dict[str, Any]) -> dict[str, Any]:
+ """OpenAI chat completion -> Anthropic messages response."""
+ choice = (response.get("choices") or [{}])[0]
+ message = choice.get("message") or {}
+
+ content: list[dict[str, Any]] = []
+ if message.get("content"):
+ content.append({"type": "text", "text": message["content"]})
+ for call in message.get("tool_calls") or []:
+ function = call.get("function") or {}
+ content.append(
+ {
+ "type": "tool_use",
+ "id": call.get("id", ""),
+ "name": function.get("name", ""),
+ "input": _parse_arguments(function.get("arguments")),
+ }
+ )
+
+ usage = response.get("usage") or {}
+ stop_reason = _STOP_REASONS.get(str(choice.get("finish_reason")), "end_turn")
+ if any(block["type"] == "tool_use" for block in content):
+ stop_reason = "tool_use"
+
+ return {
+ "id": response.get("id", ""),
+ "type": "message",
+ "role": "assistant",
+ "model": response.get("model", ""),
+ "content": content,
+ "stop_reason": stop_reason,
+ "stop_sequence": None,
+ "usage": {
+ "input_tokens": int(usage.get("prompt_tokens", 0)),
+ "output_tokens": int(usage.get("completion_tokens", 0)),
+ },
+ }
diff --git a/src/meridian/runtime/proxy/server.py b/src/meridian/runtime/proxy/server.py
index 75d26dc..4fb3168 100644
--- a/src/meridian/runtime/proxy/server.py
+++ b/src/meridian/runtime/proxy/server.py
@@ -23,6 +23,7 @@
from meridian.config import ProxyMode
from meridian.runtime.proxy.budget import BudgetExceeded, BudgetLedger, TrialLimits
from meridian.runtime.proxy.cassette import CassetteMiss, CassetteStore, request_key
+from meridian.runtime.proxy.openai_compat import from_openai_response, to_openai_request
from meridian.runtime.secrets import ProviderCredential, load_credential
TASK_HEADER = "x-meridian-task"
@@ -33,6 +34,12 @@
UPSTREAM_TIMEOUT_SECONDS = 120.0
+# Which wire format the upstream speaks. The agent always speaks Anthropic
+# messages to the proxy; "openai" makes the proxy translate to chat completions
+# (Groq, OpenAI, vLLM, Ollama, ...) so the key and the cassette format stay put.
+UPSTREAM_API_ANTHROPIC = "anthropic"
+UPSTREAM_API_OPENAI = "openai"
+
@dataclass
class ProxyConfig:
@@ -41,6 +48,11 @@ class ProxyConfig:
limits: dict[str, TrialLimits]
run_budget_cents: int | None = None
upstream_base_url: str = ANTHROPIC_BASE_URL
+ upstream_api: str = UPSTREAM_API_ANTHROPIC
+ # Harness-chosen model, applied by the proxy. The agent's own model name is
+ # ignored upstream, so a run measures the model the operator picked and the
+ # thing under test cannot swap it.
+ upstream_model: str | None = None
credential: ProviderCredential | None = None
# Whether the upstream needs authenticating at all. A real provider does; the
# deterministic stand-in provider used to record cassettes offline does not,
@@ -77,8 +89,8 @@ async def _call_upstream(
task_slug: str = "",
trial_index: int = 0,
) -> dict[str, Any]:
+ openai = state.config.upstream_api == UPSTREAM_API_OPENAI
headers = {
- "anthropic-version": ANTHROPIC_VERSION,
"content-type": "application/json",
# Forwarded so a stand-in provider can vary its behaviour per trial. A
# real provider ignores them; without them every trial of a task would
@@ -86,16 +98,24 @@ async def _call_upstream(
TASK_HEADER: task_slug,
TRIAL_HEADER: str(trial_index),
}
+ if not openai:
+ headers["anthropic-version"] = ANTHROPIC_VERSION
if state.config.requires_credential:
credential = state.config.credential or load_credential()
headers[credential.header_name] = credential.header_value
+ base = state.config.upstream_base_url.rstrip("/")
+ url = f"{base}/chat/completions" if openai else f"{base}/v1/messages"
+ if openai:
+ payload = to_openai_request(body, model=state.config.upstream_model)
+ elif state.config.upstream_model:
+ payload = {**body, "model": state.config.upstream_model}
+ else:
+ payload = body
async with httpx.AsyncClient(timeout=UPSTREAM_TIMEOUT_SECONDS) as client:
- response = await client.post(
- f"{state.config.upstream_base_url}/v1/messages", json=body, headers=headers
- )
+ response = await client.post(url, json=payload, headers=headers)
response.raise_for_status()
parsed: dict[str, Any] = response.json()
- return parsed
+ return from_openai_response(parsed) if openai else parsed
async def messages(request: Request) -> Response:
@@ -266,15 +286,20 @@ def config_from_env(env: dict[str, str] | None = None) -> ProxyConfig:
}
run_budget = environment.get("MERIDIAN_RUN_BUDGET_CENTS")
upstream = environment.get("MERIDIAN_UPSTREAM_BASE_URL", ANTHROPIC_BASE_URL)
+ upstream_api = environment.get("MERIDIAN_UPSTREAM_API", UPSTREAM_API_ANTHROPIC)
+ upstream_model = environment.get("MERIDIAN_UPSTREAM_MODEL") or None
return ProxyConfig(
mode=ProxyMode(environment.get("MERIDIAN_PROXY_MODE", "replay")),
cassette_dir=Path(environment.get("MERIDIAN_CASSETTE_DIR", "/cassettes")),
limits=limits,
run_budget_cents=int(run_budget) if run_budget else None,
upstream_base_url=upstream,
- # Only the real provider is authenticated. Pointing upstream elsewhere is
- # an explicit act, and it is how the offline recording path works.
- requires_credential=upstream == ANTHROPIC_BASE_URL,
+ upstream_api=upstream_api,
+ upstream_model=upstream_model,
+ # Only a real provider is authenticated. Pointing upstream elsewhere is
+ # an explicit act, and it is how the offline recording path works; an
+ # OpenAI-compatible upstream is always a real provider.
+ requires_credential=upstream == ANTHROPIC_BASE_URL or upstream_api == UPSTREAM_API_OPENAI,
)
diff --git a/src/meridian/runtime/proxy/session.py b/src/meridian/runtime/proxy/session.py
index 0a9698e..6176f16 100644
--- a/src/meridian/runtime/proxy/session.py
+++ b/src/meridian/runtime/proxy/session.py
@@ -166,6 +166,8 @@ def start_proxy(
upstream_base_url: str | None = None,
use_stub_provider: bool = False,
stub_slip_every: int | None = None,
+ upstream_api: str | None = None,
+ upstream_model: str | None = None,
) -> ProxyHandle:
"""Start the proxy and the internal trial network for one run."""
network_name = f"meridian-trial-{run_id}"
@@ -186,6 +188,10 @@ def start_proxy(
environment["MERIDIAN_RUN_BUDGET_CENTS"] = str(run_budget_cents)
if upstream_base_url:
environment["MERIDIAN_UPSTREAM_BASE_URL"] = upstream_base_url
+ if upstream_api and not use_stub_provider:
+ environment["MERIDIAN_UPSTREAM_API"] = upstream_api
+ if upstream_model and not use_stub_provider:
+ environment["MERIDIAN_UPSTREAM_MODEL"] = upstream_model
if mode is not ProxyMode.REPLAY and not use_stub_provider:
# The credential exists here and in no other container. Replay never
diff --git a/src/meridian/runtime/secrets.py b/src/meridian/runtime/secrets.py
index f50eaa8..d0bd383 100644
--- a/src/meridian/runtime/secrets.py
+++ b/src/meridian/runtime/secrets.py
@@ -14,10 +14,10 @@
import os
from dataclasses import dataclass
-PROVIDER_ENV_VARS = ("ANTHROPIC_API_KEY", "OPENAI_API_KEY")
+PROVIDER_ENV_VARS = ("ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GROQ_API_KEY")
# Anything matching one of these prefixes is redacted before it can be printed.
-SECRET_PREFIXES = ("sk-ant-", "sk-", "gho_", "ghp_")
+SECRET_PREFIXES = ("sk-ant-", "sk-", "gsk_", "gho_", "ghp_")
REDACTION = "«redacted»"
diff --git a/tests/unit/test_openai_compat.py b/tests/unit/test_openai_compat.py
new file mode 100644
index 0000000..b88eb8a
--- /dev/null
+++ b/tests/unit/test_openai_compat.py
@@ -0,0 +1,240 @@
+"""Anthropic <-> OpenAI translation for the proxy — pure functions, no network."""
+
+from __future__ import annotations
+
+import json
+
+import httpx
+import pytest
+
+from meridian.config import ProxyMode
+from meridian.runtime.proxy import server as server_module
+from meridian.runtime.proxy.openai_compat import from_openai_response, to_openai_request
+from meridian.runtime.proxy.server import (
+ UPSTREAM_API_OPENAI,
+ ProxyConfig,
+ config_from_env,
+ create_app,
+)
+from meridian.runtime.secrets import redact
+
+pytestmark = pytest.mark.unit
+
+ANTHROPIC_REQUEST = {
+ "model": "llama-3.3-70b-versatile",
+ "max_tokens": 256,
+ "temperature": 0,
+ "system": [{"type": "text", "text": "You are a checkout agent."}],
+ "tools": [
+ {
+ "name": "apply_coupon",
+ "description": "Apply a coupon",
+ "input_schema": {"type": "object", "properties": {"code": {"type": "string"}}},
+ }
+ ],
+ "tool_choice": {"type": "any"},
+ "messages": [
+ {"role": "user", "content": "apply SAVE10"},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "text", "text": "ok"},
+ {
+ "type": "tool_use",
+ "id": "t1",
+ "name": "apply_coupon",
+ "input": {"code": "SAVE10"},
+ },
+ ],
+ },
+ {
+ "role": "user",
+ "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "applied"}],
+ },
+ ],
+}
+
+
+def test_request_translation_maps_system_tools_and_results() -> None:
+ out = to_openai_request(ANTHROPIC_REQUEST)
+ assert out["messages"][0] == {"role": "system", "content": "You are a checkout agent."}
+ assert out["messages"][1] == {"role": "user", "content": "apply SAVE10"}
+ assistant = out["messages"][2]
+ assert assistant["role"] == "assistant"
+ assert assistant["tool_calls"][0]["function"]["name"] == "apply_coupon"
+ assert json.loads(assistant["tool_calls"][0]["function"]["arguments"]) == {"code": "SAVE10"}
+ assert out["messages"][3] == {"role": "tool", "tool_call_id": "t1", "content": "applied"}
+ assert out["tools"][0]["function"]["parameters"]["type"] == "object"
+ assert out["tool_choice"] == "required"
+ assert out["max_tokens"] == 256 and out["temperature"] == 0
+
+
+def test_request_translation_is_deterministic() -> None:
+ assert to_openai_request(ANTHROPIC_REQUEST) == to_openai_request(ANTHROPIC_REQUEST)
+
+
+def test_response_translation_text_and_usage() -> None:
+ out = from_openai_response(
+ {
+ "id": "c1",
+ "model": "m",
+ "choices": [{"message": {"content": "done"}, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 11, "completion_tokens": 3},
+ }
+ )
+ assert out["content"] == [{"type": "text", "text": "done"}]
+ assert out["stop_reason"] == "end_turn"
+ assert out["usage"] == {"input_tokens": 11, "output_tokens": 3}
+
+
+def test_response_translation_tool_call_sets_tool_use() -> None:
+ out = from_openai_response(
+ {
+ "choices": [
+ {
+ "message": {
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "x",
+ "type": "function",
+ "function": {"name": "f", "arguments": '{"a": 1}'},
+ }
+ ],
+ },
+ "finish_reason": "stop",
+ }
+ ]
+ }
+ )
+ assert out["stop_reason"] == "tool_use"
+ assert out["content"] == [{"type": "tool_use", "id": "x", "name": "f", "input": {"a": 1}}]
+
+
+def test_malformed_tool_arguments_are_surfaced_not_dropped() -> None:
+ out = from_openai_response(
+ {
+ "choices": [
+ {
+ "message": {
+ "tool_calls": [{"id": "x", "function": {"name": "f", "arguments": "{oops"}}]
+ },
+ "finish_reason": "tool_calls",
+ }
+ ]
+ }
+ )
+ assert out["content"][0]["input"] == {"_raw_arguments": "{oops"}
+
+
+def test_openai_upstream_requires_a_credential() -> None:
+ config = config_from_env(
+ {
+ "MERIDIAN_UPSTREAM_API": "openai",
+ "MERIDIAN_UPSTREAM_BASE_URL": "https://api.groq.com/openai/v1",
+ }
+ )
+ assert config.upstream_api == UPSTREAM_API_OPENAI
+ assert config.requires_credential
+
+
+def test_groq_keys_are_redacted() -> None:
+ assert "gsk_abcdefghijklmnop" not in redact("upstream rejected gsk_abcdefghijklmnop")
+
+
+async def test_proxy_translates_on_the_wire(
+ tmp_path: pytest.TempPathFactory, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ seen: dict[str, object] = {}
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ seen["url"] = str(request.url)
+ seen["auth"] = request.headers.get("authorization")
+ seen["body"] = json.loads(request.content)
+ return httpx.Response(
+ 200,
+ json={
+ "choices": [{"message": {"content": "hi"}, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 5, "completion_tokens": 2},
+ },
+ )
+
+ real_client = httpx.AsyncClient
+ monkeypatch.setattr(
+ server_module.httpx,
+ "AsyncClient",
+ lambda **kw: real_client(**{"transport": httpx.MockTransport(handler), **kw}),
+ )
+ from meridian.runtime.secrets import ProviderCredential
+
+ app = create_app(
+ ProxyConfig(
+ mode=ProxyMode.RECORD,
+ cassette_dir=tmp_path, # type: ignore[arg-type]
+ limits={},
+ upstream_base_url="https://api.groq.com/openai/v1",
+ upstream_api=UPSTREAM_API_OPENAI,
+ upstream_model="llama-3.3-70b-versatile",
+ credential=ProviderCredential(env_var="GROQ_API_KEY", value="gsk_test_key_value"),
+ )
+ )
+ async with httpx.AsyncClient(
+ transport=httpx.ASGITransport(app=app), base_url="http://proxy"
+ ) as client:
+ response = await client.post(
+ "/v1/messages",
+ json={"model": "m", "max_tokens": 8, "messages": [{"role": "user", "content": "x"}]},
+ headers={"x-meridian-task": "happy-path", "x-meridian-trial": "0"},
+ )
+ assert response.status_code == 200
+ assert response.json()["content"] == [{"type": "text", "text": "hi"}]
+ assert seen["url"] == "https://api.groq.com/openai/v1/chat/completions"
+ assert seen["auth"] == "Bearer gsk_test_key_value"
+ assert seen["body"]["model"] == "llama-3.3-70b-versatile" # type: ignore[index]
+ assert seen["body"]["messages"] == [{"role": "user", "content": "x"}] # type: ignore[index]
+
+
+async def test_anthropic_upstream_model_override(
+ tmp_path: pytest.TempPathFactory, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ seen: dict[str, object] = {}
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ seen["url"] = str(request.url)
+ seen["body"] = json.loads(request.content)
+ return httpx.Response(
+ 200,
+ json={
+ "type": "message",
+ "content": [],
+ "usage": {"input_tokens": 1, "output_tokens": 1},
+ },
+ )
+
+ real_client = httpx.AsyncClient
+ monkeypatch.setattr(
+ server_module.httpx,
+ "AsyncClient",
+ lambda **kw: real_client(**{"transport": httpx.MockTransport(handler), **kw}),
+ )
+ from meridian.runtime.secrets import ProviderCredential
+
+ app = create_app(
+ ProxyConfig(
+ mode=ProxyMode.RECORD,
+ cassette_dir=tmp_path, # type: ignore[arg-type]
+ limits={},
+ upstream_model="claude-haiku-4-5-20251001",
+ credential=ProviderCredential(env_var="ANTHROPIC_API_KEY", value="sk-ant-test-value"),
+ )
+ )
+ async with httpx.AsyncClient(
+ transport=httpx.ASGITransport(app=app), base_url="http://proxy"
+ ) as client:
+ await client.post(
+ "/v1/messages",
+ json={"model": "agent-chose-this", "max_tokens": 8, "messages": []},
+ headers={"x-meridian-task": "happy-path", "x-meridian-trial": "0"},
+ )
+ assert seen["url"] == "https://api.anthropic.com/v1/messages"
+ assert seen["body"]["model"] == "claude-haiku-4-5-20251001" # type: ignore[index]