Disclose stub self-eval; add OpenAI-compatible upstream and real-model benchmark workflow

.env.example

@@ -9,3 +9,10 @@ MERIDIAN_DATABASE_URL=postgresql://meridian:meridian@localhost:5433/meridian
 # Proxy mode: record | replay | passthrough
 # `passthrough` is rejected by config validation in gate mode.
 MERIDIAN_PROXY_MODE=replay
+
+# OpenAI-compatible providers (Groq, OpenAI, vLLM, Ollama): the proxy translates
+# the agent's Anthropic-format calls, so the agent and cassettes do not change.
+# GROQ_API_KEY=gsk-replace-me
+# MERIDIAN_UPSTREAM_API=openai
+# MERIDIAN_UPSTREAM_BASE_URL=https://api.groq.com/openai/v1
+# MERIDIAN_UPSTREAM_MODEL=llama-3.3-70b-versatile

.github/workflows/real-model-benchmark.yml

@@ -0,0 +1,87 @@
+name: real-model-benchmark
+
+# Records and scores the checkout-agent suite against a REAL model, through the
+# Meridian proxy, and publishes pass^k, the minimum detectable effect and the
+# verdicts as an artifact. Unlike self-eval, this one spends money: it needs the
+# GROQ_API_KEY or ANTHROPIC_API_KEY repository secret and is manual-only, with a hard budget cap
+# enforced by the proxy.
+
+on:
+  workflow_dispatch:
+    inputs:
+      provider:
+        description: Which provider to measure (needs its repository secret)
+        type: choice
+        options: [groq, anthropic]
+        default: groq
+      model:
+        description: Model id at the provider (e.g. llama-3.3-70b-versatile or claude-haiku-4-5-20251001)
+        default: llama-3.3-70b-versatile
+      trials:
+        description: Trials per task (n)
+        default: "10"
+      k:
+        description: The k in pass^k
+        default: "3"
+      budget_cents:
+        description: Hard run budget in cents (the proxy stops spending at this)
+        default: "200"
+
+permissions:
+  contents: read
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    env:
+      # Only the chosen provider's key is exported. The proxy takes the first
+      # credential it finds, so exporting both could send one provider's key to
+      # the other.
+      GROQ_API_KEY: ${{ inputs.provider == 'groq' && secrets.GROQ_API_KEY || '' }}
+      ANTHROPIC_API_KEY: ${{ inputs.provider == 'anthropic' && secrets.ANTHROPIC_API_KEY || '' }}
+      MERIDIAN_UPSTREAM_API: ${{ inputs.provider == 'groq' && 'openai' || 'anthropic' }}
+      MERIDIAN_UPSTREAM_BASE_URL: ${{ inputs.provider == 'groq' && 'https://api.groq.com/openai/v1' || 'https://api.anthropic.com' }}
+      MERIDIAN_UPSTREAM_MODEL: ${{ inputs.model }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - run: uv sync --frozen
+      - name: Check the secret exists
+        run: test -n "$GROQ_API_KEY$ANTHROPIC_API_KEY" || { echo "no API key secret for provider ${{ inputs.provider }}"; exit 1; }
+      - name: Build images and repin the suite
+        run: |
+          uv run meridian snapshot build ./envs/checkout --tag checkout:dev \
+            --write-ref --update-suite ./suites/checkout-agent
+          uv run meridian snapshot build ./envs/proxy --tag meridian-proxy:dev --context .
+      - name: Record and score against the real model
+        run: |
+          mkdir -p out cassettes
+          uv run meridian run \
+            --suite ./suites/checkout-agent \
+            --n "${{ inputs.trials }}" --k "${{ inputs.k }}" \
+            --proxy-mode record --live-provider \
+            --cassettes ./cassettes --budget "${{ inputs.budget_cents }}" \
+            --out out/result.json --html out/report.html
+      - name: Summarise
+        if: always()
+        run: |
+          {
+            echo "## Real-model benchmark (${{ inputs.provider }} / ${{ inputs.model }})"
+            echo
+            echo "n=${{ inputs.trials }} k=${{ inputs.k }} budget=${{ inputs.budget_cents }}c"
+            echo
+            if [ -f out/result.json ]; then echo '```json'; cat out/result.json; echo '```'; fi
+          } >> "$GITHUB_STEP_SUMMARY"
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: real-model-benchmark
+          path: |
+            out/
+            cassettes/
+      - name: Sweep
+        if: always()
+        run: uv run meridian sweep

README.md

@@ -16,7 +16,18 @@ So here is Meridian measured by Meridian, regenerated by CI on every push to
 | **Replay fidelity** | **5 / 5** | archived runs re-materialized from their manifests alone and reproduced exactly. |
 | **Contamination probe** | **pass / fail** | passes with isolation on, **fails** with `--unsafe-shared-env`. A probe that cannot fail proves nothing. |
 
-<sub>`n=5` · `k=3` · tolerance `0.03` · baseline suite pass^3 `0.486` · mean run 32.4s / 51¢ · Meridian `0.1.0` · [`benchmarks/self-eval.json`](benchmarks/self-eval.json), regenerate with `make self-eval` (~30 min)</sub>
+> **Read this before the numbers.** The self-eval above runs against a
+> deterministic **stub provider** ([`stub_provider.py`](src/meridian/runtime/proxy/stub_provider.py)),
+> not a real LLM. The stub reproduces the Anthropic wire format and "slips" one
+> trial in five on a schedule, so the *pipeline* (record → replay → gate →
+> report) is exercised for free and reproducibly. These numbers show the gate's
+> **statistical machinery behaves as designed on a controlled source of
+> variance**. They are **not** evidence about how any real model or agent
+> behaves. A real-model benchmark is tracked in the roadmap and will be
+> published separately, with its own pass^k, minimum detectable effect, and
+> inconclusive verdicts.
+
+<sub>`n=5` · `k=3` · tolerance `0.03` · baseline suite pass^3 `0.486` · mean run 32.4s / 51¢ notional (token usage × default price table, not real spend) · Meridian `0.1.0` · [`benchmarks/self-eval.json`](benchmarks/self-eval.json), regenerate with `make self-eval` (~30 min)</sub>
 
 **The miss, stated plainly.** An arithmetic bug seeded into the tool that writes
 the invoice dropped suite pass^3 by 0.114 — a real regression, correctly measured

src/meridian/runtime/orchestrator.py

@@ -8,6 +8,7 @@
 
 from __future__ import annotations
 
+import os
 import time
 import uuid
 from collections.abc import Callable, Sequence
@@ -104,6 +105,12 @@ async def execute(client: DockerClient, request: RunRequest) -> RunOutcome:
             run_budget_cents=config.execution.budget_cents,
             use_stub_provider=request.use_stub_provider,
             stub_slip_every=request.stub_slip_every,
+            # A live run reaches its provider only through the proxy. Set
+            # MERIDIAN_UPSTREAM_API=openai and MERIDIAN_UPSTREAM_BASE_URL to use an
+            # OpenAI-compatible provider (e.g. https://api.groq.com/openai/v1).
+            upstream_base_url=os.environ.get("MERIDIAN_UPSTREAM_BASE_URL") or None,
+            upstream_api=os.environ.get("MERIDIAN_UPSTREAM_API") or None,
+            upstream_model=os.environ.get("MERIDIAN_UPSTREAM_MODEL") or None,
         )
         progress(f"proxy up in {config.execution.proxy_mode} mode on {handle.network_name}")
 

src/meridian/runtime/proxy/openai_compat.py

@@ -0,0 +1,191 @@
+"""Translate Anthropic messages <-> OpenAI chat completions.
+
+The agent and the cassettes speak the Anthropic messages shape. Providers that
+only offer an OpenAI-compatible endpoint (Groq, OpenAI, Together, vLLM, Ollama)
+are reached by translating at the proxy, so the agent, the cassette format and
+the isolation model do not change, and the credential still lives only in the
+proxy container.
+
+Pure functions: no I/O, no HTTP. Unit tested without a daemon.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+# OpenAI finish_reason -> Anthropic stop_reason.
+_STOP_REASONS = {
+    "stop": "end_turn",
+    "length": "max_tokens",
+    "tool_calls": "tool_use",
+    "function_call": "tool_use",
+    "content_filter": "end_turn",
+}
+
+
+def _flatten_text(content: Any) -> str:
+    """Anthropic allows a string or a list of text blocks for system/user text."""
+    if isinstance(content, str):
+        return content
+    parts = [str(b.get("text", "")) for b in content or [] if b.get("type") == "text"]
+    return "\n".join(parts)
+
+
+def _tool_result_text(content: Any) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        return _flatten_text(content)
+    return "" if content is None else json.dumps(content, sort_keys=True)
+
+
+def _translate_message(message: dict[str, Any]) -> list[dict[str, Any]]:
+    role = message["role"]
+    content = message.get("content")
+    if isinstance(content, str):
+        return [{"role": role, "content": content}]
+
+    blocks = content or []
+    if role == "assistant":
+        text = _flatten_text(blocks)
+        tool_calls = [
+            {
+                "id": block["id"],
+                "type": "function",
+                "function": {
+                    "name": block["name"],
+                    "arguments": json.dumps(block.get("input", {}), sort_keys=True),
+                },
+            }
+            for block in blocks
+            if block.get("type") == "tool_use"
+        ]
+        out: dict[str, Any] = {"role": "assistant", "content": text or None}
+        if tool_calls:
+            out["tool_calls"] = tool_calls
+        return [out]
+
+    # user turn: tool_result blocks become their own `tool` messages, which OpenAI
+    # requires to directly follow the assistant tool_calls message.
+    translated: list[dict[str, Any]] = []
+    for block in blocks:
+        if block.get("type") == "tool_result":
+            translated.append(
+                {
+                    "role": "tool",
+                    "tool_call_id": block["tool_use_id"],
+                    "content": _tool_result_text(block.get("content")),
+                }
+            )
+    text = _flatten_text([b for b in blocks if b.get("type") == "text"])
+    if text:
+        translated.append({"role": "user", "content": text})
+    return translated
+
+
+def _translate_tool_choice(choice: Any) -> Any:
+    if not isinstance(choice, dict):
+        return None
+    kind = choice.get("type")
+    if kind == "auto":
+        return "auto"
+    if kind == "any":
+        return "required"
+    if kind == "none":
+        return "none"
+    if kind == "tool":
+        return {"type": "function", "function": {"name": choice["name"]}}
+    return None
+
+
+def to_openai_request(body: dict[str, Any], *, model: str | None = None) -> dict[str, Any]:
+    """Anthropic /v1/messages request -> OpenAI /chat/completions request."""
+    messages: list[dict[str, Any]] = []
+    system = body.get("system")
+    if system:
+        messages.append({"role": "system", "content": _flatten_text(system)})
+    for message in body.get("messages", []):
+        messages.extend(_translate_message(message))
+
+    request: dict[str, Any] = {
+        "model": model or body.get("model", ""),
+        "messages": messages,
+    }
+    if "max_tokens" in body:
+        request["max_tokens"] = body["max_tokens"]
+    for field in ("temperature", "top_p"):
+        if field in body:
+            request[field] = body[field]
+    if body.get("stop_sequences"):
+        request["stop"] = body["stop_sequences"]
+    tools = body.get("tools")
+    if tools:
+        request["tools"] = [
+            {
+                "type": "function",
+                "function": {
+                    "name": tool["name"],
+                    "description": tool.get("description", ""),
+                    "parameters": tool.get("input_schema", {"type": "object", "properties": {}}),
+                },
+            }
+            for tool in tools
+        ]
+        choice = _translate_tool_choice(body.get("tool_choice"))
+        if choice is not None:
+            request["tool_choice"] = choice
+    return request
+
+
+def _parse_arguments(raw: Any) -> dict[str, Any]:
+    if isinstance(raw, dict):
+        return raw
+    if not raw:
+        return {}
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError:
+        # Models occasionally emit malformed tool JSON. Surface it rather than
+        # dropping the call: the agent's handling of it is part of what is measured.
+        return {"_raw_arguments": str(raw)}
+    return parsed if isinstance(parsed, dict) else {"_value": parsed}
+
+
+def from_openai_response(response: dict[str, Any]) -> dict[str, Any]:
+    """OpenAI chat completion -> Anthropic messages response."""
+    choice = (response.get("choices") or [{}])[0]
+    message = choice.get("message") or {}
+
+    content: list[dict[str, Any]] = []
+    if message.get("content"):
+        content.append({"type": "text", "text": message["content"]})
+    for call in message.get("tool_calls") or []:
+        function = call.get("function") or {}
+        content.append(
+            {
+                "type": "tool_use",
+                "id": call.get("id", ""),
+                "name": function.get("name", ""),
+                "input": _parse_arguments(function.get("arguments")),
+            }
+        )
+
+    usage = response.get("usage") or {}
+    stop_reason = _STOP_REASONS.get(str(choice.get("finish_reason")), "end_turn")
+    if any(block["type"] == "tool_use" for block in content):
+        stop_reason = "tool_use"
+
+    return {
+        "id": response.get("id", ""),
+        "type": "message",
+        "role": "assistant",
+        "model": response.get("model", ""),
+        "content": content,
+        "stop_reason": stop_reason,
+        "stop_sequence": None,
+        "usage": {
+            "input_tokens": int(usage.get("prompt_tokens", 0)),
+            "output_tokens": int(usage.get("completion_tokens", 0)),
+        },
+    }