Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 22 additions & 22 deletions py/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -284,39 +284,39 @@ conflicts = [
[tool.braintrust.matrix]

[tool.braintrust.matrix.openai]
latest = "openai==2.33.0"
latest = "openai==3.42.1"

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟠 High · Latest integration replays fail

When CI runs the updated latest-version integration sessions before matching cassettes are re-recorded, _run_tests selects cassettes/latest/, and VCR's record_mode="none" cannot record updated interactions; cassette mismatches fail the tests or leave them exercising stale behavior.

Re-record and commit the latest cassettes for the updated SDK interactions.

Also at py/pyproject.toml:293, py/pyproject.toml:297, py/pyproject.toml:301, py/pyproject.toml:306, py/pyproject.toml:310, py/pyproject.toml:314, py/pyproject.toml:319, py/pyproject.toml:331, py/pyproject.toml:335, py/pyproject.toml:339-340, py/pyproject.toml:344, py/pyproject.toml:348, py/pyproject.toml:352, py/pyproject.toml:359, py/pyproject.toml:363, py/pyproject.toml:367, py/pyproject.toml:371, py/pyproject.toml:375, py/pyproject.toml:379.

Prompt for AI agents
In `py/pyproject.toml` at line 287:

**Latest integration replays fail**

When CI runs the updated latest-version integration sessions before matching cassettes are re-recorded, `_run_tests` selects `cassettes/latest/`, and VCR's `record_mode="none"` cannot record updated interactions; cassette mismatches fail the tests or leave them exercising stale behavior.

Re-record and commit the `latest` cassettes for the updated SDK interactions.

Also at `py/pyproject.toml:293`, `py/pyproject.toml:297`, `py/pyproject.toml:301`, `py/pyproject.toml:306`, `py/pyproject.toml:310`, `py/pyproject.toml:314`, `py/pyproject.toml:319`, `py/pyproject.toml:331`, `py/pyproject.toml:335`, `py/pyproject.toml:339-340`, `py/pyproject.toml:344`, `py/pyproject.toml:348`, `py/pyproject.toml:352`, `py/pyproject.toml:359`, `py/pyproject.toml:363`, `py/pyproject.toml:367`, `py/pyproject.toml:371`, `py/pyproject.toml:375`, `py/pyproject.toml:379`.

From AGENTS.md L30-34:

6. **Prefer real integration coverage over mocks.**
   - For provider/integration behavior, prefer VCR-backed tests with checked-in cassettes.
   - This includes bugs in tracing/span shaping that happen after the SDK returns a real provider payload. If the behavior depends on the provider's actual response shape, treat it as VCR-first work, not mock-first work.
   - Be actively skeptical of mock/fake tests for provider integrations. Do not reach for mocks just because they are faster or easier to write.
   - Avoid mocks/fakes unless the code is purely local or there is no practical cassette-based option.

"1.92.0" = "openai==1.92.0"
"1.77.0" = "openai==1.77.0"
"1.71.0" = "openai==1.71.0"

[tool.braintrust.matrix.anthropic]
latest = "anthropic==0.97.0"
latest = "anthropic==2.3.0"
"0.48.0" = "anthropic==0.48.0"

[tool.braintrust.matrix.cohere]
latest = "cohere==6.1.0"
latest = "cohere==7.3.0"
"5.0.0" = "cohere==5.0.0"

[tool.braintrust.matrix.openai-agents]
latest = "openai-agents==0.15.0"
latest = "openai-agents==0.24.0"
"0.13.6" = "openai-agents==0.13.6"
"0.0.19" = "openai-agents==0.0.19"

[tool.braintrust.matrix.litellm]
latest = "litellm==1.83.14"
latest = "litellm==1.106.6"
"1.74.0" = "litellm==1.74.0"

[tool.braintrust.matrix.claude-agent-sdk]
latest = "claude-agent-sdk==0.1.72"
latest = "claude-agent-sdk==0.2.176"
"0.1.10" = "claude-agent-sdk==0.1.10"

[tool.braintrust.matrix.agno]
latest = "agno==2.6.4"
latest = "agno==3.1.12"
"2.4.0" = "agno==2.4.0"
"2.1.0" = "agno==2.1.0"

[tool.braintrust.matrix.agentscope]
latest = "agentscope==1.0.19.post1"
latest = "agentscope==2.0.11"
"1.0.0" = "agentscope==1.0.0"

[tool.braintrust.matrix.autogen-agentchat]
Expand All @@ -328,64 +328,64 @@ latest = "autogen-ext[openai]==0.7.5"
"0.7.0" = "autogen-ext[openai]==0.7.0"

[tool.braintrust.matrix.strands-agents]
latest = "strands-agents==1.38.0"
latest = "strands-agents==1.63.0"
"1.20.0" = "strands-agents==1.20.0"

[tool.braintrust.matrix.pydantic-ai-integration]
latest = "pydantic-ai==1.89.0"
latest = "pydantic-ai==2.64.0"
"1.10.0" = "pydantic-ai==1.10.0"

[tool.braintrust.matrix.pydantic-ai-wrap-openai]
latest = "pydantic-ai==1.89.0"
latest = "pydantic-ai==2.64.0"
"1.0.1" = "pydantic-ai==1.0.1"
"0.1.9" = "pydantic-ai==0.1.9"

[tool.braintrust.matrix.autoevals]
latest = "autoevals==0.2.0"
latest = "autoevals==0.4.1"
"0.0.129" = "autoevals==0.0.129"

[tool.braintrust.matrix.google-genai]
latest = "google-genai==1.74.0"
latest = "google-genai==2.34.0"
"1.30.0" = "google-genai==1.30.0"

[tool.braintrust.matrix.dspy]
latest = "dspy==3.2.0"
latest = "dspy==3.4.1"
"2.6.0" = "dspy==2.6.0"

[tool.braintrust.matrix.crewai]
# 1.13.0 is the first release with the full causal-id surface (event_id /
# parent_event_id / started_event_id on BaseEvent) plus the ``usage`` field on
# LLMCallCompletedEvent that the Braintrust CrewAI integration depends on.
latest = "crewai==1.14.4"
latest = "crewai==1.15.34"
"1.13.0" = "crewai==1.13.0"

[tool.braintrust.matrix.google-adk]
latest = "google-adk==1.32.0"
latest = "google-adk==2.13.0"
"1.14.1" = "google-adk==1.14.1"

[tool.braintrust.matrix.langchain-core]
latest = "langchain-core==1.3.2"
latest = "langchain-core==1.7.3"
"0.3.28" = "langchain-core==0.3.28"

[tool.braintrust.matrix.llama-index-core]
latest = "llama-index-core==0.14.21"
latest = "llama-index-core==0.14.25"
"0.13.0" = "llama-index-core==0.13.0"

[tool.braintrust.matrix.openrouter]
latest = "openrouter==0.9.1"
latest = "openrouter==1.3.106"
"0.6.0" = "openrouter==0.6.0"

[tool.braintrust.matrix.mistralai]
latest = "mistralai==2.4.4"
latest = "mistralai==3.7.0"
"1.12.4" = "mistralai==1.12.4"

[tool.braintrust.matrix.temporalio]
latest = "temporalio==1.27.0"
latest = "temporalio==1.35.0"
"1.20.0" = "temporalio==1.20.0"
"1.19.0" = "temporalio==1.19.0"

[tool.braintrust.matrix.pytest-matrix]
latest = "pytest==9.0.3"
latest = "pytest==9.2.0"
"8.4.2" = "pytest==8.4.2"

[tool.braintrust.matrix.braintrust-core]
Expand Down
Loading