From 5f4411aa9c762e66684406e828d5bf879b3c8cc1 Mon Sep 17 00:00:00 2001 From: Taylor Dolezal <117500+onlydole@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:35:29 -0800 Subject: [PATCH 1/5] fix(mcp): background wait fails when none of its task IDs exist t3_background_wait returned a normal result when no requested ID matched a background task, so agents ended their turn believing they were waiting and were never woken. A wait that matches nothing now fails with invalid_request and says what to pass instead. Partial matches and clears are unchanged. Fixes #29772 Co-Authored-By: Claude Opus 5.5 --- .../src/mcp/toolkits/orchestrator/handlers.ts | 5 ++++- .../ThreadManagementService.test.ts | 22 +++++++++++++++++-- .../ThreadManagementService.ts | 10 ++++++++- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/apps/server/src/mcp/toolkits/orchestrator/handlers.ts b/apps/server/src/mcp/toolkits/orchestrator/handlers.ts index 3db297bc0d303..1602bb3f1133e 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/handlers.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/handlers.ts @@ -74,7 +74,10 @@ const handlers = { Effect.mapError( (cause) => new OrchestratorMcpFailure({ - code: cause.reason === "unsupported" ? "invalid_request" : "orchestration_error", + code: + cause.reason === "unsupported" || cause.reason === "unmatched" + ? "invalid_request" + : "orchestration_error", message: cause.message, }), ), diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts index 9285f72d46f56..64d5cd28cfd12 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.test.ts @@ -461,7 +461,13 @@ it.effect("waitForThread returns the cut run once its continuation is declined", }), ); -it.effect.each(["live", "no-session", "unsupported-runtime"] as const)( +it.effect.each([ + "live", + "no-match", + "no-match-clear", + "no-session", + "unsupported-runtime", +] as const)( "marks background work awaited on the thread's active provider thread with state %s", (state) => Effect.gen(function* () { @@ -537,7 +543,11 @@ it.effect.each(["live", "no-session", "unsupported-runtime"] as const)( const result = yield* Effect.gen(function* () { const service = yield* ThreadManagementService.ThreadManagementService; return yield* service - .awaitBackgroundTasks({ threadId, taskIds: ["tests", "finished"], awaited: true }) + .awaitBackgroundTasks({ + threadId, + taskIds: state.startsWith("no-match") ? ["finished"] : ["tests", "finished"], + awaited: state !== "no-match-clear", + }) .pipe(Effect.result); }).pipe(Effect.provide(Layer.merge(layerTest, ThreadCommandExecutor.layer))); @@ -546,6 +556,14 @@ it.effect.each(["live", "no-session", "unsupported-runtime"] as const)( success: { matched: ["tests"], unmatched: ["finished"] }, }); expect(calls).toEqual([{ providerThreadId, awaited: true }]); + } else if (state === "no-match") { + expect(result).toMatchObject({ + failure: { _tag: "ThreadManagementBackgroundWaitError", reason: "unmatched" }, + }); + expect(calls).toEqual([{ providerThreadId, awaited: true }]); + } else if (state === "no-match-clear") { + expect(result).toMatchObject({ success: { matched: [], unmatched: ["finished"] } }); + expect(calls).toEqual([{ providerThreadId, awaited: false }]); } else { expect(result).toMatchObject({ failure: { diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index f0346b2bec4ec..392e93dac83b8 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -326,7 +326,7 @@ export class ThreadManagementBackgroundWaitError extends Schema.TaggedError !matchedIds.has(taskId)) }; }); From dace9dc1aa11cd95af0bc4b4050708a02209d31c Mon Sep 17 00:00:00 2001 From: Taylor Dolezal <117500+onlydole@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:58:21 -0800 Subject: [PATCH 2/5] test(evals): background wait eval covers task IDs that match nothing The mcp-tool-use eval answered t3_background_wait with a generic { ok: true }, so it could not check how an agent handles a wait that matches no background work. A case can now list the background tasks running on its Claude session in world.backgroundTasks. The eval then answers t3_background_wait from T3's own handler and ThreadManagementService, so the result or error is the checkout's. A prior wait with result "default" gets the same treatment. background-wait-unmatched-id seeds the "placeholder" wait from #29772 while the tests run in the background, and passes when the agent waits again on the real task ID. Sonnet 5.5 and Opus 5.5 recover from main's success result in replay too, so it is a guard, not a reproduction. background-wait-matched-id is its control, with a wait that matched. The unmatched message also no longer says nothing would wake the thread. A task awaited earlier still can, so it now says the request registers no wait for these IDs. Co-Authored-By: Claude Opus 5.5 --- .../ThreadManagementService.ts | 2 +- evals/mcp-tool-use/README.md | 2 +- evals/mcp-tool-use/backgroundWait.ts | 152 ++++++++++++++++++ evals/mcp-tool-use/cases/orchestration.ndjson | 2 + evals/mcp-tool-use/run.ts | 50 +++++- 5 files changed, 201 insertions(+), 7 deletions(-) create mode 100644 evals/mcp-tool-use/backgroundWait.ts diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 392e93dac83b8..b9827deeff14d 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -339,7 +339,7 @@ export class ThreadManagementBackgroundWaitError extends Schema.TaggedError`. Then run it on the text before your fix, where it must fail, and on the text after, where it must pass. diff --git a/evals/mcp-tool-use/backgroundWait.ts b/evals/mcp-tool-use/backgroundWait.ts new file mode 100644 index 0000000000000..8635423ca9950 --- /dev/null +++ b/evals/mcp-tool-use/backgroundWait.ts @@ -0,0 +1,152 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * T3's real `t3_background_wait` handler and ThreadManagementService over a Claude session. + * Only the adapter is the eval's: like Claude's, it matches task IDs against the session's + * running background work, which the case lists in `world.backgroundTasks`. + */ +import { + EnvironmentId, + type OrchestrationV2ProviderThread, + type OrchestrationV2ThreadProjection, + ProviderDriverKind, + ProviderInstanceId, + ProviderSessionId, + ProviderThreadId, + ThreadId, +} from "@t3tools/contracts"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as References from "effect/References"; +// oxlint-disable-next-line t3code/no-raw-mcp-registration -- the eval calls the tool McpHttpServer registered and registers none. +import { McpSchema, McpServer } from "effect/ai"; + +import * as McpHttpServer from "../../apps/server/src/mcp/McpHttpServer.ts"; +import * as McpInvocationContext from "../../apps/server/src/mcp/McpInvocationContext.ts"; +import { liveThreadShell } from "../../apps/server/src/mcp/McpToolAccess.testkit.ts"; +import * as OrchestratorMcpService from "../../apps/server/src/mcp/OrchestratorMcpService.ts"; +import * as ThreadMetadataMcpService from "../../apps/server/src/mcp/ThreadMetadataMcpService.ts"; +import * as OrchestratorHandlers from "../../apps/server/src/mcp/toolkits/orchestrator/handlers.ts"; +import { OrchestratorToolkit } from "../../apps/server/src/mcp/toolkits/orchestrator/tools.ts"; +import * as CommandReceiptStore from "../../apps/server/src/orchestration-v2/CommandReceiptStore.ts"; +import * as Orchestrator from "../../apps/server/src/orchestration-v2/Orchestrator.ts"; +import * as ProviderSessionManager from "../../apps/server/src/orchestration-v2/ProviderSessionManager.ts"; +import * as ThreadManagementService from "../../apps/server/src/orchestration-v2/ThreadManagementService.ts"; +import * as PeerForwarding from "../../apps/server/src/peer/PeerForwarding.ts"; +import * as PeerLinkRequests from "../../apps/server/src/peer/PeerLinkRequests.ts"; +import * as PeerLinks from "../../apps/server/src/peer/PeerLinks.ts"; +import * as SqlitePersistence from "../../apps/server/src/persistence/Sqlite.ts"; +import * as ServerSettings from "../../apps/server/src/serverSettings.ts"; + +const threadId = ThreadId.make("mcp-tool-use-eval"); +const providerThreadId = ProviderThreadId.make("provider-thread:mcp-tool-use-eval"); +const providerSessionId = ProviderSessionId.make("provider-session:mcp-tool-use-eval"); +const providerInstanceId = ProviderInstanceId.make("claudeAgent"); +const driver = ProviderDriverKind.make("claudeAgent"); + +/** Calls T3's `t3_background_wait` with `running` as the session's running background task IDs. */ +export const makeBackgroundWait = Effect.fn("evaluateMcpToolUse.makeBackgroundWait")(function* ( + running: ReadonlyArray, +) { + const now = yield* DateTime.now; + const providerThread = { + id: providerThreadId, + driver, + providerInstanceId, + providerSessionId, + appThreadId: threadId, + nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" }, + pendingBackgroundTasks: running.map((taskId) => ({ taskId, kind: "command" })), + createdAt: now, + updatedAt: now, + } as unknown as OrchestrationV2ProviderThread; + const runtime = { + driver, + providerSessionId, + awaitBackgroundTasks: (input: { readonly taskIds: ReadonlyArray }) => + Effect.succeed({ matched: [...new Set(input.taskIds)].filter((id) => running.includes(id)) }), + } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime; + const real = yield* Layer.build( + ThreadManagementService.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadRecords: () => + Effect.succeed({ + thread: { id: threadId, deletedAt: null, activeProviderThreadId: providerThreadId }, + providerThreads: [providerThread], + } as unknown as OrchestrationV2ThreadProjection), + }), + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ + get: () => Effect.succeedSome(runtime), + }), + ServerSettings.layerTest(), + CommandReceiptStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory)), + ), + ), + ), + // The receipt store's in-memory migrations log every step. + ).pipe(Effect.provideService(References.MinimumLogLevel, "Error")); + const service = Context.get(real, ThreadManagementService.ThreadManagementService); + // The caller's thread is live on the Claude instance, mid-turn. + const threads = Layer.mock(ThreadManagementService.ThreadManagementService)({ + ...service, + getThreadShell: () => Effect.succeed({ ...liveThreadShell(threadId), providerInstanceId }), + }); + const context = yield* Layer.build( + McpHttpServer.toolkitRegistration(OrchestratorToolkit, OrchestratorHandlers.layer).pipe( + Layer.provideMerge(McpHttpServer.layerInProcessMcpServer), + Layer.provide(Layer.mock(OrchestratorMcpService.OrchestratorMcpService)({})), + Layer.provide(Layer.mock(ThreadMetadataMcpService.ThreadMetadataMcpService)({})), + Layer.provide(threads), + Layer.provide(Layer.mock(PeerForwarding.PeerForwarding)({})), + Layer.provide(Layer.mock(PeerLinkRequests.PeerLinkRequests)({})), + Layer.provide(Layer.mock(PeerLinks.PeerLinks)({})), + ), + ); + type Entry = Parameters[0]; + // The registry keeps each tool's handler; its public type lists only definitions. + const tools = Context.get(context, McpServer.McpServer).tools as ReadonlyArray; + const entry = tools.find((tool) => tool.tool.name === "t3_background_wait"); + if (entry === undefined) return yield* Effect.die(new Error("T3 has no t3_background_wait.")); + const scope: McpInvocationContext.McpThreadInvocationScope = { + environmentId: EnvironmentId.make("00000000-0000-4000-8000-0000000000e0"), + requestNamespace: "mcp-tool-use-eval", + thread: { threadId, providerSessionId: "mcp-tool-use-eval", providerInstanceId }, + client: undefined, + capabilities: new Set(["orchestration"]), + issuedAt: 0, + }; + const request = McpSchema.McpRequestContext.of({ + clientId: 0, + protocolVersion: "2025-06-18", + clientCapabilities: {}, + }); + return Effect.fn("evaluateMcpToolUse.backgroundWait")(function* (input: unknown) { + const result = yield* entry.handle(input).pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, scope), + Effect.provideService(McpSchema.McpRequestContext, request), + // An MCP client shows a protocol error, such as bad parameters, as the tool's error. + Effect.catch((error) => + Effect.succeed( + new McpSchema.CallToolResult({ + isError: true, + content: [{ type: "text", text: error.message }], + }), + ), + ), + ); + if (!("content" in result)) + return yield* Effect.die(new Error("Background wait asked for input.")); + const text = result.content + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join("\n"); + if (text === "Tool execution failed due to an internal server error.") { + return yield* Effect.die(new Error("t3_background_wait hit a defect in T3 or the eval.")); + } + return { text, isError: result.isError === true }; + }); +}); +export type BackgroundWait = Effect.Success>; diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index dc79eb774df7e..c69102d2412b0 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -34,3 +34,5 @@ {"id": "two-children-same-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for two Sol xhigh subagents, an Astra parent started one and refused the second, citing 'One child per named model' (#24967).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}}, "prompt": "Launch two Sol xhigh subagents. Give each a trivial task that only replies OK.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "gpt-6.1-sol", "target.options.reasoningEffort": "xhigh"}}], "minCalls": {"delegate_task": 2}, "callsMustNot": [{"tool": "delegate_task", "args": {"target.model": "~muse"}}, {"tool": "delegate_task", "args": {"target.model": "~claude"}}]}, "notes": "From #24967. One child plus the parent doing the work fails, and so does a second model in place of Sol."} {"id": "worker-counts-per-model", "area": "orchestration", "kind": "bad", "pattern": "named-model rule read as a cap", "experience": "Asked for 20 Sol workers and 20 Luna workers, the agent started one worker per model (#25359).", "rule": "Before delegate_task ... One child per named model. (intended: every named model gets a child; read as a cap of one child per model)", "privacy": "commit-safe", "setup": {"role": "top", "mocks": {"orchestrator_capabilities": {"parentThreadId": "33333333-3333-4333-8333-333333333333", "inheritedProviderInstanceId": "claudeAgent", "inheritedModel": "claude-sonnet-5-5", "inheritedOptions": [{"id": "effort", "value": "high"}], "runtimeMode": "full-access", "interactionMode": "default", "features": {"appOwnedSubagents": true, "asyncPolling": true, "cancellation": true, "batchThreadCreation": true, "threadManagement": true, "incrementalThreadRead": true, "scheduledTasks": true}, "providers": [{"providerInstanceId": "claudeAgent", "driverKind": "claudeAgent", "displayName": "Claude", "models": [{"id": "claude-opus-5-5", "label": "Claude Opus 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "fastMode", "type": "boolean", "default": false}]}, {"id": "claude-sonnet-5-5", "label": "Claude Sonnet 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}, {"id": "claude-haiku-5-5", "label": "Claude Haiku 5.5", "options": [{"id": "effort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "codex", "driverKind": "codex", "displayName": "Codex", "models": [{"id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "options": [{"id": "reasoningEffort", "values": ["low", "medium", "high", "xhigh", "max"], "default": "high"}, {"id": "serviceTier", "values": ["default", "priority"], "default": "default"}]}]}, {"providerInstanceId": "muse", "driverKind": "muse", "displayName": "Muse Code", "runtimeModes": ["approval-required", "full-access"], "models": [{"id": "muse-wonder-full", "label": "Muse Wonder Full", "options": [{"id": "reasoningEffort", "values": ["minimal", "low", "medium", "high", "xhigh", "max"], "default": "high"}]}]}, {"providerInstanceId": "grok", "driverKind": "grok", "displayName": "Grok", "models": [{"id": "grok-4.7", "label": "Grok 4.7", "options": []}]}]}, "delegate_task": "default"}, "world": {"files": {"tests/flaky/retry.test.ts": "it('retry is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/upload.test.ts": "it('upload is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/session.test.ts": "it('session is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/search.test.ts": "it('search is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/billing.test.ts": "it('billing is flaky', async () => { /* fails 1 in 5 runs */ });\n", "tests/flaky/webhook.test.ts": "it('webhook is flaky', async () => { /* fails 1 in 5 runs */ });\n"}}}, "prompt": "Six tests in tests/flaky/ fail now and then. Start 3 Muse Wonder Full workers and 3 Grok workers, one test file each, to find and fix the flake.", "expect": {"calls": [{"tool": "delegate_task", "args": {"target.model": "muse-wonder-full"}}, {"tool": "delegate_task", "args": {"target.model": "grok-4.7"}}], "minCalls": {"delegate_task": 6}}, "notes": "Scaled down from the real 20 Sol + 20 Luna ask. Muse and Grok need delegate_task from every parent, so a native Sol helper cannot pass or fail it."} {"id": "linked-thread-chip-reads-there", "area": "orchestration", "kind": "guard", "pattern": "thread chip from a linked machine read on this one", "experience": "A thread chip from a linked machine names its environmentId; an agent that reads it without passing that id reads this machine, where the thread does not exist.", "rule": "The thread chip names its environmentId; t3_thread_read takes it.", "privacy": "commit-safe", "setup": {"role": "top", "linkedEnvironments": true, "mocks": {"t3_environment_links": {"environmentId": "environment-laptop", "links": [{"environmentId": "environment-vps", "label": "Hetzner VPS", "status": "reachable", "access": "auto-accept-edits", "expiresAt": "2026-12-01T00:00:00.000Z", "lastError": null}], "notListed": "Not listed: not linked yet. Call t3_environment_link with a hint from the user's words."}, "t3_thread_read": {"thread": {"threadId": "thread-deploy-api", "title": "Deploy the API", "status": "idle"}, "items": [{"position": 1, "createdBy": "user", "text": "Deploy the API to the VPS and check the health endpoint."}, {"position": 2, "createdBy": "assistant", "text": "Deployed. /health returns 200, but /health/db times out after 5s: the API can't reach Postgres on 10.0.0.5:5432."}], "nextPosition": 3, "hasMore": false}}}, "prompt": "Pick up where [Deploy the API](t3-context://v1/thread/thread_deploy-api) left off. What is still failing?", "promptContext": {"records": [{"version": 1, "kind": "thread", "contextId": "thread_deploy-api", "label": "Deploy the API", "environmentId": "environment-vps", "threadId": "thread-deploy-api", "title": "Deploy the API"}]}, "expect": {"calls": [{"tool": "t3_thread_read", "args": {"threadId": "thread-deploy-api", "environmentId": "environment-vps"}}], "notCalls": ["t3_environment_link"]}, "source": {"request": "akj on #17130: attach a linked machine's thread without dragging"}} +{"id": "background-wait-unmatched-id", "area": "orchestration", "kind": "guard", "pattern": "waits on a task ID that matches no background work, then ends the turn as if waiting", "experience": "One agent called t3_background_wait with the literal ID 'placeholder', another with a Claude subagent ID. The tool returned a normal result listing the ID under `unmatched`, so the agent said it would continue when the work came back and ended its turn. Nothing ever woke it.", "rule": "To wait for a background command or Monitor you started (tests, a build), call `t3_background_wait` with its task ID before you end your turn.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "claude", "priorTurns": [{"role": "user", "text": "Run the full test suite in the background and fix whatever fails. It takes about 15 minutes."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "vp test run", "description": "Run the full test suite", "run_in_background": true}, "result": "Command running in background with ID: bq7m2x9k. Output is being written to: /tmp/claude/tasks/bq7m2x9k.output"}, {"tool": "t3_background_wait", "input": {"taskIds": ["placeholder"]}, "result": "default"}]}], "world": {"backgroundTasks": ["bq7m2x9k"], "tests": "The suite started a minute ago and runs for about 15 minutes. Until then /tmp/claude/tasks/bq7m2x9k.output lists 38 passing test files and no failures yet."}}, "prompt": "I'm heading out for a bit. Tell me what failed when it's done.", "expect": {"calls": [{"tool": "t3_background_wait", "args": {"taskIds": ["bq7m2x9k"]}}]}, "source": {"issue": "#29772"}, "notes": "The seeded wait gets T3's real result for an ID the session does not know. The tests are still running, so the only way to keep the thread waiting is a wait on bq7m2x9k. In replay, Sonnet 5.5 and Opus 5.5 also recover from main's success result, so this guards the recovery rather than reproducing the incident. Claude only: t3_background_wait is a Claude tool.", "maxSteps": 8} +{"id": "background-wait-matched-id", "area": "orchestration", "kind": "control", "pattern": "waits on the right task ID, then ends the turn", "experience": "With the background command's own ID registered, ending the turn is correct: T3 keeps the thread working and wakes it when the tests finish.", "rule": "To wait for a background command or Monitor you started (tests, a build), call `t3_background_wait` with its task ID before you end your turn.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "claude", "priorTurns": [{"role": "user", "text": "Run the full test suite in the background and fix whatever fails. It takes about 15 minutes."}, {"role": "assistant", "toolCalls": [{"tool": "Bash", "input": {"command": "vp test run", "description": "Run the full test suite", "run_in_background": true}, "result": "Command running in background with ID: bq7m2x9k. Output is being written to: /tmp/claude/tasks/bq7m2x9k.output"}, {"tool": "t3_background_wait", "input": {"taskIds": ["bq7m2x9k"]}, "result": "default"}]}], "world": {"backgroundTasks": ["bq7m2x9k"], "tests": "The suite started a minute ago and runs for about 15 minutes. Until then /tmp/claude/tasks/bq7m2x9k.output lists 38 passing test files and no failures yet."}}, "prompt": "I'm heading out for a bit. Tell me what failed when it's done.", "expect": {"maxCalls": {"t3_background_wait": 1, "Bash": 3}, "notCalls": ["ScheduleWakeup"]}, "source": {"issue": "#29772"}, "notes": "Same thread as background-wait-unmatched-id, with a wait T3 matched. Guards against a fix that makes agents distrust a good wait and poll; a few looks at the output are ordinary. Claude only.", "maxSteps": 8} diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 3e6bb84f697be..bbe8676d79d01 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -67,6 +67,7 @@ import { } from "../../apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts"; import { proxyConfig } from "../shared/proxy.ts"; import { changeRequestsCreatedBy } from "../../apps/server/src/orchestration-v2/PullRequestSyncReactor.ts"; +import { type BackgroundWait, makeBackgroundWait } from "./backgroundWait.ts"; import { DeviceExpect, gradeDevice } from "./device-grading.ts"; import { AGENT_DEVICE_SHELL, @@ -138,7 +139,10 @@ const Rest = [Schema.Record(Schema.String, Schema.Unknown)] as const; const PriorToolCall = Schema.Struct({ tool: Schema.String, input: Schema.Unknown, - /** For a device tool, `"default"` runs the call on the world's devices for T3's result. */ + /** + * For a device tool or t3_background_wait, `"default"` runs the call on the world's devices or + * background tasks for T3's result. + */ result: Schema.Unknown, }); @@ -223,6 +227,11 @@ const McpToolUseWorld = Schema.StructWithRest( deviceInvalidCapture: Schema.optional(Schema.Boolean), /** Devices another thread's agent-device session holds, by device id: that thread's id. */ deviceOwners: Schema.optional(Schema.Record(Schema.String, Schema.String)), + /** + * Task IDs of the background commands and Monitors still running on the agent's Claude + * session. With them, T3's own t3_background_wait handler answers from this world. + */ + backgroundTasks: Schema.optional(Schema.Array(Schema.String)), /** PRs already linked to the thread, with the state T3 last saw. */ linkedPullRequests: Schema.optional( Schema.Array( @@ -757,6 +766,10 @@ type Sample = { devices?: DeviceTools; /** T3's results for prior device calls whose case result is "default". */ readonly priorResults: Map; + /** Prior calls whose T3 result is an error, so the agent sees it as one. */ + readonly priorErrors: Set; + /** T3's t3_background_wait, once `withBackgroundWait` builds it for `world.backgroundTasks`. */ + backgroundWait?: BackgroundWait; }; const SHELL_TOOLS = new Set(["Bash", "Read", "Edit", "Write", "WebFetch"]); @@ -799,6 +812,7 @@ export const makeSample = (testCase: McpToolUseCase, base: Base): Sample => { nextTask: Math.max(0, ...taskNumbers) + 1, threads: new Map(), priorResults: new Map(), + priorErrors: new Set(), }; }; @@ -821,6 +835,25 @@ export const withDevices = Effect.fn("evaluateMcpToolUse.withDevices")(function* } }); +/** + * Builds T3's t3_background_wait for a world with background tasks, and answers prior waits whose + * result is "default" with T3's result. + */ +export const withBackgroundWait = Effect.fn("evaluateMcpToolUse.withBackgroundWait")(function* ( + sample: Sample, +) { + const running = sample.testCase.setup.world?.backgroundTasks; + if (running === undefined) return; + const backgroundWait = yield* makeBackgroundWait(running); + sample.backgroundWait = backgroundWait; + for (const call of (sample.testCase.setup.priorTurns ?? []).flatMap((t) => t.toolCalls ?? [])) { + if (call.result !== "default" || call.tool !== "t3_background_wait") continue; + const result = yield* backgroundWait(call.input); + sample.priorResults.set(call, result.text); + if (result.isError) sample.priorErrors.add(call); + } +}); + // Main's history under every case's world. const MAIN_COMMITS = [ { sha: "9f2c1ab", subject: "feat(web): show the model name in the thread header" }, @@ -895,8 +928,9 @@ export const buildMessages = (sample: Sample) => { const calls = (turn.toolCalls ?? []).map((call) => { const tool = findTool(base, call.tool); if (tool === undefined) throw new Error(`${testCase.id}: unknown prior tool ${call.tool}.`); - if (call.result === "default" && isDeviceTool(call.tool) && !sample.priorResults.has(call)) { - throw new Error(`${testCase.id}: prior ${call.tool} needs T3's result; run withDevices.`); + const fromT3 = isDeviceTool(call.tool) || call.tool === "t3_background_wait"; + if (call.result === "default" && fromT3 && !sample.priorResults.has(call)) { + throw new Error(`${testCase.id}: prior ${call.tool} needs T3's result from its world.`); } return { ...call, id: `toolu_prior_${nextId++}`, name: tool.name, tool, prior: call }; }); @@ -918,7 +952,8 @@ export const buildMessages = (sample: Sample) => { normalizeToolName(tool.name) === "orchestrator_capabilities" ? capabilities(sample, result) : (sample.priorResults.get(prior) ?? result); - push("user", toolResult(id, replayed)); + const answer = toolResult(id, replayed); + push("user", sample.priorErrors.has(prior) ? { ...answer, is_error: true } : answer); } if (turn.text) push(turn.role, text(turn.text)); } @@ -1438,6 +1473,10 @@ const answerToolCall = Effect.fn("evaluateMcpToolUse.answerToolCall")(function* const result = yield* sample.devices.call(name, input, failure); return result.isError ? failed(result.text) : toolResult(call.id, result.text); } + if (sample.backgroundWait && isT3Name(call.name) && name === "t3_background_wait" && fromWorld) { + const result = yield* sample.backgroundWait(input); + return result.isError ? failed(result.text) : toolResult(call.id, result.text); + } if (mock !== undefined && mock !== "default") return toolResult( call.id, @@ -1823,6 +1862,7 @@ const runSample = Effect.fn("evaluateMcpToolUse.runSample")(function* ( ) { const sample = { ...makeSample(testCase, base), shellUsage: spent.shellUsage }; yield* withDevices(sample); + yield* withBackgroundWait(sample); const runPromise = Effect.runPromiseWith(yield* Effect.context()); const messages = buildMessages(sample); const input = base.codex ? codexMessages(messages) : undefined; @@ -2197,7 +2237,7 @@ const main = Effect.gen(function* () { // One JSON line per case. Without --case this also checks that every case builds. for (const testCase of selected) { const sample = makeSample(testCase, baseFor(testCase)); - yield* Effect.scoped(withDevices(sample)); + yield* Effect.scoped(Effect.andThen(withDevices(sample), withBackgroundWait(sample))); const messages = buildMessages(sample); const body = sample.base.codex ? codexRequest(sample.base, codexMessages(messages)) From 44f0a6e211db54ea8b09f559c408ef2fd7ae73ba Mon Sep 17 00:00:00 2001 From: Taylor Dolezal <117500+onlydole@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:15:32 -0800 Subject: [PATCH 3/5] fix(evals): prior-result error names the setup step to run The background wait change merged the device and t3_background_wait checks into one message and dropped the hint naming the function that builds T3's results. Name withDevices or withBackgroundWait again, which restores the device test's expectation. Co-Authored-By: Claude Opus 5.5 --- evals/mcp-tool-use/run.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index bbe8676d79d01..d3d09fdb178fc 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -928,9 +928,13 @@ export const buildMessages = (sample: Sample) => { const calls = (turn.toolCalls ?? []).map((call) => { const tool = findTool(base, call.tool); if (tool === undefined) throw new Error(`${testCase.id}: unknown prior tool ${call.tool}.`); - const fromT3 = isDeviceTool(call.tool) || call.tool === "t3_background_wait"; - if (call.result === "default" && fromT3 && !sample.priorResults.has(call)) { - throw new Error(`${testCase.id}: prior ${call.tool} needs T3's result from its world.`); + const builder = isDeviceTool(call.tool) + ? "withDevices" + : call.tool === "t3_background_wait" + ? "withBackgroundWait" + : undefined; + if (call.result === "default" && builder && !sample.priorResults.has(call)) { + throw new Error(`${testCase.id}: prior ${call.tool} needs T3's result; run ${builder}.`); } return { ...call, id: `toolu_prior_${nextId++}`, name: tool.name, tool, prior: call }; }); From 18de7a091bc04781205388b8f0354ef9dc62d01b Mon Sep 17 00:00:00 2001 From: Taylor Dolezal <117500+onlydole@users.noreply.github.com> Date: Mon, 2 Nov 2026 17:19:02 -0800 Subject: [PATCH 4/5] test(evals): guard case fails on main without a model Models recover from both the old success and the new error, so the background wait guard case passes either way. Check in run.test.ts that its seeded wait on an unknown ID gets T3's error, and that the control's matched wait doesn't. The check fails on main's server code and passes with this fix. Co-Authored-By: Claude Opus 5.5 --- evals/mcp-tool-use/run.test.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 217e505f5fcf5..10b746cc2a31e 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -28,6 +28,7 @@ import { t3Default, t3SystemAppend, t3SubagentContext, + withBackgroundWait, withDevices, } from "./run.ts"; import { briefRoute, gradeDevice, shellInvocations } from "./device-grading.ts"; @@ -569,6 +570,33 @@ it.effect("builds device cases from product-backed worlds and synthetic prior tu }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); +// Models recover from either result, so this keeps the guard case failing on the old success. +it.effect("seeds T3's error for a prior background wait whose IDs match nothing", () => + Effect.gen(function* () { + const cases = NodeFS.readFileSync( + new URL("./cases/orchestration.ndjson", import.meta.url), + "utf8", + ) + .trim() + .split("\n") + .map((line) => decodeDeviceCase(line)); + const base = buildBase(template, { model: "claude-sonnet-5-5", keepUserInstructions: false }); + for (const [id, isError] of [ + ["background-wait-unmatched-id", true], + ["background-wait-matched-id", false], + ] as const) { + const testCase = cases.find((c) => c.id === id)!; + const [wait] = (testCase.setup.priorTurns ?? []) + .flatMap((turn) => turn.toolCalls ?? []) + .filter((call) => call.tool === "t3_background_wait"); + assert.isDefined(wait, id); + const sample = makeSample(testCase, base); + yield* withBackgroundWait(sample); + assert.equal(sample.priorErrors.has(wait), isError, id); + } + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); + it.each([ ["Do not use SSH for UI but use device_open and its returned command.", "managed"], ["Use device_open instead of SSH for screenshots.", "managed"], From 94f13ebb5a2ce464f9b7051f9e9fb06264e3f41b Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 20:15:07 +0000 Subject: [PATCH 5/5] fix(evals): background wait harness provides RemoteDelegation after merging main AI-assisted; verified with the tests and tools listed above. --- evals/mcp-tool-use/backgroundWait.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/evals/mcp-tool-use/backgroundWait.ts b/evals/mcp-tool-use/backgroundWait.ts index 8635423ca9950..24377c4247f8a 100644 --- a/evals/mcp-tool-use/backgroundWait.ts +++ b/evals/mcp-tool-use/backgroundWait.ts @@ -37,6 +37,7 @@ import * as ThreadManagementService from "../../apps/server/src/orchestration-v2 import * as PeerForwarding from "../../apps/server/src/peer/PeerForwarding.ts"; import * as PeerLinkRequests from "../../apps/server/src/peer/PeerLinkRequests.ts"; import * as PeerLinks from "../../apps/server/src/peer/PeerLinks.ts"; +import * as RemoteDelegation from "../../apps/server/src/peer/RemoteDelegation.ts"; import * as SqlitePersistence from "../../apps/server/src/persistence/Sqlite.ts"; import * as ServerSettings from "../../apps/server/src/serverSettings.ts"; @@ -104,6 +105,7 @@ export const makeBackgroundWait = Effect.fn("evaluateMcpToolUse.makeBackgroundWa Layer.provide(Layer.mock(PeerForwarding.PeerForwarding)({})), Layer.provide(Layer.mock(PeerLinkRequests.PeerLinkRequests)({})), Layer.provide(Layer.mock(PeerLinks.PeerLinks)({})), + Layer.provide(Layer.mock(RemoteDelegation.RemoteDelegation)({})), ), ); type Entry = Parameters[0];