From 8d18c0cc291adcf069c7666ca28984c41800536d Mon Sep 17 00:00:00 2001 From: Josh Gimenes Date: Fri, 6 Nov 2026 10:04:40 -0700 Subject: [PATCH 1/9] fix(server): agents learn when the user moves their thread to another account Switching a thread between accounts of the same provider (for example two Claude subscriptions) resumes the same native conversation, so the agent never learns it moved. It keeps acting on earlier capability results and usage limits, and can pin subagents to the old account. The first prompted turn on a new provider instance now carries a one-line note naming both instances (display name and id). It says earlier provider facts may be stale and that leaving providerInstanceId unset keeps subagents on the new one. Compaction and /goal turns stay verbatim; the note waits for the next prompted turn. Co-Authored-By: Claude Opus 5.5 --- .../ProviderTurnStartService.ts | 82 +++++++++++++++---- .../SelectionRestart.integration.test.ts | 12 ++- .../ProviderSwitch.integration.test.ts | 21 ++++- 3 files changed, 96 insertions(+), 19 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 9880da3408a77..8aca54e5e083a 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -12,7 +12,9 @@ import { type OrchestrationV2TurnItem, isCompactionCommandMessage, parseGoalCommand, + type ProviderInstanceId, RunId, + type ServerSettings as ServerSettingsValue, ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; @@ -116,6 +118,19 @@ const refusedBeforePrompt = (error: unknown): boolean => PROMPT_REFUSAL_TAGS.some((tag) => Predicate.isTagged(error, tag)) || (Predicate.hasProperty(error, "cause") && refusedBeforePrompt(error.cause)); +// A same-driver account switch resumes the native conversation, so nothing else tells the agent. +export const providerSwitchNote = ( + instances: ServerSettingsValue["providerInstances"], + from: ProviderInstanceId, + to: ProviderInstanceId, +): string => { + const name = (id: ProviderInstanceId) => { + const displayName = instances[id]?.displayName; + return displayName === undefined ? `\`${id}\`` : `${displayName} (\`${id}\`)`; + }; + return `Note: the user moved this thread from ${name(from)} to ${name(to)}. Earlier provider choices and usage limits in this conversation may be out of date. To keep subagents on ${name(to)}, leave \`providerInstanceId\` unset.`; +}; + export interface ProviderTurnStartServiceV2Shape { readonly start: (input: { readonly threadId: ThreadId; @@ -164,6 +179,7 @@ export const layer: Layer.Layer< const runExecution = yield* RunExecutionService.RunExecutionServiceV2; const runtimePolicy = yield* RuntimePolicy.RuntimePolicyV2; const turnItemPositions = yield* TurnItemPositionStore.TurnItemPositionStoreV2; + const serverSettings = yield* ServerSettings.ServerSettingsService; const isManagedWorktreePath = yield* makeIsManagedWorktreePath; const resolveRecreateBase = Effect.fn("ProviderTurnStartService.resolveRecreateBase")( @@ -859,25 +875,63 @@ export const layer: Layer.Layer< projection.providerTurns, projection.attempts, ); + const commandMessageIds = new Set( + projection.messages + .filter( + (candidate) => + isCompactionCommandMessage(candidate) || + (goalDriver && parseGoalCommand(candidate) !== null), + ) + .map((candidate) => candidate.id), + ); const restartCancelledWork = pendingRestartCancelledBackgroundWork({ runs: projection.runs, providerTurns: projection.providerTurns, - compactionMessageIds: new Set( - projection.messages - .filter( - (candidate) => - isCompactionCommandMessage(candidate) || - (goalDriver && parseGoalCommand(candidate) !== null), - ) - .map((candidate) => candidate.id), - ), + compactionMessageIds: commandMessageIds, run, attempts: projection.attempts, }); - const restartNote = + // A restart of the active run moves that run to the new account, so compare attempts. + // Commands go verbatim, so the note waits for the next prompted turn. + const promptedRuns = new Map( + projection.runs + .filter((source) => !commandMessageIds.has(source.userMessageId)) + .map((source) => [source.id, source.ordinal]), + ); + const previousAttempt = projection.attempts + .filter( + (source) => + source.id !== attempt.id && + source.startedAt !== null && + (promptedRuns.get(source.runId) ?? Infinity) <= run.ordinal, + ) + .toSorted( + (a, b) => + promptedRuns.get(a.runId)! - promptedRuns.get(b.runId)! || + a.attemptOrdinal - b.attemptOrdinal, + ) + .at(-1); + const switchNote = + commandMessageIds.has(message.id) || + previousAttempt === undefined || + previousAttempt.providerInstanceId === run.providerInstanceId + ? "" + : providerSwitchNote( + yield* serverSettings.getSettings.pipe( + Effect.map((settings) => settings.providerInstances), + Effect.orElseSucceed(() => ({})), + ), + previousAttempt.providerInstanceId, + run.providerInstanceId, + ); + const turnNote = [ restartCancelledWork.length === 0 ? "" - : restartCancelledBackgroundWorkNote(restartCancelledWork); + : restartCancelledBackgroundWorkNote(restartCancelledWork), + switchNote, + ] + .filter((note) => note !== "") + .join("\n\n"); const tokenCap = yield* handoffTokenCapConfig.pipe( Effect.orElseSucceed(() => DEFAULT_HANDOFF_TOKEN_CAP), ); @@ -1023,7 +1077,7 @@ export const layer: Layer.Layer< return handoffBudget({ tokenCap, modelContextWindow, - userText: restartNote === "" ? userText : `${restartNote}\n\n${userText}`, + userText: turnNote === "" ? userText : `${turnNote}\n\n${userText}`, attachments: message.attachments, providerThread: budgetProviderThread, nativeContextEstimate: @@ -1668,7 +1722,7 @@ export const layer: Layer.Layer< sideChatHandoff === undefined || compact || goalCommand ? "" : sideChatHandoff.summaryText; - const context = [delivery.context, sideChatContext, restartNote] + const context = [delivery.context, sideChatContext, turnNote] .filter((part) => part !== "") .join("\n\n"); const { restartContinuationOfRunId: _resumedRunId, ...promptedInput } = turnInput; @@ -1735,7 +1789,7 @@ export const layer: Layer.Layer< effectiveHandoffs.length === 0 && retryHandoff.length === 0 && sideChatHandoff === undefined && - restartNote === "" && + turnNote === "" && !noteContinuation ? session : makeDeliverySession(session, startWithHandoffs); diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index ba49f0bcfba53..f3e803ff0445c 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -31,6 +31,8 @@ import * as EventSink from "./EventSink.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import { withInlineContext } from "./ContextHandoffDelivery.ts"; +import { providerSwitchNote } from "./ProviderTurnStartService.ts"; import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; @@ -988,7 +990,9 @@ it.live.each(["active", "idle", "selection-command", "pooled", "separate-home"] assert.deepEqual(resumes, [ { instanceId: targetId, nativeId: originalThread.nativeThreadRef?.nativeId }, ]); - assert.deepEqual(messages, ["first", "second"]); + const switched = (from: typeof targetId, to: typeof targetId, text: string) => + withInlineContext(providerSwitchNote({}, from, to), text); + assert.deepEqual(messages, ["first", switched(providerInstanceId, targetId, "second")]); assert.equal((yield* Ref.get(state)).closedSessionCount, mode === "pooled" ? 0 : 1); if (mode === "pooled") { assert.isFalse( @@ -1005,7 +1009,11 @@ it.live.each(["active", "idle", "selection-command", "pooled", "separate-home"] assert.deepEqual(returnedThread.nativeThreadRef, originalThread.nativeThreadRef); assert.equal(returnedThread.providerInstanceId, providerInstanceId); assert.isEmpty(third.contextHandoffs); - assert.deepEqual(messages, ["first", "second", "third"]); + assert.deepEqual(messages, [ + "first", + switched(providerInstanceId, targetId, "second"), + switched(targetId, providerInstanceId, "third"), + ]); }).pipe(Effect.provide(ProviderReplayHarness.layerWithRegistry({ name }, layerRegistry))); }), ), diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index 728e4f2189c26..94c7ca172fb71 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -1021,7 +1021,16 @@ describe("orchestration v2 provider switching", () => { ); } else { assert.equal(handoff.delivery?.status, "injected"); - assert.equal((yield* Ref.get(capturedTurns)).at(-1)!.text, current); + const text = (yield* Ref.get(capturedTurns)).at(-1)!.text; + if ( + scenario === "large-current-input" || + scenario === "screenshot-reported-capacity-native" + ) { + assert.match(text, /^Note: the user moved this thread /); + assert.isTrue(text.endsWith(`\n\nUser message:\n${current}`)); + } else { + assert.equal(text, current); + } assert.include( yield* encodeJson(yield* Ref.get(injectedHistory)), "Original request with constraints", @@ -1749,7 +1758,10 @@ describe("orchestration v2 provider switching", () => { assert.include(deliveredHistory, partialResponse); assert.include(deliveredHistory, `run-status=${status}`); if (native) { - assert.equal(delivered.text, "Continue"); + assert.match( + delivered.text, + /^Note: the user moved this thread .+\n\nUser message:\nContinue$/s, + ); assert.notInclude(deliveredHistory, '"text":"Continue"'); assert.include(deliveredHistory, '"role":"assistant"'); assert.include(deliveredHistory, '"role":"user"'); @@ -2300,7 +2312,10 @@ describe("orchestration v2 provider switching", () => { assert.lengthOf(projection.contextHandoffs, 0); assert.deepEqual( (yield* Ref.get(capturedTurns)).map((turn) => turn.text), - ["First account turn", "Alternate account turn"], + [ + "First account turn", + "Note: the user moved this thread from `codex` to `codex-alternate`. Earlier provider choices and usage limits in this conversation may be out of date. To keep subagents on `codex-alternate`, leave `providerInstanceId` unset.\n\nUser message:\nAlternate account turn", + ], ); }), ), From e78be2c8153048950387c4beff5eee8aa183f2d3 Mon Sep 17 00:00:00 2001 From: Josh Gimenes Date: Fri, 6 Nov 2026 10:35:00 -0700 Subject: [PATCH 2/9] fix(server): shorten the account switch note and keep it after refused prompts The note now reads "Note: this thread now runs on Extra (`claude-extra`), switched by the user from Work (`claudeAgent`)." An attempt's startedAt is set before the provider accepts the prompt, so a refused first prompt on the new account consumed the note. The previous attempt now has to have a provider turn, the same proof the restart note uses. Adds mcp-tool-use cases: delegates follow an account switch (bad) and the user can still name the old account (control). Co-Authored-By: Claude Opus 5.5 --- .../ProviderTurnStartService.ts | 33 ++-- .../SelectionRestart.integration.test.ts | 2 +- .../ProviderSwitch.integration.test.ts | 148 +++++++++++++++++- evals/mcp-tool-use/cases/orchestration.ndjson | 2 + evals/mcp-tool-use/run.ts | 17 +- 5 files changed, 176 insertions(+), 26 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 8aca54e5e083a..364da3272d095 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -14,7 +14,6 @@ import { parseGoalCommand, type ProviderInstanceId, RunId, - type ServerSettings as ServerSettingsValue, ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; @@ -119,16 +118,11 @@ const refusedBeforePrompt = (error: unknown): boolean => (Predicate.hasProperty(error, "cause") && refusedBeforePrompt(error.cause)); // A same-driver account switch resumes the native conversation, so nothing else tells the agent. -export const providerSwitchNote = ( - instances: ServerSettingsValue["providerInstances"], - from: ProviderInstanceId, - to: ProviderInstanceId, -): string => { - const name = (id: ProviderInstanceId) => { - const displayName = instances[id]?.displayName; - return displayName === undefined ? `\`${id}\`` : `${displayName} (\`${id}\`)`; - }; - return `Note: the user moved this thread from ${name(from)} to ${name(to)}. Earlier provider choices and usage limits in this conversation may be out of date. To keep subagents on ${name(to)}, leave \`providerInstanceId\` unset.`; +type NamedInstance = { readonly id: string; readonly displayName?: string | undefined }; +export const providerSwitchNote = (from: NamedInstance, to: NamedInstance): string => { + const name = ({ id, displayName }: NamedInstance) => + displayName === undefined ? `\`${id}\`` : `${displayName} (\`${id}\`)`; + return `Note: this thread now runs on ${name(to)}, switched by the user from ${name(from)}.`; }; export interface ProviderTurnStartServiceV2Shape { @@ -898,11 +892,13 @@ export const layer: Layer.Layer< .filter((source) => !commandMessageIds.has(source.userMessageId)) .map((source) => [source.id, source.ordinal]), ); + // Only a provider turn proves the prompt arrived; a refused prompt keeps the note pending. + const acceptedAttemptIds = new Set(projection.providerTurns.map((turn) => turn.runAttemptId)); const previousAttempt = projection.attempts .filter( (source) => source.id !== attempt.id && - source.startedAt !== null && + acceptedAttemptIds.has(source.id) && (promptedRuns.get(source.runId) ?? Infinity) <= run.ordinal, ) .toSorted( @@ -911,18 +907,19 @@ export const layer: Layer.Layer< a.attemptOrdinal - b.attemptOrdinal, ) .at(-1); + const instances = yield* serverSettings.getSettings.pipe( + Effect.map((settings) => settings.providerInstances), + Effect.orElseSucceed(() => undefined), + ); + const named = (id: ProviderInstanceId) => ({ id, displayName: instances?.[id]?.displayName }); const switchNote = commandMessageIds.has(message.id) || previousAttempt === undefined || previousAttempt.providerInstanceId === run.providerInstanceId ? "" : providerSwitchNote( - yield* serverSettings.getSettings.pipe( - Effect.map((settings) => settings.providerInstances), - Effect.orElseSucceed(() => ({})), - ), - previousAttempt.providerInstanceId, - run.providerInstanceId, + named(previousAttempt.providerInstanceId), + named(run.providerInstanceId), ); const turnNote = [ restartCancelledWork.length === 0 diff --git a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts index f3e803ff0445c..a62a3675a7556 100644 --- a/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts +++ b/apps/server/src/orchestration-v2/SelectionRestart.integration.test.ts @@ -991,7 +991,7 @@ it.live.each(["active", "idle", "selection-command", "pooled", "separate-home"] { instanceId: targetId, nativeId: originalThread.nativeThreadRef?.nativeId }, ]); const switched = (from: typeof targetId, to: typeof targetId, text: string) => - withInlineContext(providerSwitchNote({}, from, to), text); + withInlineContext(providerSwitchNote({ id: from }, { id: to }), text); assert.deepEqual(messages, ["first", switched(providerInstanceId, targetId, "second")]); assert.equal((yield* Ref.get(state)).closedSessionCount, mode === "pooled" ? 0 : 1); if (mode === "pooled") { diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index 94c7ca172fb71..efa8b83bc1c54 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -62,6 +62,8 @@ import { GROK_MODEL_SELECTION, } from "./fixtures/shared.ts"; import * as ProviderReplayHarness from "./ProviderReplayHarness.ts"; +import { withInlineContext } from "../ContextHandoffDelivery.ts"; +import { providerSwitchNote } from "../ProviderTurnStartService.ts"; import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; @@ -1024,9 +1026,10 @@ describe("orchestration v2 provider switching", () => { const text = (yield* Ref.get(capturedTurns)).at(-1)!.text; if ( scenario === "large-current-input" || - scenario === "screenshot-reported-capacity-native" + scenario === "screenshot-reported-capacity-native" || + scenario === "screenshot-option-change-retry-native" ) { - assert.match(text, /^Note: the user moved this thread /); + assert.match(text, /^Note: this thread now runs on /); assert.isTrue(text.endsWith(`\n\nUser message:\n${current}`)); } else { assert.equal(text, current); @@ -1179,7 +1182,14 @@ describe("orchestration v2 provider switching", () => { yield* wait(3, "completed"); const retried = yield* orchestrator.getThreadProjection(threadId); const lastTurn = (yield* Ref.get(capturedTurns)).at(-1)!; - assert.equal(lastTurn.text, retryText); + // The failed target turn never reached the provider, so the retry carries the note. + assert.equal( + lastTurn.text, + withInlineContext( + providerSwitchNote({ id: "codex" }, { id: "claudeAgent" }), + retryText, + ), + ); if (failure !== "injection" && failure !== "exhausted-target-retry") { const delta = yield* encodeJson( (yield* Ref.get(injectedHistory)).slice(historyBeforeRetry), @@ -1760,7 +1770,7 @@ describe("orchestration v2 provider switching", () => { if (native) { assert.match( delivered.text, - /^Note: the user moved this thread .+\n\nUser message:\nContinue$/s, + /^Note: this thread now runs on .+\n\nUser message:\nContinue$/s, ); assert.notInclude(deliveredHistory, '"text":"Continue"'); assert.include(deliveredHistory, '"role":"assistant"'); @@ -2264,6 +2274,27 @@ describe("orchestration v2 provider switching", () => { occurredAt: now, payload: { ...queued.runs[0]!, status: "completed", completedAt: now }, }, + // A real provider records its turn; the held fixture turn stops before that. + { + id: EventId.make("event:queued-account-switch:first-turn"), + type: "provider-turn.updated", + threadId: queuedThreadId, + runId: queued.runs[0]!.id, + nodeId: queued.runs[0]!.rootNodeId!, + providerInstanceId: CODEX_MODEL_SELECTION.instanceId, + occurredAt: now, + payload: { + id: ProviderTurnId.make("provider-turn:queued-account-switch:first"), + providerThreadId: queued.runs[0]!.providerThreadId!, + nodeId: queued.runs[0]!.rootNodeId!, + runAttemptId: queued.runs[0]!.activeAttemptId, + nativeTurnRef: null, + ordinal: 1, + status: "completed", + startedAt: now, + completedAt: now, + }, + }, ], }); yield* orchestrator.resumeQueuedRuns; @@ -2314,13 +2345,120 @@ describe("orchestration v2 provider switching", () => { (yield* Ref.get(capturedTurns)).map((turn) => turn.text), [ "First account turn", - "Note: the user moved this thread from `codex` to `codex-alternate`. Earlier provider choices and usage limits in this conversation may be out of date. To keep subagents on `codex-alternate`, leave `providerInstanceId` unset.\n\nUser message:\nAlternate account turn", + "Note: this thread now runs on `codex-alternate`, switched by the user from `codex`.\n\nUser message:\nAlternate account turn", ], ); }), ), ); + it.live("keeps the account switch note pending after a refused prompt", () => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace("refused-account-switch"); + const capturedTurns = yield* Ref.make>([]); + const refuseStarts = yield* Ref.make(0); + const alternateSelection: ModelSelection = { + ...CLAUDE_MODEL_SELECTION, + instanceId: ProviderInstanceId.make("claude-alternate"), + }; + const adapters = [CLAUDE_MODEL_SELECTION, alternateSelection].map((modelSelection) => + makeTestAdapter({ + instanceId: modelSelection.instanceId, + driver: CLAUDE_DRIVER, + capabilities: ClaudeProviderCapabilitiesV2, + modelSelection, + responseByRunOrdinal: {}, + capturedTurns, + refuseStarts, + }), + ); + const layerRegistry = Layer.succeed( + ProviderAdapterRegistry.ProviderAdapterRegistryV2, + ProviderAdapterRegistry.ProviderAdapterRegistryV2.of({ + get: (instanceId) => + Effect.succeed(adapters.find((adapter) => adapter.instanceId === instanceId)!), + list: () => Effect.succeed(adapters.map((adapter) => adapter.instanceId)), + getMetadata: () => + Effect.succeed({ + driver: CLAUDE_DRIVER, + continuationKey: "claude:shared-native-account-history", + enabled: true, + capabilities: ClaudeProviderCapabilitiesV2, + }), + }), + ); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const run = Effect.fn("run")(function* (ordinal: number, modelSelection: ModelSelection) { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`refused-account-switch:${ordinal}`), + threadId, + messageId: MessageId.make(`refused-account-switch:${ordinal}`), + createdBy: "user", + creationSource: "web", + text: `Request ${ordinal}`, + attachments: [], + modelSelection, + dispatchMode: { type: "start_immediately" }, + }); + yield* orchestrator.streamStoredEvents.pipe( + Stream.filter( + ({ event }) => + event.type === "run.updated" && + event.payload.ordinal === ordinal && + (event.payload.status === "completed" || event.payload.status === "failed"), + ), + Stream.runHead, + ); + yield* worker.drain(); + return (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status; + }); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("refused-account-switch:create"), + threadId, + projectId, + createdBy: "user", + creationSource: "web", + title: "Refused account switch", + modelSelection: CLAUDE_MODEL_SELECTION, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }); + assert.equal(yield* run(1, CLAUDE_MODEL_SELECTION), "completed"); + yield* Ref.set(refuseStarts, 1); + assert.equal(yield* run(2, alternateSelection), "failed"); + assert.equal(yield* run(3, alternateSelection), "completed"); + const turns = yield* Ref.get(capturedTurns); + assert.lengthOf(turns, 2); + assert.include( + turns[1]!.text, + "Note: this thread now runs on `claude-alternate`, switched by the user from `claudeAgent`.", + ); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { + name: "refused-account-switch", + runtimePolicyOverride: { + cwd, + approvalPolicy: "never", + sandboxPolicy: { type: "readOnly" }, + }, + }, + layerRegistry, + ), + ), + ); + }), + ), + ); + it.live("finishes earlier queued Codex turns before handing context to queued Claude", () => Effect.scoped( Effect.gen(function* () { diff --git a/evals/mcp-tool-use/cases/orchestration.ndjson b/evals/mcp-tool-use/cases/orchestration.ndjson index 520bd2abf10d3..26fc417773462 100644 --- a/evals/mcp-tool-use/cases/orchestration.ndjson +++ b/evals/mcp-tool-use/cases/orchestration.ndjson @@ -40,3 +40,5 @@ {"id": "settle-after-delayed-merge-waits", "area": "orchestration", "kind": "guard", "pattern": "settle before delayed work finishes", "experience": "Asked to merge a PR once its checks pass and then settle this thread, an agent following the #26749 settle bullet can settle in the first turn; a settled thread refuses watch_pull_request and drops PR-watch wakes, so the merge never happens.", "rule": "Asked to settle ... a thread, including this one: call t3_thread_organize. (the request is to settle after the merge, not now)", "privacy": "commit-safe", "notes": "From the CodeRabbit finding on #26765. Settling this thread applies at turn end, and Orchestrator.ts rejects watch wakes on a settled thread. Good: watch #123 and end the turn without settling. gh pr merge --auto plus a settle would still lose the wake that confirms the merge.", "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Okay this looks good to me. Any risk merging it?"}, {"role": "assistant", "text": "Low risk. [#123](https://github.com/acme/app/pull/123) only adds an accessible title to the HTML preview frame, and the new test covers it. Two checks are still running: Check and Test Web.", "toolCalls": [{"tool": "Bash", "input": {"command": "gh pr view 123 --json title,files --jq '.title, .files[].path'"}, "result": "fix(web): HTML preview frame gets an accessible title\napps/web/src/HtmlPreview.tsx\napps/web/src/HtmlPreview.test.tsx"}, {"tool": "Bash", "input": {"command": "gh pr checks 123"}, "result": "Lint\tpass\t41s\nTypecheck\tpass\t1m2s\nCheck\tpending\t0\nTest Web\tpending\t0"}, {"tool": "link_pull_request", "input": {"url": "https://github.com/acme/app/pull/123"}, "result": {"host": "github.com", "repository": "acme/app", "number": 123, "url": "https://github.com/acme/app/pull/123", "alreadyLinked": false}}]}], "mocks": {"watch_pull_request": "default", "link_pull_request": "default", "t3_thread_organize": {"settlesWhenTurnEnds": true}}, "world": {"branch": "fix/html-preview-title", "commits": [{"sha": "abc1234", "subject": "fix(web): HTML preview frame gets an accessible title"}], "diff": "diff --git a/apps/web/src/HtmlPreview.tsx b/apps/web/src/HtmlPreview.tsx\n--- a/apps/web/src/HtmlPreview.tsx\n+++ b/apps/web/src/HtmlPreview.tsx\n@@ -1,3 +1,3 @@\n export function HtmlPreview({ html }: { html: string }) {\n- return