diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 84c78b91453c2..3dbeb80a13c3b 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -29,6 +29,7 @@ import { ProviderTurnId, RunAttemptId, RunId, + RuntimeRequestId, ThreadId, TurnItemId, } from "@t3tools/contracts"; @@ -1836,29 +1837,28 @@ describe("CodexAdapterV2 post-settle continuation", () => { event.type === "message.updated" && event.message.role === "assistant", ); - it.effect("keeps an asynchronous Codex question actionable after the turn completes", () => - Effect.scoped( - Effect.gen(function* () { - const nativeThreadId = "async-question-thread"; - const nativeTurnId = "async-question-turn"; - const usage = { - totalTokens: 15, - inputTokens: 10, - cachedInputTokens: 2, - outputTokens: 5, - reasoningOutputTokens: 1, - }; - const transcript = makeCodexReplayTranscript({ - scenario: "async-question-and-billed-usage", - entries: [ - ...codexReplayPreamble({ - nativeThreadId, - nativeTurnId, - prompt: "Continue while I decide.", - }), - { + it.effect.each([ + { terminalStatus: "completed", restored: false }, + { terminalStatus: "interrupted", restored: false }, + { terminalStatus: "completed", restored: true }, + ] as const)( + "marks an asynchronous Codex question awaited until the turn is $terminalStatus (restored: $restored)", + ({ terminalStatus, restored }) => + Effect.scoped( + Effect.gen(function* () { + const nativeThreadId = "async-question-thread"; + const nativeTurnId = "async-question-turn"; + const usage = { + totalTokens: 15, + inputTokens: 10, + cachedInputTokens: 2, + outputTokens: 5, + reasoningOutputTokens: 1, + }; + const question = (label: string) => + ({ type: "emit_inbound", - label: "question", + label, frame: { method: "item/completed", params: { @@ -1866,120 +1866,178 @@ describe("CodexAdapterV2 post-settle continuation", () => { turnId: nativeTurnId, item: { type: "agentMessage", - id: "async-question-item", + id: restored ? "prior-question" : "async-question-item", text: "Which branch?", delivery: "async", questions: [{ title: "Which branch?", options: ["main", "dev"] }], }, }, }, - }, - { + }) as const; + const sleep = (method: "item/started" | "item/completed") => + ({ type: "emit_inbound", - label: "usage", + label: `sleep ${method}`, frame: { - method: "thread/tokenUsage/updated", + method, params: { threadId: nativeThreadId, turnId: nativeTurnId, - tokenUsage: { total: usage, last: usage, modelContextWindow: 200_000 }, + startedAtMs: 1782622441000, + completedAtMs: 1782622501000, + item: { type: "sleep", id: "sleep-item", durationMs: 60_000 }, }, }, - }, - { - type: "emit_inbound", - label: "complete", - frame: { - method: "turn/completed", - params: { - threadId: nativeThreadId, - turn: makeCodexReplayTurn({ id: nativeTurnId, status: "completed" }), + }) as const; + const transcript = makeCodexReplayTranscript({ + scenario: "async-question-and-billed-usage", + entries: [ + ...codexReplayPreamble({ + nativeThreadId, + nativeTurnId, + prompt: "Continue while I decide.", + }), + ...(restored ? [] : [question("question")]), + sleep("item/started"), + // A repeated completion of a tracked question must not reopen or replace it. + question("repeated question"), + // An interrupted turn ends without completing the sleep item. + ...(terminalStatus === "completed" ? [sleep("item/completed")] : []), + { + type: "emit_inbound", + label: "usage", + frame: { + method: "thread/tokenUsage/updated", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + tokenUsage: { total: usage, last: usage, modelContextWindow: 200_000 }, + }, }, }, + { + type: "emit_inbound", + label: "complete", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: terminalStatus }), + }, + }, + }, + ], + }); + const harness = yield* makeCodexReplayHarness(transcript); + const now = yield* DateTime.now; + const restoredRequest: OrchestrationV2RuntimeRequest = { + id: RuntimeRequestId.make("restored-question"), + nodeId: NodeId.make("restored-question-node"), + providerTurnId: null, + nativeRequestRef: { + driver: CodexAdapterV2.CODEX_DRIVER_KIND, + nativeId: "async:prior-question", + strength: "strong", }, - ], - }); - const harness = yield* makeCodexReplayHarness(transcript); - yield* harness.runtime.startTurn( - makeCodexTestTurnInput({ - threadId: harness.threadId, - providerThread: harness.providerThread, - now: yield* DateTime.now, - attemptId: RunAttemptId.make("async-question-attempt"), - text: "Continue while I decide.", - }), - ); - yield* harness.firstTerminal; - const requests = harness.events.flatMap((event) => - event.type === "runtime_request.updated" ? [event.runtimeRequest] : [], - ); - assert.lengthOf(requests, 1); - assert.equal(requests[0]?.status, "pending"); - assert.deepEqual(requests[0]?.responseCapability, { type: "message" }); - const questionItem = harness.events.find( - (event) => - event.type === "turn_item.updated" && event.turnItem.type === "user_input_request", - ); - assert.equal(questionItem?.type, "turn_item.updated"); - if ( - questionItem?.type === "turn_item.updated" && - questionItem.turnItem.type === "user_input_request" - ) { + kind: "user_input", + status: "pending", + responseCapability: { type: "message" }, + createdAt: now, + resolvedAt: null, + }; + const turnInput = { + ...makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("async-question-attempt"), + text: "Continue while I decide.", + }), + pendingUserInputRequests: restored ? [restoredRequest] : [], + }; + yield* harness.runtime.startTurn(turnInput); + yield* harness.firstTerminal; + const requests = harness.events.flatMap((event) => + event.type === "runtime_request.updated" ? [event.runtimeRequest] : [], + ); assert.deepEqual( - questionItem.turnItem.questions[0]?.options.map((option) => option.label), - ["main", "dev"], + requests.map((request) => [request.status, request.responseCapability]), + restored + ? [ + ["pending", { type: "message", agentWaiting: true }], + ["pending", { type: "message" }], + ] + : [ + ["pending", { type: "message" }], + ["pending", { type: "message", agentWaiting: true }], + ["pending", { type: "message" }], + ], + ); + const questionItem = harness.events.find( + (event) => + event.type === "turn_item.updated" && event.turnItem.type === "user_input_request", + ); + assert.equal(questionItem?.type, restored ? undefined : "turn_item.updated"); + if ( + questionItem?.type === "turn_item.updated" && + questionItem.turnItem.type === "user_input_request" + ) { + assert.deepEqual( + questionItem.turnItem.questions[0]?.options.map((option) => option.label), + ["main", "dev"], + ); + assert.equal(questionItem.turnItem.responseMode, "message"); + } + const questionNode = harness.events.find( + (event) => event.type === "node.updated" && event.node.id === requests[0]?.nodeId, ); - assert.equal(questionItem.turnItem.responseMode, "message"); - } - const questionNode = harness.events.find( - (event) => event.type === "node.updated" && event.node.id === requests[0]?.nodeId, - ); - assert.equal( - questionNode?.type === "node.updated" && questionNode.node.countsForRun, - false, - ); - const contextReport = harness.events.find( - (event) => - event.type === "provider_turn.updated" && event.providerTurn.tokenUsage !== undefined, - ); - assert.equal(contextReport?.type, "provider_turn.updated"); - if (contextReport?.type === "provider_turn.updated") { assert.equal( - contextReport.providerTurn.runAttemptId, - RunAttemptId.make("async-question-attempt"), + questionNode?.type === "node.updated" && questionNode.node.countsForRun, + false, ); - assert.equal(contextReport.providerTurn.providerThreadId, harness.providerThread.id); - assert.equal(contextReport.providerTurn.tokenUsage?.usedTokens, 15); - assert.equal(contextReport.providerTurn.tokenUsage?.maxTokens, 200_000); - } - const completed = harness.events.find( - (event) => - event.type === "provider_turn.updated" && event.providerTurn.status === "completed", - ); - assert.equal(completed?.type, "provider_turn.updated"); - if (completed?.type === "provider_turn.updated") { - assert.deepEqual(completed.providerTurn.turnTokenUsage, { - usageStatus: "complete", - usageScope: "main_agent", - hasSubagents: false, - inputTokens: 10, - cachedInputTokens: 2, - outputTokens: 5, - reasoningTokens: 1, - }); - } - assert.isEmpty(assistantMessages(harness.events)); - }).pipe( - Effect.provide( - Layer.mergeAll( - IdAllocator.layer, - McpProviderSessions.layer, - NodeServices.layer, - NodeCrypto.layer, + const contextReport = harness.events.find( + (event) => + event.type === "provider_turn.updated" && event.providerTurn.tokenUsage !== undefined, + ); + assert.equal(contextReport?.type, "provider_turn.updated"); + if (contextReport?.type === "provider_turn.updated") { + assert.equal( + contextReport.providerTurn.runAttemptId, + RunAttemptId.make("async-question-attempt"), + ); + assert.equal(contextReport.providerTurn.providerThreadId, harness.providerThread.id); + assert.equal(contextReport.providerTurn.tokenUsage?.usedTokens, 15); + assert.equal(contextReport.providerTurn.tokenUsage?.maxTokens, 200_000); + } + const completed = harness.events.find( + (event) => + event.type === "provider_turn.updated" && + event.providerTurn.status === terminalStatus, + ); + assert.equal(completed?.type, "provider_turn.updated"); + if (completed?.type === "provider_turn.updated") { + assert.deepEqual(completed.providerTurn.turnTokenUsage, { + usageStatus: terminalStatus === "completed" ? "complete" : "partial", + usageScope: "main_agent", + hasSubagents: false, + inputTokens: 10, + cachedInputTokens: 2, + outputTokens: 5, + reasoningTokens: 1, + }); + } + assert.isEmpty(assistantMessages(harness.events)); + }).pipe( + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), ), ), ), - ), ); it.effect("stops asking for commands the user allowed for the session", () => diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 07c4e4e72eca7..4339a1bffde3c 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -1204,6 +1204,7 @@ interface ActiveCodexTurnContext { readonly runningCompactionItemIds: Set; // Modern approval requests omit paths; keep the kinds without retaining patch bodies. readonly fileChangeKindsByItem: Map; + readonly asyncQuestions: Map; readonly itemPositions: Map< string, { readonly ordinal: number; readonly startedAt: DateTime.Utc } @@ -2128,6 +2129,11 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( startedAt: input.startedAt, itemPositions: new Map(), fileChangeKindsByItem: new Map(), + asyncQuestions: new Map( + (input.turnInput.pendingUserInputRequests ?? []) + .filter((request) => request.nativeRequestRef?.driver === CODEX_PROVIDER) + .map((request) => [String(request.id), request]), + ), runningCompactionItemIds: new Set(), }; const nativeThreadId = input.turnInput.providerThread.nativeThreadRef?.nativeId; @@ -2734,6 +2740,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( startedAt: turn.startedAt, itemPositions: new Map(), fileChangeKindsByItem: new Map(), + asyncQuestions: new Map(), runningCompactionItemIds: new Set(), }; beginTurnTokenUsage(activeContext); @@ -4848,6 +4855,32 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }).pipe(Effect.orDie, turnTerminalizationPermit.withPermits(1)), ); + // Codex agents wait on an async question's answer with the interruptible clock.sleep tool. + const markAsyncQuestionsAwaited = Effect.fn("CodexAdapterV2.markAsyncQuestionsAwaited")( + function* (context: ActiveCodexTurnContext, agentWaiting: boolean) { + for (const [id, request] of context.asyncQuestions) { + if ( + request.responseCapability.type === "message" && + (request.responseCapability.agentWaiting === true) === agentWaiting + ) + continue; + const runtimeRequest: OrchestrationV2RuntimeRequest = { + ...request, + responseCapability: agentWaiting + ? { type: "message", agentWaiting } + : { type: "message" }, + }; + context.asyncQuestions.set(id, runtimeRequest); + yield* emitProviderEvent({ + type: "runtime_request.updated", + driver: CODEX_PROVIDER, + threadId: context.projectionThreadId, + runtimeRequest, + }); + } + }, + ); + yield* client.handleServerNotification("item/started", (payload) => Effect.gen(function* () { const context = yield* awaitActiveTurn(payload.turnId); @@ -4884,6 +4917,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } yield* completeProviderRetry(context, yield* DateTime.now); + if (payload.item.type === "sleep") { + yield* markAsyncQuestionsAwaited(context, true); + return; + } if (payload.item.type !== "reasoning") { yield* resolveItemPosition( context, @@ -5017,6 +5054,11 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } const { context, settled } = resolved; + if (payload.item.type === "sleep") { + yield* markAsyncQuestionsAwaited(context, false); + return; + } + if (payload.item.type === "reasoning") { yield* completeReasoning(payload.turnId, payload.item); return; @@ -5223,10 +5265,15 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } if (payload.item.delivery === "async" && payload.item.questions?.length) { + const nativeRequestId = `async:${payload.item.id}`; + // Request ids are random, so a repeated completion would add a second question. + for (const request of context.asyncQuestions.values()) { + if (request.nativeRequestRef?.nativeId === nativeRequestId) return; + } const artifacts = yield* buildUserInputRequestArtifacts({ context, nativeItemId: payload.item.id, - nativeRequestId: `async:${payload.item.id}`, + nativeRequestId, responseMode: "message", questions: payload.item.questions.map((question, index) => ({ id: String(index), @@ -5251,6 +5298,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( threadId: artifacts.node.threadId, runtimeRequest: artifacts.request, }); + context.asyncQuestions.set(String(artifacts.request.id), artifacts.request); yield* Ref.update(finalAnswerItemIdsByTurn, (current) => { const ids = current.get(payload.turnId); if (!ids?.has(payload.item.id)) return current; @@ -6125,6 +6173,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( return false; } input.context.fileChangeKindsByItem.clear(); + // Stop or failure can end the turn before the sleep item completes. + yield* markAsyncQuestionsAwaited(input.context, false); + input.context.asyncQuestions.clear(); const providerRetry = yield* Ref.modify(providerRetries, (current) => { const retry = current.get(input.context.providerTurnId); if (retry === undefined) { @@ -6620,6 +6671,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( startedAt: now, itemPositions: new Map(), fileChangeKindsByItem: new Map(), + asyncQuestions: new Map(), runningCompactionItemIds: new Set(), }; const providerTurn = { diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 81e64ddb70920..c5e1adfb51d69 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -176,6 +176,7 @@ export interface EventSinkV2Shape { }; /** Like writeWithEffects' updateDelegatedCompletionOnly, for this run's run.updated only. */ readonly updateDelegatedCompletionOnlyRunId?: RunId; + readonly guardPendingUserInputCancellations?: boolean; }) => Effect.Effect< { readonly receipt: CommandReceiptStore.CommandReceiptV2; @@ -312,23 +313,31 @@ const layerBase: Layer.Layer< }); // A user can answer after terminal normalization reads the pending request. - // Recheck inside the write transaction so stale cleanup cannot erase answers. + // Recheck inside the write transaction so stale cleanup cannot erase answers, + // and so a provider repeating a pending question cannot reopen it. const guardUserInputCancellations = /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced(function* (events: ReadonlyArray) { const staleRequests = new Set(); const staleNodes = new Set(); + const settledRequests = new Set(); for (const event of events) { if ( event.type !== "runtime-request.updated" || event.payload.kind !== "user_input" || - event.payload.status !== "cancelled" + (event.payload.status !== "cancelled" && event.payload.status !== "pending") ) continue; const current = yield* projectionStore.getRuntimeRequest( event.threadId, event.payload.id, ); + if (event.payload.status === "pending") { + if (current !== undefined && current.status !== "pending") { + settledRequests.add(event.payload.id); + } + continue; + } if ( current?.status !== "pending" || current.kind !== "user_input" || @@ -342,7 +351,9 @@ const layerBase: Layer.Layer< return events.filter((event) => { switch (event.type) { case "runtime-request.updated": - return event.payload.status !== "cancelled" || !staleRequests.has(event.payload.id); + return event.payload.status === "pending" + ? !settledRequests.has(event.payload.id) + : event.payload.status !== "cancelled" || !staleRequests.has(event.payload.id); case "node.updated": return event.payload.status !== "cancelled" || !staleNodes.has(event.payload.id); case "turn-item.updated": @@ -682,19 +693,24 @@ const layerBase: Layer.Layer< } const runId = input.updateDelegatedCompletionOnlyRunId; - const normalized = yield* normalizeEvents( + const events = runId === undefined ? input.events - : yield* updateDelegatedCompletionOnly(input.events, (id) => id === runId), + : yield* updateDelegatedCompletionOnly(input.events, (id) => id === runId); + const normalized = yield* normalizeEvents( + input.guardPendingUserInputCancellations === true + ? yield* guardUserInputCancellations(events) + : events, ); const storedEvents = yield* eventStore.append({ commandId: input.commandId, events: normalized, }); - // A Stop that changes nothing can still queue the stop of its delegated tasks. + // A Stop that changes nothing can still queue the stop of its delegated tasks, + // and the user-input guard can drop every event. const sequence = storedEvents.at(-1)?.sequence ?? - (input.effects.length === 0 + (input.effects.length === 0 && input.guardPendingUserInputCancellations !== true ? undefined : yield* eventStore.latestSequence({ threadId: input.threadId })); if (sequence === undefined) { diff --git a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts index 6b1388f0ccfec..c96d0cef8184e 100644 --- a/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionRecovery.test.ts @@ -55,6 +55,62 @@ const layerRecoveryTest = ProviderRuntimeRecovery.layer.pipe( ), Layer.provide(ServerSettings.layerTest()), ); +// Answers a question right after recovery reads its thread, like a respond already in flight at shutdown. +const layerAnswerDuringRecoveryTest = (answeredRequestId: RuntimeRequestId) => + ProviderRuntimeRecovery.layer.pipe( + Layer.provide( + Layer.effect( + ProjectionStore.ProjectionStoreV2, + Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const eventSink = yield* EventSink.EventSinkV2; + return ProjectionStore.ProjectionStoreV2.of({ + ...projections, + getRuntimeRecoveryProjection: (threadId) => + projections.getRuntimeRecoveryProjection(threadId).pipe( + Effect.tap((projection) => + Effect.forEach( + projection.runtimeRequests.filter( + (request) => request.id === answeredRequestId, + ), + (request) => + eventSink.write({ + events: [ + { + id: EventId.make(`event:${request.id}:answered`), + type: "runtime-request.updated", + threadId, + nodeId: request.nodeId, + occurredAt: request.createdAt, + payload: { + ...request, + status: "resolved", + resolvedAt: request.createdAt, + }, + }, + ], + }), + ).pipe(Effect.orDie), + ), + ), + }); + }), + ), + ), + Layer.provideMerge( + EventSink.layer.pipe( + Layer.provideMerge( + Layer.mergeAll( + EventStore.layer, + ProjectionStore.layer, + EffectOutbox.layer, + IdAllocator.layer, + ).pipe(Layer.provideMerge(SqlitePersistence.layerMemory)), + ), + ), + ), + Layer.provide(ServerSettings.layerTest()), + ); const providerInstanceId = ProviderInstanceId.make("codex"); const modelSelection = { instanceId: providerInstanceId, model: "gpt-5.4" }; const driver = ProviderDriverKind.make("codex"); @@ -212,6 +268,61 @@ it.effect.each(["startup", "shutdown"] as const)( assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []); }).pipe(Effect.provide(layerRecoveryTest)), ); +it.effect( + "shutdown stops awaiting async questions without reopening one answered meanwhile", + () => { + const answeredRequestId = RuntimeRequestId.make("request:recovery:awaited-answered"); + return Effect.gen(function* () { + const projections = yield* ProjectionStore.ProjectionStoreV2; + const eventSink = yield* EventSink.EventSinkV2; + const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService; + const now = yield* DateTime.now; + const threads = [] as Array; + for (const requestId of [ + RuntimeRequestId.make("request:recovery:awaited"), + answeredRequestId, + ]) { + const threadId = yield* createThread(requestId); + const nodeId = NodeId.make(`node:${requestId}`); + yield* eventSink.write({ + events: [ + { + id: EventId.make(`event:${requestId}`), + type: "runtime-request.updated", + threadId, + nodeId, + occurredAt: now, + payload: { + id: requestId, + nodeId, + providerTurnId: null, + nativeRequestRef: null, + kind: "user_input", + status: "pending", + responseCapability: { type: "message", agentWaiting: true }, + createdAt: now, + resolvedAt: null, + }, + }, + ], + }); + threads.push([threadId, requestId]); + } + yield* recovery.reconcile("shutdown"); + const requests = [] as Array; + for (const [threadId, requestId] of threads) { + const request = (yield* projections.getThreadProjection(threadId)).runtimeRequests.find( + (candidate) => candidate.id === requestId, + ); + requests.push([request?.status, request?.responseCapability]); + } + assert.deepEqual(requests, [ + ["pending", { type: "message" }], + ["resolved", { type: "message", agentWaiting: true }], + ]); + }).pipe(Effect.provide(layerAnswerDuringRecoveryTest(answeredRequestId))); + }, +); it.effect.each(["startup", "shutdown"] as const)( "stops a detached session and its orphaned active provider thread on %s", diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts index 9ae1bc4882d82..739eaa62f8deb 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts @@ -5298,10 +5298,13 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => { const requestId = RuntimeRequestId.make("request:projection-shell-question"); for (const [responseCapability, blocking] of [ [{ type: "message" }, false], + [{ type: "message", agentWaiting: true }, true], [{ type: "live", providerSessionId: ProviderSessionId.make("session:question") }, true], ] as const) { yield* projectionStore.apply({ - id: EventId.make(`event:projection-shell-question:${responseCapability.type}`), + id: EventId.make( + `event:projection-shell-question:${responseCapability.type}:${blocking}`, + ), type: "runtime-request.updated", threadId, nodeId: rootNodeId, diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 2f8c972e7ded9..2a12d35b5e7c1 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1496,7 +1496,9 @@ function pendingRuntimeRequestSummary( id: request.id, kind: request.kind, createdAt: request.createdAt, - blocking: request.responseCapability.type !== "message", + blocking: + request.responseCapability.type !== "message" || + request.responseCapability.agentWaiting === true, }; } diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 222747c413982..42a2535a8d8d8 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -257,6 +257,12 @@ export const make = Effect.gen(function* () { ) && !isProviderNativeSubagentThread(projection.thread), ); + // The agent that was waiting on these questions is gone; they stay answerable. + const awaitedQuestions = pendingRequests.filter( + (request) => + request.responseCapability.type === "message" && + request.responseCapability.agentWaiting === true, + ); const recoveredRequestIds = new Set(questions.map((request) => request.id)); const messageRequestNodeIds = new Set( pendingRequests @@ -442,7 +448,7 @@ export const make = Effect.gen(function* () { }, }); } - for (const question of questions) { + for (const question of [...questions, ...awaitedQuestions]) { events.push({ id: yield* allocateEventId(), type: "runtime-request.updated", @@ -793,6 +799,8 @@ export const make = Effect.gen(function* () { acceptedAt: now, events, effects, + // A respond already past the shutdown gate can answer after this read. + guardPendingUserInputCancellations: true, cancelUnsettledEffects: { effectTypes: EffectOutbox.PROCESS_BOUND_EFFECT_TYPES, reason: detail, diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 54b16d7aa2aef..69f616c8d1e60 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -685,9 +685,10 @@ export const layer: Layer.Layer< runs: current.runs, turnItems: current.turnItems, }); - const inheritedBackgroundTurnItems = yield* projectionStore - .getRuntimeRecoveryProjection(projection.thread.id) - .pipe(Effect.map(selectInheritedBackgroundItems)); + const runtimeProjection = yield* projectionStore.getRuntimeRecoveryProjection( + projection.thread.id, + ); + const inheritedBackgroundTurnItems = selectInheritedBackgroundItems(runtimeProjection); const providerSessionId = providerThread.providerSessionId; const runControls = makeRunControls({ threadId: projection.thread.id, @@ -1671,6 +1672,11 @@ export const layer: Layer.Layer< !noteContinuation ? session : makeDeliverySession(session, startWithHandoffs); + const providerTurnIds = new Set( + projection.providerTurns + .filter((turn) => turn.providerThreadId === runningProviderThread.id) + .map((turn) => turn.id), + ); return runExecution.startRootRun({ commandId: CommandId.make(`command:effect:provider-turn.start:${run.id}`), appThread: projection.thread, @@ -1680,6 +1686,14 @@ export const layer: Layer.Layer< rootNode: runningRootNode, checkpointScope, providerThread: runningProviderThread, + pendingUserInputRequests: runtimeProjection.runtimeRequests.filter( + (request) => + request.kind === "user_input" && + request.status === "pending" && + request.responseCapability.type === "message" && + request.providerTurnId !== null && + providerTurnIds.has(request.providerTurnId), + ), attempt: runningAttempt, attemptId: attempt.id, loadInheritedBackgroundTurnItems: runControls.loadInheritedBackgroundTurnItems, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 5bfb4fe34dc6a..81bfbeaed70f3 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -23,6 +23,7 @@ import { ProviderTurnId, RunAttemptId, RunId, + RuntimeRequestId, ServerSettingsError, ThreadId, TurnItemId, @@ -513,6 +514,57 @@ it("does not route a superseded attempt through a reused provider thread", () => providerTurnId: null, }); assert.isFalse(RunExecutionService.routeProviderEvent(oldTurnEvent, newAttempt, newState)[0]); + + const requestId = RuntimeRequestId.make("request:shared-runtime:restart"); + const restoredState = RunExecutionService.makeProviderEventRoutingState({ + identity: newAttempt, + providerTurnId: null, + inheritedUserInputRequestIds: [requestId], + }); + const requestEvent: Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "runtime_request.updated" } + > = { + type: "runtime_request.updated", + driver, + threadId, + runtimeRequest: { + id: requestId, + nodeId: NodeId.make("node:shared-runtime:question"), + providerTurnId: oldTurnEvent.providerTurn.id, + nativeRequestRef: null, + kind: "user_input", + status: "pending", + responseCapability: { type: "message", agentWaiting: true }, + createdAt: DateTime.makeUnsafe("2026-10-30T00:00:00Z"), + resolvedAt: null, + }, + }; + assert.isFalse(RunExecutionService.routeProviderEvent(requestEvent, newAttempt, newState)[0]); + assert.isTrue(RunExecutionService.routeProviderEvent(requestEvent, newAttempt, restoredState)[0]); + assert.isFalse( + RunExecutionService.routeProviderEvent( + { + ...requestEvent, + threadId: ThreadId.make("thread:other"), + }, + newAttempt, + restoredState, + )[0], + ); + assert.isFalse( + RunExecutionService.routeProviderEvent( + { + ...requestEvent, + runtimeRequest: { + ...requestEvent.runtimeRequest, + id: RuntimeRequestId.make("request:other"), + }, + }, + newAttempt, + restoredState, + )[0], + ); }); it("routes only exact same-thread background items inherited from settled runs", () => { diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 304eeaba43421..d5104bd3a4117 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -15,12 +15,14 @@ import { type OrchestrationV2ProviderTurn, type OrchestrationV2Run, type OrchestrationV2RunAttempt, + type OrchestrationV2RuntimeRequest, type OrchestrationV2Subagent, type OrchestrationV2TurnItem, type ProviderSessionId, type ProviderThreadId, type ProviderTurnId, type RunAttemptId, + type RuntimeRequestId, type ThreadId, type TurnItemId, } from "@t3tools/contracts"; @@ -64,6 +66,7 @@ export interface ProviderEventRoutingState { readonly ownedProviderThreadIds: ReadonlySet; readonly ownedProviderTurns: ReadonlyMap; readonly inheritedBackgroundTurnItems: ReadonlyMap; + readonly inheritedUserInputRequestIds: ReadonlySet; readonly rootProviderTurnId: ProviderTurnId | null; } @@ -329,6 +332,7 @@ function finalProviderThreadStatus( export function makeProviderEventRoutingState(input: { readonly identity: ProviderEventRouteIdentity; readonly inheritedBackgroundTurnItems?: ReadonlyArray; + readonly inheritedUserInputRequestIds?: ReadonlyArray; readonly providerTurnId: ProviderTurnId | null; readonly relatedThreadIds?: ReadonlyArray; readonly relatedProviderThreadIds?: ReadonlyArray; @@ -348,6 +352,7 @@ export function makeProviderEventRoutingState(input: { inheritedBackgroundTurnItems: new Map( (input.inheritedBackgroundTurnItems ?? []).map((item) => [item.id, item.runId]), ), + inheritedUserInputRequestIds: new Set(input.inheritedUserInputRequestIds), rootProviderTurnId: input.providerTurnId, }; } @@ -505,6 +510,8 @@ export function routeProviderEvent( case "runtime_request.updated": return [ (event.threadId !== undefined && ownsChildThread(event.threadId)) || + (event.threadId === input.threadId && + state.inheritedUserInputRequestIds.has(event.runtimeRequest.id)) || (event.runtimeRequest.providerTurnId !== null && state.ownedProviderTurns.has(event.runtimeRequest.providerTurnId)), state, @@ -565,6 +572,7 @@ export interface RunExecutionServiceV2StartRootRunInput { readonly attemptId: RunAttemptId; readonly providerTurnOrdinal: number; readonly nativeThreadHasTurns?: boolean; + readonly pendingUserInputRequests?: ReadonlyArray; readonly loadInheritedBackgroundTurnItems?: () => Effect.Effect< ReadonlyArray, unknown @@ -1053,6 +1061,9 @@ export const layer: Layer.Layer< makeProviderEventRoutingState({ identity: routeIdentity, inheritedBackgroundTurnItems, + inheritedUserInputRequestIds: (input.pendingUserInputRequests ?? []).map( + (request) => request.id, + ), providerTurnId: input.attempt.providerTurnId, ...(input.relatedThreadIds === undefined ? {} @@ -1697,6 +1708,7 @@ export const layer: Layer.Layer< runId: input.run.id, runOrdinal: input.run.ordinal, providerTurnOrdinal: input.providerTurnOrdinal, + pendingUserInputRequests: input.pendingUserInputRequests ?? [], ...(input.nativeThreadHasTurns === undefined ? {} : { nativeThreadHasTurns: input.nativeThreadHasTurns }), diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts index b90f491dfa94e..5b17d4f4a6071 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts @@ -1734,6 +1734,32 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => { .pipe(Effect.result); assert.equal(duplicate._tag, "Failure"); assert.equal((yield* orchestrator.getThreadProjection(threadId)).messages.length, 1); + + // Codex repeats the question as pending when the sleep the answer interrupted ends. + const answeredRequest = answered.runtimeRequests[0]!; + yield* eventSink.write({ + guardPendingUserInputCancellations: true, + commandId: CommandId.make("runtime-async-question-repeat"), + events: [ + { + id: EventId.make("runtime-async-question-repeat"), + type: "runtime-request.updated", + threadId, + nodeId, + occurredAt: yield* DateTime.now, + payload: { + ...answeredRequest, + status: "pending", + resolvedAt: null, + responseCapability: { type: "message" }, + }, + }, + ], + }); + assert.deepEqual( + (yield* orchestrator.getThreadProjection(threadId)).runtimeRequests, + answered.runtimeRequests, + ); }), ); diff --git a/docs/internals/providers.md b/docs/internals/providers.md index 3a31a9dac93cd..c0f95bcafe6c5 100644 --- a/docs/internals/providers.md +++ b/docs/internals/providers.md @@ -112,6 +112,11 @@ no pending RPC response to send. The `user_input_request` turn items and runtime requests with `responseCapability: { type: "message" }`. Their execution nodes do not block the run. Web, desktop, and mobile use their normal question panels, and requests remain pending after a turn finishes, a provider exits, or the server restarts. +The agent waits for an answer by calling the interruptible `clock.sleep` tool, so the adapter sets +`agentWaiting` on the request while a sleep item runs; status surfaces then show the thread waiting +for input instead of Working. The turn ending and server recovery both clear the flag, since no +agent is left waiting. The event sink drops the request's later `pending` repeats once it is +answered. `runtime-request.respond` reads the persisted request and question item, validates required answers, and commits the resolution and a user message in one transaction. Repeating the same diff --git a/docs/user/providers-codex.md b/docs/user/providers-codex.md index bdc87902a63b2..c32152994ed7c 100644 --- a/docs/user/providers-codex.md +++ b/docs/user/providers-codex.md @@ -79,7 +79,8 @@ contains a copied Codex setup. Use a fresh shadow directory and sign in again. ## Answer questions while Codex works Codex can ask a question and keep working. Answer it in the thread's question -panel. The answer becomes a new message: it reaches the active turn, or starts +panel. While Codex stops to wait for your answer, the thread shows that it needs +input and you get the usual notification. The answer becomes a new message: it reaches the active turn, or starts another turn if Codex has finished. Unanswered questions survive reconnects. If you do not want to answer, dismiss the question from its panel. Dismissing closes it without sending anything to Codex. This requires a Codex version that diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 461ec5e4b3f3e..2d439d250da64 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1228,7 +1228,11 @@ export const OrchestrationV2RuntimeRequest = Schema.Struct({ providerSessionId: ProviderSessionId, supportsFeedback: Schema.optional(Schema.Boolean), }), - Schema.Struct({ type: Schema.Literal("message") }), + Schema.Struct({ + type: Schema.Literal("message"), + // The agent has stopped working to wait for this answer. + agentWaiting: Schema.optional(Schema.Boolean), + }), Schema.Struct({ type: Schema.Literal("not_resumable"), reason: Schema.String }), ]), createdAt: Schema.DateTimeUtc, diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 9ea7597e6d087..9679678ab5b93 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -481,6 +481,7 @@ export interface ProviderAdapterV2TurnInput { readonly runId: RunId; readonly runOrdinal: number; readonly nativeThreadHasTurns?: boolean; + readonly pendingUserInputRequests?: ReadonlyArray; readonly providerTurnOrdinal: number; readonly restartContinuationOfRunId?: RunId; readonly attemptId: RunAttemptId;