From 04366c6b5bec79cbc153ecbf105d5952b7b5dd75 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Mon, 14 Sep 2026 11:50:37 +0100 Subject: [PATCH 1/2] fix: preserve headless tool failure truth --- packages/cli/src/session/prompt.ts | 11 +++ packages/cli/src/tool/task.ts | 15 ++- .../cli/test/session/prompt-tool-loop.test.ts | 54 +++++++++++ packages/cli/test/tool/task.test.ts | 92 +++++++++++++++++++ 4 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 packages/cli/test/session/prompt-tool-loop.test.ts create mode 100644 packages/cli/test/tool/task.test.ts diff --git a/packages/cli/src/session/prompt.ts b/packages/cli/src/session/prompt.ts index 96f8ff9..e22ac47 100644 --- a/packages/cli/src/session/prompt.ts +++ b/packages/cli/src/session/prompt.ts @@ -62,6 +62,15 @@ const STRUCTURED_OUTPUT_SYSTEM_PROMPT = `IMPORTANT: The user has requested struc export namespace SessionPrompt { const log = Log.create({ service: "session.prompt" }) + export function hasToolCalls(parts: MessageV2.Part[]) { + return parts.some( + (part) => + part.type === "tool" && + !part.metadata?.providerExecuted && + !(part.state.status === "error" && part.state.metadata?.interrupted === true), + ) + } + const state = Instance.state( () => { const data: Record< @@ -332,9 +341,11 @@ export namespace SessionPrompt { } if (!lastUser) throw new Error("No user message found in stream. This should never happen.") + const lastAssistantMsg = msgs.findLast((msg) => msg.info.id === lastAssistant?.id) if ( lastAssistant?.finish && !["tool-calls", "unknown"].includes(lastAssistant.finish) && + !hasToolCalls(lastAssistantMsg?.parts ?? []) && lastUser.id < lastAssistant.id ) { log.info("exiting loop", { sessionID }) diff --git a/packages/cli/src/tool/task.ts b/packages/cli/src/tool/task.ts index 5e1a0ae..cb21ba7 100644 --- a/packages/cli/src/tool/task.ts +++ b/packages/cli/src/tool/task.ts @@ -25,6 +25,19 @@ const parameters = z.object({ command: z.string().describe("The command that triggered this task").optional(), }) +export function taskResultText(result: MessageV2.WithParts, sessionID: string) { + if (result.info.role === "assistant" && result.info.error) { + const data = result.info.error.data + const message = "message" in data && typeof data.message === "string" ? data.message : result.info.error.name + throw new Error(`Subagent failed (task_id: ${sessionID}): ${message}`) + } + const failed = result.parts.findLast((part) => part.type === "tool" && part.state.status === "error") + if (failed?.type === "tool" && failed.state.status === "error") { + throw new Error(`Subagent failed (task_id: ${sessionID}): ${failed.state.error}`) + } + return result.parts.findLast((part) => part.type === "text")?.text ?? "" +} + export const TaskTool = Tool.define("task", async (ctx) => { const agents = await Agent.list().then((x) => x.filter((a) => a.mode !== "primary")) @@ -150,7 +163,7 @@ export const TaskTool = Tool.define("task", async (ctx) => { parts: promptParts, }) - const text = result.parts.findLast((x) => x.type === "text")?.text ?? "" + const text = taskResultText(result, session.id) await Plugin.trigger( "agent.subtask.complete", diff --git a/packages/cli/test/session/prompt-tool-loop.test.ts b/packages/cli/test/session/prompt-tool-loop.test.ts new file mode 100644 index 0000000..9fa76d8 --- /dev/null +++ b/packages/cli/test/session/prompt-tool-loop.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "bun:test" +import { SessionPrompt } from "../../src/session/prompt" +import type { MessageV2 } from "../../src/session/message-v2" + +function tool(input: { + status?: "pending" | "running" | "completed" | "error" + providerExecuted?: boolean + interrupted?: boolean +}) { + const status = input.status ?? "completed" + return { + id: "part_1", + sessionID: "session_1", + messageID: "message_1", + type: "tool", + callID: "call_1", + tool: "read", + metadata: input.providerExecuted ? { providerExecuted: true } : undefined, + state: + status === "pending" + ? { status, input: {}, raw: "{}" } + : status === "running" + ? { status, input: {}, time: { start: 1 } } + : status === "completed" + ? { + status, + input: {}, + output: "ok", + title: "read", + metadata: {}, + time: { start: 1, end: 2 }, + } + : { + status, + input: {}, + error: "failed", + metadata: input.interrupted ? { interrupted: true } : undefined, + time: { start: 1, end: 2 }, + }, + } as MessageV2.ToolPart +} + +describe("session prompt tool-call continuation", () => { + test("detects non-provider-executed tool calls that require another model turn", () => { + expect(SessionPrompt.hasToolCalls([tool({ status: "pending" })])).toBe(true) + expect(SessionPrompt.hasToolCalls([tool({ status: "completed" })])).toBe(true) + expect(SessionPrompt.hasToolCalls([tool({ status: "error" })])).toBe(true) + }) + + test("ignores provider-executed and cleanup-interrupted tool calls", () => { + expect(SessionPrompt.hasToolCalls([tool({ providerExecuted: true })])).toBe(false) + expect(SessionPrompt.hasToolCalls([tool({ status: "error", interrupted: true })])).toBe(false) + }) +}) diff --git a/packages/cli/test/tool/task.test.ts b/packages/cli/test/tool/task.test.ts new file mode 100644 index 0000000..e6e6f2c --- /dev/null +++ b/packages/cli/test/tool/task.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, test } from "bun:test" +import { MessageV2 } from "../../src/session/message-v2" +import { taskResultText } from "../../src/tool/task" + +function result(input: { error?: MessageV2.Assistant["error"]; parts?: MessageV2.Part[] }): MessageV2.WithParts { + return { + info: { + id: "message_1", + sessionID: "child_1", + role: "assistant", + time: { created: 1 }, + error: input.error, + parentID: "user_1", + modelID: "model", + providerID: "provider", + mode: "build", + agent: "build", + path: { cwd: "/workspace", root: "/workspace" }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + }, + parts: input.parts ?? [], + } +} + +describe("task tool child result", () => { + test("surfaces a child session error with child attribution", () => { + expect(() => + taskResultText( + result({ + error: new MessageV2.APIError({ + message: "provider unavailable", + isRetryable: false, + }).toObject(), + }), + "child_1", + ), + ).toThrow("Subagent failed (task_id: child_1): provider unavailable") + }) + + test("surfaces the last failed child tool instead of returning empty success", () => { + expect(() => + taskResultText( + result({ + parts: [ + { + id: "part_1", + sessionID: "child_1", + messageID: "message_1", + type: "tool", + callID: "call_1", + tool: "read", + state: { + status: "error", + input: {}, + error: "permission denied", + time: { start: 1, end: 2 }, + }, + }, + ], + }), + "child_1", + ), + ).toThrow("Subagent failed (task_id: child_1): permission denied") + }) + + test("returns the last child text when execution succeeds", () => { + expect( + taskResultText( + result({ + parts: [ + { + id: "part_1", + sessionID: "child_1", + messageID: "message_1", + type: "text", + text: "first", + }, + { + id: "part_2", + sessionID: "child_1", + messageID: "message_1", + type: "text", + text: "done", + }, + ], + }), + "child_1", + ), + ).toBe("done") + }) +}) From 532f00f4f032bc82b90fe02bc7d5e63b8c038319 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Mon, 14 Sep 2026 12:03:12 +0100 Subject: [PATCH 2/2] fix: continue structured output after tool calls --- packages/cli/src/session/processor.ts | 5 +- packages/cli/src/session/prompt.ts | 10 +- .../session/processor-tool-metadata.test.ts | 96 +++++++++++++ .../cli/test/session/prompt-tool-loop.test.ts | 132 ++++++++++++++++-- 4 files changed, 227 insertions(+), 16 deletions(-) create mode 100644 packages/cli/test/session/processor-tool-metadata.test.ts diff --git a/packages/cli/src/session/processor.ts b/packages/cli/src/session/processor.ts index 2577169..f7eba32 100644 --- a/packages/cli/src/session/processor.ts +++ b/packages/cli/src/session/processor.ts @@ -145,7 +145,10 @@ export namespace SessionProcessor { start: Date.now(), }, }, - metadata: value.providerMetadata, + metadata: { + ...value.providerMetadata, + ...(value.providerExecuted ? { providerExecuted: true } : {}), + }, }) toolcalls[value.toolCallId] = part as MessageV2.ToolPart diff --git a/packages/cli/src/session/prompt.ts b/packages/cli/src/session/prompt.ts index e22ac47..9265f89 100644 --- a/packages/cli/src/session/prompt.ts +++ b/packages/cli/src/session/prompt.ts @@ -63,12 +63,7 @@ export namespace SessionPrompt { const log = Log.create({ service: "session.prompt" }) export function hasToolCalls(parts: MessageV2.Part[]) { - return parts.some( - (part) => - part.type === "tool" && - !part.metadata?.providerExecuted && - !(part.state.status === "error" && part.state.metadata?.interrupted === true), - ) + return parts.some((part) => part.type === "tool" && !part.metadata?.providerExecuted) } const state = Instance.state( @@ -720,8 +715,9 @@ export namespace SessionPrompt { // Check if model finished (finish reason is not "tool-calls" or "unknown") const modelFinished = processor.message.finish && !["tool-calls", "unknown"].includes(processor.message.finish) + const hasCurrentToolCalls = hasToolCalls(await MessageV2.parts(processor.message.id)) - if (modelFinished && !processor.message.error) { + if (modelFinished && !hasCurrentToolCalls && !processor.message.error) { if (format.type === "json_schema") { // Model stopped without calling StructuredOutput tool processor.message.error = new MessageV2.StructuredOutputError({ diff --git a/packages/cli/test/session/processor-tool-metadata.test.ts b/packages/cli/test/session/processor-tool-metadata.test.ts new file mode 100644 index 0000000..72c8ce0 --- /dev/null +++ b/packages/cli/test/session/processor-tool-metadata.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, spyOn, test } from "bun:test" +import { Agent } from "../../src/agent/agent" +import { Identifier } from "../../src/id/id" +import { Instance } from "../../src/project/instance" +import { Provider } from "../../src/provider/provider" +import { Session } from "../../src/session" +import { LLM } from "../../src/session/llm" +import { SessionProcessor } from "../../src/session/processor" +import type { MessageV2 } from "../../src/session/message-v2" +import { tmpdir } from "../fixture/fixture" + +describe("session processor tool metadata", () => { + test("persists provider-executed attribution from the stream", async () => { + await using tmp = await tmpdir({ + config: { + enabled_providers: ["alibaba"], + provider: { alibaba: { options: { apiKey: "test-key" } } }, + }, + }) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const session = await Session.create({ title: "Provider tool metadata fixture" }) + const agent = await Agent.get("build") + const model = await Provider.getModel("alibaba", "qwen-plus") + const user = (await Session.updateMessage({ + id: Identifier.ascending("message"), + sessionID: session.id, + role: "user", + time: { created: Date.now() }, + agent: agent.name, + model: { providerID: model.providerID, modelID: model.id }, + })) as MessageV2.User + const assistant = (await Session.updateMessage({ + id: Identifier.ascending("message"), + sessionID: session.id, + role: "assistant", + parentID: user.id, + modelID: model.id, + providerID: model.providerID, + mode: agent.name, + agent: agent.name, + path: { cwd: tmp.path, root: tmp.path }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: Date.now() }, + })) as MessageV2.Assistant + + const stream = spyOn(LLM, "stream").mockResolvedValue({ + fullStream: (async function* () { + yield { type: "tool-input-start", id: "call_1", toolName: "server_tool" } + yield { + type: "tool-call", + toolCallId: "call_1", + toolName: "server_tool", + input: {}, + providerExecuted: true, + } + yield { + type: "finish-step", + finishReason: "stop", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + } + })(), + } as unknown as Awaited>) + + try { + const processor = SessionProcessor.create({ + assistantMessage: assistant, + sessionID: session.id, + model, + abort: new AbortController().signal, + }) + await processor.process({ + user, + sessionID: session.id, + model, + agent, + abort: new AbortController().signal, + system: [], + messages: [], + tools: {}, + }) + + const part = (await Session.messages({ sessionID: session.id })) + .flatMap((message) => message.parts) + .find((item) => item.type === "tool" && item.callID === "call_1") + expect(part?.type === "tool" ? part.metadata?.providerExecuted : undefined).toBe(true) + } finally { + stream.mockRestore() + } + }, + }) + }) +}) diff --git a/packages/cli/test/session/prompt-tool-loop.test.ts b/packages/cli/test/session/prompt-tool-loop.test.ts index 9fa76d8..8cfba1c 100644 --- a/packages/cli/test/session/prompt-tool-loop.test.ts +++ b/packages/cli/test/session/prompt-tool-loop.test.ts @@ -1,12 +1,12 @@ import { describe, expect, test } from "bun:test" +import path from "path" +import { Instance } from "../../src/project/instance" +import { Session } from "../../src/session" import { SessionPrompt } from "../../src/session/prompt" import type { MessageV2 } from "../../src/session/message-v2" +import { tmpdir } from "../fixture/fixture" -function tool(input: { - status?: "pending" | "running" | "completed" | "error" - providerExecuted?: boolean - interrupted?: boolean -}) { +function tool(input: { status?: "pending" | "running" | "completed" | "error"; providerExecuted?: boolean }) { const status = input.status ?? "completed" return { id: "part_1", @@ -34,12 +34,23 @@ function tool(input: { status, input: {}, error: "failed", - metadata: input.interrupted ? { interrupted: true } : undefined, time: { start: 1, end: 2 }, }, } as MessageV2.ToolPart } +function stream(chunks: unknown[]) { + const body = + chunks + .map((chunk) => `data: ${JSON.stringify(chunk)}`) + .concat("data: [DONE]") + .join("\n\n") + "\n\n" + return new Response(body, { + status: 200, + headers: { "Content-Type": "text/event-stream" }, + }) +} + describe("session prompt tool-call continuation", () => { test("detects non-provider-executed tool calls that require another model turn", () => { expect(SessionPrompt.hasToolCalls([tool({ status: "pending" })])).toBe(true) @@ -47,8 +58,113 @@ describe("session prompt tool-call continuation", () => { expect(SessionPrompt.hasToolCalls([tool({ status: "error" })])).toBe(true) }) - test("ignores provider-executed and cleanup-interrupted tool calls", () => { + test("ignores provider-executed tool calls", () => { expect(SessionPrompt.hasToolCalls([tool({ providerExecuted: true })])).toBe(false) - expect(SessionPrompt.hasToolCalls([tool({ status: "error", interrupted: true })])).toBe(false) }) + + test("continues structured output after a provider reports stop with a local tool call", async () => { + const requests: Record[] = [] + const server = Bun.serve({ + port: 0, + async fetch(request) { + requests.push((await request.json()) as Record) + const call = requests.length === 1 ? "invalid" : "StructuredOutput" + const args = + requests.length === 1 ? { tool: "missing", error: "fixture tool call" } : { result: "follow-up reached" } + return stream([ + { + id: `chatcmpl-${requests.length}`, + object: "chat.completion.chunk", + choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], + }, + { + id: `chatcmpl-${requests.length}`, + object: "chat.completion.chunk", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + id: `call_${requests.length}`, + type: "function", + function: { name: call, arguments: JSON.stringify(args) }, + }, + ], + }, + finish_reason: null, + }, + ], + }, + { + id: `chatcmpl-${requests.length}`, + object: "chat.completion.chunk", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }, + ]) + }, + }) + + try { + await using tmp = await tmpdir({ + init: async (dir) => { + await Bun.write( + path.join(dir, "aictrl.json"), + JSON.stringify({ + $schema: "https://aictrl.ai/config.json", + enabled_providers: ["alibaba"], + provider: { + alibaba: { + options: { + apiKey: "test-key", + baseURL: `${server.url.origin}/v1`, + }, + }, + }, + }), + ) + }, + }) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const session = await Session.create({ title: "Tool continuation fixture" }) + const result = await SessionPrompt.prompt({ + sessionID: session.id, + model: { providerID: "alibaba", modelID: "qwen-plus" }, + parts: [{ type: "text", text: "Return structured output after using a tool." }], + format: { + type: "json_schema", + schema: { + type: "object", + properties: { result: { type: "string" } }, + required: ["result"], + }, + retryCount: 0, + }, + }) + + expect(requests).toHaveLength(2) + expect(result.info.role).toBe("assistant") + if (result.info.role !== "assistant") throw new Error("Expected assistant result") + expect(result.info.structured).toEqual({ result: "follow-up reached" }) + expect(result.info.error).toBeUndefined() + + const messages = await Session.messages({ sessionID: session.id }) + const first = messages.find( + (message) => + message.info.role === "assistant" && + message.parts.some((part) => part.type === "tool" && part.tool === "invalid"), + ) + expect(first?.info.role === "assistant" ? first.info.finish : undefined).toBe("stop") + expect(first?.info.role === "assistant" ? first.info.error : undefined).toBeUndefined() + }, + }) + } finally { + server.stop() + } + }, 15_000) })