-
Notifications
You must be signed in to change notification settings - Fork 0
fix: preserve headless tool-call failure truth #116
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -145,7 +145,10 @@ export namespace SessionProcessor { | |
| start: Date.now(), | ||
| }, | ||
| }, | ||
| metadata: value.providerMetadata, | ||
| metadata: { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 providerMetadata can forge providerExecuted flag. 🤖 Fix with your agentWhy this mattersWhen start: Date.now(),
},
},
metadata: {
...value.providerMetadata,
...(value.providerExecuted ? { providerExecuted: true } : {}),
},
})
toolcalls[value.toolCallId] = part as MessageV2.ToolPart |
||
| ...value.providerMetadata, | ||
| ...(value.providerExecuted ? { providerExecuted: true } : {}), | ||
| }, | ||
| }) | ||
| toolcalls[value.toolCallId] = part as MessageV2.ToolPart | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -62,6 +62,10 @@ const STRUCTURED_OUTPUT_SYSTEM_PROMPT = `IMPORTANT: The user has requested struc | |||||
| export namespace SessionPrompt { | ||||||
| const log = Log.create({ service: "session.prompt" }) | ||||||
|
|
||||||
| export function hasToolCalls(parts: MessageV2.Part[]) { | ||||||
| return parts.some((part) => part.type === "tool" && !part.metadata?.providerExecuted) | ||||||
| } | ||||||
|
|
||||||
| const state = Instance.state( | ||||||
| () => { | ||||||
| const data: Record< | ||||||
|
|
@@ -332,9 +336,11 @@ export namespace SessionPrompt { | |||||
| } | ||||||
|
|
||||||
| if (!lastUser) throw new Error("No user message found in stream. This should never happen.") | ||||||
| const lastAssistantMsg = msgs.findLast((msg) => msg.info.id === lastAssistant?.id) | ||||||
| if ( | ||||||
| lastAssistant?.finish && | ||||||
| !["tool-calls", "unknown"].includes(lastAssistant.finish) && | ||||||
| !hasToolCalls(lastAssistantMsg?.parts ?? []) && | ||||||
| lastUser.id < lastAssistant.id | ||||||
| ) { | ||||||
| log.info("exiting loop", { sessionID }) | ||||||
|
|
@@ -709,8 +715,9 @@ export namespace SessionPrompt { | |||||
|
|
||||||
| // Check if model finished (finish reason is not "tool-calls" or "unknown") | ||||||
| const modelFinished = processor.message.finish && !["tool-calls", "unknown"].includes(processor.message.finish) | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ⚪ Finish-reason filter duplicated; extract helper.
Suggested change
🤖 Fix with your agentWhy this mattersThis PR extracts hasToolCalls as a shared helper, but the adjacent finish-reason filter // Check if model finished (finish reason is not "tool-calls" or "unknown")
const modelFinished = processor.message.finish && !["tool-calls", "unknown"].includes(processor.message.finish)
const hasCurrentToolCalls = hasToolCalls(await MessageV2.parts(processor.message.id))
if (modelFinished && !hasCurrentToolCalls && !processor.message.error) { |
||||||
| const hasCurrentToolCalls = hasToolCalls(await MessageV2.parts(processor.message.id)) | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 Parts query runs even when model not finished. 🤖 Fix with your agentWhy this mattershasCurrentToolCalls awaits // Check if model finished (finish reason is not "tool-calls" or "unknown")
const modelFinished = processor.message.finish && !["tool-calls", "unknown"].includes(processor.message.finish)
const hasCurrentToolCalls = hasToolCalls(await MessageV2.parts(processor.message.id))
if (modelFinished && !hasCurrentToolCalls && !processor.message.error) { |
||||||
|
|
||||||
| if (modelFinished && !processor.message.error) { | ||||||
| if (modelFinished && !hasCurrentToolCalls && !processor.message.error) { | ||||||
| if (format.type === "json_schema") { | ||||||
| // Model stopped without calling StructuredOutput tool | ||||||
| processor.message.error = new MessageV2.StructuredOutputError({ | ||||||
|
|
||||||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -25,6 +25,19 @@ const parameters = z.object({ | |||||
| command: z.string().describe("The command that triggered this task").optional(), | ||||||
| }) | ||||||
|
|
||||||
| export function taskResultText(result: MessageV2.WithParts, sessionID: string) { | ||||||
| if (result.info.role === "assistant" && result.info.error) { | ||||||
| const data = result.info.error.data | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟠 taskResultText crashes when error.data is undefined. 🤖 Fix with your agentWhy this matterstaskResultText reads export function taskResultText(result: MessageV2.WithParts, sessionID: string) {
if (result.info.role === "assistant" && result.info.error) {
const data = result.info.error.data
const message = "message" in data && typeof data.message === "string" ? data.message : result.info.error.name
throw new Error(`Subagent failed (task_id: ${sessionID}): ${message}`)
}
const failed = result.parts.findLast((part) => part.type === "tool" && part.state.status === "error") |
||||||
| const message = "message" in data && typeof data.message === "string" ? data.message : result.info.error.name | ||||||
| throw new Error(`Subagent failed (task_id: ${sessionID}): ${message}`) | ||||||
| } | ||||||
| const failed = result.parts.findLast((part) => part.type === "tool" && part.state.status === "error") | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ⚪ Duplicated tool error check; use type predicate. 🤖 Fix with your agentWhy this mattersThe findLast predicate and the immediately following if condition repeat the same compound check ( throw new Error(`Subagent failed (task_id: ${sessionID}): ${message}`)
}
const failed = result.parts.findLast((part) => part.type === "tool" && part.state.status === "error")
if (failed?.type === "tool" && failed.state.status === "error") {
throw new Error(`Subagent failed (task_id: ${sessionID}): ${failed.state.error}`)
}
return result.parts.findLast((part) => part.type === "text")?.text ?? "" |
||||||
| if (failed?.type === "tool" && failed.state.status === "error") { | ||||||
| throw new Error(`Subagent failed (task_id: ${sessionID}): ${failed.state.error}`) | ||||||
| } | ||||||
| return result.parts.findLast((part) => part.type === "text")?.text ?? "" | ||||||
| } | ||||||
|
|
||||||
| export const TaskTool = Tool.define("task", async (ctx) => { | ||||||
| const agents = await Agent.list().then((x) => x.filter((a) => a.mode !== "primary")) | ||||||
|
|
||||||
|
|
@@ -150,7 +163,7 @@ export const TaskTool = Tool.define("task", async (ctx) => { | |||||
| parts: promptParts, | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 agent.subtask.complete skipped on child failure.
Suggested change
🤖 Fix with your agentWhy this matterstaskResultText now throws on child failure before parts: promptParts,
})
const text = taskResultText(result, session.id)
await Plugin.trigger(
"agent.subtask.complete", |
||||||
| }) | ||||||
|
|
||||||
| const text = result.parts.findLast((x) => x.type === "text")?.text ?? "" | ||||||
| const text = taskResultText(result, session.id) | ||||||
|
|
||||||
| await Plugin.trigger( | ||||||
| "agent.subtask.complete", | ||||||
|
|
||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| import { describe, expect, spyOn, test } from "bun:test" | ||
| import { Agent } from "../../src/agent/agent" | ||
| import { Identifier } from "../../src/id/id" | ||
| import { Instance } from "../../src/project/instance" | ||
| import { Provider } from "../../src/provider/provider" | ||
| import { Session } from "../../src/session" | ||
| import { LLM } from "../../src/session/llm" | ||
| import { SessionProcessor } from "../../src/session/processor" | ||
| import type { MessageV2 } from "../../src/session/message-v2" | ||
| import { tmpdir } from "../fixture/fixture" | ||
|
|
||
| describe("session processor tool metadata", () => { | ||
| test("persists provider-executed attribution from the stream", async () => { | ||
| await using tmp = await tmpdir({ | ||
| config: { | ||
| enabled_providers: ["alibaba"], | ||
| provider: { alibaba: { options: { apiKey: "test-key" } } }, | ||
| }, | ||
| }) | ||
|
|
||
| await Instance.provide({ | ||
| directory: tmp.path, | ||
| fn: async () => { | ||
| const session = await Session.create({ title: "Provider tool metadata fixture" }) | ||
| const agent = await Agent.get("build") | ||
| const model = await Provider.getModel("alibaba", "qwen-plus") | ||
| const user = (await Session.updateMessage({ | ||
| id: Identifier.ascending("message"), | ||
| sessionID: session.id, | ||
| role: "user", | ||
| time: { created: Date.now() }, | ||
| agent: agent.name, | ||
| model: { providerID: model.providerID, modelID: model.id }, | ||
| })) as MessageV2.User | ||
| const assistant = (await Session.updateMessage({ | ||
| id: Identifier.ascending("message"), | ||
| sessionID: session.id, | ||
| role: "assistant", | ||
| parentID: user.id, | ||
| modelID: model.id, | ||
| providerID: model.providerID, | ||
| mode: agent.name, | ||
| agent: agent.name, | ||
| path: { cwd: tmp.path, root: tmp.path }, | ||
| cost: 0, | ||
| tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, | ||
| time: { created: Date.now() }, | ||
| })) as MessageV2.Assistant | ||
|
|
||
| const stream = spyOn(LLM, "stream").mockResolvedValue({ | ||
| fullStream: (async function* () { | ||
| yield { type: "tool-input-start", id: "call_1", toolName: "server_tool" } | ||
| yield { | ||
| type: "tool-call", | ||
| toolCallId: "call_1", | ||
| toolName: "server_tool", | ||
| input: {}, | ||
| providerExecuted: true, | ||
| } | ||
| yield { | ||
| type: "finish-step", | ||
| finishReason: "stop", | ||
| usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, | ||
| } | ||
| })(), | ||
| } as unknown as Awaited<ReturnType<typeof LLM.stream>>) | ||
|
|
||
| try { | ||
| const processor = SessionProcessor.create({ | ||
| assistantMessage: assistant, | ||
| sessionID: session.id, | ||
| model, | ||
| abort: new AbortController().signal, | ||
| }) | ||
| await processor.process({ | ||
| user, | ||
| sessionID: session.id, | ||
| model, | ||
| agent, | ||
| abort: new AbortController().signal, | ||
| system: [], | ||
| messages: [], | ||
| tools: {}, | ||
| }) | ||
|
|
||
| const part = (await Session.messages({ sessionID: session.id })) | ||
| .flatMap((message) => message.parts) | ||
| .find((item) => item.type === "tool" && item.callID === "call_1") | ||
| expect(part?.type === "tool" ? part.metadata?.providerExecuted : undefined).toBe(true) | ||
| } finally { | ||
| stream.mockRestore() | ||
| } | ||
| }, | ||
| }) | ||
| }) | ||
| }) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,170 @@ | ||
| import { describe, expect, test } from "bun:test" | ||
| import path from "path" | ||
| import { Instance } from "../../src/project/instance" | ||
| import { Session } from "../../src/session" | ||
| import { SessionPrompt } from "../../src/session/prompt" | ||
| import type { MessageV2 } from "../../src/session/message-v2" | ||
| import { tmpdir } from "../fixture/fixture" | ||
|
|
||
| function tool(input: { status?: "pending" | "running" | "completed" | "error"; providerExecuted?: boolean }) { | ||
| const status = input.status ?? "completed" | ||
| return { | ||
| id: "part_1", | ||
| sessionID: "session_1", | ||
| messageID: "message_1", | ||
| type: "tool", | ||
| callID: "call_1", | ||
| tool: "read", | ||
| metadata: input.providerExecuted ? { providerExecuted: true } : undefined, | ||
| state: | ||
| status === "pending" | ||
| ? { status, input: {}, raw: "{}" } | ||
| : status === "running" | ||
| ? { status, input: {}, time: { start: 1 } } | ||
| : status === "completed" | ||
| ? { | ||
| status, | ||
| input: {}, | ||
| output: "ok", | ||
| title: "read", | ||
| metadata: {}, | ||
| time: { start: 1, end: 2 }, | ||
| } | ||
| : { | ||
| status, | ||
| input: {}, | ||
| error: "failed", | ||
| time: { start: 1, end: 2 }, | ||
| }, | ||
| } as MessageV2.ToolPart | ||
| } | ||
|
|
||
| function stream(chunks: unknown[]) { | ||
| const body = | ||
| chunks | ||
| .map((chunk) => `data: ${JSON.stringify(chunk)}`) | ||
| .concat("data: [DONE]") | ||
| .join("\n\n") + "\n\n" | ||
| return new Response(body, { | ||
| status: 200, | ||
| headers: { "Content-Type": "text/event-stream" }, | ||
| }) | ||
| } | ||
|
|
||
| describe("session prompt tool-call continuation", () => { | ||
| test("detects non-provider-executed tool calls that require another model turn", () => { | ||
| expect(SessionPrompt.hasToolCalls([tool({ status: "pending" })])).toBe(true) | ||
| expect(SessionPrompt.hasToolCalls([tool({ status: "completed" })])).toBe(true) | ||
| expect(SessionPrompt.hasToolCalls([tool({ status: "error" })])).toBe(true) | ||
| }) | ||
|
|
||
| test("ignores provider-executed tool calls", () => { | ||
| expect(SessionPrompt.hasToolCalls([tool({ providerExecuted: true })])).toBe(false) | ||
| }) | ||
|
|
||
| test("continues structured output after a provider reports stop with a local tool call", async () => { | ||
| const requests: Record<string, unknown>[] = [] | ||
| const server = Bun.serve({ | ||
| port: 0, | ||
| async fetch(request) { | ||
| requests.push((await request.json()) as Record<string, unknown>) | ||
| const call = requests.length === 1 ? "invalid" : "StructuredOutput" | ||
| const args = | ||
| requests.length === 1 ? { tool: "missing", error: "fixture tool call" } : { result: "follow-up reached" } | ||
| return stream([ | ||
| { | ||
| id: `chatcmpl-${requests.length}`, | ||
| object: "chat.completion.chunk", | ||
| choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], | ||
| }, | ||
| { | ||
| id: `chatcmpl-${requests.length}`, | ||
| object: "chat.completion.chunk", | ||
| choices: [ | ||
| { | ||
| index: 0, | ||
| delta: { | ||
| tool_calls: [ | ||
| { | ||
| index: 0, | ||
| id: `call_${requests.length}`, | ||
| type: "function", | ||
| function: { name: call, arguments: JSON.stringify(args) }, | ||
| }, | ||
| ], | ||
| }, | ||
| finish_reason: null, | ||
| }, | ||
| ], | ||
| }, | ||
| { | ||
| id: `chatcmpl-${requests.length}`, | ||
| object: "chat.completion.chunk", | ||
| choices: [{ index: 0, delta: {}, finish_reason: "stop" }], | ||
| usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, | ||
| }, | ||
| ]) | ||
| }, | ||
| }) | ||
|
|
||
| try { | ||
| await using tmp = await tmpdir({ | ||
| init: async (dir) => { | ||
| await Bun.write( | ||
| path.join(dir, "aictrl.json"), | ||
| JSON.stringify({ | ||
| $schema: "https://aictrl.ai/config.json", | ||
| enabled_providers: ["alibaba"], | ||
| provider: { | ||
| alibaba: { | ||
| options: { | ||
| apiKey: "test-key", | ||
| baseURL: `${server.url.origin}/v1`, | ||
| }, | ||
| }, | ||
| }, | ||
| }), | ||
| ) | ||
| }, | ||
| }) | ||
|
|
||
| await Instance.provide({ | ||
| directory: tmp.path, | ||
| fn: async () => { | ||
| const session = await Session.create({ title: "Tool continuation fixture" }) | ||
| const result = await SessionPrompt.prompt({ | ||
| sessionID: session.id, | ||
| model: { providerID: "alibaba", modelID: "qwen-plus" }, | ||
| parts: [{ type: "text", text: "Return structured output after using a tool." }], | ||
| format: { | ||
| type: "json_schema", | ||
| schema: { | ||
| type: "object", | ||
| properties: { result: { type: "string" } }, | ||
| required: ["result"], | ||
| }, | ||
| retryCount: 0, | ||
| }, | ||
| }) | ||
|
|
||
| expect(requests).toHaveLength(2) | ||
| expect(result.info.role).toBe("assistant") | ||
| if (result.info.role !== "assistant") throw new Error("Expected assistant result") | ||
| expect(result.info.structured).toEqual({ result: "follow-up reached" }) | ||
| expect(result.info.error).toBeUndefined() | ||
|
|
||
| const messages = await Session.messages({ sessionID: session.id }) | ||
| const first = messages.find( | ||
| (message) => | ||
| message.info.role === "assistant" && | ||
| message.parts.some((part) => part.type === "tool" && part.tool === "invalid"), | ||
| ) | ||
| expect(first?.info.role === "assistant" ? first.info.finish : undefined).toBe("stop") | ||
| expect(first?.info.role === "assistant" ? first.info.error : undefined).toBeUndefined() | ||
| }, | ||
| }) | ||
| } finally { | ||
| server.stop() | ||
| } | ||
| }, 15_000) | ||
| }) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🟡 providerExecuted magic string lacks shared constant.
🤖 Fix with your agent
Why this matters
The
providerExecutedkey is written into tool-part metadata in processor.ts and read back with a magic string in prompt.ts (part.metadata?.providerExecuted, packages/cli/src/session/prompt.ts:66), creating an implicit cross-file contract with no shared constant or typed field. The new tests also hardcode the string. A typo or rename on either side would silently disable the headless-truth logic with no compiler error.