Skip to content

Commit c22be2d

Browse files
Tom-R-MainTom-R-Main
committed
Add multimodal content + reasoning-effort support to adapters
- Introduce ChatContent (text + image parts) and content.ts helpers; ChatMessage.content and ChatAgent.chat() now accept multimodal input. - All adapters (Anthropic, Gemini, OpenAI, xAI) encode image parts to their provider-native shapes. - Add normalized ReasoningEffort: OpenAI/OpenRouter map to reasoning.effort, Anthropic maps to extended-thinking budget_tokens. - Preserve Anthropic thinking/redacted_thinking blocks across tool-call turns so follow-up requests don't 400. - Normalize exf_tasks_list status to the real enum (synonyms + drop unknown).
1 parent 7fb3762 commit c22be2d

12 files changed

Lines changed: 327 additions & 49 deletions

File tree

‎src/framework/adapters/anthropic.ts‎

Lines changed: 74 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,26 @@
55
* Default model: claude-sonnet-4-6
66
*/
77

8-
import type { AIAdapter, AdapterConfig, ChatMessage, ChatOptions, AdapterResponse } from "./types.js";
8+
import type { AIAdapter, AdapterConfig, ChatMessage, ChatOptions, AdapterResponse, ReasoningEffort } from "./types.js";
99
import type { ToolRegistry } from "../registry.js";
1010
import { safeJsonParse } from "./util.js";
11+
import { chatContentToText, imageBase64, imageMime, normalizeChatContent } from "./content.js";
12+
13+
/**
14+
* Anthropic has no `effort` knob — extended thinking is controlled by a token
15+
* budget. Translate the normalized effort level to a budget; `none`/`minimal`
16+
* mean "no thinking". Budgets must be >= 1024 and strictly < max_tokens.
17+
*/
18+
const THINKING_BUDGET: Record<ReasoningEffort, number> = {
19+
none: 0,
20+
minimal: 0,
21+
low: 2048,
22+
medium: 6144,
23+
high: 12288,
24+
xhigh: 24576,
25+
};
26+
/** Output headroom reserved for the answer on top of the thinking budget. */
27+
const ANSWER_HEADROOM = 8192;
1128

1229
export function createAnthropicAdapter(config?: Partial<AdapterConfig>): AIAdapter {
1330
const apiKey = config?.apiKey ?? process.env.ANTHROPIC_API_KEY;
@@ -19,6 +36,8 @@ export function createAnthropicAdapter(config?: Partial<AdapterConfig>): AIAdapt
1936

2037
const model = config?.model ?? "claude-sonnet-4-6";
2138
const systemPrompt = config?.systemPrompt ?? "You are a helpful assistant with access to tools. Use tools when they're relevant.";
39+
const thinkingBudget = config?.reasoningEffort ? THINKING_BUDGET[config.reasoningEffort] : 0;
40+
const thinkingEnabled = thinkingBudget > 0;
2241

2342
return {
2443
name: config?.name ?? "Claude",
@@ -32,24 +51,30 @@ export function createAnthropicAdapter(config?: Partial<AdapterConfig>): AIAdapt
3251
if (msg.role === "user") {
3352
anthropicMessages.push({
3453
role: "user",
35-
content: msg.content,
54+
content: anthropicContent(msg.content),
3655
});
3756
} else if (msg.role === "assistant") {
3857
// If this assistant message preceded a tool call, we need the tool_use block
3958
if (msg.toolCallId && msg.toolName) {
59+
const toolUse = {
60+
type: "tool_use",
61+
id: msg.toolCallId,
62+
name: msg.toolName,
63+
input: safeJsonParse(chatContentToText(msg.content), {}),
64+
};
65+
// Extended thinking REQUIRES the original thinking block(s) to lead
66+
// the assistant message that contains the tool_use, replayed
67+
// verbatim (signatures are validated). The agent loop stashes them
68+
// on `thinkingBlocks`; without this the follow-up turn 400s.
69+
const preserved = Array.isArray(msg.thinkingBlocks) ? msg.thinkingBlocks : [];
4070
anthropicMessages.push({
4171
role: "assistant",
42-
content: [{
43-
type: "tool_use",
44-
id: msg.toolCallId,
45-
name: msg.toolName,
46-
input: safeJsonParse(msg.content, {}),
47-
}],
72+
content: preserved.length ? [...preserved, toolUse] : [toolUse],
4873
});
4974
} else {
5075
anthropicMessages.push({
5176
role: "assistant",
52-
content: msg.content,
77+
content: chatContentToText(msg.content),
5378
});
5479
}
5580
} else if (msg.role === "tool") {
@@ -58,7 +83,7 @@ export function createAnthropicAdapter(config?: Partial<AdapterConfig>): AIAdapt
5883
content: [{
5984
type: "tool_result",
6085
tool_use_id: msg.toolCallId!,
61-
content: msg.content,
86+
content: chatContentToText(msg.content),
6287
}],
6388
});
6489
}
@@ -67,12 +92,18 @@ export function createAnthropicAdapter(config?: Partial<AdapterConfig>): AIAdapt
6792
const tools = registry.toAnthropicFormat();
6893
const body: Record<string, unknown> = {
6994
model,
70-
max_tokens: 2048,
95+
// Extended thinking requires max_tokens > budget_tokens; reserve answer
96+
// headroom on top of the budget when thinking is on.
97+
max_tokens: thinkingEnabled ? thinkingBudget + ANSWER_HEADROOM : 2048,
7198
system: options?.systemPrompt ?? systemPrompt,
7299
messages: anthropicMessages,
73100
tools,
74101
};
75102

103+
if (thinkingEnabled) {
104+
body.thinking = { type: "enabled", budget_tokens: thinkingBudget };
105+
}
106+
76107
// Tool choice + disable parallel tool use.
77108
// Without disable_parallel_tool_use, Anthropic may emit multiple
78109
// tool_use blocks in one assistant turn. Our adapter only returns
@@ -86,11 +117,16 @@ export function createAnthropicAdapter(config?: Partial<AdapterConfig>): AIAdapt
86117
type: "auto",
87118
disable_parallel_tool_use: true,
88119
};
89-
if (options?.toolChoice === "required") {
90-
toolChoice.type = "any";
91-
} else if (typeof options?.toolChoice === "object") {
92-
toolChoice.type = "tool";
93-
toolChoice.name = options.toolChoice.name;
120+
// Extended thinking is incompatible with forced tool use (type "any"
121+
// or "tool") — Anthropic requires "auto". Honor the caller's forcing
122+
// only when thinking is off.
123+
if (!thinkingEnabled) {
124+
if (options?.toolChoice === "required") {
125+
toolChoice.type = "any";
126+
} else if (typeof options?.toolChoice === "object") {
127+
toolChoice.type = "tool";
128+
toolChoice.name = options.toolChoice.name;
129+
}
94130
}
95131
body.tool_choice = toolChoice;
96132
}
@@ -122,17 +158,39 @@ export function createAnthropicAdapter(config?: Partial<AdapterConfig>): AIAdapt
122158
const toolUseBlock = blocks.find((b) => b.type === "tool_use");
123159

124160
if (toolUseBlock) {
161+
// Preserve thinking/redacted_thinking blocks verbatim so the agent loop
162+
// can replay them on the assistant turn that continues this tool call.
163+
const thinking = blocks.filter(
164+
(b) => b.type === "thinking" || b.type === "redacted_thinking"
165+
);
125166
return {
126167
toolCall: {
127168
id: toolUseBlock.id!,
128169
name: toolUseBlock.name!,
129170
args: (toolUseBlock.input as Record<string, unknown>) ?? {},
130171
},
131172
...(textBlock?.text && { text: textBlock.text }),
173+
...(thinking.length && { thinking }),
132174
};
133175
}
134176

135177
return { text: textBlock?.text ?? "(no response)" };
136178
},
137179
};
138180
}
181+
182+
function anthropicContent(content: ChatMessage["content"]): unknown {
183+
if (typeof content === "string") return content;
184+
return normalizeChatContent(content).map((part) =>
185+
part.type === "text"
186+
? { type: "text", text: part.text }
187+
: {
188+
type: "image",
189+
source: {
190+
type: "base64",
191+
media_type: imageMime(part),
192+
data: imageBase64(part),
193+
},
194+
}
195+
);
196+
}

‎src/framework/adapters/content.ts‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
import { readFileSync } from "node:fs";
2+
import { extname } from "node:path";
3+
import type { ChatContent, ContentPart, ImageContentPart } from "./types.js";
4+
5+
export function textPart(text: string): ContentPart {
6+
return { type: "text", text };
7+
}
8+
9+
export function normalizeChatContent(content: ChatContent): ContentPart[] {
10+
return typeof content === "string" ? [textPart(content)] : content;
11+
}
12+
13+
export function chatContentToText(content: ChatContent): string {
14+
if (typeof content === "string") return content;
15+
return content
16+
.map((part, index) => {
17+
if (part.type === "text") return part.text;
18+
return `[image ${index + 1}${part.mime ? ` · ${part.mime}` : ""}]`;
19+
})
20+
.join("");
21+
}
22+
23+
export function imageDataUrl(part: ImageContentPart): string {
24+
if (part.dataUrl) return part.dataUrl;
25+
if (!part.path) throw new Error("image content part requires dataUrl or path");
26+
const bytes = readFileSync(part.path);
27+
return `data:${part.mime || mimeFromPath(part.path)};base64,${bytes.toString("base64")}`;
28+
}
29+
30+
export function imageBase64(part: ImageContentPart): string {
31+
const dataUrl = imageDataUrl(part);
32+
const marker = ";base64,";
33+
const idx = dataUrl.indexOf(marker);
34+
if (!dataUrl.startsWith("data:") || idx < 0) {
35+
throw new Error("image dataUrl must be a base64 data URL");
36+
}
37+
return dataUrl.slice(idx + marker.length);
38+
}
39+
40+
export function imageMime(part: ImageContentPart): string {
41+
if (part.mime) return part.mime;
42+
if (part.dataUrl?.startsWith("data:")) {
43+
const end = part.dataUrl.indexOf(";");
44+
if (end > 5) return part.dataUrl.slice(5, end);
45+
}
46+
return part.path ? mimeFromPath(part.path) : "image/png";
47+
}
48+
49+
export function mimeFromPath(path: string): string {
50+
switch (extname(path).toLowerCase()) {
51+
case ".jpg":
52+
case ".jpeg":
53+
return "image/jpeg";
54+
case ".gif":
55+
return "image/gif";
56+
case ".webp":
57+
return "image/webp";
58+
case ".png":
59+
default:
60+
return "image/png";
61+
}
62+
}

‎src/framework/adapters/gemini.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import type { AIAdapter, AdapterConfig, ChatMessage, ChatOptions, AdapterResponse } from "./types.js";
99
import type { ToolRegistry } from "../registry.js";
1010
import { safeJsonParse } from "./util.js";
11+
import { chatContentToText, imageBase64, imageMime, normalizeChatContent } from "./content.js";
1112

1213
export function createGeminiAdapter(config?: Partial<AdapterConfig>): AIAdapter {
1314
const apiKey = config?.apiKey ?? process.env.GEMINI_API_KEY;
@@ -33,14 +34,18 @@ export function createGeminiAdapter(config?: Partial<AdapterConfig>): AIAdapter
3334
parts: [{
3435
functionResponse: {
3536
name: msg.toolName!,
36-
response: safeJsonParse(msg.content, { result: msg.content }),
37+
response: safeJsonParse(chatContentToText(msg.content), { result: chatContentToText(msg.content) }),
3738
},
3839
}],
3940
};
4041
}
4142
return {
4243
role: msg.role === "assistant" ? "model" as const : "user" as const,
43-
parts: [{ text: msg.content }],
44+
parts: normalizeChatContent(msg.content).map((part) =>
45+
part.type === "text"
46+
? { text: part.text }
47+
: { inlineData: { mimeType: imageMime(part), data: imageBase64(part) } }
48+
),
4449
};
4550
});
4651

‎src/framework/adapters/openai.ts‎

Lines changed: 39 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
* Any OpenAI-compatible provider can be added with createChatCompletionsAdapter().
88
*/
99

10-
import type { AIAdapter, AdapterConfig, ChatMessage, ChatOptions, AdapterResponse } from "./types.js";
10+
import type { AIAdapter, AdapterConfig, ChatMessage, ChatOptions, AdapterResponse, ReasoningEffort } from "./types.js";
1111
import type { ToolRegistry } from "../registry.js";
12+
import { chatContentToText, imageDataUrl, normalizeChatContent } from "./content.js";
1213

1314
// ─── OpenAI Responses API ──────────────────────────────────────────────────
1415

@@ -50,18 +51,18 @@ export function createOpenAIAdapter(config?: Partial<AdapterConfig>): AIAdapter
5051
input = [{
5152
type: "function_call_output",
5253
call_id: lastMsg.toolCallId!,
53-
output: lastMsg.content,
54+
output: chatContentToText(lastMsg.content),
5455
}];
5556
} else {
5657
// User follow-up after a completed turn. Keep the
5758
// previous_response_id so OpenAI threads this turn onto the
5859
// prior conversation server-side. Previously we discarded
5960
// the id here, which restarted context every user message.
60-
input = lastMsg.content;
61+
input = responsesInputContent(lastMsg.content);
6162
}
6263
} else {
6364
const lastUser = messages.filter((m) => m.role === "user").pop();
64-
input = lastUser?.content ?? "";
65+
input = lastUser ? responsesInputContent(lastUser.content) : "";
6566
}
6667

6768
const tools = registry.getAll().map((tool) => ({
@@ -145,14 +146,15 @@ interface ChatCompletionsConfig {
145146
apiKey: string;
146147
baseUrl: string;
147148
systemPrompt?: string;
149+
reasoningEffort?: ReasoningEffort;
148150
}
149151

150152
/**
151153
* Generic adapter for any OpenAI-compatible Chat Completions API.
152154
* Used by OpenRouter and any other compatible provider.
153155
*/
154156
function createChatCompletionsAdapter(config: ChatCompletionsConfig): AIAdapter {
155-
const { name, model, apiKey, baseUrl } = config;
157+
const { name, model, apiKey, baseUrl, reasoningEffort } = config;
156158
const sysPrompt = config.systemPrompt ?? "You are a helpful assistant with access to tools. Use tools when they're relevant.";
157159

158160
return {
@@ -170,7 +172,7 @@ function createChatCompletionsAdapter(config: ChatCompletionsConfig): AIAdapter
170172
openaiMessages.push({
171173
role: "tool",
172174
tool_call_id: msg.toolCallId!,
173-
content: msg.content,
175+
content: chatContentToText(msg.content),
174176
});
175177
} else if (msg.role === "assistant" && msg.toolCallId) {
176178
openaiMessages.push({
@@ -181,14 +183,14 @@ function createChatCompletionsAdapter(config: ChatCompletionsConfig): AIAdapter
181183
type: "function",
182184
function: {
183185
name: msg.toolName!,
184-
arguments: msg.content,
186+
arguments: chatContentToText(msg.content),
185187
},
186188
}],
187189
});
188190
} else {
189191
openaiMessages.push({
190192
role: msg.role,
191-
content: msg.content,
193+
content: chatCompletionsContent(msg.content),
192194
});
193195
}
194196
}
@@ -203,6 +205,13 @@ function createChatCompletionsAdapter(config: ChatCompletionsConfig): AIAdapter
203205
parallel_tool_calls: false,
204206
};
205207

208+
// Reasoning effort. OpenRouter's unified `reasoning` field normalizes
209+
// across model families: OpenAI-style `effort`, Anthropic-style thinking
210+
// budgets, and Gemini all accept `{ effort }`. `none` disables it.
211+
if (reasoningEffort && reasoningEffort !== "none") {
212+
body.reasoning = { effort: reasoningEffort };
213+
}
214+
206215
// Tool choice support
207216
if (options?.toolChoice === "required") {
208217
body.tool_choice = "required";
@@ -244,6 +253,27 @@ function createChatCompletionsAdapter(config: ChatCompletionsConfig): AIAdapter
244253
};
245254
}
246255

256+
function responsesInputContent(content: ChatMessage["content"]): unknown {
257+
if (typeof content === "string") return content;
258+
return [{
259+
role: "user",
260+
content: content.map((part) =>
261+
part.type === "text"
262+
? { type: "input_text", text: part.text }
263+
: { type: "input_image", image_url: imageDataUrl(part), detail: part.detail ?? "auto" }
264+
),
265+
}];
266+
}
267+
268+
function chatCompletionsContent(content: ChatMessage["content"]): unknown {
269+
if (typeof content === "string") return content;
270+
return normalizeChatContent(content).map((part) =>
271+
part.type === "text"
272+
? { type: "text", text: part.text }
273+
: { type: "image_url", image_url: { url: imageDataUrl(part), detail: part.detail ?? "auto" } }
274+
);
275+
}
276+
247277
// ─── OpenRouter ────────────────────────────────────────────────────────────
248278

249279
/**
@@ -267,6 +297,6 @@ export function createOpenRouterAdapter(config?: Partial<AdapterConfig>): AIAdap
267297
apiKey,
268298
baseUrl: "https://openrouter.ai/api/v1",
269299
systemPrompt: config?.systemPrompt,
300+
reasoningEffort: config?.reasoningEffort,
270301
});
271302
}
272-

0 commit comments

Comments
 (0)