diff --git a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts index f9819070fd885..a0b6e941e8a9e 100644 --- a/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts @@ -47,6 +47,7 @@ import { OPENCODE_PROVIDER, t3McpServerName, } from "@t3tools/provider-opencode/testing"; +import { toolOutputImages } from "@t3tools/shared/toolOutput"; import { resolveAttachmentPath } from "../../attachmentStore.ts"; import { openCode2ReplayRuntime } from "./OpenCode2AdapterV2.testkit.ts"; import { @@ -2296,6 +2297,10 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { active: Record, ended?: "succeeded", assistant: Record = {}, + tool: { readonly name: string; readonly content: ReadonlyArray } = { + name: "shell", + content: [{ type: "text", text: "GAP" }], + }, ) => [ { type: "runtime_exit", status: "success" } as const, out("event.subscribe"), @@ -2318,12 +2323,12 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { { type: "tool", id: "call_gap", - name: "shell", + name: tool.name, executed: true, state: { status: "completed", input: { command: "echo GAP" }, - content: [{ type: "text", text: "GAP" }], + content: tool.content, metadata: {}, }, time: { created: 2, ran: 2, completed: 2 }, @@ -2448,6 +2453,158 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => { }).pipe(Effect.scoped), ); + const PNG = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR4nGNgAAIAAAUAAXpeqz8AAAAASUVORK5CYII="; + const generated = { + type: "file", + uri: `data:image/png;base64,${PNG}`, + mime: "image/png", + name: "generated.png", + }; + const generatedBlock = { type: "image", mimeType: "image/png", data: PNG }; + const caption = { type: "text", text: "Generated a cat." }; + + const completedToolOutputs = (collected: ReadonlyArray) => + collected.flatMap((event) => + event.type === "turn_item.updated" && + event.turnItem.type === "dynamic_tool" && + event.turnItem.status === "completed" + ? [event.turnItem.output] + : [], + ); + const untilTerminal = (runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime) => + runtime.events.pipe( + Stream.takeUntil((event) => event.type === "turn.terminal"), + Stream.runCollect, + Effect.forkScoped, + ); + + const liveToolOutputs = (content: ReadonlyArray) => + Effect.gen(function* () { + const tool = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call_image" }; + const { runtime, thread } = yield* resumed([ + out("session.prompt", { sessionID: SESSION, text: "" }), + promptAccepted, + event("session.execution.started", { sessionID: SESSION }), + event("session.tool.input.started", { ...tool, name: "image_generation" }), + event("session.tool.called", { + ...tool, + name: "image_generation", + input: { prompt: "a cat" }, + executed: true, + }), + { + type: "emit_inbound", + frame: { + type: "sdk.event", + event: { + id: "evt_toolsuccess_call_image", + created: 1, + type: "session.tool.success", + data: { ...tool, content, metadata: {}, executed: true }, + durable: { aggregateID: SESSION, seq: 1, version: 2 }, + }, + }, + }, + event("session.execution.succeeded", { sessionID: SESSION }), + ]); + const events = yield* untilTerminal(runtime); + yield* runtime.startTurn(turnInput(thread)); + return completedToolOutputs(yield* Fiber.join(events)); + }).pipe(Effect.scoped); + + const historyToolOutputs = (content: ReadonlyArray) => + Effect.gen(function* () { + const { runtime, thread } = yield* resumed([ + out("session.prompt", { sessionID: SESSION, id: PROMPT_ID, text: "" }), + promptAccepted, + ...reconnected({}, "succeeded", {}, { name: "image_generation", content }), + ]); + const events = yield* untilTerminal(runtime); + yield* runtime.startTurn(turnInput(thread)); + return completedToolOutputs(yield* Fiber.join(events)); + }).pipe(Effect.scoped); + + const toolResultSources = [ + { source: "a live event", toolOutputs: liveToolOutputs }, + { source: "the session history", toolOutputs: historyToolOutputs }, + ]; + + it.effect.each(toolResultSources)( + "keeps an image a tool returned alone, read from $source", + ({ toolOutputs }) => + Effect.gen(function* () { + const outputs = yield* toolOutputs([generated]); + assert.deepEqual(outputs, [{ content: [generatedBlock] }]); + assert.deepEqual(toolOutputImages(outputs[0]), [{ mimeType: "image/png", data: PNG }]); + }), + ); + + it.effect.each(toolResultSources)( + "keeps both the text and the image of a tool result, read from $source", + ({ toolOutputs }) => + Effect.gen(function* () { + assert.deepEqual(yield* toolOutputs([caption, generated]), [ + { content: [caption, generatedBlock] }, + ]); + }), + ); + + it.effect.each(toolResultSources)( + "keeps the order of interleaved text and images in a tool result, read from $source", + ({ toolOutputs }) => + Effect.gen(function* () { + const JPEG = btoa("jpeg"); + const second = { type: "file", uri: `data:image/jpeg;base64,${JPEG}`, mime: "image/jpeg" }; + const secondBlock = { type: "image", mimeType: "image/jpeg", data: JPEG }; + const closing = { type: "text", text: "And a dog." }; + const unsupported = { type: "file", uri: "file:///work/cat.png", mime: "image/png" }; + assert.deepEqual(yield* toolOutputs([generated, caption, unsupported, second, closing]), [ + { content: [generatedBlock, caption, secondBlock, closing] }, + ]); + }), + ); + + it.effect.each(toolResultSources)( + "keeps a text-only tool result as text, read from $source", + ({ toolOutputs }) => + Effect.gen(function* () { + assert.deepEqual(yield* toolOutputs([caption]), ["Generated a cat."]); + }), + ); + + it.effect.each(toolResultSources)( + "leaves files it cannot serve out of a tool result, read from $source", + ({ toolOutputs }) => + Effect.gen(function* () { + const svg = `data:image/svg+xml;base64,${btoa("")}`; + assert.deepEqual( + yield* toolOutputs([ + caption, + { type: "file", uri: svg, mime: "image/svg+xml" }, + { type: "file", uri: "file:///work/cat.png", mime: "image/png" }, + ]), + ["Generated a cat."], + ); + }), + ); + + it.effect.each(toolResultSources)( + "leaves out an image whose data URI type conflicts with its mime, read from $source", + ({ toolOutputs }) => + Effect.gen(function* () { + const svg = btoa(""); + assert.deepEqual( + yield* toolOutputs([ + caption, + { type: "file", uri: `data:image/svg+xml;base64,${svg}`, mime: "image/png" }, + { type: "file", uri: `data:;base64,${PNG}`, mime: "image/png" }, + ]), + ["Generated a cat."], + ); + }), + ); + it.effect("finishes a workspace command whose stream dropped before its inbox event", () => Effect.gen(function* () { // session.command returns 204 without an inbox id; the stream drops before session.inbox.enqueued provides it. diff --git a/packages/provider-opencode/src/server/toolItems.ts b/packages/provider-opencode/src/server/toolItems.ts index c4188914e5b25..28de86ecd7c87 100644 --- a/packages/provider-opencode/src/server/toolItems.ts +++ b/packages/provider-opencode/src/server/toolItems.ts @@ -5,6 +5,10 @@ import { formatReadToolLabel, formatSearchToolLabel } from "@t3tools/shared/tool // Search results stay on the timeline wire, so keep their text a preview. const SEARCH_PREVIEW_MAX_CHARS = 8_000; +export type OpenCodeToolContentBlock = + | { readonly type: "text"; readonly text: string } + | { readonly type: "image"; readonly mimeType: string; readonly data: string }; + type ToolItemBase = Omit< Extract, "type" | "toolName" | "input" | "output" @@ -98,9 +102,11 @@ export function openCodeToolTurnItem( readonly input: Record; readonly output: string | undefined; readonly completedMetadata: unknown; + /** Result blocks in provider order. Only a dynamic tool can show images; other kinds stay text. */ + readonly content?: ReadonlyArray; }, ): OrchestrationV2TurnItem { - const { input, output } = tool; + const { input, output, content } = tool; switch (openCodeToolProjectionKind(tool.name)) { case "command_execution": { const exitCode = recordNumber(tool.completedMetadata, "exit", "exitCode"); @@ -169,6 +175,9 @@ export function openCodeToolTurnItem( const readPath = recordString(input, "filePath", "path", "file"); const cleanedOutput = tool.name === "read" && output !== undefined ? nativeReadOutput(output) : output; + // Text-only results stay a string; images take the block shape clients and + // the server already read tool-output images from. + const result = content?.some((block) => block.type === "image") ? { content } : cleanedOutput; return { ...base, title: @@ -178,7 +187,7 @@ export function openCodeToolTurnItem( type: "dynamic_tool", toolName: tool.name, input, - ...(cleanedOutput === undefined ? {} : { output: cleanedOutput }), + ...(result === undefined ? {} : { output: result }), }; } } diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index 81871b4b69829..98cadaad2503f 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -64,6 +64,7 @@ import { SKILL_MENTION_PATTERN } from "@t3tools/shared/composerInlineTokens"; import * as KeyedLock from "@t3tools/shared/KeyedLock"; import { getModelSelectionStringOptionValue, modelSelectionsEqual } from "@t3tools/shared/model"; import { causeErrorTag } from "@t3tools/shared/observability"; +import { readToolOutputImage } from "@t3tools/shared/toolOutput"; import { isProviderNativeImageAttachment, @@ -87,7 +88,11 @@ import { import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import { turnScopedSelectionTransition } from "@t3tools/provider-core/server/selectionTransition"; import { OPENCODE_PROVIDER, openCodePermissionRequestKind } from "../adapter.ts"; -import { openCodeCommandFailed, openCodeToolTurnItem } from "../toolItems.ts"; +import { + openCodeCommandFailed, + openCodeToolTurnItem, + type OpenCodeToolContentBlock, +} from "../toolItems.ts"; const OpenCode2ProviderCapabilities = { sessions: { @@ -501,6 +506,34 @@ const sessionIdOf = (providerThread: OrchestrationV2ProviderThread) => { const textOf = (content: ReadonlyArray<{ readonly type: string; readonly text?: string }>) => content.flatMap((part) => (part.type === "text" && part.text ? [part.text] : [])).join("\n"); +const BASE64_DATA_URI = /^data:([^;,]*)[^,]*;base64,(.+)$/s; + +/** + * An image tool such as `image_generation` answers with a file block, not text. + * Only raster images inlined as base64 are kept; anything else can't be served. + * The data URI's media type must match the block's mime, so a conflicting file isn't relabeled. + * Blocks keep the provider's order so text and images stay interleaved. + */ +const toolContentOf = ( + content: ReadonlyArray<{ + readonly type: string; + readonly text?: string; + readonly uri?: string; + readonly mime?: string; + }>, +) => + content.flatMap((part): ReadonlyArray => { + if (part.type === "text") return part.text ? [{ type: "text", text: part.text }] : []; + const match = part.type === "file" ? BASE64_DATA_URI.exec(part.uri ?? "") : null; + const image = + match == null || match[1]!.toLowerCase() !== part.mime?.toLowerCase() + ? null + : readToolOutputImage({ type: "image", mimeType: part.mime, data: match[2] }); + return image?.data === undefined + ? [] + : [{ type: "image", mimeType: image.mimeType, data: image.data }]; + }); + const stringField = (record: Readonly> | undefined, key: string) => { const value = record?.[key]; return typeof value === "string" && value.trim().length > 0 ? value : undefined; @@ -996,7 +1029,11 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr turn: ActiveTurn, id: string, status: "running" | "completed" | "failed" | "interrupted", - result?: { readonly output: string | undefined; readonly metadata: unknown }, + result?: { + readonly output: string | undefined; + readonly content?: ReadonlyArray; + readonly metadata: unknown; + }, ) { const tool = turn.tools.get(id); if (tool === undefined) return; @@ -1018,6 +1055,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr name: tool.name, input: tool.input, output: result?.output, + ...(result?.content === undefined ? {} : { content: result.content }), completedMetadata: status === "completed" ? result?.metadata : undefined, }, ), @@ -2094,9 +2132,9 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr } return yield* settleCall(call, "completed", subagentOutput(textOf(event.data.content))); } - const output = textOf(event.data.content); yield* emitTool(state, turn, event.data.id, "completed", { - output, + output: textOf(event.data.content), + content: toolContentOf(event.data.content), metadata: event.data.metadata, }); turn.tools.delete(event.data.id); @@ -2555,6 +2593,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr if (part.state.status === "completed") { yield* emitTool(state, turn, part.id, "completed", { output: textOf(part.state.content), + content: toolContentOf(part.state.content), metadata: part.state.metadata, }); turn.tools.delete(part.id);