diff --git a/CHANGELOG.md b/CHANGELOG.md index fecf8c25..247110a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,10 +37,12 @@ opt-out. - `lousho doctor --ping` checks OpenRouter at `/api/v1/key`, which needs the key (its `/models` answers without one, so a wrong key passed); a key the provider rejects is now a failure, and a ping of a custom `OPENAI_BASE_URL` says it shows reachability only. - `lousho doctor` fix lines use PowerShell syntax (`$env:NAME=""`) on Windows. - The `lousho eval` table shows the scores of passing scored gates (such as `t.check('words', n, atLeast(3))`), not only in `--json`. +- **Behavior change.** An agent with `output` and tools no longer sends the strict `responseFormat` schema on calls that offer tools: `openai/gpt-4o-mini` via OpenRouter then called tools on every step until `maxSteps` and never answered. `responseFormat` now goes only on calls without tools (an agent without tools, the "answer now" call at `maxSteps`, the repair step); tool-offering calls answer from the schema in the prompt, with the existing parse and repair. `output: { schema, responseFormat: 'always' }` restores the old behavior, and is the default with `promptSchema: false`. `object.delta` events still stream on every step of an `output` run. See [Structured output](docs/structured-output.md#how-it-works). ### Documentation - Tool idempotency: `toolCallId` and `ctx.approval.id` identify one call; after a crash the model may retry with a new call, so side effects that must happen once need a key from your own data. New guidance for ending a run after a rejected approval, and for extract-then-approve-then-act with structured output. +- [Durable execution](docs/durable-execution.md#recovering-after-a-crash): `agent.resume()` resolves with a result, not a stream (stream a resumed run with `agent.stream([], { sessionId })`), and `agent.pending()` also lists runs still active in another process. [Installation](docs/installation.md#typescript-settings): TypeScript 6+ projects need `"types": ["node"]` for the SDK's declarations. ## [1.0.0-alpha.23] - 2026-10-16 diff --git a/api/executor.api.md b/api/executor.api.md index 5a20eca2..80e3247b 100644 --- a/api/executor.api.md +++ b/api/executor.api.md @@ -1539,6 +1539,7 @@ interface OutputError { // @public interface OutputSpec { promptSchema?: boolean; + responseFormat?: 'final' | 'always'; schema: TSchema; } diff --git a/api/index.api.md b/api/index.api.md index b31574c5..79c2aac8 100644 --- a/api/index.api.md +++ b/api/index.api.md @@ -4442,6 +4442,7 @@ export interface OutputError { // @public export interface OutputSpec { promptSchema?: boolean; + responseFormat?: 'final' | 'always'; schema: TSchema; } diff --git a/api/worker.api.md b/api/worker.api.md index d3a0ca56..6e43e7e2 100644 --- a/api/worker.api.md +++ b/api/worker.api.md @@ -2604,6 +2604,7 @@ interface OutputError { // @public interface OutputSpec { promptSchema?: boolean; + responseFormat?: 'final' | 'always'; schema: TSchema; } diff --git a/docs/durable-execution.md b/docs/durable-execution.md index 60beeaae..3732b6c7 100644 --- a/docs/durable-execution.md +++ b/docs/durable-execution.md @@ -174,6 +174,19 @@ for (const run of await agent.pending()) { } ``` +`agent.resume()` resolves with the finished `ExecutionResult` (or `null` when +nothing was pending); it does not return a stream. To watch a resumed run's +events, stream it instead: `agent.stream([], { sessionId })` resumes an +unfinished `kind: 'run'` checkpoint without new input (see +[above](#what-happens-when-you-send-again-with-the-same-sessionid)) and +returns the usual `AgentRun`. A paused run streams with +`agent.approvals.streamResolve()`. + +`pending()` reads the shared checkpoint store, so it also lists runs that +are still running right now in another process or replica (as +`'in-progress'`, without `lastError`) - not only crashed ones. See below for +why resuming one of those is safe. + Each entry is `{ sessionId, kind, checkpointId, status, approvalId?, approvalKind?, step, lastError?, attempts? }`: diff --git a/docs/installation.md b/docs/installation.md index 04072848..8721a83c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -183,6 +183,21 @@ code import those types (error TS1479 inside the SDK's declarations). Use add `"type": "module"`, or keep `skipLibCheck: true` (what `lousho init` projects use). +The root entry's declarations use Node types (`Buffer`, and imports from +`node:crypto`, `node:http` and `node:net`), so install `@types/node`. +TypeScript 6 no longer loads every `@types` package by default (`types` +defaults to `[]`), so on TypeScript 6+ also list it in `tsconfig.json`, +or `skipLibCheck: false` reports `Cannot find name 'Buffer'` and +`Cannot find module 'node:http'` inside the SDK's declarations: + +```json +{ + "compilerOptions": { + "types": ["node"] + } +} +``` + ## Installing from a local build To try an unreleased commit, build and pack the SDK from a checkout of this diff --git a/docs/structured-output.md b/docs/structured-output.md index 549cb14e..efbea3c4 100644 --- a/docs/structured-output.md +++ b/docs/structured-output.md @@ -176,10 +176,18 @@ console.log(object?.note); // typed: string | undefined 1. The system prompt gets an `## Output format` section asking for the final answer as only a JSON object matching the schema, rendered as JSON Schema. -2. Every model call carries `responseFormat: { type: 'json', schema }` on - `GenerateOptions`. It is a hint: the built-in `ai`-SDK providers turn on - the model's JSON mode (with the schema, for models that support structured - outputs); a custom provider may use it or ignore it. +2. Model calls that offer no tools carry `responseFormat: { type: 'json', + schema }` on `GenerateOptions`: every call of an agent without tools, the + "answer now" call when steps run out (see below), and the repair step. It + is a hint: the built-in `ai`-SDK providers turn on the model's JSON mode + (with the schema, for models that support structured outputs; OpenRouter + and the `pi` provider send a strict `json_schema` response format); a + custom provider may use it or ignore it. A call that offers tools goes + without it, and the model answers from the schema in the prompt: with a + strict response format next to tools, some models (`openai/gpt-4o-mini` + via OpenRouter) keep calling tools until `maxSteps` instead of answering. + `toolChoice: 'none'` counts as no tools offered. To send it on every + call anyway, pass `output: { schema, responseFormat: 'always' }`. 3. When the model replies without tool calls, the reply is parsed as JSON (a ```` ```json ```` code fence around it is tolerated) and validated with the schema. A near miss is read without a repair call: when the whole @@ -221,13 +229,15 @@ call only and is not kept in `result.messages` or the session transcript. ## The schema in the prompt -Steps 1 and 2 send the schema twice: once inside the `## Output format` -system-prompt block, and once on `responseFormat`. When the provider enforces -`responseFormat` (OpenAI structured outputs, OpenRouter, the `pi` provider) -the prompt copy is redundant - on a large schema it costs around a thousand -tokens per call. Pass `output` as `{ schema, promptSchema: false }` to leave -the schema out of the prompt; the JSON-only ask stays and `responseFormat` is -sent unchanged: +Steps 1 and 2 send the schema twice on a call without tools: once inside the +`## Output format` system-prompt block, and once on `responseFormat`. When the +provider enforces `responseFormat` (OpenAI structured outputs, OpenRouter, the +`pi` provider) the prompt copy is redundant - on a large schema it costs +around a thousand tokens per call. Pass `output` as `{ schema, promptSchema: +false }` to leave the schema out of the prompt; the JSON-only ask stays, and +since `responseFormat` is then the only copy of the schema it goes on every +call, tool-offering ones included (`responseFormat` defaults to `'always'` +here; set `'final'` to keep it off those calls): ```ts import { z } from 'zod'; @@ -245,6 +255,15 @@ provider - or model - treats `responseFormat` as a JSON-mode hint only (the built-in `ai`-SDK providers drop the schema for models without structured-outputs support; a custom provider may ignore it entirely). The prompt copy is then the only place the model sees the shape to answer with. +Keep it too for an agent with tools: its tool-offering calls carry no +`responseFormat` by default, so the prompt copy is what they answer from. + +| `output` | Schema in the prompt | `responseFormat` on calls offering tools | on calls without tools | +| --- | --- | --- | --- | +| `schema` or `{ schema }` | yes | no | yes | +| `{ schema, responseFormat: 'always' }` | yes | yes | yes | +| `{ schema, promptSchema: false }` | no | yes | yes | +| `{ schema, promptSchema: false, responseFormat: 'final' }` | no | no | yes | ## Repaired output and business rules diff --git a/docs/upgrading.md b/docs/upgrading.md index f90b0cc7..ce2473f4 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -258,6 +258,7 @@ there is one. A change marked "no opt-out" closed a security or data-loss bug. | An aborted run ends `finishReason: 'aborted'` at once, even when the provider ignores the signal. See [Cancellation](./runs.md#cancellation). | no opt-out | | The `openrouter` provider created without `apiKey` reads `OPENROUTER_API_KEY` (it sent `OPENAI_API_KEY`). | pass `apiKey` | | The `@modelcontextprotocol/sdk` peer range is `^1.31.0`. | no opt-out | +| With `output`, `responseFormat` is sent only on model calls that offer no tools (an agent without tools, the "answer now" call at `maxSteps`, the repair step); a call that offers tools relies on the schema in the prompt. A strict `json_schema` next to tools made `openai/gpt-4o-mini` via OpenRouter call tools until `maxSteps`. See [How it works](./structured-output.md#how-it-works). | `output: { schema, responseFormat: 'always' }` (already the default with `promptSchema: false`) | ### Tools, MCP and tracing diff --git a/llms-full.txt b/llms-full.txt index a5b4fc27..ecaa6ecd 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -542,6 +542,21 @@ code import those types (error TS1479 inside the SDK's declarations). Use add `"type": "module"`, or keep `skipLibCheck: true` (what `lousho init` projects use). +The root entry's declarations use Node types (`Buffer`, and imports from +`node:crypto`, `node:http` and `node:net`), so install `@types/node`. +TypeScript 6 no longer loads every `@types` package by default (`types` +defaults to `[]`), so on TypeScript 6+ also list it in `tsconfig.json`, +or `skipLibCheck: false` reports `Cannot find name 'Buffer'` and +`Cannot find module 'node:http'` inside the SDK's declarations: + +```json +{ + "compilerOptions": { + "types": ["node"] + } +} +``` + ## Installing from a local build To try an unreleased commit, build and pack the SDK from a checkout of this @@ -7389,6 +7404,19 @@ for (const run of await agent.pending()) { } ``` +`agent.resume()` resolves with the finished `ExecutionResult` (or `null` when +nothing was pending); it does not return a stream. To watch a resumed run's +events, stream it instead: `agent.stream([], { sessionId })` resumes an +unfinished `kind: 'run'` checkpoint without new input (see +[above](#what-happens-when-you-send-again-with-the-same-sessionid)) and +returns the usual `AgentRun`. A paused run streams with +`agent.approvals.streamResolve()`. + +`pending()` reads the shared checkpoint store, so it also lists runs that +are still running right now in another process or replica (as +`'in-progress'`, without `lastError`) - not only crashed ones. See below for +why resuming one of those is safe. + Each entry is `{ sessionId, kind, checkpointId, status, approvalId?, approvalKind?, step, lastError?, attempts? }`: @@ -17317,10 +17345,18 @@ console.log(object?.note); // typed: string | undefined 1. The system prompt gets an `## Output format` section asking for the final answer as only a JSON object matching the schema, rendered as JSON Schema. -2. Every model call carries `responseFormat: { type: 'json', schema }` on - `GenerateOptions`. It is a hint: the built-in `ai`-SDK providers turn on - the model's JSON mode (with the schema, for models that support structured - outputs); a custom provider may use it or ignore it. +2. Model calls that offer no tools carry `responseFormat: { type: 'json', + schema }` on `GenerateOptions`: every call of an agent without tools, the + "answer now" call when steps run out (see below), and the repair step. It + is a hint: the built-in `ai`-SDK providers turn on the model's JSON mode + (with the schema, for models that support structured outputs; OpenRouter + and the `pi` provider send a strict `json_schema` response format); a + custom provider may use it or ignore it. A call that offers tools goes + without it, and the model answers from the schema in the prompt: with a + strict response format next to tools, some models (`openai/gpt-4o-mini` + via OpenRouter) keep calling tools until `maxSteps` instead of answering. + `toolChoice: 'none'` counts as no tools offered. To send it on every + call anyway, pass `output: { schema, responseFormat: 'always' }`. 3. When the model replies without tool calls, the reply is parsed as JSON (a ```` ```json ```` code fence around it is tolerated) and validated with the schema. A near miss is read without a repair call: when the whole @@ -17362,13 +17398,15 @@ call only and is not kept in `result.messages` or the session transcript. ## The schema in the prompt -Steps 1 and 2 send the schema twice: once inside the `## Output format` -system-prompt block, and once on `responseFormat`. When the provider enforces -`responseFormat` (OpenAI structured outputs, OpenRouter, the `pi` provider) -the prompt copy is redundant - on a large schema it costs around a thousand -tokens per call. Pass `output` as `{ schema, promptSchema: false }` to leave -the schema out of the prompt; the JSON-only ask stays and `responseFormat` is -sent unchanged: +Steps 1 and 2 send the schema twice on a call without tools: once inside the +`## Output format` system-prompt block, and once on `responseFormat`. When the +provider enforces `responseFormat` (OpenAI structured outputs, OpenRouter, the +`pi` provider) the prompt copy is redundant - on a large schema it costs +around a thousand tokens per call. Pass `output` as `{ schema, promptSchema: +false }` to leave the schema out of the prompt; the JSON-only ask stays, and +since `responseFormat` is then the only copy of the schema it goes on every +call, tool-offering ones included (`responseFormat` defaults to `'always'` +here; set `'final'` to keep it off those calls): ```ts import { z } from 'zod'; @@ -17386,6 +17424,15 @@ provider - or model - treats `responseFormat` as a JSON-mode hint only (the built-in `ai`-SDK providers drop the schema for models without structured-outputs support; a custom provider may ignore it entirely). The prompt copy is then the only place the model sees the shape to answer with. +Keep it too for an agent with tools: its tool-offering calls carry no +`responseFormat` by default, so the prompt copy is what they answer from. + +| `output` | Schema in the prompt | `responseFormat` on calls offering tools | on calls without tools | +| --- | --- | --- | --- | +| `schema` or `{ schema }` | yes | no | yes | +| `{ schema, responseFormat: 'always' }` | yes | yes | yes | +| `{ schema, promptSchema: false }` | no | yes | yes | +| `{ schema, promptSchema: false, responseFormat: 'final' }` | no | no | yes | ## Repaired output and business rules @@ -19677,6 +19724,7 @@ there is one. A change marked "no opt-out" closed a security or data-loss bug. | An aborted run ends `finishReason: 'aborted'` at once, even when the provider ignores the signal. See [Cancellation](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/runs.md#cancellation). | no opt-out | | The `openrouter` provider created without `apiKey` reads `OPENROUTER_API_KEY` (it sent `OPENAI_API_KEY`). | pass `apiKey` | | The `@modelcontextprotocol/sdk` peer range is `^1.31.0`. | no opt-out | +| With `output`, `responseFormat` is sent only on model calls that offer no tools (an agent without tools, the "answer now" call at `maxSteps`, the repair step); a call that offers tools relies on the schema in the prompt. A strict `json_schema` next to tools made `openai/gpt-4o-mini` via OpenRouter call tools until `maxSteps`. See [How it works](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/structured-output.md#how-it-works). | `output: { schema, responseFormat: 'always' }` (already the default with `promptSchema: false`) | ### Tools, MCP and tracing diff --git a/src/createAgent.ts b/src/createAgent.ts index ebbd257f..ebe76931 100644 --- a/src/createAgent.ts +++ b/src/createAgent.ts @@ -535,7 +535,10 @@ export interface CreateAgentBase recordStepUsage(state.usage, measured)); - const generateRequest = await prepareGenerateRequest(options, state.messages, tools, callSignal, sideGenerate); + // The repair step's call carries the `output` schema as `responseFormat` even when it offers tools. + const answerOnly = state.outputRepair === true; + state.outputRepair = undefined; + const generateRequest = await prepareGenerateRequest(options, state.messages, tools, callSignal, sideGenerate, answerOnly); callSignal?.throwIfAborted(); called = true; const generated = await generateInSpan(options, generateRequest, state.messages, agentSpanId, callSignal, inputCheck, partial); diff --git a/src/execution/agentRun.ts b/src/execution/agentRun.ts index ff67d75d..c70821d4 100644 --- a/src/execution/agentRun.ts +++ b/src/execution/agentRun.ts @@ -187,9 +187,17 @@ export interface RunEventSink { /** * Obtains one model step - streamed when the provider can; `onOutput` before its first text or tool call is reported. * With `hold`, the step's `text.delta`, `reasoning.*` and provider-run tool events wait for it: released - * in order when it resolves `true`, dropped when `false`. + * in order when it resolves `true`, dropped when `false`. `objectDeltas` (an `output` run) also reports + * the reply parsed so far as `object.delta` events. */ - generate(provider: LLMProvider, request: GenerateOptions, onOutput?: () => void, hold?: Promise, onText?: (text: string) => void): Promise; + generate( + provider: LLMProvider, + request: GenerateOptions, + onOutput?: () => void, + hold?: Promise, + onText?: (text: string) => void, + objectDeltas?: boolean + ): Promise; /** The sink for a sub-agent's run, whose events carry `subagent`. */ forSubagent(subagent: SubagentInfo): RunEventSink; } @@ -509,9 +517,9 @@ class RunEvents { agentDrift: (drift) => this.emit({ type: 'agent.drift', ...drift }, subagent), handoff: (handoff) => this.emit({ type: 'handoff', ...handoff }, subagent), guardrail: (event) => this.emit(event, subagent), - generate: async (provider, request, onOutput, hold, onText) => { + generate: async (provider, request, onOutput, hold, onText, objectDeltas) => { const call = withProviderEvents(request, this.providerEvents(subagent)); - const generated = settleHostedFinish(await this.generateStep(provider, call, subagent, onOutput, hold, onText)); + const generated = settleHostedFinish(await this.generateStep(provider, call, subagent, onOutput, hold, onText, objectDeltas)); const measured = measureUsage(generated.servedBy?.model ?? request.model ?? provider.name, request.messages, generated); const measuredStep = { finishReason: generated.finishReason, ...measured, usage: measured.usage }; // A step a parallel input guardrail blocked reports only the usage the provider reported. @@ -553,14 +561,15 @@ class RunEvents { subagent: SubagentInfo | undefined, onOutput?: () => void, hold?: Promise, - onText?: (text: string) => void + onText?: (text: string) => void, + objectDeltas = Boolean(request.responseFormat) ): Promise { const held = heldUntil(hold); // A provider that ignores an abort may stream on after the run stopped waiting; that output is not reported. const report = (event: () => void) => { if (!request.signal?.aborted) held(event); }; - const objectDelta = request.responseFormat ? partialObjects() : undefined; + const objectDelta = objectDeltas ? partialObjects() : undefined; const sink: StepSink = { onTextDelta: (text) => { onText?.(text); diff --git a/src/execution/agentRunState.ts b/src/execution/agentRunState.ts index 6357fd99..6e797180 100644 --- a/src/execution/agentRunState.ts +++ b/src/execution/agentRunState.ts @@ -98,6 +98,12 @@ export interface AgentRunState { * of the finished checkpoint (the session transcript). */ requestOnly?: Set; + /** + * Set when the structured-output repair step is queued: its model call + * sends the `output` schema as `responseFormat` even with tools offered. + * Cleared by that call. + */ + outputRepair?: boolean; /** The run lease the run holds on its checkpoint id, when the store has leases (see `runLease.ts`). */ lease?: RunLease; /** What each sub-agent run still running has spent so far (not in `usage` until it ends); checkpointed. */ diff --git a/src/execution/generateStep.ts b/src/execution/generateStep.ts index f5e463a2..6b3e3c82 100644 --- a/src/execution/generateStep.ts +++ b/src/execution/generateStep.ts @@ -138,14 +138,17 @@ function resolveModel(options: ExecuteOptions): string | undefined { /** * Builds the next provider.generate() request and runs the onLLMRequest * callback and preGenerate hooks for it. Errors thrown here are NOT - * provider failures and are never compacted. + * provider failures and are never compacted. `answerOnly` (the + * structured-output repair step) sends the `output` schema as + * `responseFormat` even when the call offers tools. */ export async function prepareGenerateRequest( options: ExecuteOptions, messages: Message[], tools: ToolDefinition[], callSignal?: AbortSignal, - generate?: GenerateHookContext['generate'] + generate?: GenerateHookContext['generate'], + answerOnly = false ): Promise { const { temperature, maxTokens, onLLMRequest, hooks } = options; // The agent's / call's settings, then the run's own `temperature` / `maxTokens`; an unset key is not sent at all. @@ -157,6 +160,9 @@ export async function prepareGenerateRequest( // The tools of this call, from the transcript as it is now (deferred tools load through `tool_search`). const deferral = deferralOf(options); const sent = visibleTools(tools, messages, deferral); + // Tools the model may call on this call: function tools not disabled by `toolChoice: 'none'`, or hosted ones. + const offersTools = !answerOnly && ((sent.length > 0 && settings.toolChoice !== 'none') || Boolean(hostedTools?.length)); + const responseFormat = options.output ? outputResponseFormat(options.output, offersTools) : undefined; const generateRequest: GenerateOptions = { // agent.settings.model > the model the provider was configured with > @@ -167,8 +173,9 @@ export async function prepareGenerateRequest( tools: sent.length > 0 ? sent : undefined, // The provider's own tools, sent with every call of the run (plan mode: read-only ones only). ...(hostedTools?.length && { hostedTools }), - // A JSON-mode hint for runs with an `output` schema. - ...(options.output ? { responseFormat: outputResponseFormat(options.output) } : {}), + // A JSON-mode hint for runs with an `output` schema - by default only on calls that offer no tools + // (a strict response format next to tools makes some models call tools forever instead of answering). + ...(responseFormat && { responseFormat }), // The providers send it only to models that accept it. ...(options.reasoning !== undefined && { reasoning: options.reasoning }), // The providers mark Anthropic cache breakpoints unless it is `false`. @@ -293,7 +300,7 @@ export function generateInSpan( const request = recordingRetries(generateRequest, llmSpan); const generated = settleHostedFinish( await abortable( - runEvents ? runEvents.generate(provider, request, onOutput, hold, partial && ((text) => (partial.text += text))) : provider.generate(request), + runEvents ? runEvents.generate(provider, request, onOutput, hold, partial && ((text) => (partial.text += text)), Boolean(options.output)) : provider.generate(request), // The run's signal too, so a provider that ignores it does not hold an aborted run. anySignal(callSignal, options.signal) ) diff --git a/src/execution/structuredOutput.test.ts b/src/execution/structuredOutput.test.ts index 4f3516ea..b4b01d6f 100644 --- a/src/execution/structuredOutput.test.ts +++ b/src/execution/structuredOutput.test.ts @@ -111,7 +111,101 @@ describe('structured output', () => { expect(result.object).toEqual({ city: 'Paris', tempC: 21 }); expect(result.toolCalls).toHaveLength(1); expect(model.calls[0].tools).toHaveLength(1); - expect(model.calls[0].responseFormat?.type).toBe('json'); + // A call that offers tools carries no strict response format (some models then call tools forever); + // the schema is in the system prompt instead. + expect(model.calls[0]).not.toHaveProperty('responseFormat'); + expect(model.calls[1]).not.toHaveProperty('responseFormat'); + expect(model.calls[0].messages[0].content).toContain('"tempC":{"type":"number"}'); + }); + + describe('responseFormat and tools', () => { + const lookup = defineTool({ + name: 'lookup', + description: 'Current temperature', + input: z.object({ city: z.string() }), + execute: async () => 21, + }); + const toolCall = { toolCalls: [{ name: 'lookup', args: { city: 'Paris' } }] }; + + it('parses JSON embedded in a prose answer on a tool-offering step, with no repair', async () => { + const model = mockModel([toolCall, 'Here you go: {"city":"Paris","tempC":21}']); + const result = await createAgent({ provider: model, tools: [lookup], output: weather }).send('Weather in Paris?'); + + expect(result.object).toEqual({ city: 'Paris', tempC: 21 }); + expect(result.steps).toBe(2); + expect(result.outputRepaired).toBeUndefined(); + }); + + it('a plain-text answer on a tool-offering step is repaired by a call that does carry the response format', async () => { + const model = mockModel([toolCall, 'It is 21 degrees in Paris.', '{"city":"Paris","tempC":21}']); + const result = await createAgent({ provider: model, tools: [lookup], output: weather }).send('Weather in Paris?'); + + expect(result.object).toEqual({ city: 'Paris', tempC: 21 }); + expect(result.outputRepaired).toBe(true); + expect(model.calls).toHaveLength(3); + expect(model.calls[1]).not.toHaveProperty('responseFormat'); + // The repair step: tools still offered, the schema enforced. + expect(model.calls[2].tools).toHaveLength(1); + expect(model.calls[2].responseFormat).toMatchObject({ type: 'json', schema: { required: ['city', 'tempC'] } }); + }); + + it('a step after the repair step goes without it again', async () => { + const model = mockModel([toolCall, 'It is warm.', toolCall, '{"city":"Paris","tempC":21}']); + const result = await createAgent({ provider: model, tools: [lookup], output: weather }).send('Weather in Paris?'); + + expect(result.object).toEqual({ city: 'Paris', tempC: 21 }); + expect(model.calls[2].responseFormat?.type).toBe('json'); + expect(model.calls[3]).not.toHaveProperty('responseFormat'); + }); + + it('the forced answer at maxSteps (tools with toolChoice none) carries it', async () => { + const model = mockModel([toolCall, '{"city":"Paris","tempC":21}']); + const result = await createAgent({ provider: model, tools: [lookup], output: weather, maxSteps: 1 }).send('go'); + + expect(result.object).toEqual({ city: 'Paris', tempC: 21 }); + expect(model.calls[0]).not.toHaveProperty('responseFormat'); + expect(model.calls[1].toolChoice).toBe('none'); + expect(model.calls[1].responseFormat?.type).toBe('json'); + }); + + it("toolChoice 'none' counts as no tools offered", async () => { + const model = mockModel(['{"city":"Paris","tempC":21}']); + await createAgent({ provider: model, tools: [lookup], output: weather, modelSettings: { toolChoice: 'none' } }).send('go'); + + expect(model.calls[0].responseFormat?.type).toBe('json'); + }); + + it("output { responseFormat: 'always' } sends it on tool-offering calls too", async () => { + const model = mockModel([toolCall, '{"city":"Paris","tempC":21}']); + await createAgent({ provider: model, tools: [lookup], output: { schema: weather, responseFormat: 'always' } }).send('go'); + + expect(model.calls[0].responseFormat?.type).toBe('json'); + expect(model.calls[1].responseFormat?.type).toBe('json'); + }); + + it("promptSchema: false defaults to 'always', the response format being the only copy of the schema", async () => { + const model = mockModel([toolCall, '{"city":"Paris","tempC":21}']); + await createAgent({ provider: model, tools: [lookup], output: { schema: weather, promptSchema: false } }).send('go'); + expect(model.calls[0].responseFormat?.type).toBe('json'); + + const final = mockModel([toolCall, '{"city":"Paris","tempC":21}']); + await createAgent({ provider: final, tools: [lookup], output: { schema: weather, promptSchema: false, responseFormat: 'final' } }).send('go'); + expect(final.calls[0]).not.toHaveProperty('responseFormat'); + }); + + it('rejects an unknown responseFormat mode', () => { + expect(() => createAgent({ provider: mockModel([]), output: { schema: weather, responseFormat: 'sometimes' as never } })).toThrow( + /output\.responseFormat must be 'final' or 'always'/ + ); + }); + + it('streams object.delta for a final answer on a tool-offering step', async () => { + const run = createAgent({ provider: mockModel([toolCall, '{"city":"Paris","tempC":21}']), tools: [lookup], output: weather }).stream('go'); + const objects: unknown[] = []; + for await (const event of run) if (event.type === 'object.delta') objects.push(event.object); + + expect(objects.at(-1)).toEqual({ city: 'Paris', tempC: 21 }); + }); }); it('makes one last tool-less call for the answer when maxSteps runs out', async () => { diff --git a/src/execution/structuredOutput.ts b/src/execution/structuredOutput.ts index cb80b1a5..4ae024cb 100644 --- a/src/execution/structuredOutput.ts +++ b/src/execution/structuredOutput.ts @@ -35,12 +35,28 @@ export interface OutputError { * the default (`true`, or a bare `output: schema`) when the provider takes * `responseFormat` as a JSON-mode hint only - the prompt copy is then the * only place the model sees the shape to answer with. + * + * `responseFormat` says which model calls carry the schema as + * `GenerateOptions.responseFormat`: `'final'` only those that offer no tools + * (an agent without tools, the salvage call at `maxSteps`, the repair step), + * `'always'` every call. A strict `json_schema` response format on a call + * that also offers tools makes some models (gpt-4o-mini via OpenRouter) + * call tools forever instead of answering, so `'final'` is the default; a + * tool-offering call relies on the schema in the prompt and the repair step. + * With `promptSchema: false` the default is `'always'`, since the response + * format is then the only place the model sees the schema. */ export interface OutputSpec { /** The schema the final reply is validated into `result.object` with (what a bare `output` is). */ schema: TSchema; /** Also write the JSON Schema into the system prompt (default `true`). */ promptSchema?: boolean; + /** + * Which model calls send the schema as `responseFormat`: `'final'` (calls + * that offer no tools) or `'always'`. Default `'final'`, or `'always'` + * with `promptSchema: false`. + */ + responseFormat?: 'final' | 'always'; } /** `output` given as an {@link OutputSpec} vs a bare schema: the spec has a `schema` key and is not itself a schema. */ @@ -250,7 +266,12 @@ function assertNoDates(schema: StandardSchemaV1): void { * on the first model call. */ export function assertOutputSchema(schema: StandardSchemaV1 | OutputSpec | undefined): void { - if (schema) jsonSchemasOf(outputSchemaOf(schema)); + if (!schema) return; + const mode = isOutputSpec(schema) ? schema.responseFormat : undefined; + if (mode !== undefined && mode !== 'final' && mode !== 'always') { + throw new ConfigurationError(`output.responseFormat must be 'final' or 'always', got ${JSON.stringify(mode)}.`, 'output'); + } + jsonSchemasOf(outputSchemaOf(schema)); } function jsonSchemaOf(schema: StandardSchemaV1): Record { @@ -278,11 +299,22 @@ export function outputInstruction(output: StandardSchemaV1 | OutputSpec): string ].join('\n'); } -/** The `responseFormat` hint sent with every model call of the run. */ -export function outputResponseFormat(output: StandardSchemaV1 | OutputSpec): GenerateOptions['responseFormat'] { +/** + * The `responseFormat` hint for one model call of an `output` run, or + * undefined when the call goes without it: a call that `offersTools` sends + * it only under `responseFormat: 'always'` (see {@link OutputSpec}). + */ +export function outputResponseFormat(output: StandardSchemaV1 | OutputSpec, offersTools = false): GenerateOptions['responseFormat'] { + if (offersTools && responseFormatMode(output) === 'final') return undefined; return { type: 'json', schema: jsonSchemaOf(outputSchemaOf(output)) }; } +/** An `output` option's `responseFormat` mode, its default applied. */ +function responseFormatMode(output: StandardSchemaV1 | OutputSpec): 'final' | 'always' { + if (!isOutputSpec(output)) return 'final'; + return output.responseFormat ?? (output.promptSchema === false ? 'always' : 'final'); +} + /** `text` without surrounding whitespace and a ```/```json code fence. */ function unfence(text: string): string { const trimmed = text.trim(); diff --git a/src/execution/structuredOutputTools.live.test.ts b/src/execution/structuredOutputTools.live.test.ts new file mode 100644 index 00000000..8e6222c6 --- /dev/null +++ b/src/execution/structuredOutputTools.live.test.ts @@ -0,0 +1,38 @@ +/** + * Live test (costs money, needs OPENROUTER_API_KEY; well under 0.05 USD): the + * documented headline setup - `openrouter/openai/gpt-4o-mini`, one tool and a + * zod `output` schema. With a strict `json_schema` response format on every + * call, this model called the tool on every step until `maxSteps`; the + * response format now goes only on calls that offer no tools, and the run + * answers in a few steps with a valid object. + * Run with `npm run test:live -- src/execution/structuredOutputTools`. + */ +import { describe, expect, it } from 'vitest'; +import { z } from 'zod'; +import { createAgent } from '../createAgent'; +import '../providers'; // registers the real providers (openrouter) +import { defineTool } from '../tools/defineTool'; + +const weather = defineTool({ + name: 'get_weather', + description: 'Current temperature in Celsius for a city', + input: z.object({ city: z.string() }), + execute: async ({ city }) => ({ city, tempC: 21 }), +}); + +describe.skipIf(!process.env.OPENROUTER_API_KEY)('output schema + tools on a real model', () => { + it('answers with a valid object instead of calling tools until maxSteps', async () => { + const agent = createAgent({ + model: 'openrouter/openai/gpt-4o-mini', + tools: [weather], + output: z.object({ city: z.string(), tempC: z.number() }), + maxSteps: 6, + }); + + const result = await agent.send('What is the weather in Paris?'); + + expect(result.finishReason).toBe('stop'); + expect(result.object).toEqual({ city: 'Paris', tempC: 21 }); + expect(result.steps).toBeLessThanOrEqual(3); + }, 60_000); +});