Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/3132-output-cap-clamp.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
A model call no longer sends an output-token cap above the 12288 a provider endpoint accepts. A model row that declares more than that ceiling — or a caller asking for more — is clamped to it, so the request is answered instead of being refused whole with a 400 that lost the step's answer. A cap at or under the ceiling is sent unchanged, and a model declaring none is never given one.
24 changes: 24 additions & 0 deletions core/src/host/models.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ const ROUTE_WAIT_MS = 30_000
const TEST_TIMEOUT_MS = 30_000
const TEST_MAX_TOKENS = 16
const TEST_PROMPT = 'Reply with the single word OK.'
// What a provider's endpoint takes as an output cap: above this DashScope refuses the
// whole request with a 400 (`max_completion_tokens … expected a value <= 12288`), so the
// answer is lost even though the rest of the call was fine (#3132). A model row can
// declare more than the endpoint accepts, so the cap is clamped on the way out.
const OUTBOUND_MAX_TOKENS = 12_288

// The outcome of a connection test as a code only (#409). A provider's own text can
// echo part of a key, so none of it leaves the host: the client words each code.
Expand Down Expand Up @@ -38,6 +43,25 @@ export function install(rt) {
const reasoningCache = new Map()
ctx.on?.('llm/adapters-updated', () => { catalogCache = { at: 0, value: [] }; reasoningCache.clear() })

// The clamp sits on `llm/stream`, the seam every model call crosses — a Bot's turn and
// its checkpoint, a workflow step, the host's own request. `next` cannot take new
// options (the constraint secrets.js and images.js document), so a call that needs the
// cap set starts over with a copy and passes this listener the second time, carrying
// 12288. The caller's object is never written to: DSH compares a prepared call's config
// and throws INVALID_PREPARED_CALL when one changes underneath it.
ctx.on?.('llm/stream', (options, next) => {
const asked = options?.maxTokens
if (Number.isFinite(asked)) return asked > OUTBOUND_MAX_TOKENS ? ctx.llm.stream({ ...options, maxTokens: OUTBOUND_MAX_TOKENS }) : next()
return (async function* clampDeclaredCap() {
let declared
try { declared = (await ctx.llm.resolveModelInfo?.(options.provider, options.model))?.defaultMaxTokens } catch { declared = undefined }
// A cap at or under the ceiling, and a model that declares none, pass through:
// a clamp lowers a cap and never raises or invents one.
if (!Number.isFinite(declared) || declared <= OUTBOUND_MAX_TOKENS) { yield* next(); return }
yield* ctx.llm.stream({ ...options, maxTokens: OUTBOUND_MAX_TOKENS })
})()
})

// The thinking levels a model serves: `{efforts: [{id, name, description?}],
// defaultEffort?}`, or undefined when it has none or the provider cannot say.
async function reasoningOf(ref) {
Expand Down
1 change: 1 addition & 0 deletions core/test/install.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ test('apply registers listeners, tools, and effects in a fixed order', (t) => {
'effect ds-bot: diagnostics log',
'effect ds-bot: state lock',
'on llm/adapters-updated',
'on llm/stream',
'on session/event',
'on session/event',
'on agent/status',
Expand Down
95 changes: 95 additions & 0 deletions core/test/llm-output-cap.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
import assert from 'node:assert/strict'
import { test } from 'node:test'
import { install } from '../src/host/models.js'
import { fakeContext } from './fake-ctx.js'

// #3132: DashScope refuses `max_completion_tokens` above 12288 with a 400, and a model
// row may declare more than the endpoint takes. The request is clamped at the seam every
// model call crosses; nothing in this file asserts on the model or frontend packages.

const CEILING = 12_288
const MODEL = 'opencsitool/DeepSeek-V4.1-Flash-1017'

const user = text => ({ role: 'user', source: { kind: 'user' }, content: [{ type: 'text', text }] })
const request = (overrides = {}) => ({
provider: 'opencsitool',
model: 'DeepSeek-V4.1-Flash-1017',
messages: [user('Review this pull request')],
sessionId: 's-review',
...overrides,
})

// DSH runs `llm/stream` as a waterfall: a listener calls `next()` for the rest of the
// chain, and a listener that restarts a request calls `ctx.llm.stream(copy)`, which runs
// the chain again from the top. `next` takes no arguments, so this stand-in advances on
// `next()` only — a copy reaches the provider through `stream`, exactly as in the host.
function bench({ info = {}, resolveError } = {}) {
const ctx = fakeContext()
const requests = []
ctx.llm = {
listProviders: () => [],
resolveModelInfo: async (provider, model) => {
if (resolveError !== undefined) throw resolveError
return info[`${provider}/${model}`]
},
stream: options => run(options),
}
const run = (options, index = 0) => {
const chain = ctx.listeners.get('llm/stream') ?? []
if (index < chain.length) return chain[index](options, () => run(options, index + 1))
requests.push(options)
return (async function* answered() { yield { type: 'finish', reason: { kind: 'stop' } } })()
}
install({ ctx, state: {} })
const ask = async asked => {
for await (const chunk of ctx.llm.stream(asked)) void chunk
return { sent: requests.at(-1), count: requests.length }
}
return { ctx, requests, ask }
}

test('a model row declaring more than the endpoint takes is asked for 12288, and the call is not rewritten in place', async () => {
const { ask } = bench({ info: { [MODEL]: { defaultMaxTokens: 16_384 } } })
const asked = request()
const { sent, count } = await ask(asked)
assert.equal(sent.maxTokens, CEILING)
assert.equal(count, 1, 'one provider request, not two')
assert.equal(asked.maxTokens, undefined, 'the caller\'s object is untouched')
})

test('a model row under the ceiling keeps what it declares, and nothing is invented', async () => {
const { ask } = bench({ info: { [MODEL]: { defaultMaxTokens: 4_096 } } })
const { sent } = await ask(request())
assert.equal('maxTokens' in sent, false, 'the declared default is filled by DSH, not written here')
})

test('a caller that asks for more than the ceiling is clamped to it, and its own object keeps its value', async () => {
const { ask } = bench({ info: { [MODEL]: { defaultMaxTokens: 16_384 } } })
const asked = request({ maxTokens: 20_000 })
const { sent, count } = await ask(asked)
assert.equal(sent.maxTokens, CEILING)
assert.equal(count, 1)
assert.equal(asked.maxTokens, 20_000)
})

test('a caller at or under the ceiling is passed through unchanged', async () => {
const { ask } = bench({ info: { [MODEL]: { defaultMaxTokens: 16_384 } } })
for (const maxTokens of [CEILING, 4_096, 1]) {
const { sent } = await ask(request({ maxTokens }))
assert.equal(sent.maxTokens, maxTokens)
}
})

test('nothing declared and nothing asked: no cap appears on the request', async () => {
const { ask } = bench({ info: { [MODEL]: {} } })
const { sent, count } = await ask(request())
assert.equal('maxTokens' in sent, false)
assert.equal(count, 1)
})

test('a model lookup that fails leaves the request alone instead of breaking the call', async () => {
const { ask } = bench({ resolveError: new Error('no adapter') })
const { sent, count } = await ask(request())
assert.equal('maxTokens' in sent, false)
assert.equal(count, 1)
})
Loading
Loading