From 2b06a2a2813bbda9e511b79b670a6e137becc6b5 Mon Sep 17 00:00:00 2001 From: Quickbeard Date: Thu, 27 Aug 2026 10:05:22 +0700 Subject: [PATCH 1/5] feat(gateway): per-model context limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port the hub's per-model limit scheme (hub src/lib/model-limits.ts) into packages/codev-gateway, replacing the flat 196608-token window that assumed every gateway model was the same size — untrue once the gateway served both a 1M-token and a 200K-token model. - model-limits.ts: ModelLimits {context, trigger, output?}, resolved remote -> table -> 200K default; declaredInput() lands each model's own compaction trigger against the single global compaction.reserved (40000). - Config blocks now write per-model limit.context/input/output; the 1M model gets its true window so the context gauge and compaction stop understating it by ~5x. - fetchModelWindows() reads LiteLLM's /model_group/info (never throws); the TUI and desktop configure flows cache the result in the shared auth file under model_limits — the same key the hub writes — so hub-cached windows are honored and vice versa. Co-Authored-By: Claude Fable 5 --- packages/codev-gateway/src/auth-file.ts | 18 +++ packages/codev-gateway/src/backend.ts | 51 ++++++++ packages/codev-gateway/src/config-block.ts | 65 +++++----- packages/codev-gateway/src/const.ts | 13 +- packages/codev-gateway/src/index.ts | 3 + packages/codev-gateway/src/model-limits.ts | 116 ++++++++++++++++++ packages/codev-gateway/test/backend.test.ts | 37 +++++- .../codev-gateway/test/config-block.test.ts | 49 +++++++- .../codev-gateway/test/model-limits.test.ts | 100 +++++++++++++++ .../src/main/gateway-sso/controller.test.ts | 7 ++ .../src/main/gateway-sso/controller.ts | 11 ++ .../tui/src/component/dialog-configure.tsx | 7 ++ .../tui/test/custom-provider.view.test.tsx | 4 +- 13 files changed, 432 insertions(+), 49 deletions(-) create mode 100644 packages/codev-gateway/src/model-limits.ts create mode 100644 packages/codev-gateway/test/model-limits.test.ts diff --git a/packages/codev-gateway/src/auth-file.ts b/packages/codev-gateway/src/auth-file.ts index 9dc97f84dc..b21b5e9e55 100644 --- a/packages/codev-gateway/src/auth-file.ts +++ b/packages/codev-gateway/src/auth-file.ts @@ -2,6 +2,9 @@ import { randomBytes } from "node:crypto" import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs" import { homedir } from "node:os" import { dirname, join } from "node:path" +// Type-only: model-limits.ts imports loadModelLimits from here, so a value +// import would close a require cycle (hub src/lib/auth.ts has the same note). +import type { ModelLimits } from "./model-limits" export interface AuthData { access_token: string @@ -55,6 +58,7 @@ interface AuthFileContents { supabase_anon_key?: string gateway_url?: string skillhub_cookie?: string + model_limits?: Record } // Keyed off the hub's auth.json rather than the ~/.codev-hub/ dir (mirroring @@ -172,6 +176,20 @@ export function loadApiKey(): ApiKeyCreds | null { } } +// The gateway's per-model windows, shared with the hub (hub saveModelLimits +// writes the same key). Skips the write entirely for an empty map so a gateway +// that reports nothing — every max_input_tokens null, the case today — doesn't +// churn the auth file on every configure run. Callers go through +// model-limits.ts cacheModelLimits, which also resets its process-local cache. +export function saveModelLimits(limits: Record): void { + if (Object.keys(limits).length === 0) return + writeAuthFile({ ...(readAuthFile() ?? {}), model_limits: limits }) +} + +export function loadModelLimits(): Record | null { + return readAuthFile()?.model_limits ?? null +} + // Hub-only credential (written by `codevhub login --admin`); read here so // skill commands can ride an existing admin session, never written. export function loadSkillhubCookie(): string | null { diff --git a/packages/codev-gateway/src/backend.ts b/packages/codev-gateway/src/backend.ts index 7e68e753f3..26fa650d81 100644 --- a/packages/codev-gateway/src/backend.ts +++ b/packages/codev-gateway/src/backend.ts @@ -115,6 +115,57 @@ export async function fetchModels(apiKey: string, baseUrl: string, opts?: FetchO return ids } +// A window as the gateway reports it (hub ModelWindow). Deliberately NOT +// model-limits.ts's ModelLimits: callers convert with limitsFromWindow (via +// cacheModelLimits), which decides where the trigger belongs. +export interface ModelWindow { + context: number + output?: number +} + +// Per-model context windows from LiteLLM's /model_group/info, keyed by +// model_group id — the same id /v1/models reports. Entries whose +// max_input_tokens the gateway hasn't been told are skipped, which today is +// all of them: the field is nullable and only populated when an admin sets it +// on the model. So this returns {} on the current deployment and the static +// table in model-limits.ts carries every model; the moment an admin fills the +// field in, that model becomes gateway-driven with no CoDev Code release. +// +// Unlike fetchModels, this NEVER throws and never fail-stops the caller: a +// window is an optimization over a sane default, and configure must not break +// because a metadata endpoint 404s on some other gateway build. The endpoint +// lives at the gateway root, so a trailing /v1 is stripped before joining. +export async function fetchModelWindows( + apiKey: string, + baseUrl: string, + opts?: FetchOptions, +): Promise> { + const fetchImpl = opts?.fetchImpl ?? fetch + try { + const stripped = baseUrl.replace(/\/?v1\/?$/, "") + const res = await fetchImpl(`${stripped.endsWith("/") ? stripped : `${stripped}/`}model_group/info`, { + headers: { accept: "application/json", Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(MODELS_TIMEOUT_MS), + }) + if (!res.ok) return {} + const data = (await res.json()) as { + data?: Array<{ model_group?: string; max_input_tokens?: number | null; max_output_tokens?: number | null }> + } + const out: Record = {} + for (const entry of data.data ?? []) { + const id = entry.model_group + const context = entry.max_input_tokens + if (!id || typeof context !== "number" || context <= 0) continue + const output = + typeof entry.max_output_tokens === "number" && entry.max_output_tokens > 0 ? entry.max_output_tokens : undefined + out[id] = { context, ...(output ? { output } : {}) } + } + return out + } catch { + return {} + } +} + // Validates a key against the gateway's /key/info endpoint (LiteLLM): true on // 2xx, false on 401/403 (invalid/revoked), throws on anything else so callers // can distinguish "key is bad" from "couldn't reach the gateway". /key/info diff --git a/packages/codev-gateway/src/config-block.ts b/packages/codev-gateway/src/config-block.ts index e853620c69..8c8321314a 100644 --- a/packages/codev-gateway/src/config-block.ts +++ b/packages/codev-gateway/src/config-block.ts @@ -1,4 +1,4 @@ -import { GATEWAY_COMPACT_RESERVED, GATEWAY_CONTEXT_WINDOW, GATEWAY_MAX_OUTPUT_TOKENS } from "./const" +import { COMPACT_RESERVED, declaredInput, limitsFor, outputTokens } from "./model-limits" import { GATEWAY_PROVIDER } from "./provider" // A config model entry with no `modalities` resolves image input to false, and @@ -26,7 +26,10 @@ const MODEL_MODALITIES: { attachment: boolean; modalities: { input: Modality[]; // than that PR cannot attribute a `netgate` config. The per-model `limit` is // mandatory: without it opencode sizes the context window to 0 and disables // auto-compaction; the modality block is mandatory for the same reason (hub PR -// #219) — see MODEL_MODALITIES. +// #219) — see MODEL_MODALITIES. Each model's numbers come from +// model-limits.ts limitsFor — per-model, like the hub's — with `limit.input` +// carrying the model's own compaction trigger against the single global +// `compaction.reserved` (see declaredInput). // // Deliberately no top-level `model`, and the hub fragment dropped it in the // same round (hub PR #221): a pinned default outranks the TUI's persisted @@ -49,7 +52,7 @@ export function buildGatewayConfig(input: { apiKey?: string; baseUrl: string; mo return { compaction: { auto: true, - reserved: GATEWAY_COMPACT_RESERVED, + reserved: COMPACT_RESERVED, }, provider: { [GATEWAY_PROVIDER.id]: { @@ -59,32 +62,42 @@ export function buildGatewayConfig(input: { apiKey?: string; baseUrl: string; mo baseURL: normalizeGatewayBaseUrl(input.baseUrl), ...(input.apiKey === undefined ? {} : { apiKey: input.apiKey }), }, - models: Object.fromEntries( - ids.map((id) => [ - id, - { - name: id, - ...MODEL_MODALITIES, - limit: { - context: GATEWAY_CONTEXT_WINDOW, - output: GATEWAY_MAX_OUTPUT_TOKENS, - }, - }, - ]), - ), + models: Object.fromEntries(ids.map((id) => [id, modelEntry(id)])), }, }, } } +// One model's config entry, numbers resolved per model. `context` is the true +// window (the "% context used" gauge divides by it); `input` lands the model's +// own compaction trigger against the shared `compaction.reserved` above. +function modelEntry(id: string) { + const limits = limitsFor(id) + return { + name: id, + ...MODEL_MODALITIES, + limit: { + context: limits.context, + input: declaredInput(limits), + output: outputTokens(limits), + }, + } +} + // The codev.json fragment for a user-named custom provider connected through // the TUI's "Other" flow. Unlike buildGatewayConfig it writes no apiKey (the // credential lives in opencode's auth.json and the provider registry merges it // in by provider id at load) and no npm (the loader defaults to // @ai-sdk/openai-compatible). The per-model limit is still mandatory — without -// it opencode sizes the context window to 0 — so the gateway defaults stand in -// for the unknown endpoint's real limits, and MODEL_MODALITIES applies for the -// same reason it does to the gateway block. +// it opencode sizes the context window to 0 — so model-limits.ts stands in for +// the unknown endpoint's real limits (a known model id gets its table entry, +// anything else the safe 200K default), and MODEL_MODALITIES applies for the +// same reason it does to the gateway block. Deliberately no compaction block: +// connecting a provider must not change global compaction for unrelated ones. +// Without a configured reserve opencode subtracts its own smaller default from +// `limit.input`, so compaction fires a little later than the model's intended +// trigger but always below the true window — declaredInput's clamp keeps the +// declared budget inside `context`. export function buildCustomProviderConfig(input: { providerID: string name: string @@ -98,19 +111,7 @@ export function buildCustomProviderConfig(input: { options: { baseURL: normalizeGatewayBaseUrl(input.baseUrl), }, - models: Object.fromEntries( - input.models.map((id) => [ - id, - { - name: id, - ...MODEL_MODALITIES, - limit: { - context: GATEWAY_CONTEXT_WINDOW, - output: GATEWAY_MAX_OUTPUT_TOKENS, - }, - }, - ]), - ), + models: Object.fromEntries(input.models.map((id) => [id, modelEntry(id)])), }, }, } diff --git a/packages/codev-gateway/src/const.ts b/packages/codev-gateway/src/const.ts index d6bb46894d..4ae7e474d3 100644 --- a/packages/codev-gateway/src/const.ts +++ b/packages/codev-gateway/src/const.ts @@ -13,11 +13,8 @@ export const LOGIN_SUCCESS_URL = `${BASE_URL}/codev/oauth/success` export const CLIENT_ID = atob("bGl0ZWxsbS10ZXN0") -// The self-hosted gateway model window and the compaction reserve derived from -// firing auto-compaction at ~85% of it — identical to the hub's constants so a -// hub-written and an agent-written codev.json agree byte-for-byte. -export const GATEWAY_CONTEXT_WINDOW = 196608 -export const GATEWAY_COMPACT_PCT = 85 -export const GATEWAY_COMPACT_TRIGGER = Math.round(GATEWAY_CONTEXT_WINDOW * (GATEWAY_COMPACT_PCT / 100)) -export const GATEWAY_COMPACT_RESERVED = GATEWAY_CONTEXT_WINDOW - GATEWAY_COMPACT_TRIGGER -export const GATEWAY_MAX_OUTPUT_TOKENS = 65536 +// Context windows and compaction triggers are per-model and live in +// model-limits.ts, mirrored from the hub's src/lib/model-limits.ts. They used +// to be flat GATEWAY_CONTEXT_WINDOW / GATEWAY_COMPACT_* constants here, which +// assumed every gateway model shared one 196608-token window — untrue once the +// gateway served both a 1M-token and a 200K-token model. diff --git a/packages/codev-gateway/src/index.ts b/packages/codev-gateway/src/index.ts index fb5c9dd762..a28f04808b 100644 --- a/packages/codev-gateway/src/index.ts +++ b/packages/codev-gateway/src/index.ts @@ -15,10 +15,13 @@ export { fetchApiKey, fetchCodevConfig, fetchModels, + fetchModelWindows, supabaseExchange, validateApiKey, type AnalysisBackendSession, + type ModelWindow, } from "./backend" +export { cacheModelLimits, limitsFor, type ModelLimits } from "./model-limits" export { buildCustomProviderConfig, buildGatewayConfig, normalizeGatewayBaseUrl } from "./config-block" export { CUSTOM_FALLBACK_PROVIDER, diff --git a/packages/codev-gateway/src/model-limits.ts b/packages/codev-gateway/src/model-limits.ts new file mode 100644 index 0000000000..30276be69e --- /dev/null +++ b/packages/codev-gateway/src/model-limits.ts @@ -0,0 +1,116 @@ +import { loadModelLimits, saveModelLimits } from "./auth-file" + +// Per-model context windows and auto-compaction triggers, mirrored from the +// codev hub's src/lib/model-limits.ts so a hub-written and an agent-written +// codev.json agree on every number. The gateway serves custom models opencode +// doesn't recognize — a config model entry with no `limit` resolves to context +// 0, which mis-sizes the window gauge and disables auto-compaction outright — +// so every model written into a config block needs limits from here. + +export interface ModelLimits { + // The model's true window, written verbatim into `limit.context` — it drives + // the "% context used" gauge, so understating it misreports every session. + context: number + // Where auto-compaction should fire. Deliberately explicit rather than a + // percentage of `context`: the gap between the two is a judgement call per + // model, not a constant. + trigger: number + // Max output tokens. Absent ⇒ DEFAULT_OUTPUT_TOKENS. + output?: number +} + +export const DEFAULT_OUTPUT_TOKENS = 65536 + +// The global `compaction.reserved` — opencode's config schema has a single +// shared reserve with no per-model variant. See declaredInput() for how a +// shared reserve still yields exact per-model triggers. +export const COMPACT_RESERVED = 40000 + +// Percentage used to derive a trigger from a window we didn't pick ourselves, +// i.e. one reported by the gateway. Table entries state their trigger outright. +export const DEFAULT_COMPACT_PCT = 80 + +// Unrecognized models are treated as 200K-class: it matches the smaller of the +// models actually served, and guessing low is the safe direction — too small a +// window wastes capacity, too large overruns the model and 400s mid-session. +export const DEFAULT_LIMITS: ModelLimits = { context: 200000, trigger: 160000 } + +// The 1M-window model's id, base64 like the hub's M3_ID so the upstream +// vendor's name stays out of casual greps of the shipped bundle. +const M3_ID = atob("TWluaU1heC9NaW5pTWF4LU0z") + +// Known gateway models, keyed by the exact id `/v1/models` reports. The 200K +// sibling of M3_ID is deliberately absent: DEFAULT_LIMITS already describes it, +// and an entry restating the default is one more thing to keep in sync. +const TABLE: Record = { + [M3_ID]: { context: 1000000, trigger: 800000 }, + "zai-org/GLM-4.7-cc": { context: 200000, trigger: 160000 }, +} + +// Windows reported by the gateway, cached in the shared auth file (the hub +// writes the same key from its model-choice step). Read once per process: +// building a config block calls limitsFor once per model and none of them can +// change the file mid-run. +let remoteCache: Record | null | undefined + +function remoteLimits(): Record | null { + if (remoteCache === undefined) remoteCache = loadModelLimits() + return remoteCache +} + +// Test seam, also used by cacheModelLimits: lets a fresh write become visible +// without the once-per-process cache leaking across cases. +export function resetModelLimitsCache(): void { + remoteCache = undefined +} + +// Persist gateway-reported windows for limitsFor's remote tier. Unlike the +// hub's one-shot CLI, the TUI and desktop are long-lived processes that write +// the cache and build a config block in the same run, so the process-local +// cache is reset alongside the write. An empty map is skipped entirely — a +// gateway that reports nothing must not churn the auth file or clear limits +// the hub cached. +export function cacheModelLimits(windows: Record): void { + const entries = Object.entries(windows) + if (entries.length === 0) return + saveModelLimits(Object.fromEntries(entries.map(([id, w]) => [id, limitsFromWindow(w.context, w.output)]))) + resetModelLimitsCache() +} + +// Resolve one model's limits. Precedence is remote → table → default: the +// gateway is authoritative about its own models when it says anything at all, +// and the table carries the models it currently reports nothing for. +export function limitsFor(modelId: string): ModelLimits { + const remote = remoteLimits()?.[modelId] + if (remote) return remote + return TABLE[modelId] ?? DEFAULT_LIMITS +} + +// Turn a gateway-reported window into full limits. +export function limitsFromWindow(context: number, output?: number): ModelLimits { + return { + context, + trigger: Math.round((context * DEFAULT_COMPACT_PCT) / 100), + ...(output ? { output } : {}), + } +} + +// opencode's trigger is `limit.input − compaction.reserved`, falling back to +// `limit.context − maxOutputTokens` when `limit.input` is absent — a branch +// that ignores `reserved` entirely. So `input` is what makes the reserve +// authoritative, and the only per-model lever over a trigger that otherwise +// shares one global reserve across every model in the config. Solving +// `input − reserved = trigger` gives `input = trigger + reserved`, landing each +// model's trigger exactly on target regardless of what the others need. +// +// Clamped to the true window: above `context` the declared budget would let a +// session run past the model's real ceiling before compacting. Clamping only +// moves a trigger earlier, so a bad table entry degrades into early compaction +// rather than 400s. +export function declaredInput(limits: ModelLimits): number { + return Math.min(limits.trigger + COMPACT_RESERVED, limits.context) +} + +export function outputTokens(limits: ModelLimits): number { + return limits.output ?? DEFAULT_OUTPUT_TOKENS +} diff --git a/packages/codev-gateway/test/backend.test.ts b/packages/codev-gateway/test/backend.test.ts index fdd5fc870a..313529cbf0 100644 --- a/packages/codev-gateway/test/backend.test.ts +++ b/packages/codev-gateway/test/backend.test.ts @@ -1,5 +1,5 @@ import { afterAll, describe, expect, test } from "bun:test" -import { fetchModels, validateApiKey } from "../src/backend" +import { fetchModels, fetchModelWindows, validateApiKey } from "../src/backend" // A real loopback server (not a mock) that records the path + auth header of the // last request, so tests assert the exact endpoint fetchModels/validateApiKey @@ -52,6 +52,41 @@ describe("fetchModels", () => { }) }) +describe("fetchModelWindows", () => { + test("reads windows from /model_group/info at the gateway root", async () => { + handler = () => + Response.json({ + data: [ + { model_group: "big", max_input_tokens: 1000000, max_output_tokens: 65536 }, + { model_group: "small", max_input_tokens: 200000 }, + ], + }) + seen.length = 0 + const windows = await fetchModelWindows("sk-test", `${base}/v1`) + expect(windows).toEqual({ big: { context: 1000000, output: 65536 }, small: { context: 200000 } }) + // The endpoint lives at the root — the /v1 the base carried is stripped. + expect(seen.at(-1)).toEqual({ path: "/model_group/info", auth: "Bearer sk-test" }) + }) + + // LiteLLM's max_input_tokens is nullable and unset until an admin fills it + // in — those entries carry no information and must not shadow the table. + test("skips entries without a positive max_input_tokens", async () => { + handler = () => + Response.json({ + data: [{ model_group: "a", max_input_tokens: null }, { model_group: "b" }, { max_input_tokens: 5 }], + }) + expect(await fetchModelWindows("k", base)).toEqual({}) + }) + + // A window is an optimization over a sane default: a gateway build without + // the endpoint (404) or an unreachable one must not fail the configure flow. + test("returns {} instead of throwing on errors", async () => { + handler = () => new Response("nope", { status: 404 }) + expect(await fetchModelWindows("k", base)).toEqual({}) + expect(await fetchModelWindows("k", "http://127.0.0.1:9")).toEqual({}) + }) +}) + describe("validateApiKey", () => { test("returns true on a 2xx from /key/info at the gateway root", async () => { handler = () => Response.json({ ok: true }) diff --git a/packages/codev-gateway/test/config-block.test.ts b/packages/codev-gateway/test/config-block.test.ts index d8306798f3..f412951f33 100644 --- a/packages/codev-gateway/test/config-block.test.ts +++ b/packages/codev-gateway/test/config-block.test.ts @@ -1,5 +1,29 @@ -import { describe, expect, test } from "bun:test" +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" import { buildCustomProviderConfig, buildGatewayConfig, normalizeGatewayBaseUrl } from "../src/config-block" +import { resetModelLimitsCache } from "../src/model-limits" + +let tempDir: string +let originalTestHome: string | undefined + +// The config block resolves limits through limitsFor, whose remote tier reads +// the shared auth file — point it at an empty home so a hub-cached +// model_limits on the developer's machine can't leak into the expectations. +beforeEach(() => { + tempDir = mkdtempSync(join(tmpdir(), "codev-gateway-config-")) + originalTestHome = process.env.CODEV_TEST_HOME + process.env.CODEV_TEST_HOME = tempDir + resetModelLimitsCache() +}) + +afterEach(() => { + if (originalTestHome === undefined) delete process.env.CODEV_TEST_HOME + if (originalTestHome !== undefined) process.env.CODEV_TEST_HOME = originalTestHome + rmSync(tempDir, { recursive: true, force: true }) + resetModelLimitsCache() +}) describe("normalizeGatewayBaseUrl", () => { test("appends /v1 when missing", () => { @@ -21,7 +45,7 @@ describe("buildGatewayConfig", () => { models: ["MiniMax/MiniMax-M2.7", "other/model"], }) expect(config).toEqual({ - compaction: { auto: true, reserved: 29491 }, + compaction: { auto: true, reserved: 40000 }, provider: { netgate: { npm: "@ai-sdk/openai-compatible", @@ -32,13 +56,15 @@ describe("buildGatewayConfig", () => { name: "MiniMax/MiniMax-M2.7", attachment: true, modalities: { input: ["text", "image"], output: ["text"] }, - limit: { context: 196608, output: 65536 }, + // Unknown id ⇒ the safe 200K default; input = trigger + reserved + // clamped to context (160000 + 40000 = 200000 exactly). + limit: { context: 200000, input: 200000, output: 65536 }, }, "other/model": { name: "other/model", attachment: true, modalities: { input: ["text", "image"], output: ["text"] }, - limit: { context: 196608, output: 65536 }, + limit: { context: 200000, input: 200000, output: 65536 }, }, }, }, @@ -46,6 +72,17 @@ describe("buildGatewayConfig", () => { }) }) + // The per-model table: the 1M-window model compacts at its own trigger + // (input − reserved = 800000) while sharing the global 40000 reserve. + test("writes the 1M window for the table's known model", () => { + const config = buildGatewayConfig({ baseUrl: "https://gw.test", model: "MiniMax/MiniMax-M3" }) + expect(config.provider.netgate.models["MiniMax/MiniMax-M3"].limit).toEqual({ + context: 1000000, + input: 840000, + output: 65536, + }) + }) + // Without the declaration a custom-provider model resolves to text-only and // the session replaces an attached image with an error note (hub PR #219). test("declares image input so attachments are not stripped", () => { @@ -107,13 +144,13 @@ describe("buildCustomProviderConfig", () => { name: "m1", attachment: true, modalities: { input: ["text", "image"], output: ["text"] }, - limit: { context: 196608, output: 65536 }, + limit: { context: 200000, input: 200000, output: 65536 }, }, m2: { name: "m2", attachment: true, modalities: { input: ["text", "image"], output: ["text"] }, - limit: { context: 196608, output: 65536 }, + limit: { context: 200000, input: 200000, output: 65536 }, }, }, }, diff --git a/packages/codev-gateway/test/model-limits.test.ts b/packages/codev-gateway/test/model-limits.test.ts new file mode 100644 index 0000000000..3a4c429323 --- /dev/null +++ b/packages/codev-gateway/test/model-limits.test.ts @@ -0,0 +1,100 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { saveApiKey, saveModelLimits } from "../src/auth-file" +import { + cacheModelLimits, + DEFAULT_LIMITS, + declaredInput, + limitsFor, + limitsFromWindow, + outputTokens, + resetModelLimitsCache, +} from "../src/model-limits" + +let tempDir: string +let originalTestHome: string | undefined + +// CODEV_TEST_HOME rather than HOME: Bun's homedir() ignores runtime HOME +// changes, so overriding HOME here would silently hit the real ~/.codev. +beforeEach(() => { + tempDir = mkdtempSync(join(tmpdir(), "codev-gateway-limits-")) + originalTestHome = process.env.CODEV_TEST_HOME + process.env.CODEV_TEST_HOME = tempDir + resetModelLimitsCache() +}) + +afterEach(() => { + if (originalTestHome === undefined) delete process.env.CODEV_TEST_HOME + if (originalTestHome !== undefined) process.env.CODEV_TEST_HOME = originalTestHome + rmSync(tempDir, { recursive: true, force: true }) + resetModelLimitsCache() +}) + +describe("limitsFor", () => { + test("resolves the table's known models and defaults the rest", () => { + expect(limitsFor("MiniMax/MiniMax-M3")).toEqual({ context: 1000000, trigger: 800000 }) + expect(limitsFor("zai-org/GLM-4.7-cc")).toEqual({ context: 200000, trigger: 160000 }) + expect(limitsFor("anything/else")).toEqual(DEFAULT_LIMITS) + }) + + // The remote tier: a window the gateway (or the hub, which shares the auth + // file) cached outranks the static table. + test("prefers cached gateway-reported limits over the table", () => { + saveModelLimits({ "MiniMax/MiniMax-M3": { context: 500000, trigger: 400000 } }) + resetModelLimitsCache() + expect(limitsFor("MiniMax/MiniMax-M3")).toEqual({ context: 500000, trigger: 400000 }) + expect(limitsFor("anything/else")).toEqual(DEFAULT_LIMITS) + }) +}) + +describe("cacheModelLimits", () => { + test("persists windows as full limits and makes them visible in-process", () => { + cacheModelLimits({ "gw/model": { context: 300000, output: 32000 } }) + expect(limitsFor("gw/model")).toEqual({ context: 300000, trigger: 240000, output: 32000 }) + }) + + // A gateway that reports nothing must neither churn the auth file nor clear + // limits the hub cached there. + test("an empty map leaves an existing cache untouched", () => { + saveModelLimits({ "gw/model": { context: 300000, trigger: 240000 } }) + resetModelLimitsCache() + cacheModelLimits({}) + expect(limitsFor("gw/model")).toEqual({ context: 300000, trigger: 240000 }) + }) + + // The auth file's read-merge-write must keep the other blocks: the write + // path is shared with credentials the hub and the upload daemon depend on. + test("does not clobber the api-key block", async () => { + saveApiKey({ apiKey: "sk-1", model: "m" }) + cacheModelLimits({ "gw/model": { context: 300000 } }) + const { loadApiKey } = await import("../src/auth-file") + expect(loadApiKey()).toEqual({ apiKey: "sk-1", model: "m" }) + }) +}) + +describe("declaredInput", () => { + test("adds the reserve and clamps to the true window", () => { + expect(declaredInput({ context: 1000000, trigger: 800000 })).toBe(840000) + // DEFAULT_LIMITS lands exactly on the clamp: 160000 + 40000 = 200000. + expect(declaredInput(DEFAULT_LIMITS)).toBe(200000) + // A trigger too close to the window can only move compaction earlier, + // never let the declared budget exceed the model's real ceiling. + expect(declaredInput({ context: 100000, trigger: 90000 })).toBe(100000) + }) +}) + +describe("limitsFromWindow", () => { + test("derives the trigger at 80% and keeps output only when reported", () => { + expect(limitsFromWindow(300000)).toEqual({ context: 300000, trigger: 240000 }) + expect(limitsFromWindow(300000, 32000)).toEqual({ context: 300000, trigger: 240000, output: 32000 }) + }) +}) + +describe("outputTokens", () => { + test("falls back to the 65536 default", () => { + expect(outputTokens({ context: 1, trigger: 1 })).toBe(65536) + expect(outputTokens({ context: 1, trigger: 1, output: 32000 })).toBe(32000) + }) +}) diff --git a/packages/desktop/src/main/gateway-sso/controller.test.ts b/packages/desktop/src/main/gateway-sso/controller.test.ts index 3c318c35de..244b8c49b4 100644 --- a/packages/desktop/src/main/gateway-sso/controller.test.ts +++ b/packages/desktop/src/main/gateway-sso/controller.test.ts @@ -122,11 +122,16 @@ describe("gateway sso controller", () => { test("complete writes the auth file and returns the key beside a keyless netgate config", async () => { const savedKeys: unknown[] = [] + const cachedWindows: unknown[] = [] const controller = controllerWith({ silentSso: async () => auth, fetchCodevConfig: async () => ({ supabaseUrl: "s", supabaseAnonKey: "k", gatewayUrl: "https://gw.example.test" }), fetchApiKey: async () => "sk-live", fetchModels: async () => ["model-a", "model-b"], + fetchModelWindows: async () => ({ "model-a": { context: 300000 } }), + cacheModelLimits: (windows) => { + cachedWindows.push(windows) + }, saveCodevConfig: () => {}, saveApiKey: (creds) => { savedKeys.push(creds) @@ -140,6 +145,8 @@ describe("gateway sso controller", () => { // SSO-minted keys record no base_url (mirrors the TUI flow). expect(savedKeys).toEqual([{ apiKey: "sk-live", baseUrl: undefined, model: "model-a" }]) + // Gateway-reported windows were cached for the config block's limitsFor. + expect(cachedWindows).toEqual([{ "model-a": { context: 300000 } }]) expect(result.providerID).toBe("netgate") // The key rides beside the config for the renderer's auth.set; the config // block itself must not carry it (codev.json stays credential-free). diff --git a/packages/desktop/src/main/gateway-sso/controller.ts b/packages/desktop/src/main/gateway-sso/controller.ts index e74da6a773..4784dec046 100644 --- a/packages/desktop/src/main/gateway-sso/controller.ts +++ b/packages/desktop/src/main/gateway-sso/controller.ts @@ -1,8 +1,10 @@ import { buildGatewayConfig, + cacheModelLimits, fetchApiKey, fetchCodevConfig, fetchModels, + fetchModelWindows, GATEWAY_PROVIDER, loadApiKey, loadGatewayUrl, @@ -46,6 +48,8 @@ type GatewayFns = { fetchCodevConfig: typeof fetchCodevConfig fetchApiKey: typeof fetchApiKey fetchModels: typeof fetchModels + fetchModelWindows: typeof fetchModelWindows + cacheModelLimits: typeof cacheModelLimits saveCodevConfig: typeof saveCodevConfig saveApiKey: typeof saveApiKey loadApiKey: typeof loadApiKey @@ -62,6 +66,8 @@ export function createGatewaySsoController(deps: Deps) { fetchCodevConfig, fetchApiKey, fetchModels, + fetchModelWindows, + cacheModelLimits, saveCodevConfig, saveApiKey, loadApiKey, @@ -242,6 +248,11 @@ export function createGatewaySsoController(deps: Deps) { const url = gatewayUrl setState({ status: "Fetching models..." }) models = await withRetries("Fetching models", () => gateway.fetchModels(key, url, fetchOpts)) + // Gateway-reported windows override the static table in the config block + // complete() writes (limitsFor's remote tier). Never blocks the flow: + // fetchModelWindows returns {} on any failure and an empty map is not + // cached, so the static per-model table stands in. + gateway.cacheModelLimits(await gateway.fetchModelWindows(key, url, fetchOpts)) setState({ status: "" }) return models }, diff --git a/packages/tui/src/component/dialog-configure.tsx b/packages/tui/src/component/dialog-configure.tsx index c36698964d..88bb6bbc6c 100644 --- a/packages/tui/src/component/dialog-configure.tsx +++ b/packages/tui/src/component/dialog-configure.tsx @@ -1,8 +1,10 @@ import { buildGatewayConfig, + cacheModelLimits, fetchApiKey, fetchCodevConfig, fetchModels, + fetchModelWindows, GATEWAY_PROVIDER, login, saveApiKey, @@ -298,6 +300,11 @@ async function runModelFlow(ctx: FlowContext, creds: GatewayCreds) { // key would just fail on the first chat. Silently retry, then quit rather // than drop into an unconfigured chat. models = await withRetries("Fetching models", setStatus, () => fetchModels(creds.apiKey, creds.gatewayBaseUrl)) + // Gateway-reported windows override the static table in the config block + // written below (limitsFor's remote tier). Never blocks the flow: + // fetchModelWindows returns {} on any failure and an empty map is not + // cached, so the static per-model table stands in. + cacheModelLimits(await fetchModelWindows(creds.apiKey, creds.gatewayBaseUrl)) } catch (err) { failAndQuit( ctx, diff --git a/packages/tui/test/custom-provider.view.test.tsx b/packages/tui/test/custom-provider.view.test.tsx index 532fdb5148..6ba87b4683 100644 --- a/packages/tui/test/custom-provider.view.test.tsx +++ b/packages/tui/test/custom-provider.view.test.tsx @@ -200,13 +200,13 @@ test("Custom provider with a base URL discovers models and writes the codev.json name: "mock-small", attachment: true, modalities: { input: ["text", "image"], output: ["text"] }, - limit: { context: 196608, output: 65536 }, + limit: { context: 200000, input: 200000, output: 65536 }, }, "mock-large": { name: "mock-large", attachment: true, modalities: { input: ["text", "image"], output: ["text"] }, - limit: { context: 196608, output: 65536 }, + limit: { context: 200000, input: 200000, output: 65536 }, }, }, }, From 2044c3723bbe09fc538da52909abbbeedc690f0d Mon Sep 17 00:00:00 2001 From: Quickbeard Date: Thu, 27 Aug 2026 10:06:05 +0700 Subject: [PATCH 2/5] docs: changelog entry for per-model gateway limits Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 932e576b82..0b483cfead 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ file is the human-readable summary of what changed and why. ## Unreleased +### Gateway + +- **Per-model context limits** — configuring the gateway (the TUI `/configure` + dialog, `codev configure`, or the desktop sign-in) used to stamp every model + with the same 196,608-token window; the 1M-token model was treated as ~5× + smaller than it is, so the context gauge overstated usage and auto-compaction + fired around 167K tokens. Config blocks now carry each model's real window + and its own compaction trigger, mirroring what CoDev Hub writes: the 1M + model compacts at 800K, unknown models keep a safe 200K default, and windows + the gateway itself reports (LiteLLM `model_group/info`, cached in the shared + auth file like the hub does) override the built-in table without needing a + release. Already-written configs are not rewritten — re-running the configure + flow (or the hub) picks up the new numbers. + ([#337](https://github.com/quickbeard/codev-code/pull/337)) + ### Skills - **Skill Hub in the desktop app** — a new Skill Hub entry in the sidebar From 701308da041ccb818649952dc63766621f001bd7 Mon Sep 17 00:00:00 2001 From: Quickbeard Date: Thu, 27 Aug 2026 10:24:16 +0700 Subject: [PATCH 3/5] refactor(gateway): plain model id, uniform 90% trigger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The table now carries only context windows — the trigger is always 90% of the window regardless of source (table, default, or gateway-reported) — and the 1M model's id is a plain string instead of base64. Co-Authored-By: Claude Fable 5 --- packages/codev-gateway/src/model-limits.ts | 57 ++++++++++--------- .../codev-gateway/test/config-block.test.ts | 7 ++- .../codev-gateway/test/model-limits.test.ts | 18 +++--- 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/packages/codev-gateway/src/model-limits.ts b/packages/codev-gateway/src/model-limits.ts index 30276be69e..18881d4de3 100644 --- a/packages/codev-gateway/src/model-limits.ts +++ b/packages/codev-gateway/src/model-limits.ts @@ -1,19 +1,20 @@ import { loadModelLimits, saveModelLimits } from "./auth-file" -// Per-model context windows and auto-compaction triggers, mirrored from the -// codev hub's src/lib/model-limits.ts so a hub-written and an agent-written -// codev.json agree on every number. The gateway serves custom models opencode -// doesn't recognize — a config model entry with no `limit` resolves to context -// 0, which mis-sizes the window gauge and disables auto-compaction outright — -// so every model written into a config block needs limits from here. +// Per-model context windows and auto-compaction triggers, modeled on the +// codev hub's src/lib/model-limits.ts (same window table, same auth-file +// cache; triggers here are uniformly COMPACT_PCT of the window). The gateway +// serves custom models opencode doesn't recognize — a config model entry with +// no `limit` resolves to context 0, which mis-sizes the window gauge and +// disables auto-compaction outright — so every model written into a config +// block needs limits from here. export interface ModelLimits { // The model's true window, written verbatim into `limit.context` — it drives // the "% context used" gauge, so understating it misreports every session. context: number - // Where auto-compaction should fire. Deliberately explicit rather than a - // percentage of `context`: the gap between the two is a judgement call per - // model, not a constant. + // Where auto-compaction should fire: always COMPACT_PCT of the window here + // (see limitsFromWindow), though a cached hub-written entry may carry its + // own number. trigger: number // Max output tokens. Absent ⇒ DEFAULT_OUTPUT_TOKENS. output?: number @@ -26,26 +27,25 @@ export const DEFAULT_OUTPUT_TOKENS = 65536 // shared reserve still yields exact per-model triggers. export const COMPACT_RESERVED = 40000 -// Percentage used to derive a trigger from a window we didn't pick ourselves, -// i.e. one reported by the gateway. Table entries state their trigger outright. -export const DEFAULT_COMPACT_PCT = 80 +// The trigger is always this share of the window, whatever its source — the +// table, the default, or a gateway-reported window. +export const COMPACT_PCT = 90 + +// Known gateway models' context windows, keyed by the exact id `/v1/models` +// reports. The 200K sibling of the 1M model is deliberately absent: +// DEFAULT_WINDOW already describes it, and an entry restating the default is +// one more thing to keep in sync. +const TABLE: Record = { + "MiniMax/MiniMax-M3": 1000000, + "zai-org/GLM-4.7-cc": 200000, +} // Unrecognized models are treated as 200K-class: it matches the smaller of the // models actually served, and guessing low is the safe direction — too small a // window wastes capacity, too large overruns the model and 400s mid-session. -export const DEFAULT_LIMITS: ModelLimits = { context: 200000, trigger: 160000 } - -// The 1M-window model's id, base64 like the hub's M3_ID so the upstream -// vendor's name stays out of casual greps of the shipped bundle. -const M3_ID = atob("TWluaU1heC9NaW5pTWF4LU0z") - -// Known gateway models, keyed by the exact id `/v1/models` reports. The 200K -// sibling of M3_ID is deliberately absent: DEFAULT_LIMITS already describes it, -// and an entry restating the default is one more thing to keep in sync. -const TABLE: Record = { - [M3_ID]: { context: 1000000, trigger: 800000 }, - "zai-org/GLM-4.7-cc": { context: 200000, trigger: 160000 }, -} +const DEFAULT_WINDOW = 200000 + +export const DEFAULT_LIMITS: ModelLimits = limitsFromWindow(DEFAULT_WINDOW) // Windows reported by the gateway, cached in the shared auth file (the hub // writes the same key from its model-choice step). Read once per process: @@ -83,14 +83,15 @@ export function cacheModelLimits(windows: Record { attachment: true, modalities: { input: ["text", "image"], output: ["text"] }, // Unknown id ⇒ the safe 200K default; input = trigger + reserved - // clamped to context (160000 + 40000 = 200000 exactly). + // (180000 + 40000) clamped to the 200000 window. limit: { context: 200000, input: 200000, output: 65536 }, }, "other/model": { @@ -73,12 +73,13 @@ describe("buildGatewayConfig", () => { }) // The per-model table: the 1M-window model compacts at its own trigger - // (input − reserved = 800000) while sharing the global 40000 reserve. + // (input − reserved = 900000, 90% of the window) while sharing the global + // 40000 reserve. test("writes the 1M window for the table's known model", () => { const config = buildGatewayConfig({ baseUrl: "https://gw.test", model: "MiniMax/MiniMax-M3" }) expect(config.provider.netgate.models["MiniMax/MiniMax-M3"].limit).toEqual({ context: 1000000, - input: 840000, + input: 940000, output: 65536, }) }) diff --git a/packages/codev-gateway/test/model-limits.test.ts b/packages/codev-gateway/test/model-limits.test.ts index 3a4c429323..bd54d6bf49 100644 --- a/packages/codev-gateway/test/model-limits.test.ts +++ b/packages/codev-gateway/test/model-limits.test.ts @@ -34,9 +34,11 @@ afterEach(() => { describe("limitsFor", () => { test("resolves the table's known models and defaults the rest", () => { - expect(limitsFor("MiniMax/MiniMax-M3")).toEqual({ context: 1000000, trigger: 800000 }) - expect(limitsFor("zai-org/GLM-4.7-cc")).toEqual({ context: 200000, trigger: 160000 }) + // Triggers are always 90% of the window, whatever its source. + expect(limitsFor("MiniMax/MiniMax-M3")).toEqual({ context: 1000000, trigger: 900000 }) + expect(limitsFor("zai-org/GLM-4.7-cc")).toEqual({ context: 200000, trigger: 180000 }) expect(limitsFor("anything/else")).toEqual(DEFAULT_LIMITS) + expect(DEFAULT_LIMITS).toEqual({ context: 200000, trigger: 180000 }) }) // The remote tier: a window the gateway (or the hub, which shares the auth @@ -52,7 +54,7 @@ describe("limitsFor", () => { describe("cacheModelLimits", () => { test("persists windows as full limits and makes them visible in-process", () => { cacheModelLimits({ "gw/model": { context: 300000, output: 32000 } }) - expect(limitsFor("gw/model")).toEqual({ context: 300000, trigger: 240000, output: 32000 }) + expect(limitsFor("gw/model")).toEqual({ context: 300000, trigger: 270000, output: 32000 }) }) // A gateway that reports nothing must neither churn the auth file nor clear @@ -76,8 +78,8 @@ describe("cacheModelLimits", () => { describe("declaredInput", () => { test("adds the reserve and clamps to the true window", () => { - expect(declaredInput({ context: 1000000, trigger: 800000 })).toBe(840000) - // DEFAULT_LIMITS lands exactly on the clamp: 160000 + 40000 = 200000. + expect(declaredInput({ context: 1000000, trigger: 900000 })).toBe(940000) + // The 200K default hits the clamp: 180000 + 40000 > 200000. expect(declaredInput(DEFAULT_LIMITS)).toBe(200000) // A trigger too close to the window can only move compaction earlier, // never let the declared budget exceed the model's real ceiling. @@ -86,9 +88,9 @@ describe("declaredInput", () => { }) describe("limitsFromWindow", () => { - test("derives the trigger at 80% and keeps output only when reported", () => { - expect(limitsFromWindow(300000)).toEqual({ context: 300000, trigger: 240000 }) - expect(limitsFromWindow(300000, 32000)).toEqual({ context: 300000, trigger: 240000, output: 32000 }) + test("derives the trigger at 90% and keeps output only when reported", () => { + expect(limitsFromWindow(300000)).toEqual({ context: 300000, trigger: 270000 }) + expect(limitsFromWindow(300000, 32000)).toEqual({ context: 300000, trigger: 270000, output: 32000 }) }) }) From 2c1925d0a6e6ae7a81a55e7fbc86e534dcd71d74 Mon Sep 17 00:00:00 2001 From: Quickbeard Date: Thu, 27 Aug 2026 10:24:51 +0700 Subject: [PATCH 4/5] docs: changelog reflects 90% trigger Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b483cfead..a7c5859023 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,8 @@ file is the human-readable summary of what changed and why. with the same 196,608-token window; the 1M-token model was treated as ~5× smaller than it is, so the context gauge overstated usage and auto-compaction fired around 167K tokens. Config blocks now carry each model's real window - and its own compaction trigger, mirroring what CoDev Hub writes: the 1M - model compacts at 800K, unknown models keep a safe 200K default, and windows + and its own compaction trigger (90% of the window): the 1M model compacts + at 900K, unknown models keep a safe 200K default, and windows the gateway itself reports (LiteLLM `model_group/info`, cached in the shared auth file like the hub does) override the built-in table without needing a release. Already-written configs are not rewritten — re-running the configure From 1c01a3e288499cadee56e12cd1a8dd8bf24441c8 Mon Sep 17 00:00:00 2001 From: Quickbeard Date: Thu, 27 Aug 2026 11:16:11 +0700 Subject: [PATCH 5/5] fix(gateway): M3 window is 262144 on the current deployment The gateway serves MiniMax-M3 with a 262144-token window (resource cap), not the model's native 1M. With the 90% trigger + 40000 reserve exceeding that window, declaredInput's clamp pins limit.input at the window itself. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 18 +++++++++--------- packages/codev-gateway/src/const.ts | 4 ++-- packages/codev-gateway/src/model-limits.ts | 9 +++++---- .../codev-gateway/test/config-block.test.ts | 12 ++++++------ .../codev-gateway/test/model-limits.test.ts | 6 +++--- 5 files changed, 25 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7c5859023..4534b25b84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,15 +11,15 @@ file is the human-readable summary of what changed and why. - **Per-model context limits** — configuring the gateway (the TUI `/configure` dialog, `codev configure`, or the desktop sign-in) used to stamp every model - with the same 196,608-token window; the 1M-token model was treated as ~5× - smaller than it is, so the context gauge overstated usage and auto-compaction - fired around 167K tokens. Config blocks now carry each model's real window - and its own compaction trigger (90% of the window): the 1M model compacts - at 900K, unknown models keep a safe 200K default, and windows - the gateway itself reports (LiteLLM `model_group/info`, cached in the shared - auth file like the hub does) override the built-in table without needing a - release. Already-written configs are not rewritten — re-running the configure - flow (or the hub) picks up the new numbers. + with the same 196,608-token window, so differently sized models all showed + the same context gauge and compacted at the same point. Config blocks now + carry each model's served window and a compaction trigger at 90% of it: + MiniMax-M3 gets its deployment's 262,144-token window, unknown models keep + a safe 200K default, and windows the gateway itself reports (LiteLLM + `model_group/info`, cached in the shared auth file like the hub does) + override the built-in table without needing a release. Already-written + configs are not rewritten — re-running the configure flow (or the hub) + picks up the new numbers. ([#337](https://github.com/quickbeard/codev-code/pull/337)) ### Skills diff --git a/packages/codev-gateway/src/const.ts b/packages/codev-gateway/src/const.ts index 4ae7e474d3..17421e786e 100644 --- a/packages/codev-gateway/src/const.ts +++ b/packages/codev-gateway/src/const.ts @@ -16,5 +16,5 @@ export const CLIENT_ID = atob("bGl0ZWxsbS10ZXN0") // Context windows and compaction triggers are per-model and live in // model-limits.ts, mirrored from the hub's src/lib/model-limits.ts. They used // to be flat GATEWAY_CONTEXT_WINDOW / GATEWAY_COMPACT_* constants here, which -// assumed every gateway model shared one 196608-token window — untrue once the -// gateway served both a 1M-token and a 200K-token model. +// assumed every gateway model shared one 196608-token window — untrue once +// the gateway served differently sized models. diff --git a/packages/codev-gateway/src/model-limits.ts b/packages/codev-gateway/src/model-limits.ts index 18881d4de3..2bca0e9ab4 100644 --- a/packages/codev-gateway/src/model-limits.ts +++ b/packages/codev-gateway/src/model-limits.ts @@ -32,11 +32,12 @@ export const COMPACT_RESERVED = 40000 export const COMPACT_PCT = 90 // Known gateway models' context windows, keyed by the exact id `/v1/models` -// reports. The 200K sibling of the 1M model is deliberately absent: -// DEFAULT_WINDOW already describes it, and an entry restating the default is -// one more thing to keep in sync. +// reports. M3's window is capped at 262144 on the current deployment (the +// model supports more, the gateway's resources don't). The 200K sibling is +// deliberately absent: DEFAULT_WINDOW already describes it, and an entry +// restating the default is one more thing to keep in sync. const TABLE: Record = { - "MiniMax/MiniMax-M3": 1000000, + "MiniMax/MiniMax-M3": 262144, "zai-org/GLM-4.7-cc": 200000, } diff --git a/packages/codev-gateway/test/config-block.test.ts b/packages/codev-gateway/test/config-block.test.ts index 92ae17e7ae..70df717cfb 100644 --- a/packages/codev-gateway/test/config-block.test.ts +++ b/packages/codev-gateway/test/config-block.test.ts @@ -72,14 +72,14 @@ describe("buildGatewayConfig", () => { }) }) - // The per-model table: the 1M-window model compacts at its own trigger - // (input − reserved = 900000, 90% of the window) while sharing the global - // 40000 reserve. - test("writes the 1M window for the table's known model", () => { + // The per-model table: M3's 262144-token window (capped by the deployment, + // not the model). Its 90% trigger + the 40000 reserve exceeds the window, + // so declaredInput's clamp pins input at the window itself. + test("writes the table window for the known model", () => { const config = buildGatewayConfig({ baseUrl: "https://gw.test", model: "MiniMax/MiniMax-M3" }) expect(config.provider.netgate.models["MiniMax/MiniMax-M3"].limit).toEqual({ - context: 1000000, - input: 940000, + context: 262144, + input: 262144, output: 65536, }) }) diff --git a/packages/codev-gateway/test/model-limits.test.ts b/packages/codev-gateway/test/model-limits.test.ts index bd54d6bf49..85cd146e76 100644 --- a/packages/codev-gateway/test/model-limits.test.ts +++ b/packages/codev-gateway/test/model-limits.test.ts @@ -35,7 +35,7 @@ afterEach(() => { describe("limitsFor", () => { test("resolves the table's known models and defaults the rest", () => { // Triggers are always 90% of the window, whatever its source. - expect(limitsFor("MiniMax/MiniMax-M3")).toEqual({ context: 1000000, trigger: 900000 }) + expect(limitsFor("MiniMax/MiniMax-M3")).toEqual({ context: 262144, trigger: 235930 }) expect(limitsFor("zai-org/GLM-4.7-cc")).toEqual({ context: 200000, trigger: 180000 }) expect(limitsFor("anything/else")).toEqual(DEFAULT_LIMITS) expect(DEFAULT_LIMITS).toEqual({ context: 200000, trigger: 180000 }) @@ -44,9 +44,9 @@ describe("limitsFor", () => { // The remote tier: a window the gateway (or the hub, which shares the auth // file) cached outranks the static table. test("prefers cached gateway-reported limits over the table", () => { - saveModelLimits({ "MiniMax/MiniMax-M3": { context: 500000, trigger: 400000 } }) + saveModelLimits({ "MiniMax/MiniMax-M3": { context: 500000, trigger: 450000 } }) resetModelLimitsCache() - expect(limitsFor("MiniMax/MiniMax-M3")).toEqual({ context: 500000, trigger: 400000 }) + expect(limitsFor("MiniMax/MiniMax-M3")).toEqual({ context: 500000, trigger: 450000 }) expect(limitsFor("anything/else")).toEqual(DEFAULT_LIMITS) }) })