diff --git a/CHANGELOG.md b/CHANGELOG.md index 932e576b82..4534b25b84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ file is the human-readable summary of what changed and why. ## Unreleased +### Gateway + +- **Per-model context limits** — configuring the gateway (the TUI `/configure` + dialog, `codev configure`, or the desktop sign-in) used to stamp every model + with the same 196,608-token window, so differently sized models all showed + the same context gauge and compacted at the same point. Config blocks now + carry each model's served window and a compaction trigger at 90% of it: + MiniMax-M3 gets its deployment's 262,144-token window, unknown models keep + a safe 200K default, and windows the gateway itself reports (LiteLLM + `model_group/info`, cached in the shared auth file like the hub does) + override the built-in table without needing a release. Already-written + configs are not rewritten — re-running the configure flow (or the hub) + picks up the new numbers. + ([#337](https://github.com/quickbeard/codev-code/pull/337)) + ### Skills - **Skill Hub in the desktop app** — a new Skill Hub entry in the sidebar diff --git a/packages/codev-gateway/src/auth-file.ts b/packages/codev-gateway/src/auth-file.ts index 9dc97f84dc..b21b5e9e55 100644 --- a/packages/codev-gateway/src/auth-file.ts +++ b/packages/codev-gateway/src/auth-file.ts @@ -2,6 +2,9 @@ import { randomBytes } from "node:crypto" import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs" import { homedir } from "node:os" import { dirname, join } from "node:path" +// Type-only: model-limits.ts imports loadModelLimits from here, so a value +// import would close a require cycle (hub src/lib/auth.ts has the same note). +import type { ModelLimits } from "./model-limits" export interface AuthData { access_token: string @@ -55,6 +58,7 @@ interface AuthFileContents { supabase_anon_key?: string gateway_url?: string skillhub_cookie?: string + model_limits?: Record } // Keyed off the hub's auth.json rather than the ~/.codev-hub/ dir (mirroring @@ -172,6 +176,20 @@ export function loadApiKey(): ApiKeyCreds | null { } } +// The gateway's per-model windows, shared with the hub (hub saveModelLimits +// writes the same key). Skips the write entirely for an empty map so a gateway +// that reports nothing — every max_input_tokens null, the case today — doesn't +// churn the auth file on every configure run. Callers go through +// model-limits.ts cacheModelLimits, which also resets its process-local cache. +export function saveModelLimits(limits: Record): void { + if (Object.keys(limits).length === 0) return + writeAuthFile({ ...(readAuthFile() ?? {}), model_limits: limits }) +} + +export function loadModelLimits(): Record | null { + return readAuthFile()?.model_limits ?? null +} + // Hub-only credential (written by `codevhub login --admin`); read here so // skill commands can ride an existing admin session, never written. export function loadSkillhubCookie(): string | null { diff --git a/packages/codev-gateway/src/backend.ts b/packages/codev-gateway/src/backend.ts index 7e68e753f3..26fa650d81 100644 --- a/packages/codev-gateway/src/backend.ts +++ b/packages/codev-gateway/src/backend.ts @@ -115,6 +115,57 @@ export async function fetchModels(apiKey: string, baseUrl: string, opts?: FetchO return ids } +// A window as the gateway reports it (hub ModelWindow). Deliberately NOT +// model-limits.ts's ModelLimits: callers convert with limitsFromWindow (via +// cacheModelLimits), which decides where the trigger belongs. +export interface ModelWindow { + context: number + output?: number +} + +// Per-model context windows from LiteLLM's /model_group/info, keyed by +// model_group id — the same id /v1/models reports. Entries whose +// max_input_tokens the gateway hasn't been told are skipped, which today is +// all of them: the field is nullable and only populated when an admin sets it +// on the model. So this returns {} on the current deployment and the static +// table in model-limits.ts carries every model; the moment an admin fills the +// field in, that model becomes gateway-driven with no CoDev Code release. +// +// Unlike fetchModels, this NEVER throws and never fail-stops the caller: a +// window is an optimization over a sane default, and configure must not break +// because a metadata endpoint 404s on some other gateway build. The endpoint +// lives at the gateway root, so a trailing /v1 is stripped before joining. +export async function fetchModelWindows( + apiKey: string, + baseUrl: string, + opts?: FetchOptions, +): Promise> { + const fetchImpl = opts?.fetchImpl ?? fetch + try { + const stripped = baseUrl.replace(/\/?v1\/?$/, "") + const res = await fetchImpl(`${stripped.endsWith("/") ? stripped : `${stripped}/`}model_group/info`, { + headers: { accept: "application/json", Authorization: `Bearer ${apiKey}` }, + signal: AbortSignal.timeout(MODELS_TIMEOUT_MS), + }) + if (!res.ok) return {} + const data = (await res.json()) as { + data?: Array<{ model_group?: string; max_input_tokens?: number | null; max_output_tokens?: number | null }> + } + const out: Record = {} + for (const entry of data.data ?? []) { + const id = entry.model_group + const context = entry.max_input_tokens + if (!id || typeof context !== "number" || context <= 0) continue + const output = + typeof entry.max_output_tokens === "number" && entry.max_output_tokens > 0 ? entry.max_output_tokens : undefined + out[id] = { context, ...(output ? { output } : {}) } + } + return out + } catch { + return {} + } +} + // Validates a key against the gateway's /key/info endpoint (LiteLLM): true on // 2xx, false on 401/403 (invalid/revoked), throws on anything else so callers // can distinguish "key is bad" from "couldn't reach the gateway". /key/info diff --git a/packages/codev-gateway/src/config-block.ts b/packages/codev-gateway/src/config-block.ts index e853620c69..8c8321314a 100644 --- a/packages/codev-gateway/src/config-block.ts +++ b/packages/codev-gateway/src/config-block.ts @@ -1,4 +1,4 @@ -import { GATEWAY_COMPACT_RESERVED, GATEWAY_CONTEXT_WINDOW, GATEWAY_MAX_OUTPUT_TOKENS } from "./const" +import { COMPACT_RESERVED, declaredInput, limitsFor, outputTokens } from "./model-limits" import { GATEWAY_PROVIDER } from "./provider" // A config model entry with no `modalities` resolves image input to false, and @@ -26,7 +26,10 @@ const MODEL_MODALITIES: { attachment: boolean; modalities: { input: Modality[]; // than that PR cannot attribute a `netgate` config. The per-model `limit` is // mandatory: without it opencode sizes the context window to 0 and disables // auto-compaction; the modality block is mandatory for the same reason (hub PR -// #219) — see MODEL_MODALITIES. +// #219) — see MODEL_MODALITIES. Each model's numbers come from +// model-limits.ts limitsFor — per-model, like the hub's — with `limit.input` +// carrying the model's own compaction trigger against the single global +// `compaction.reserved` (see declaredInput). // // Deliberately no top-level `model`, and the hub fragment dropped it in the // same round (hub PR #221): a pinned default outranks the TUI's persisted @@ -49,7 +52,7 @@ export function buildGatewayConfig(input: { apiKey?: string; baseUrl: string; mo return { compaction: { auto: true, - reserved: GATEWAY_COMPACT_RESERVED, + reserved: COMPACT_RESERVED, }, provider: { [GATEWAY_PROVIDER.id]: { @@ -59,32 +62,42 @@ export function buildGatewayConfig(input: { apiKey?: string; baseUrl: string; mo baseURL: normalizeGatewayBaseUrl(input.baseUrl), ...(input.apiKey === undefined ? {} : { apiKey: input.apiKey }), }, - models: Object.fromEntries( - ids.map((id) => [ - id, - { - name: id, - ...MODEL_MODALITIES, - limit: { - context: GATEWAY_CONTEXT_WINDOW, - output: GATEWAY_MAX_OUTPUT_TOKENS, - }, - }, - ]), - ), + models: Object.fromEntries(ids.map((id) => [id, modelEntry(id)])), }, }, } } +// One model's config entry, numbers resolved per model. `context` is the true +// window (the "% context used" gauge divides by it); `input` lands the model's +// own compaction trigger against the shared `compaction.reserved` above. +function modelEntry(id: string) { + const limits = limitsFor(id) + return { + name: id, + ...MODEL_MODALITIES, + limit: { + context: limits.context, + input: declaredInput(limits), + output: outputTokens(limits), + }, + } +} + // The codev.json fragment for a user-named custom provider connected through // the TUI's "Other" flow. Unlike buildGatewayConfig it writes no apiKey (the // credential lives in opencode's auth.json and the provider registry merges it // in by provider id at load) and no npm (the loader defaults to // @ai-sdk/openai-compatible). The per-model limit is still mandatory — without -// it opencode sizes the context window to 0 — so the gateway defaults stand in -// for the unknown endpoint's real limits, and MODEL_MODALITIES applies for the -// same reason it does to the gateway block. +// it opencode sizes the context window to 0 — so model-limits.ts stands in for +// the unknown endpoint's real limits (a known model id gets its table entry, +// anything else the safe 200K default), and MODEL_MODALITIES applies for the +// same reason it does to the gateway block. Deliberately no compaction block: +// connecting a provider must not change global compaction for unrelated ones. +// Without a configured reserve opencode subtracts its own smaller default from +// `limit.input`, so compaction fires a little later than the model's intended +// trigger but always below the true window — declaredInput's clamp keeps the +// declared budget inside `context`. export function buildCustomProviderConfig(input: { providerID: string name: string @@ -98,19 +111,7 @@ export function buildCustomProviderConfig(input: { options: { baseURL: normalizeGatewayBaseUrl(input.baseUrl), }, - models: Object.fromEntries( - input.models.map((id) => [ - id, - { - name: id, - ...MODEL_MODALITIES, - limit: { - context: GATEWAY_CONTEXT_WINDOW, - output: GATEWAY_MAX_OUTPUT_TOKENS, - }, - }, - ]), - ), + models: Object.fromEntries(input.models.map((id) => [id, modelEntry(id)])), }, }, } diff --git a/packages/codev-gateway/src/const.ts b/packages/codev-gateway/src/const.ts index d6bb46894d..17421e786e 100644 --- a/packages/codev-gateway/src/const.ts +++ b/packages/codev-gateway/src/const.ts @@ -13,11 +13,8 @@ export const LOGIN_SUCCESS_URL = `${BASE_URL}/codev/oauth/success` export const CLIENT_ID = atob("bGl0ZWxsbS10ZXN0") -// The self-hosted gateway model window and the compaction reserve derived from -// firing auto-compaction at ~85% of it — identical to the hub's constants so a -// hub-written and an agent-written codev.json agree byte-for-byte. -export const GATEWAY_CONTEXT_WINDOW = 196608 -export const GATEWAY_COMPACT_PCT = 85 -export const GATEWAY_COMPACT_TRIGGER = Math.round(GATEWAY_CONTEXT_WINDOW * (GATEWAY_COMPACT_PCT / 100)) -export const GATEWAY_COMPACT_RESERVED = GATEWAY_CONTEXT_WINDOW - GATEWAY_COMPACT_TRIGGER -export const GATEWAY_MAX_OUTPUT_TOKENS = 65536 +// Context windows and compaction triggers are per-model and live in +// model-limits.ts, mirrored from the hub's src/lib/model-limits.ts. They used +// to be flat GATEWAY_CONTEXT_WINDOW / GATEWAY_COMPACT_* constants here, which +// assumed every gateway model shared one 196608-token window — untrue once +// the gateway served differently sized models. diff --git a/packages/codev-gateway/src/index.ts b/packages/codev-gateway/src/index.ts index fb5c9dd762..a28f04808b 100644 --- a/packages/codev-gateway/src/index.ts +++ b/packages/codev-gateway/src/index.ts @@ -15,10 +15,13 @@ export { fetchApiKey, fetchCodevConfig, fetchModels, + fetchModelWindows, supabaseExchange, validateApiKey, type AnalysisBackendSession, + type ModelWindow, } from "./backend" +export { cacheModelLimits, limitsFor, type ModelLimits } from "./model-limits" export { buildCustomProviderConfig, buildGatewayConfig, normalizeGatewayBaseUrl } from "./config-block" export { CUSTOM_FALLBACK_PROVIDER, diff --git a/packages/codev-gateway/src/model-limits.ts b/packages/codev-gateway/src/model-limits.ts new file mode 100644 index 0000000000..2bca0e9ab4 --- /dev/null +++ b/packages/codev-gateway/src/model-limits.ts @@ -0,0 +1,118 @@ +import { loadModelLimits, saveModelLimits } from "./auth-file" + +// Per-model context windows and auto-compaction triggers, modeled on the +// codev hub's src/lib/model-limits.ts (same window table, same auth-file +// cache; triggers here are uniformly COMPACT_PCT of the window). The gateway +// serves custom models opencode doesn't recognize — a config model entry with +// no `limit` resolves to context 0, which mis-sizes the window gauge and +// disables auto-compaction outright — so every model written into a config +// block needs limits from here. + +export interface ModelLimits { + // The model's true window, written verbatim into `limit.context` — it drives + // the "% context used" gauge, so understating it misreports every session. + context: number + // Where auto-compaction should fire: always COMPACT_PCT of the window here + // (see limitsFromWindow), though a cached hub-written entry may carry its + // own number. + trigger: number + // Max output tokens. Absent ⇒ DEFAULT_OUTPUT_TOKENS. + output?: number +} + +export const DEFAULT_OUTPUT_TOKENS = 65536 + +// The global `compaction.reserved` — opencode's config schema has a single +// shared reserve with no per-model variant. See declaredInput() for how a +// shared reserve still yields exact per-model triggers. +export const COMPACT_RESERVED = 40000 + +// The trigger is always this share of the window, whatever its source — the +// table, the default, or a gateway-reported window. +export const COMPACT_PCT = 90 + +// Known gateway models' context windows, keyed by the exact id `/v1/models` +// reports. M3's window is capped at 262144 on the current deployment (the +// model supports more, the gateway's resources don't). The 200K sibling is +// deliberately absent: DEFAULT_WINDOW already describes it, and an entry +// restating the default is one more thing to keep in sync. +const TABLE: Record = { + "MiniMax/MiniMax-M3": 262144, + "zai-org/GLM-4.7-cc": 200000, +} + +// Unrecognized models are treated as 200K-class: it matches the smaller of the +// models actually served, and guessing low is the safe direction — too small a +// window wastes capacity, too large overruns the model and 400s mid-session. +const DEFAULT_WINDOW = 200000 + +export const DEFAULT_LIMITS: ModelLimits = limitsFromWindow(DEFAULT_WINDOW) + +// Windows reported by the gateway, cached in the shared auth file (the hub +// writes the same key from its model-choice step). Read once per process: +// building a config block calls limitsFor once per model and none of them can +// change the file mid-run. +let remoteCache: Record | null | undefined + +function remoteLimits(): Record | null { + if (remoteCache === undefined) remoteCache = loadModelLimits() + return remoteCache +} + +// Test seam, also used by cacheModelLimits: lets a fresh write become visible +// without the once-per-process cache leaking across cases. +export function resetModelLimitsCache(): void { + remoteCache = undefined +} + +// Persist gateway-reported windows for limitsFor's remote tier. Unlike the +// hub's one-shot CLI, the TUI and desktop are long-lived processes that write +// the cache and build a config block in the same run, so the process-local +// cache is reset alongside the write. An empty map is skipped entirely — a +// gateway that reports nothing must not churn the auth file or clear limits +// the hub cached. +export function cacheModelLimits(windows: Record): void { + const entries = Object.entries(windows) + if (entries.length === 0) return + saveModelLimits(Object.fromEntries(entries.map(([id, w]) => [id, limitsFromWindow(w.context, w.output)]))) + resetModelLimitsCache() +} + +// Resolve one model's limits. Precedence is remote → table → default: the +// gateway is authoritative about its own models when it says anything at all, +// and the table carries the models it currently reports nothing for. +export function limitsFor(modelId: string): ModelLimits { + const remote = remoteLimits()?.[modelId] + if (remote) return remote + const window = TABLE[modelId] + return window ? limitsFromWindow(window) : DEFAULT_LIMITS +} + +// Turn a context window into full limits, wherever the window came from. +export function limitsFromWindow(context: number, output?: number): ModelLimits { + return { + context, + trigger: Math.round((context * COMPACT_PCT) / 100), + ...(output ? { output } : {}), + } +} + +// opencode's trigger is `limit.input − compaction.reserved`, falling back to +// `limit.context − maxOutputTokens` when `limit.input` is absent — a branch +// that ignores `reserved` entirely. So `input` is what makes the reserve +// authoritative, and the only per-model lever over a trigger that otherwise +// shares one global reserve across every model in the config. Solving +// `input − reserved = trigger` gives `input = trigger + reserved`, landing each +// model's trigger exactly on target regardless of what the others need. +// +// Clamped to the true window: above `context` the declared budget would let a +// session run past the model's real ceiling before compacting. Clamping only +// moves a trigger earlier, so a bad table entry degrades into early compaction +// rather than 400s. +export function declaredInput(limits: ModelLimits): number { + return Math.min(limits.trigger + COMPACT_RESERVED, limits.context) +} + +export function outputTokens(limits: ModelLimits): number { + return limits.output ?? DEFAULT_OUTPUT_TOKENS +} diff --git a/packages/codev-gateway/test/backend.test.ts b/packages/codev-gateway/test/backend.test.ts index fdd5fc870a..313529cbf0 100644 --- a/packages/codev-gateway/test/backend.test.ts +++ b/packages/codev-gateway/test/backend.test.ts @@ -1,5 +1,5 @@ import { afterAll, describe, expect, test } from "bun:test" -import { fetchModels, validateApiKey } from "../src/backend" +import { fetchModels, fetchModelWindows, validateApiKey } from "../src/backend" // A real loopback server (not a mock) that records the path + auth header of the // last request, so tests assert the exact endpoint fetchModels/validateApiKey @@ -52,6 +52,41 @@ describe("fetchModels", () => { }) }) +describe("fetchModelWindows", () => { + test("reads windows from /model_group/info at the gateway root", async () => { + handler = () => + Response.json({ + data: [ + { model_group: "big", max_input_tokens: 1000000, max_output_tokens: 65536 }, + { model_group: "small", max_input_tokens: 200000 }, + ], + }) + seen.length = 0 + const windows = await fetchModelWindows("sk-test", `${base}/v1`) + expect(windows).toEqual({ big: { context: 1000000, output: 65536 }, small: { context: 200000 } }) + // The endpoint lives at the root — the /v1 the base carried is stripped. + expect(seen.at(-1)).toEqual({ path: "/model_group/info", auth: "Bearer sk-test" }) + }) + + // LiteLLM's max_input_tokens is nullable and unset until an admin fills it + // in — those entries carry no information and must not shadow the table. + test("skips entries without a positive max_input_tokens", async () => { + handler = () => + Response.json({ + data: [{ model_group: "a", max_input_tokens: null }, { model_group: "b" }, { max_input_tokens: 5 }], + }) + expect(await fetchModelWindows("k", base)).toEqual({}) + }) + + // A window is an optimization over a sane default: a gateway build without + // the endpoint (404) or an unreachable one must not fail the configure flow. + test("returns {} instead of throwing on errors", async () => { + handler = () => new Response("nope", { status: 404 }) + expect(await fetchModelWindows("k", base)).toEqual({}) + expect(await fetchModelWindows("k", "http://127.0.0.1:9")).toEqual({}) + }) +}) + describe("validateApiKey", () => { test("returns true on a 2xx from /key/info at the gateway root", async () => { handler = () => Response.json({ ok: true }) diff --git a/packages/codev-gateway/test/config-block.test.ts b/packages/codev-gateway/test/config-block.test.ts index d8306798f3..70df717cfb 100644 --- a/packages/codev-gateway/test/config-block.test.ts +++ b/packages/codev-gateway/test/config-block.test.ts @@ -1,5 +1,29 @@ -import { describe, expect, test } from "bun:test" +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" import { buildCustomProviderConfig, buildGatewayConfig, normalizeGatewayBaseUrl } from "../src/config-block" +import { resetModelLimitsCache } from "../src/model-limits" + +let tempDir: string +let originalTestHome: string | undefined + +// The config block resolves limits through limitsFor, whose remote tier reads +// the shared auth file — point it at an empty home so a hub-cached +// model_limits on the developer's machine can't leak into the expectations. +beforeEach(() => { + tempDir = mkdtempSync(join(tmpdir(), "codev-gateway-config-")) + originalTestHome = process.env.CODEV_TEST_HOME + process.env.CODEV_TEST_HOME = tempDir + resetModelLimitsCache() +}) + +afterEach(() => { + if (originalTestHome === undefined) delete process.env.CODEV_TEST_HOME + if (originalTestHome !== undefined) process.env.CODEV_TEST_HOME = originalTestHome + rmSync(tempDir, { recursive: true, force: true }) + resetModelLimitsCache() +}) describe("normalizeGatewayBaseUrl", () => { test("appends /v1 when missing", () => { @@ -21,7 +45,7 @@ describe("buildGatewayConfig", () => { models: ["MiniMax/MiniMax-M2.7", "other/model"], }) expect(config).toEqual({ - compaction: { auto: true, reserved: 29491 }, + compaction: { auto: true, reserved: 40000 }, provider: { netgate: { npm: "@ai-sdk/openai-compatible", @@ -32,13 +56,15 @@ describe("buildGatewayConfig", () => { name: "MiniMax/MiniMax-M2.7", attachment: true, modalities: { input: ["text", "image"], output: ["text"] }, - limit: { context: 196608, output: 65536 }, + // Unknown id ⇒ the safe 200K default; input = trigger + reserved + // (180000 + 40000) clamped to the 200000 window. + limit: { context: 200000, input: 200000, output: 65536 }, }, "other/model": { name: "other/model", attachment: true, modalities: { input: ["text", "image"], output: ["text"] }, - limit: { context: 196608, output: 65536 }, + limit: { context: 200000, input: 200000, output: 65536 }, }, }, }, @@ -46,6 +72,18 @@ describe("buildGatewayConfig", () => { }) }) + // The per-model table: M3's 262144-token window (capped by the deployment, + // not the model). Its 90% trigger + the 40000 reserve exceeds the window, + // so declaredInput's clamp pins input at the window itself. + test("writes the table window for the known model", () => { + const config = buildGatewayConfig({ baseUrl: "https://gw.test", model: "MiniMax/MiniMax-M3" }) + expect(config.provider.netgate.models["MiniMax/MiniMax-M3"].limit).toEqual({ + context: 262144, + input: 262144, + output: 65536, + }) + }) + // Without the declaration a custom-provider model resolves to text-only and // the session replaces an attached image with an error note (hub PR #219). test("declares image input so attachments are not stripped", () => { @@ -107,13 +145,13 @@ describe("buildCustomProviderConfig", () => { name: "m1", attachment: true, modalities: { input: ["text", "image"], output: ["text"] }, - limit: { context: 196608, output: 65536 }, + limit: { context: 200000, input: 200000, output: 65536 }, }, m2: { name: "m2", attachment: true, modalities: { input: ["text", "image"], output: ["text"] }, - limit: { context: 196608, output: 65536 }, + limit: { context: 200000, input: 200000, output: 65536 }, }, }, }, diff --git a/packages/codev-gateway/test/model-limits.test.ts b/packages/codev-gateway/test/model-limits.test.ts new file mode 100644 index 0000000000..85cd146e76 --- /dev/null +++ b/packages/codev-gateway/test/model-limits.test.ts @@ -0,0 +1,102 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test" +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { saveApiKey, saveModelLimits } from "../src/auth-file" +import { + cacheModelLimits, + DEFAULT_LIMITS, + declaredInput, + limitsFor, + limitsFromWindow, + outputTokens, + resetModelLimitsCache, +} from "../src/model-limits" + +let tempDir: string +let originalTestHome: string | undefined + +// CODEV_TEST_HOME rather than HOME: Bun's homedir() ignores runtime HOME +// changes, so overriding HOME here would silently hit the real ~/.codev. +beforeEach(() => { + tempDir = mkdtempSync(join(tmpdir(), "codev-gateway-limits-")) + originalTestHome = process.env.CODEV_TEST_HOME + process.env.CODEV_TEST_HOME = tempDir + resetModelLimitsCache() +}) + +afterEach(() => { + if (originalTestHome === undefined) delete process.env.CODEV_TEST_HOME + if (originalTestHome !== undefined) process.env.CODEV_TEST_HOME = originalTestHome + rmSync(tempDir, { recursive: true, force: true }) + resetModelLimitsCache() +}) + +describe("limitsFor", () => { + test("resolves the table's known models and defaults the rest", () => { + // Triggers are always 90% of the window, whatever its source. + expect(limitsFor("MiniMax/MiniMax-M3")).toEqual({ context: 262144, trigger: 235930 }) + expect(limitsFor("zai-org/GLM-4.7-cc")).toEqual({ context: 200000, trigger: 180000 }) + expect(limitsFor("anything/else")).toEqual(DEFAULT_LIMITS) + expect(DEFAULT_LIMITS).toEqual({ context: 200000, trigger: 180000 }) + }) + + // The remote tier: a window the gateway (or the hub, which shares the auth + // file) cached outranks the static table. + test("prefers cached gateway-reported limits over the table", () => { + saveModelLimits({ "MiniMax/MiniMax-M3": { context: 500000, trigger: 450000 } }) + resetModelLimitsCache() + expect(limitsFor("MiniMax/MiniMax-M3")).toEqual({ context: 500000, trigger: 450000 }) + expect(limitsFor("anything/else")).toEqual(DEFAULT_LIMITS) + }) +}) + +describe("cacheModelLimits", () => { + test("persists windows as full limits and makes them visible in-process", () => { + cacheModelLimits({ "gw/model": { context: 300000, output: 32000 } }) + expect(limitsFor("gw/model")).toEqual({ context: 300000, trigger: 270000, output: 32000 }) + }) + + // A gateway that reports nothing must neither churn the auth file nor clear + // limits the hub cached there. + test("an empty map leaves an existing cache untouched", () => { + saveModelLimits({ "gw/model": { context: 300000, trigger: 240000 } }) + resetModelLimitsCache() + cacheModelLimits({}) + expect(limitsFor("gw/model")).toEqual({ context: 300000, trigger: 240000 }) + }) + + // The auth file's read-merge-write must keep the other blocks: the write + // path is shared with credentials the hub and the upload daemon depend on. + test("does not clobber the api-key block", async () => { + saveApiKey({ apiKey: "sk-1", model: "m" }) + cacheModelLimits({ "gw/model": { context: 300000 } }) + const { loadApiKey } = await import("../src/auth-file") + expect(loadApiKey()).toEqual({ apiKey: "sk-1", model: "m" }) + }) +}) + +describe("declaredInput", () => { + test("adds the reserve and clamps to the true window", () => { + expect(declaredInput({ context: 1000000, trigger: 900000 })).toBe(940000) + // The 200K default hits the clamp: 180000 + 40000 > 200000. + expect(declaredInput(DEFAULT_LIMITS)).toBe(200000) + // A trigger too close to the window can only move compaction earlier, + // never let the declared budget exceed the model's real ceiling. + expect(declaredInput({ context: 100000, trigger: 90000 })).toBe(100000) + }) +}) + +describe("limitsFromWindow", () => { + test("derives the trigger at 90% and keeps output only when reported", () => { + expect(limitsFromWindow(300000)).toEqual({ context: 300000, trigger: 270000 }) + expect(limitsFromWindow(300000, 32000)).toEqual({ context: 300000, trigger: 270000, output: 32000 }) + }) +}) + +describe("outputTokens", () => { + test("falls back to the 65536 default", () => { + expect(outputTokens({ context: 1, trigger: 1 })).toBe(65536) + expect(outputTokens({ context: 1, trigger: 1, output: 32000 })).toBe(32000) + }) +}) diff --git a/packages/desktop/src/main/gateway-sso/controller.test.ts b/packages/desktop/src/main/gateway-sso/controller.test.ts index 3c318c35de..244b8c49b4 100644 --- a/packages/desktop/src/main/gateway-sso/controller.test.ts +++ b/packages/desktop/src/main/gateway-sso/controller.test.ts @@ -122,11 +122,16 @@ describe("gateway sso controller", () => { test("complete writes the auth file and returns the key beside a keyless netgate config", async () => { const savedKeys: unknown[] = [] + const cachedWindows: unknown[] = [] const controller = controllerWith({ silentSso: async () => auth, fetchCodevConfig: async () => ({ supabaseUrl: "s", supabaseAnonKey: "k", gatewayUrl: "https://gw.example.test" }), fetchApiKey: async () => "sk-live", fetchModels: async () => ["model-a", "model-b"], + fetchModelWindows: async () => ({ "model-a": { context: 300000 } }), + cacheModelLimits: (windows) => { + cachedWindows.push(windows) + }, saveCodevConfig: () => {}, saveApiKey: (creds) => { savedKeys.push(creds) @@ -140,6 +145,8 @@ describe("gateway sso controller", () => { // SSO-minted keys record no base_url (mirrors the TUI flow). expect(savedKeys).toEqual([{ apiKey: "sk-live", baseUrl: undefined, model: "model-a" }]) + // Gateway-reported windows were cached for the config block's limitsFor. + expect(cachedWindows).toEqual([{ "model-a": { context: 300000 } }]) expect(result.providerID).toBe("netgate") // The key rides beside the config for the renderer's auth.set; the config // block itself must not carry it (codev.json stays credential-free). diff --git a/packages/desktop/src/main/gateway-sso/controller.ts b/packages/desktop/src/main/gateway-sso/controller.ts index e74da6a773..4784dec046 100644 --- a/packages/desktop/src/main/gateway-sso/controller.ts +++ b/packages/desktop/src/main/gateway-sso/controller.ts @@ -1,8 +1,10 @@ import { buildGatewayConfig, + cacheModelLimits, fetchApiKey, fetchCodevConfig, fetchModels, + fetchModelWindows, GATEWAY_PROVIDER, loadApiKey, loadGatewayUrl, @@ -46,6 +48,8 @@ type GatewayFns = { fetchCodevConfig: typeof fetchCodevConfig fetchApiKey: typeof fetchApiKey fetchModels: typeof fetchModels + fetchModelWindows: typeof fetchModelWindows + cacheModelLimits: typeof cacheModelLimits saveCodevConfig: typeof saveCodevConfig saveApiKey: typeof saveApiKey loadApiKey: typeof loadApiKey @@ -62,6 +66,8 @@ export function createGatewaySsoController(deps: Deps) { fetchCodevConfig, fetchApiKey, fetchModels, + fetchModelWindows, + cacheModelLimits, saveCodevConfig, saveApiKey, loadApiKey, @@ -242,6 +248,11 @@ export function createGatewaySsoController(deps: Deps) { const url = gatewayUrl setState({ status: "Fetching models..." }) models = await withRetries("Fetching models", () => gateway.fetchModels(key, url, fetchOpts)) + // Gateway-reported windows override the static table in the config block + // complete() writes (limitsFor's remote tier). Never blocks the flow: + // fetchModelWindows returns {} on any failure and an empty map is not + // cached, so the static per-model table stands in. + gateway.cacheModelLimits(await gateway.fetchModelWindows(key, url, fetchOpts)) setState({ status: "" }) return models }, diff --git a/packages/tui/src/component/dialog-configure.tsx b/packages/tui/src/component/dialog-configure.tsx index c36698964d..88bb6bbc6c 100644 --- a/packages/tui/src/component/dialog-configure.tsx +++ b/packages/tui/src/component/dialog-configure.tsx @@ -1,8 +1,10 @@ import { buildGatewayConfig, + cacheModelLimits, fetchApiKey, fetchCodevConfig, fetchModels, + fetchModelWindows, GATEWAY_PROVIDER, login, saveApiKey, @@ -298,6 +300,11 @@ async function runModelFlow(ctx: FlowContext, creds: GatewayCreds) { // key would just fail on the first chat. Silently retry, then quit rather // than drop into an unconfigured chat. models = await withRetries("Fetching models", setStatus, () => fetchModels(creds.apiKey, creds.gatewayBaseUrl)) + // Gateway-reported windows override the static table in the config block + // written below (limitsFor's remote tier). Never blocks the flow: + // fetchModelWindows returns {} on any failure and an empty map is not + // cached, so the static per-model table stands in. + cacheModelLimits(await fetchModelWindows(creds.apiKey, creds.gatewayBaseUrl)) } catch (err) { failAndQuit( ctx, diff --git a/packages/tui/test/custom-provider.view.test.tsx b/packages/tui/test/custom-provider.view.test.tsx index 532fdb5148..6ba87b4683 100644 --- a/packages/tui/test/custom-provider.view.test.tsx +++ b/packages/tui/test/custom-provider.view.test.tsx @@ -200,13 +200,13 @@ test("Custom provider with a base URL discovers models and writes the codev.json name: "mock-small", attachment: true, modalities: { input: ["text", "image"], output: ["text"] }, - limit: { context: 196608, output: 65536 }, + limit: { context: 200000, input: 200000, output: 65536 }, }, "mock-large": { name: "mock-large", attachment: true, modalities: { input: ["text", "image"], output: ["text"] }, - limit: { context: 196608, output: 65536 }, + limit: { context: 200000, input: 200000, output: 65536 }, }, }, },