feat(gateway): per-model context limits

CHANGELOG.md

@@ -7,6 +7,21 @@ file is the human-readable summary of what changed and why.
 
 ## Unreleased
 
+### Gateway
+
+- **Per-model context limits** — configuring the gateway (the TUI `/configure`
+  dialog, `codev configure`, or the desktop sign-in) used to stamp every model
+  with the same 196,608-token window, so differently sized models all showed
+  the same context gauge and compacted at the same point. Config blocks now
+  carry each model's served window and a compaction trigger at 90% of it:
+  MiniMax-M3 gets its deployment's 262,144-token window, unknown models keep
+  a safe 200K default, and windows the gateway itself reports (LiteLLM
+  `model_group/info`, cached in the shared auth file like the hub does)
+  override the built-in table without needing a release. Already-written
+  configs are not rewritten — re-running the configure flow (or the hub)
+  picks up the new numbers.
+  ([#337](https://github.com/quickbeard/codev-code/pull/337))
+
 ### Skills
 
 - **Skill Hub in the desktop app** — a new Skill Hub entry in the sidebar

packages/codev-gateway/src/auth-file.ts

@@ -2,6 +2,9 @@ import { randomBytes } from "node:crypto"
 import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"
 import { homedir } from "node:os"
 import { dirname, join } from "node:path"
+// Type-only: model-limits.ts imports loadModelLimits from here, so a value
+// import would close a require cycle (hub src/lib/auth.ts has the same note).
+import type { ModelLimits } from "./model-limits"
 
 export interface AuthData {
   access_token: string
@@ -55,6 +58,7 @@ interface AuthFileContents {
   supabase_anon_key?: string
   gateway_url?: string
   skillhub_cookie?: string
+  model_limits?: Record<string, ModelLimits>
 }
 
 // Keyed off the hub's auth.json rather than the ~/.codev-hub/ dir (mirroring
@@ -172,6 +176,20 @@ export function loadApiKey(): ApiKeyCreds | null {
   }
 }
 
+// The gateway's per-model windows, shared with the hub (hub saveModelLimits
+// writes the same key). Skips the write entirely for an empty map so a gateway
+// that reports nothing — every max_input_tokens null, the case today — doesn't
+// churn the auth file on every configure run. Callers go through
+// model-limits.ts cacheModelLimits, which also resets its process-local cache.
+export function saveModelLimits(limits: Record<string, ModelLimits>): void {
+  if (Object.keys(limits).length === 0) return
+  writeAuthFile({ ...(readAuthFile() ?? {}), model_limits: limits })
+}
+
+export function loadModelLimits(): Record<string, ModelLimits> | null {
+  return readAuthFile()?.model_limits ?? null
+}
+
 // Hub-only credential (written by `codevhub login --admin`); read here so
 // skill commands can ride an existing admin session, never written.
 export function loadSkillhubCookie(): string | null {

packages/codev-gateway/src/backend.ts

@@ -115,6 +115,57 @@ export async function fetchModels(apiKey: string, baseUrl: string, opts?: FetchO
   return ids
 }
 
+// A window as the gateway reports it (hub ModelWindow). Deliberately NOT
+// model-limits.ts's ModelLimits: callers convert with limitsFromWindow (via
+// cacheModelLimits), which decides where the trigger belongs.
+export interface ModelWindow {
+  context: number
+  output?: number
+}
+
+// Per-model context windows from LiteLLM's /model_group/info, keyed by
+// model_group id — the same id /v1/models reports. Entries whose
+// max_input_tokens the gateway hasn't been told are skipped, which today is
+// all of them: the field is nullable and only populated when an admin sets it
+// on the model. So this returns {} on the current deployment and the static
+// table in model-limits.ts carries every model; the moment an admin fills the
+// field in, that model becomes gateway-driven with no CoDev Code release.
+//
+// Unlike fetchModels, this NEVER throws and never fail-stops the caller: a
+// window is an optimization over a sane default, and configure must not break
+// because a metadata endpoint 404s on some other gateway build. The endpoint
+// lives at the gateway root, so a trailing /v1 is stripped before joining.
+export async function fetchModelWindows(
+  apiKey: string,
+  baseUrl: string,
+  opts?: FetchOptions,
+): Promise<Record<string, ModelWindow>> {
+  const fetchImpl = opts?.fetchImpl ?? fetch
+  try {
+    const stripped = baseUrl.replace(/\/?v1\/?$/, "")
+    const res = await fetchImpl(`${stripped.endsWith("/") ? stripped : `${stripped}/`}model_group/info`, {
+      headers: { accept: "application/json", Authorization: `Bearer ${apiKey}` },
+      signal: AbortSignal.timeout(MODELS_TIMEOUT_MS),
+    })
+    if (!res.ok) return {}
+    const data = (await res.json()) as {
+      data?: Array<{ model_group?: string; max_input_tokens?: number | null; max_output_tokens?: number | null }>
+    }
+    const out: Record<string, ModelWindow> = {}
+    for (const entry of data.data ?? []) {
+      const id = entry.model_group
+      const context = entry.max_input_tokens
+      if (!id || typeof context !== "number" || context <= 0) continue
+      const output =
+        typeof entry.max_output_tokens === "number" && entry.max_output_tokens > 0 ? entry.max_output_tokens : undefined
+      out[id] = { context, ...(output ? { output } : {}) }
+    }
+    return out
+  } catch {
+    return {}
+  }
+}
+
 // Validates a key against the gateway's /key/info endpoint (LiteLLM): true on
 // 2xx, false on 401/403 (invalid/revoked), throws on anything else so callers
 // can distinguish "key is bad" from "couldn't reach the gateway". /key/info

packages/codev-gateway/src/config-block.ts

@@ -1,4 +1,4 @@
-import { GATEWAY_COMPACT_RESERVED, GATEWAY_CONTEXT_WINDOW, GATEWAY_MAX_OUTPUT_TOKENS } from "./const"
+import { COMPACT_RESERVED, declaredInput, limitsFor, outputTokens } from "./model-limits"
 import { GATEWAY_PROVIDER } from "./provider"
 
 // A config model entry with no `modalities` resolves image input to false, and
@@ -26,7 +26,10 @@ const MODEL_MODALITIES: { attachment: boolean; modalities: { input: Modality[];
 // than that PR cannot attribute a `netgate` config. The per-model `limit` is
 // mandatory: without it opencode sizes the context window to 0 and disables
 // auto-compaction; the modality block is mandatory for the same reason (hub PR
-// #219) — see MODEL_MODALITIES.
+// #219) — see MODEL_MODALITIES. Each model's numbers come from
+// model-limits.ts limitsFor — per-model, like the hub's — with `limit.input`
+// carrying the model's own compaction trigger against the single global
+// `compaction.reserved` (see declaredInput).
 //
 // Deliberately no top-level `model`, and the hub fragment dropped it in the
 // same round (hub PR #221): a pinned default outranks the TUI's persisted
@@ -49,7 +52,7 @@ export function buildGatewayConfig(input: { apiKey?: string; baseUrl: string; mo
   return {
     compaction: {
       auto: true,
-      reserved: GATEWAY_COMPACT_RESERVED,
+      reserved: COMPACT_RESERVED,
     },
     provider: {
       [GATEWAY_PROVIDER.id]: {
@@ -59,32 +62,42 @@ export function buildGatewayConfig(input: { apiKey?: string; baseUrl: string; mo
           baseURL: normalizeGatewayBaseUrl(input.baseUrl),
           ...(input.apiKey === undefined ? {} : { apiKey: input.apiKey }),
         },
-        models: Object.fromEntries(
-          ids.map((id) => [
-            id,
-            {
-              name: id,
-              ...MODEL_MODALITIES,
-              limit: {
-                context: GATEWAY_CONTEXT_WINDOW,
-                output: GATEWAY_MAX_OUTPUT_TOKENS,
-              },
-            },
-          ]),
-        ),
+        models: Object.fromEntries(ids.map((id) => [id, modelEntry(id)])),
       },
     },
   }
 }
 
+// One model's config entry, numbers resolved per model. `context` is the true
+// window (the "% context used" gauge divides by it); `input` lands the model's
+// own compaction trigger against the shared `compaction.reserved` above.
+function modelEntry(id: string) {
+  const limits = limitsFor(id)
+  return {
+    name: id,
+    ...MODEL_MODALITIES,
+    limit: {
+      context: limits.context,
+      input: declaredInput(limits),
+      output: outputTokens(limits),
+    },
+  }
+}
+
 // The codev.json fragment for a user-named custom provider connected through
 // the TUI's "Other" flow. Unlike buildGatewayConfig it writes no apiKey (the
 // credential lives in opencode's auth.json and the provider registry merges it
 // in by provider id at load) and no npm (the loader defaults to
 // @ai-sdk/openai-compatible). The per-model limit is still mandatory — without
-// it opencode sizes the context window to 0 — so the gateway defaults stand in
-// for the unknown endpoint's real limits, and MODEL_MODALITIES applies for the
-// same reason it does to the gateway block.
+// it opencode sizes the context window to 0 — so model-limits.ts stands in for
+// the unknown endpoint's real limits (a known model id gets its table entry,
+// anything else the safe 200K default), and MODEL_MODALITIES applies for the
+// same reason it does to the gateway block. Deliberately no compaction block:
+// connecting a provider must not change global compaction for unrelated ones.
+// Without a configured reserve opencode subtracts its own smaller default from
+// `limit.input`, so compaction fires a little later than the model's intended
+// trigger but always below the true window — declaredInput's clamp keeps the
+// declared budget inside `context`.
 export function buildCustomProviderConfig(input: {
   providerID: string
   name: string
@@ -98,19 +111,7 @@ export function buildCustomProviderConfig(input: {
         options: {
           baseURL: normalizeGatewayBaseUrl(input.baseUrl),
         },
-        models: Object.fromEntries(
-          input.models.map((id) => [
-            id,
-            {
-              name: id,
-              ...MODEL_MODALITIES,
-              limit: {
-                context: GATEWAY_CONTEXT_WINDOW,
-                output: GATEWAY_MAX_OUTPUT_TOKENS,
-              },
-            },
-          ]),
-        ),
+        models: Object.fromEntries(input.models.map((id) => [id, modelEntry(id)])),
       },
     },
   }

packages/codev-gateway/src/const.ts

@@ -13,11 +13,8 @@ export const LOGIN_SUCCESS_URL = `${BASE_URL}/codev/oauth/success`
 
 export const CLIENT_ID = atob("bGl0ZWxsbS10ZXN0")
 
-// The self-hosted gateway model window and the compaction reserve derived from
-// firing auto-compaction at ~85% of it — identical to the hub's constants so a
-// hub-written and an agent-written codev.json agree byte-for-byte.
-export const GATEWAY_CONTEXT_WINDOW = 196608
-export const GATEWAY_COMPACT_PCT = 85
-export const GATEWAY_COMPACT_TRIGGER = Math.round(GATEWAY_CONTEXT_WINDOW * (GATEWAY_COMPACT_PCT / 100))
-export const GATEWAY_COMPACT_RESERVED = GATEWAY_CONTEXT_WINDOW - GATEWAY_COMPACT_TRIGGER
-export const GATEWAY_MAX_OUTPUT_TOKENS = 65536
+// Context windows and compaction triggers are per-model and live in
+// model-limits.ts, mirrored from the hub's src/lib/model-limits.ts. They used
+// to be flat GATEWAY_CONTEXT_WINDOW / GATEWAY_COMPACT_* constants here, which
+// assumed every gateway model shared one 196608-token window — untrue once
+// the gateway served differently sized models.

packages/codev-gateway/src/index.ts

@@ -15,10 +15,13 @@ export {
   fetchApiKey,
   fetchCodevConfig,
   fetchModels,
+  fetchModelWindows,
   supabaseExchange,
   validateApiKey,
   type AnalysisBackendSession,
+  type ModelWindow,
 } from "./backend"
+export { cacheModelLimits, limitsFor, type ModelLimits } from "./model-limits"
 export { buildCustomProviderConfig, buildGatewayConfig, normalizeGatewayBaseUrl } from "./config-block"
 export {
   CUSTOM_FALLBACK_PROVIDER,

packages/codev-gateway/src/model-limits.ts

@@ -0,0 +1,118 @@
+import { loadModelLimits, saveModelLimits } from "./auth-file"
+
+// Per-model context windows and auto-compaction triggers, modeled on the
+// codev hub's src/lib/model-limits.ts (same window table, same auth-file
+// cache; triggers here are uniformly COMPACT_PCT of the window). The gateway
+// serves custom models opencode doesn't recognize — a config model entry with
+// no `limit` resolves to context 0, which mis-sizes the window gauge and
+// disables auto-compaction outright — so every model written into a config
+// block needs limits from here.
+
+export interface ModelLimits {
+  // The model's true window, written verbatim into `limit.context` — it drives
+  // the "% context used" gauge, so understating it misreports every session.
+  context: number
+  // Where auto-compaction should fire: always COMPACT_PCT of the window here
+  // (see limitsFromWindow), though a cached hub-written entry may carry its
+  // own number.
+  trigger: number
+  // Max output tokens. Absent ⇒ DEFAULT_OUTPUT_TOKENS.
+  output?: number
+}
+
+export const DEFAULT_OUTPUT_TOKENS = 65536
+
+// The global `compaction.reserved` — opencode's config schema has a single
+// shared reserve with no per-model variant. See declaredInput() for how a
+// shared reserve still yields exact per-model triggers.
+export const COMPACT_RESERVED = 40000
+
+// The trigger is always this share of the window, whatever its source — the
+// table, the default, or a gateway-reported window.
+export const COMPACT_PCT = 90
+
+// Known gateway models' context windows, keyed by the exact id `/v1/models`
+// reports. M3's window is capped at 262144 on the current deployment (the
+// model supports more, the gateway's resources don't). The 200K sibling is
+// deliberately absent: DEFAULT_WINDOW already describes it, and an entry
+// restating the default is one more thing to keep in sync.
+const TABLE: Record<string, number> = {
+  "MiniMax/MiniMax-M3": 262144,
+  "zai-org/GLM-4.7-cc": 200000,
+}
+
+// Unrecognized models are treated as 200K-class: it matches the smaller of the
+// models actually served, and guessing low is the safe direction — too small a
+// window wastes capacity, too large overruns the model and 400s mid-session.
+const DEFAULT_WINDOW = 200000
+
+export const DEFAULT_LIMITS: ModelLimits = limitsFromWindow(DEFAULT_WINDOW)
+
+// Windows reported by the gateway, cached in the shared auth file (the hub
+// writes the same key from its model-choice step). Read once per process:
+// building a config block calls limitsFor once per model and none of them can
+// change the file mid-run.
+let remoteCache: Record<string, ModelLimits> | null | undefined
+
+function remoteLimits(): Record<string, ModelLimits> | null {
+  if (remoteCache === undefined) remoteCache = loadModelLimits()
+  return remoteCache
+}
+
+// Test seam, also used by cacheModelLimits: lets a fresh write become visible
+// without the once-per-process cache leaking across cases.
+export function resetModelLimitsCache(): void {
+  remoteCache = undefined
+}
+
+// Persist gateway-reported windows for limitsFor's remote tier. Unlike the
+// hub's one-shot CLI, the TUI and desktop are long-lived processes that write
+// the cache and build a config block in the same run, so the process-local
+// cache is reset alongside the write. An empty map is skipped entirely — a
+// gateway that reports nothing must not churn the auth file or clear limits
+// the hub cached.
+export function cacheModelLimits(windows: Record<string, { context: number; output?: number }>): void {
+  const entries = Object.entries(windows)
+  if (entries.length === 0) return
+  saveModelLimits(Object.fromEntries(entries.map(([id, w]) => [id, limitsFromWindow(w.context, w.output)])))
+  resetModelLimitsCache()
+}
+
+// Resolve one model's limits. Precedence is remote → table → default: the
+// gateway is authoritative about its own models when it says anything at all,
+// and the table carries the models it currently reports nothing for.
+export function limitsFor(modelId: string): ModelLimits {
+  const remote = remoteLimits()?.[modelId]
+  if (remote) return remote
+  const window = TABLE[modelId]
+  return window ? limitsFromWindow(window) : DEFAULT_LIMITS
+}
+
+// Turn a context window into full limits, wherever the window came from.
+export function limitsFromWindow(context: number, output?: number): ModelLimits {
+  return {
+    context,
+    trigger: Math.round((context * COMPACT_PCT) / 100),
+    ...(output ? { output } : {}),
+  }
+}
+
+// opencode's trigger is `limit.input − compaction.reserved`, falling back to
+// `limit.context − maxOutputTokens` when `limit.input` is absent — a branch
+// that ignores `reserved` entirely. So `input` is what makes the reserve
+// authoritative, and the only per-model lever over a trigger that otherwise
+// shares one global reserve across every model in the config. Solving
+// `input − reserved = trigger` gives `input = trigger + reserved`, landing each
+// model's trigger exactly on target regardless of what the others need.
+//
+// Clamped to the true window: above `context` the declared budget would let a
+// session run past the model's real ceiling before compacting. Clamping only
+// moves a trigger earlier, so a bad table entry degrades into early compaction
+// rather than 400s.
+export function declaredInput(limits: ModelLimits): number {
+  return Math.min(limits.trigger + COMPACT_RESERVED, limits.context)
+}
+
+export function outputTokens(limits: ModelLimits): number {
+  return limits.output ?? DEFAULT_OUTPUT_TOKENS
+}