diff --git a/apps/server/src/usage/usageAggregation.test.ts b/apps/server/src/usage/usageAggregation.test.ts index 08b3f3ae1b8c8..48a1a7742cef7 100644 --- a/apps/server/src/usage/usageAggregation.test.ts +++ b/apps/server/src/usage/usageAggregation.test.ts @@ -19,6 +19,7 @@ const rates: RateTable = new Map([ cacheCreationCostPerToken: 2.5e-5, }, ultrafast: null, + contextTiers: [], }, ], ]); diff --git a/apps/server/src/usage/usagePricing.test.ts b/apps/server/src/usage/usagePricing.test.ts index dc0f37cf1376e..48f007c1b6d7d 100644 --- a/apps/server/src/usage/usagePricing.test.ts +++ b/apps/server/src/usage/usagePricing.test.ts @@ -195,6 +195,104 @@ describe("usage pricing", () => { expect(cost("gpt-6-sol", "ultrafast")).toBe(cost("gpt-6-sol", "standard")); }); + it("prices a request above the model's long-context threshold at its above-tier rates", () => { + const table = parseRateTable({ + "claude-haiku-5-5": { + input_cost_per_token: 1e-6, + output_cost_per_token: 5e-6, + cache_read_input_token_cost: 1e-7, + cache_creation_input_token_cost: 1.25e-6, + input_cost_per_token_above_100k_tokens: 2e-6, + output_cost_per_token_above_100k_tokens: 7.5e-6, + cache_read_input_token_cost_above_100k_tokens: 2e-7, + cache_creation_input_token_cost_above_100k_tokens: 2.5e-6, + }, + "gpt-6-sol": { + ...rate(2e-6, 2e-7), + input_cost_per_token_priority: 4e-6, + output_cost_per_token_priority: 2e-5, + cache_read_input_token_cost_priority: 4e-7, + input_cost_per_token_above_272k_tokens_priority: 8e-6, + output_cost_per_token_above_272k_tokens_priority: 3e-5, + cache_read_input_token_cost_above_272k_tokens_priority: 8e-7, + }, + }); + const longRequest = ( + model: string, + speed: UsageSpeed, + totals: { uncached: number; cacheRead: number; cacheWrite: number; output: number }, + ) => ({ + model, + speed, + reportedCostUsd: null, + totals: { + uncachedInputTokens: totals.uncached, + cachedInputTokens: totals.cacheRead, + cacheCreationTokens: totals.cacheWrite, + outputTokens: totals.output, + reasoningTokens: 0, + }, + }); + + // The Claude prompt is input + cache read + cache write: 120K, above the 100K tier. + expect + .soft( + priceUsage( + table, + longRequest("claude-haiku-5-5", "standard", { + uncached: 10_000, + cacheRead: 100_000, + cacheWrite: 10_000, + output: 1_000, + }), + ).costUsd, + ) + .toBeCloseTo(10_000 * 2e-6 + 100_000 * 2e-7 + 10_000 * 2.5e-6 + 1_000 * 7.5e-6, 12); + expect( + priceUsage( + table, + longRequest("gpt-6-sol", "fast", { + uncached: 20_000, + cacheRead: 280_000, + cacheWrite: 0, + output: 1_000, + }), + ).costUsd, + ).toBeCloseTo(20_000 * 8e-6 + 280_000 * 8e-7 + 1_000 * 3e-5, 12); + }); + + it("bills the long-context tier only above its threshold, filling missing tier fields from base", () => { + const table = parseRateTable({ + "gpt-6-sol": { + input_cost_per_token: 2e-6, + output_cost_per_token: 1e-5, + cache_read_input_token_cost: 2e-7, + input_cost_per_token_priority: 4e-6, + output_cost_per_token_priority: 2e-5, + input_cost_per_token_above_272k_tokens: 4e-6, + }, + }); + const cost = (prompt: number, speed: UsageSpeed = "standard") => + priceUsage(table, { + model: "gpt-6-sol", + speed, + reportedCostUsd: null, + totals: { + uncachedInputTokens: prompt - 200_000, + cachedInputTokens: 200_000, + cacheCreationTokens: 0, + outputTokens: 1_000, + reasoningTokens: 0, + }, + }).costUsd; + + expect(cost(272_000)).toBeCloseTo(72_000 * 2e-6 + 200_000 * 2e-7 + 1_000 * 1e-5, 12); + // Output keeps the base rate; cache reads keep the base discount relative to input. + expect(cost(272_001)).toBeCloseTo(72_001 * 4e-6 + 200_000 * 4e-7 + 1_000 * 1e-5, 12); + // Without a priority long-context rate, priority requests keep the priority rate. + expect(cost(272_001, "fast")).toBeCloseTo(72_001 * 4e-6 + 200_000 * 4e-7 + 1_000 * 2e-5, 12); + }); + it("keeps the canonical Fable rate separate from DeepInfra in either order", () => { const canonical = ["claude-fable-5", rate(1e-5, 1e-6)] as const; const deepInfra = ["deepinfra/anthropic/claude-fable-5", rate(1e-5)] as const; @@ -239,8 +337,12 @@ describe("usage pricing", () => { ...rate(1), input_cost_per_token_priority: 2, output_cost_per_token_priority: 10, + input_cost_per_token_above_272k_tokens: 2, + input_cost_per_token_above_272k_tokens_priority: 4, + output_cost_per_token_above_272k_tokens_priority: 20, }, - "provider-b/example-model": rate(1), + "provider-b/example-model": { ...rate(1), input_cost_per_token_above_272k_tokens: 2 }, + "provider-c/example-model": rate(1), }); expect(lookupRate(table, "example-model")).toEqual( diff --git a/apps/server/src/usage/usagePricing.ts b/apps/server/src/usage/usagePricing.ts index 609a573f54f63..184b22c7810b3 100644 --- a/apps/server/src/usage/usagePricing.ts +++ b/apps/server/src/usage/usagePricing.ts @@ -14,12 +14,23 @@ export interface TokenRates { readonly cacheCreationCostPerToken: number; } -/** Omits LiteLLM's above_272k_tokens, flex and batches variants, which transcripts do not record. */ -export interface ModelRate extends TokenRates { +interface SpeedRates extends TokenRates { readonly fast: TokenRates | null; readonly ultrafast: TokenRates | null; } +/** A request whose prompt exceeds `thresholdTokens` bills every token at this tier. */ +interface ContextTier { + readonly thresholdTokens: number; + readonly rate: SpeedRates; +} + +/** Omits LiteLLM's flex and batches variants, which transcripts do not record. */ +export interface ModelRate extends SpeedRates { + /** Sorted by descending threshold. */ + readonly contextTiers: readonly ContextTier[]; +} + export type RateTable = ReadonlyMap; export function createOverrideRateTable( @@ -37,6 +48,7 @@ export function createOverrideRateTable( (prices.cacheWriteCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000, fast: null, ultrafast: null, + contextTiers: [], }, ]), ); @@ -53,9 +65,10 @@ function readTokenRates( entry: LiteLlmEntry, suffix: string, standard?: TokenRates, + outputFallback?: number, ): TokenRates | null { const input = finiteNumber(entry[`input_cost_per_token${suffix}`]); - const output = finiteNumber(entry[`output_cost_per_token${suffix}`]); + const output = finiteNumber(entry[`output_cost_per_token${suffix}`]) ?? outputFallback ?? null; if (input === null || output === null) return null; const cacheRate = (name: string, field: "cacheReadCostPerToken" | "cacheCreationCostPerToken") => finiteNumber(entry[`${name}${suffix}`]) ?? @@ -89,6 +102,40 @@ function fastMultiplier(entry: LiteLlmEntry): number | null { return fast !== null && fast > 0 ? fast : null; } +function readSpeedRates(entry: LiteLlmEntry, standard: TokenRates, suffix = ""): SpeedRates { + const multiple = fastMultiplier(entry); + return { + ...standard, + fast: + multiple === null + ? readTokenRates(entry, `${suffix}_priority`, standard) + : scaleTokenRates(standard, multiple), + ultrafast: readTokenRates(entry, `${suffix}_ultrafast`, standard), + }; +} + +const CONTEXT_TIER_INPUT_KEY = + /^input_cost_per_token_above_(\d{1,7})k_tokens(?:_priority|_ultrafast)?$/; + +// Fields a tier omits fall back to base rates, so a priority-only tier still prices standard requests at base. +function readContextTiers(entry: LiteLlmEntry, base: TokenRates): ContextTier[] { + const thresholds = new Set(); + for (const key of Object.keys(entry)) { + const match = CONTEXT_TIER_INPUT_KEY.exec(key); + if (match !== null) thresholds.add(match[1]!); + } + return [...thresholds] + .map((threshold) => { + const suffix = `_above_${threshold}k_tokens`; + const standard = readTokenRates(entry, suffix, base, base.outputCostPerToken) ?? base; + return { + thresholdTokens: Number(threshold) * 1_000, + rate: readSpeedRates(entry, standard, suffix), + }; + }) + .toSorted((a, b) => b.thresholdTokens - a.thresholdTokens); +} + // LiteLLM prefixes for the model makers' own APIs; their price beats a cloud's or reseller's for a bare name. const FIRST_PARTY_PROVIDERS = new Set([ "anthropic", @@ -113,14 +160,9 @@ export function parseRateTable(document: unknown): RateTable { const key = normalizeRateKey(name); if (key.length === 0) continue; - const multiple = fastMultiplier(entry); table.set(key, { - ...standard, - fast: - multiple === null - ? readTokenRates(entry, "_priority", standard) - : scaleTokenRates(standard, multiple), - ultrafast: readTokenRates(entry, "_ultrafast", standard), + ...readSpeedRates(entry, standard), + contextTiers: readContextTiers(entry, standard), }); } @@ -154,8 +196,8 @@ function sameTokenRates(a: TokenRates | null, b: TokenRates | null): boolean { ); } -// Resellers often omit speed tiers, so a missing tier fills from the other entry instead of conflicting. -function mergeAliasRate(a: ModelRate, b: ModelRate): ModelRate | null { +// Resellers often omit speed and context tiers, so a missing tier fills from the other entry instead of conflicting. +function mergeSpeedRates(a: SpeedRates, b: SpeedRates): SpeedRates | null { const fast = a.fast ?? b.fast; const ultrafast = a.ultrafast ?? b.ultrafast; if ( @@ -168,8 +210,39 @@ function mergeAliasRate(a: ModelRate, b: ModelRate): ModelRate | null { return { ...a, fast, ultrafast }; } -function ratesAt(rate: ModelRate, speed: UsageSpeed): TokenRates { - return (speed === "standard" ? null : rate[speed]) ?? rate; +function mergeContextTiers( + a: readonly ContextTier[], + b: readonly ContextTier[], +): readonly ContextTier[] | null { + if (a.length === 0) return b; + if (b.length === 0) return a; + if (a.length !== b.length) return null; + const merged: ContextTier[] = []; + for (const [index, tier] of a.entries()) { + const other = b[index]!; + const rate = mergeSpeedRates(tier.rate, other.rate); + if (tier.thresholdTokens !== other.thresholdTokens || rate === null) return null; + merged.push({ thresholdTokens: tier.thresholdTokens, rate }); + } + return merged; +} + +function mergeAliasRate(a: ModelRate, b: ModelRate): ModelRate | null { + const speeds = mergeSpeedRates(a, b); + const contextTiers = mergeContextTiers(a.contextTiers, b.contextTiers); + return speeds === null || contextTiers === null ? null : { ...speeds, contextTiers }; +} + +function promptTokens(totals: UsageTokenTotals): number { + return totals.uncachedInputTokens + totals.cachedInputTokens + totals.cacheCreationTokens; +} + +// A tier without the requested speed keeps the base tier's rate for that speed. +function ratesAt(rate: ModelRate, speed: UsageSpeed, totals: UsageTokenTotals): TokenRates { + const prompt = promptTokens(totals); + const tier = rate.contextTiers.find((tier) => prompt > tier.thresholdTokens)?.rate; + if (speed === "standard") return tier ?? rate; + return tier?.[speed] ?? rate[speed] ?? tier ?? rate; } function normalizeRateKey(model: string): string { @@ -274,11 +347,13 @@ export function priceUsage( return reported === null ? unsplit(0, "unpriced") : unsplit(reported, "providerReported"); } - const listCost = costByCategory(totals, ratesAt(rate, record.speed)); + const listCost = costByCategory(totals, ratesAt(rate, record.speed, totals)); const listCostUsd = sumCategories(listCost); if (reported !== null && listCostUsd <= 0) return unsplit(reported, "providerReported"); const premiumUsd = - record.speed === "standard" ? 0 : listCostUsd - sumCategories(costByCategory(totals, rate)); + record.speed === "standard" + ? 0 + : listCostUsd - sumCategories(costByCategory(totals, ratesAt(rate, "standard", totals))); const scale = reported === null ? 1 : reported / listCostUsd; return { costUsd: reported ?? listCostUsd, @@ -301,6 +376,6 @@ export function cacheSavingsUsd( const rate = overrides?.get(record.model.trim()) ?? lookupRate(table, record.rateModel ?? record.model); if (rate === null) return 0; - const rates = ratesAt(rate, record.speed); + const rates = ratesAt(rate, record.speed, record.totals); return record.totals.cachedInputTokens * (rates.inputCostPerToken - rates.cacheReadCostPerToken); }