diff --git a/apps/server/src/provider/Drivers/claudeUsage.ts b/apps/server/src/provider/Drivers/claudeUsage.ts index aca7696d31df5..be49dfc3ba307 100644 --- a/apps/server/src/provider/Drivers/claudeUsage.ts +++ b/apps/server/src/provider/Drivers/claudeUsage.ts @@ -56,6 +56,11 @@ function parseClaudeRecord(parsed: unknown): readonly UsageRecord[] { messageId === null && requestId === null ? null : `${messageId ?? ""}:${requestId ?? ""}`; const cost = record["costUSD"]; + const cacheCreation = usageRecord["cache_creation"]; + const cacheCreation1hTokens = + typeof cacheCreation === "object" && cacheCreation !== null + ? tokenCount((cacheCreation as Record)["ephemeral_1h_input_tokens"]) + : 0; const executor: UsageRecord = { provider: "claude", @@ -70,6 +75,7 @@ function parseClaudeRecord(parsed: unknown): readonly UsageRecord[] { // Anthropic folds thinking tokens into output and does not break them out. reasoningTokens: 0, }, + ...(cacheCreation1hTokens > 0 ? { cacheCreation1hTokens } : {}), reportedCostUsd: typeof cost === "number" && Number.isFinite(cost) ? cost : null, speed: usageRecord["speed"] === "fast" ? "fast" : "standard", dedupeKey, diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index a1de34fac38bb..740e86dfbbe81 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -2687,7 +2687,7 @@ describe("UsageService", () => { 60 + 20 * (version === 4 ? 1 : 2), ); yield* upgraded.awaitPersisted; - assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v11.json"); + assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v12.json"); assert.deepStrictEqual( (yield* Effect.promise(() => NodeFSP.readdir(stateDir))) .filter((name) => name.startsWith("usage-scan-cache")) diff --git a/apps/server/src/usage/usageAggregation.test.ts b/apps/server/src/usage/usageAggregation.test.ts index 08b3f3ae1b8c8..3ff1e98df521c 100644 --- a/apps/server/src/usage/usageAggregation.test.ts +++ b/apps/server/src/usage/usageAggregation.test.ts @@ -12,11 +12,13 @@ const rates: RateTable = new Map([ outputCostPerToken: 5e-5, cacheReadCostPerToken: 1e-6, cacheCreationCostPerToken: 1.25e-5, + cacheCreation1hCostPerToken: 1.25e-5, fast: { inputCostPerToken: 2e-5, outputCostPerToken: 1e-4, cacheReadCostPerToken: 2e-6, cacheCreationCostPerToken: 2.5e-5, + cacheCreation1hCostPerToken: 2.5e-5, }, ultrafast: null, }, diff --git a/apps/server/src/usage/usagePricing.test.ts b/apps/server/src/usage/usagePricing.test.ts index dc0f37cf1376e..fe9f056b06a7f 100644 --- a/apps/server/src/usage/usagePricing.test.ts +++ b/apps/server/src/usage/usagePricing.test.ts @@ -47,7 +47,13 @@ describe("usage pricing", () => { }); for (const reportedCostUsd of [null, 99]) { - expect(priceUsage(table, record("example-model", reportedCostUsd), overrides)).toMatchObject({ + expect( + priceUsage( + table, + { ...record("example-model", reportedCostUsd, "fast"), cacheCreation1hTokens: 750_000 }, + overrides, + ), + ).toMatchObject({ costUsd: 13.5, costSource: "modelPriced", }); @@ -137,14 +143,17 @@ describe("usage pricing", () => { ); }); - it("splits cost by category and prices the speed premium", () => { + it("splits mixed-TTL cache costs and speed premiums while preserving reported totals", () => { const table = parseRateTable({ "claude-opus-5-5": { ...rate(4e-6, 4e-7), cache_creation_input_token_cost: 5e-6, + cache_creation_input_token_cost_above_1hr: 8e-6, provider_specific_entry: { fast: 2 }, }, + "claude-sonnet-5-5": { ...rate(2e-6), cache_creation_input_token_cost: 2.5e-6 }, }); + const mixed = { ...record("claude-opus-5-5", null, "fast"), cacheCreation1hTokens: 750_000 }; const split = (input: number, cacheRead: number, cacheWrite: number, output: number) => ({ input: expect.closeTo(input), cacheRead: expect.closeTo(cacheRead), @@ -152,18 +161,22 @@ describe("usage pricing", () => { output: expect.closeTo(output), }); - expect(priceUsage(table, record("claude-opus-5-5", null, "fast"))).toEqual({ - costUsd: expect.closeTo(58.8), + expect(priceUsage(table, mixed)).toEqual({ + costUsd: expect.closeTo(63.3), costSource: "modelPriced", - categoryCostUsd: split(8, 0.8, 10, 40), - speedPremiumUsd: expect.closeTo(29.4), + categoryCostUsd: split(8, 0.8, 14.5, 40), + speedPremiumUsd: expect.closeTo(31.65), }); - expect(priceUsage(table, record("claude-opus-5-5", 29.4, "fast"))).toEqual({ - costUsd: 29.4, + expect(priceUsage(table, { ...mixed, reportedCostUsd: 31.65 })).toEqual({ + costUsd: 31.65, costSource: "providerReported", - categoryCostUsd: split(4, 0.4, 5, 20), - speedPremiumUsd: expect.closeTo(14.7), + categoryCostUsd: split(4, 0.4, 7.25, 20), + speedPremiumUsd: expect.closeTo(15.825), }); + expect( + priceUsage(table, { ...mixed, model: "claude-sonnet-5-5", speed: "standard" }).categoryCostUsd + ?.cacheWrite, + ).toBeCloseTo(2.5); expect(priceUsage(table, record("unknown-model", 29.4, "fast"))).toMatchObject({ costUsd: 29.4, categoryCostUsd: null, @@ -175,6 +188,8 @@ describe("usage pricing", () => { const table = parseRateTable({ "gpt-6-astra": { ...rate(1e-5, 1e-6), + cache_creation_input_token_cost: 5e-6, + cache_creation_input_token_cost_above_1hr: 8e-6, input_cost_per_token_priority: 2e-5, output_cost_per_token_priority: 1e-4, cache_read_input_token_cost_priority: 2e-6, @@ -184,7 +199,11 @@ describe("usage pricing", () => { "gpt-6-sol": rate(2e-6, 2e-7), }); const cost = (model: string, speed: UsageSpeed) => - priceUsage(table, record(model, null, speed)).costUsd; + priceUsage(table, { ...record(model, null, speed), cacheCreation1hTokens: 750_000 }).costUsd; + expect( + priceUsage(table, { ...record("gpt-6-astra", null, "fast"), cacheCreation1hTokens: 750_000 }) + .categoryCostUsd?.cacheWrite, + ).toBeCloseTo(14.5); const standard = cost("gpt-6-astra", "standard"); expect(cost("gpt-6-astra", "fast")).toBeCloseTo(2 * standard); diff --git a/apps/server/src/usage/usagePricing.ts b/apps/server/src/usage/usagePricing.ts index 609a573f54f63..4f08f8a6056b6 100644 --- a/apps/server/src/usage/usagePricing.ts +++ b/apps/server/src/usage/usagePricing.ts @@ -2,7 +2,6 @@ import type { UsageCategoryCost, UsageCostSource, UsageModelPriceOverride, - UsageTokenTotals, } from "@t3tools/contracts"; import type { UsageRecord, UsageSpeed } from "@t3tools/provider-core/server/usage"; @@ -12,6 +11,8 @@ export interface TokenRates { readonly outputCostPerToken: number; readonly cacheReadCostPerToken: number; readonly cacheCreationCostPerToken: number; + /** Cache writes with a 1-hour TTL. Equals the 5-minute rate when unpublished. */ + readonly cacheCreation1hCostPerToken: number; } /** Omits LiteLLM's above_272k_tokens, flex and batches variants, which transcripts do not record. */ @@ -26,19 +27,23 @@ export function createOverrideRateTable( overrides: Readonly>, ): RateTable { return new Map( - Object.entries(overrides).map(([model, prices]) => [ - model.trim(), - { - inputCostPerToken: prices.inputCostPerMillionTokens / 1_000_000, - outputCostPerToken: prices.outputCostPerMillionTokens / 1_000_000, - cacheReadCostPerToken: - (prices.cacheReadCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000, - cacheCreationCostPerToken: - (prices.cacheWriteCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000, - fast: null, - ultrafast: null, - }, - ]), + Object.entries(overrides).map(([model, prices]) => { + const cacheCreationCostPerToken = + (prices.cacheWriteCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000; + return [ + model.trim(), + { + inputCostPerToken: prices.inputCostPerMillionTokens / 1_000_000, + outputCostPerToken: prices.outputCostPerMillionTokens / 1_000_000, + cacheReadCostPerToken: + (prices.cacheReadCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000, + cacheCreationCostPerToken, + cacheCreation1hCostPerToken: cacheCreationCostPerToken, + fast: null, + ultrafast: null, + }, + ]; + }), ); } @@ -48,7 +53,7 @@ function finiteNumber(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } -// Missing cache rates fall back to input pricing; Anthropic normally discounts reads and charges extra for writes. +// Missing cache rates fall back to input pricing, and missing 1-hour write rates to the 5-minute rate. function readTokenRates( entry: LiteLlmEntry, suffix: string, @@ -57,18 +62,29 @@ function readTokenRates( const input = finiteNumber(entry[`input_cost_per_token${suffix}`]); const output = finiteNumber(entry[`output_cost_per_token${suffix}`]); if (input === null || output === null) return null; - const cacheRate = (name: string, field: "cacheReadCostPerToken" | "cacheCreationCostPerToken") => + const cacheRate = ( + name: string, + field: "cacheReadCostPerToken" | "cacheCreationCostPerToken" | "cacheCreation1hCostPerToken", + fallback: number, + ) => finiteNumber(entry[`${name}${suffix}`]) ?? (standard !== undefined && standard.inputCostPerToken > 0 ? (standard[field] / standard.inputCostPerToken) * input - : input); + : fallback); + const cacheCreationCostPerToken = cacheRate( + "cache_creation_input_token_cost", + "cacheCreationCostPerToken", + input, + ); return { inputCostPerToken: input, outputCostPerToken: output, - cacheReadCostPerToken: cacheRate("cache_read_input_token_cost", "cacheReadCostPerToken"), - cacheCreationCostPerToken: cacheRate( - "cache_creation_input_token_cost", - "cacheCreationCostPerToken", + cacheReadCostPerToken: cacheRate("cache_read_input_token_cost", "cacheReadCostPerToken", input), + cacheCreationCostPerToken, + cacheCreation1hCostPerToken: cacheRate( + "cache_creation_input_token_cost_above_1hr", + "cacheCreation1hCostPerToken", + cacheCreationCostPerToken, ), }; } @@ -79,6 +95,7 @@ function scaleTokenRates(rates: TokenRates, multiple: number): TokenRates { outputCostPerToken: rates.outputCostPerToken * multiple, cacheReadCostPerToken: rates.cacheReadCostPerToken * multiple, cacheCreationCostPerToken: rates.cacheCreationCostPerToken * multiple, + cacheCreation1hCostPerToken: rates.cacheCreation1hCostPerToken * multiple, }; } @@ -150,7 +167,8 @@ function sameTokenRates(a: TokenRates | null, b: TokenRates | null): boolean { a.inputCostPerToken === b.inputCostPerToken && a.outputCostPerToken === b.outputCostPerToken && a.cacheReadCostPerToken === b.cacheReadCostPerToken && - a.cacheCreationCostPerToken === b.cacheCreationCostPerToken + a.cacheCreationCostPerToken === b.cacheCreationCostPerToken && + a.cacheCreation1hCostPerToken === b.cacheCreation1hCostPerToken ); } @@ -228,7 +246,7 @@ function resolveRate(table: RateTable, model: string): ModelRate | null { export type PricedRecord = Pick< UsageRecord, - "model" | "rateModel" | "totals" | "speed" | "reportedCostUsd" + "model" | "rateModel" | "totals" | "cacheCreation1hTokens" | "speed" | "reportedCostUsd" >; export interface PricedUsage { @@ -238,11 +256,18 @@ export interface PricedUsage { readonly speedPremiumUsd: number; } -function costByCategory(totals: UsageTokenTotals, rates: TokenRates): UsageCategoryCost { +function costByCategory(record: PricedRecord, rates: TokenRates): UsageCategoryCost { + const { totals } = record; + const cacheCreation1hTokens = Math.min( + record.cacheCreation1hTokens ?? 0, + totals.cacheCreationTokens, + ); return { input: totals.uncachedInputTokens * rates.inputCostPerToken, cacheRead: totals.cachedInputTokens * rates.cacheReadCostPerToken, - cacheWrite: totals.cacheCreationTokens * rates.cacheCreationCostPerToken, + cacheWrite: + (totals.cacheCreationTokens - cacheCreation1hTokens) * rates.cacheCreationCostPerToken + + cacheCreation1hTokens * rates.cacheCreation1hCostPerToken, output: totals.outputTokens * rates.outputCostPerToken, }; } @@ -257,7 +282,7 @@ export function priceUsage( record: PricedRecord, overrides?: RateTable, ): PricedUsage { - const { model, totals, reportedCostUsd } = record; + const { model, reportedCostUsd } = record; const override = overrides?.get(model.trim()); const reported = override === undefined && reportedCostUsd !== null && Number.isFinite(reportedCostUsd) @@ -274,11 +299,11 @@ export function priceUsage( return reported === null ? unsplit(0, "unpriced") : unsplit(reported, "providerReported"); } - const listCost = costByCategory(totals, ratesAt(rate, record.speed)); + const listCost = costByCategory(record, ratesAt(rate, record.speed)); const listCostUsd = sumCategories(listCost); if (reported !== null && listCostUsd <= 0) return unsplit(reported, "providerReported"); const premiumUsd = - record.speed === "standard" ? 0 : listCostUsd - sumCategories(costByCategory(totals, rate)); + record.speed === "standard" ? 0 : listCostUsd - sumCategories(costByCategory(record, rate)); const scale = reported === null ? 1 : reported / listCostUsd; return { costUsd: reported ?? listCostUsd, diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index d0e5eaceec269..5ba14779df51b 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -65,7 +65,16 @@ describe("scan cache round trip", () => { [ "/a.jsonl", 100, - [record(), record({ dedupeKey: "msg_2:", model: "claude-opus-5-5", speed: "fast" })], + [ + record(), + record({ + dedupeKey: "msg_2:", + model: "anthropic/claude-opus-5-5", + rateModel: "claude-opus-5-5", + speed: "fast", + cacheCreation1hTokens: 8, + }), + ], ], [ "/b.jsonl", @@ -217,15 +226,11 @@ describe("scan cache round trip", () => { }; const restored = decodeScanCache(JSON.parse(JSON.stringify(legacy)), TEST_FORMAT_MAP); - expect(restored.get("/gone.jsonl")).toEqual( - version >= 10 - ? original.get("/gone.jsonl") - : { - ...original.get("/gone.jsonl"), - size: -1, - position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, - }, - ); + expect(restored.get("/gone.jsonl")).toEqual({ + ...original.get("/gone.jsonl"), + size: -1, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }); expect(restored.get("/gone-codex.jsonl")).toEqual({ ...original.get("/gone-codex.jsonl"), size: -1, @@ -248,11 +253,11 @@ describe("scan cache round trip", () => { }, ); - it("re-parses a Claude transcript cached before advisor usage was counted", () => { + it("re-parses a Claude transcript cached before 1-hour cache writes were counted", () => { const original = cacheWith([["/claude.jsonl", 100, [record()]]]); - const v9 = { ...parse(makeScanCacheWriter()(original, {})), version: 9 }; + const v11 = { ...parse(makeScanCacheWriter()(original, {})), version: 11 }; - const restored = decodeScanCache(JSON.parse(JSON.stringify(v9)), TEST_FORMAT_MAP); + const restored = decodeScanCache(JSON.parse(JSON.stringify(v11)), TEST_FORMAT_MAP); expect(restored.get("/claude.jsonl")).toEqual({ ...original.get("/claude.jsonl"), size: -1, diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index c2cdc1cbf5abd..452b5ef52b779 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -9,7 +9,7 @@ import * as Schema from "effect/Schema"; import { CLAUDE_SYNTHETIC_MODEL } from "../provider/Drivers/claudeUsage.ts"; import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts"; -const USAGE_SCAN_CACHE_VERSION = 11 as const; +const USAGE_SCAN_CACHE_VERSION = 12 as const; const SPEED_COMPATIBLE_SINCE_VERSION = 4; /** @@ -17,8 +17,9 @@ const SPEED_COMPATIBLE_SINCE_VERSION = 4; * replace it, dropping saved usage for deleted transcripts. Separate files keep * the newer one; older files are removed once it is saved. */ -export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v11.json"; +export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v12.json"; export const LEGACY_SCAN_CACHE_FILE_NAMES = [ + "usage-scan-cache-v11.json", "usage-scan-cache-v10.json", "usage-scan-cache-v9.json", "usage-scan-cache-v8.json", @@ -57,7 +58,8 @@ type SerializedRecord = readonly [ reportedCostUsd: number | null, speed: number, rateModelIndex?: number | null, - unresolvedServiceTierTurnId?: string, + unresolvedServiceTierTurnId?: string | null, + cacheCreation1hTokens?: number, ]; interface SerializedFile { @@ -116,9 +118,11 @@ function serializeFile(entry: CachedFile, tables: InternTables): SerializedFile record.rateModel === undefined ? null : intern(tables.models, tables.modelIndex, record.rateModel), - ...(record.unresolvedServiceTierTurnId === undefined - ? ([] as const) - : ([record.unresolvedServiceTierTurnId] as const)), + ...(record.cacheCreation1hTokens !== undefined + ? ([record.unresolvedServiceTierTurnId ?? null, record.cacheCreation1hTokens] as const) + : record.unresolvedServiceTierTurnId !== undefined + ? ([record.unresolvedServiceTierTurnId] as const) + : ([] as const)), ]; return { s: entry.size, @@ -209,6 +213,7 @@ export function decodeScanCache( speedIndex, rateModelIndex, unresolvedServiceTierTurnId, + cacheCreation1h = 0, ] = row as SerializedRecord; const speed = typeof speedIndex === "number" ? SPEEDS[speedIndex] : undefined; @@ -218,14 +223,14 @@ export function decodeScanCache( typeof timestampMs !== "number" || !Number.isFinite(timestampMs) || model === undefined || - (unresolvedServiceTierTurnId !== undefined && - typeof unresolvedServiceTierTurnId !== "string") || + (unresolvedServiceTierTurnId != null && typeof unresolvedServiceTierTurnId !== "string") || (rateModelIndex != null && rateModel === undefined) || !Number.isFinite(uncached) || !Number.isFinite(cached) || !Number.isFinite(cacheCreation) || !Number.isFinite(output) || !Number.isFinite(reasoning) || + !Number.isFinite(cacheCreation1h) || speed === undefined ) { return null; @@ -248,9 +253,10 @@ export function decodeScanCache( outputTokens: output, reasoningTokens: reasoning, }, + ...(cacheCreation1h > 0 ? { cacheCreation1hTokens: cacheCreation1h } : {}), reportedCostUsd: typeof reportedCostUsd === "number" ? reportedCostUsd : null, speed, - ...(version >= 7 && unresolvedServiceTierTurnId !== undefined + ...(version >= 7 && unresolvedServiceTierTurnId != null ? { unresolvedServiceTierTurnId } : {}), dedupeKey: typeof dedupeKey === "string" ? dedupeKey : null, @@ -289,9 +295,9 @@ export function decodeScanCache( } // Keep history for deleted rollouts, but re-parse incompatible reducer state. // v9 and v11 repair skipped Codex requests; clean v8+ entries stay served while unchanged. - // v10 adds Claude advisor calls, which no older Claude entry holds. + // v10 adds Claude advisor calls and v12 Claude 1-hour cache writes. const legacy = - (entry.p === "claude" && version < 10) || + (entry.p === "claude" && version < 12) || (format.state !== undefined && (version < 6 || (entry.p === "codex" && (version < 8 || (version < 11 && malformedRecords > 0))))); diff --git a/apps/server/src/usage/usageTranscriptStreaming.test.ts b/apps/server/src/usage/usageTranscriptStreaming.test.ts index 856ba2ac5b9d7..f65e22456991a 100644 --- a/apps/server/src/usage/usageTranscriptStreaming.test.ts +++ b/apps/server/src/usage/usageTranscriptStreaming.test.ts @@ -45,6 +45,7 @@ const claude = (id = "m1", output = 99) => ({ output_tokens: output, cache_read_input_tokens: 20, cache_creation_input_tokens: 5, + cache_creation: { ephemeral_1h_input_tokens: 3, ephemeral_5m_input_tokens: 2 }, speed: "fast", }, }, @@ -213,6 +214,7 @@ describe("large usage records", () => { cacheCreationTokens: 5, reasoningTokens: 0, }, + cacheCreation1hTokens: 3, reportedCostUsd: 0.25, speed: "fast", dedupeKey: "m1:r-m1", diff --git a/packages/provider-core/src/server/usage.ts b/packages/provider-core/src/server/usage.ts index 3915ce7b2d9fa..1ba5af077759f 100644 --- a/packages/provider-core/src/server/usage.ts +++ b/packages/provider-core/src/server/usage.ts @@ -18,6 +18,8 @@ export interface UsageRecord { readonly rateModel?: string; readonly sessionId: string; readonly totals: UsageTokenTotals; + /** Subset of cacheCreationTokens written with a 1-hour TTL. */ + readonly cacheCreation1hTokens?: number; readonly reportedCostUsd: number | null; readonly speed: UsageSpeed; readonly unresolvedServiceTierTurnId?: string;