Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apps/server/src/provider/Drivers/claudeUsage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ function claudeLine(overrides: {
usage: {
input_tokens: 2,
cache_creation_input_tokens: 66818,
cache_creation: { ephemeral_5m_input_tokens: 818, ephemeral_1h_input_tokens: 66000 },
cache_read_input_tokens: 1000,
output_tokens: overrides.outputTokens ?? 286,
...(overrides.speed === undefined ? {} : { speed: overrides.speed }),
Expand All @@ -44,6 +45,7 @@ describe("parseClaudeLine", () => {
outputTokens: 286,
reasoningTokens: 0,
});
expect(record?.oneHourCacheCreationTokens).toBe(66000);
expect(record?.dedupeKey).toBe("msg_1:");
expect(record?.speed).toBe("standard");
});
Expand Down
12 changes: 11 additions & 1 deletion apps/server/src/provider/Drivers/claudeUsage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,15 @@ function parseClaudeRecord(parsed: unknown): UsageRecord | null {
messageId === null && requestId === null ? null : `${messageId ?? ""}:${requestId ?? ""}`;

const cost = record["costUSD"];
const cacheCreationTokens = tokenCount(usageRecord["cache_creation_input_tokens"]);
const cacheCreation = usageRecord["cache_creation"];
const oneHourCacheCreationTokens =
typeof cacheCreation === "object" && cacheCreation !== null
? Math.min(
tokenCount((cacheCreation as Record<string, unknown>)["ephemeral_1h_input_tokens"]),
cacheCreationTokens,
)
: 0;

return {
provider: "claude",
Expand All @@ -65,11 +74,12 @@ function parseClaudeRecord(parsed: unknown): UsageRecord | null {
totals: {
uncachedInputTokens: tokenCount(usageRecord["input_tokens"]),
cachedInputTokens: tokenCount(usageRecord["cache_read_input_tokens"]),
cacheCreationTokens: tokenCount(usageRecord["cache_creation_input_tokens"]),
cacheCreationTokens,
outputTokens: tokenCount(usageRecord["output_tokens"]),
// Anthropic folds thinking tokens into output and does not break them out.
reasoningTokens: 0,
},
...(oneHourCacheCreationTokens > 0 ? { oneHourCacheCreationTokens } : {}),
reportedCostUsd: typeof cost === "number" && Number.isFinite(cost) ? cost : null,
speed: usageRecord["speed"] === "fast" ? "fast" : "standard",
dedupeKey,
Expand Down
2 changes: 1 addition & 1 deletion apps/server/src/usage/UsageService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1971,7 +1971,7 @@ describe("UsageService", () => {
60 + 20 * (version === 4 ? 1 : 2),
);
yield* upgraded.awaitPersisted;
assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v8.json");
assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v9.json");
assert.strictEqual(
yield* Effect.promise(() => NodeFSP.readFile(legacyPath, "utf8")),
legacy,
Expand Down
2 changes: 2 additions & 0 deletions apps/server/src/usage/usageAggregation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ const rates: RateTable = new Map([
outputCostPerToken: 5e-5,
cacheReadCostPerToken: 1e-6,
cacheCreationCostPerToken: 1.25e-5,
oneHourCacheCreationCostPerToken: 2e-5,
fast: {
inputCostPerToken: 2e-5,
outputCostPerToken: 1e-4,
cacheReadCostPerToken: 2e-6,
cacheCreationCostPerToken: 2.5e-5,
oneHourCacheCreationCostPerToken: 4e-5,
},
ultrafast: null,
},
Expand Down
25 changes: 25 additions & 0 deletions apps/server/src/usage/usagePricing.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,31 @@ describe("usage pricing", () => {
});
});

it("prices one-hour cache writes at the one-hour write rate", () => {
const table = parseRateTable({
"claude-opus-5-5": {
...rate(4e-6, 4e-7),
cache_creation_input_token_cost: 5e-6,
cache_creation_input_token_cost_above_1hr: 8e-6,
input_cost_per_token_priority: 8e-6,
output_cost_per_token_priority: 40e-6,
cache_creation_input_token_cost_priority: 12e-6,
},
});
const writes = (speed: UsageSpeed) => ({
...record("claude-opus-5-5", null, speed),
oneHourCacheCreationTokens: 600_000,
});

expect(priceUsage(table, writes("standard"))).toMatchObject({
categoryCostUsd: { cacheWrite: expect.closeTo(0.4 * 5 + 0.6 * 8) },
});
// The priority tier publishes no one-hour rate, so it keeps the standard 8/5 premium.
expect(priceUsage(table, writes("fast"))).toMatchObject({
categoryCostUsd: { cacheWrite: expect.closeTo(0.4 * 12 + 0.6 * 12 * 1.6) },
});
});

it("prices Codex priority and ultrafast requests at their published tier rates", () => {
const table = parseRateTable({
"gpt-6-astra": {
Expand Down
38 changes: 26 additions & 12 deletions apps/server/src/usage/usagePricing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import type {
UsageCategoryCost,
UsageCostSource,
UsageModelPriceOverride,
UsageTokenTotals,
} from "@t3tools/contracts";

import type { UsageRecord, UsageSpeed } from "@t3tools/provider-core/server/usage";
Expand All @@ -12,6 +11,7 @@ export interface TokenRates {
readonly outputCostPerToken: number;
readonly cacheReadCostPerToken: number;
readonly cacheCreationCostPerToken: number;
readonly oneHourCacheCreationCostPerToken: number;
}

/**
Expand All @@ -38,6 +38,8 @@ export function createOverrideRateTable(
(prices.cacheReadCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000,
cacheCreationCostPerToken:
(prices.cacheWriteCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000,
oneHourCacheCreationCostPerToken:
(prices.cacheWriteCostPerMillionTokens ?? prices.inputCostPerMillionTokens) / 1_000_000,
fast: null,
ultrafast: null,
},
Expand Down Expand Up @@ -68,14 +70,20 @@ function readTokenRates(
(standard !== undefined && standard.inputCostPerToken > 0
? (standard[field] / standard.inputCostPerToken) * input
: input);
const cacheCreation = cacheRate("cache_creation_input_token_cost", "cacheCreationCostPerToken");
// Without its own one-hour rate, a tier keeps the standard tier's one-hour premium over writes.
const oneHourCacheCreation =
finiteNumber(entry[`cache_creation_input_token_cost_above_1hr${suffix}`]) ??
(standard !== undefined && standard.cacheCreationCostPerToken > 0
? (standard.oneHourCacheCreationCostPerToken / standard.cacheCreationCostPerToken) *
cacheCreation
: cacheCreation);
return {
inputCostPerToken: input,
outputCostPerToken: output,
cacheReadCostPerToken: cacheRate("cache_read_input_token_cost", "cacheReadCostPerToken"),
cacheCreationCostPerToken: cacheRate(
"cache_creation_input_token_cost",
"cacheCreationCostPerToken",
),
cacheCreationCostPerToken: cacheCreation,
oneHourCacheCreationCostPerToken: oneHourCacheCreation,
};
}

Expand All @@ -85,6 +93,7 @@ function scaleTokenRates(rates: TokenRates, multiple: number): TokenRates {
outputCostPerToken: rates.outputCostPerToken * multiple,
cacheReadCostPerToken: rates.cacheReadCostPerToken * multiple,
cacheCreationCostPerToken: rates.cacheCreationCostPerToken * multiple,
oneHourCacheCreationCostPerToken: rates.oneHourCacheCreationCostPerToken * multiple,
};
}

Expand Down Expand Up @@ -146,7 +155,8 @@ function sameTokenRates(a: TokenRates | null, b: TokenRates | null): boolean {
a.inputCostPerToken === b.inputCostPerToken &&
a.outputCostPerToken === b.outputCostPerToken &&
a.cacheReadCostPerToken === b.cacheReadCostPerToken &&
a.cacheCreationCostPerToken === b.cacheCreationCostPerToken
a.cacheCreationCostPerToken === b.cacheCreationCostPerToken &&
a.oneHourCacheCreationCostPerToken === b.oneHourCacheCreationCostPerToken
);
}

Expand Down Expand Up @@ -224,7 +234,7 @@ function resolveRate(table: RateTable, model: string): ModelRate | null {

export type PricedRecord = Pick<
UsageRecord,
"model" | "rateModel" | "totals" | "speed" | "reportedCostUsd"
"model" | "rateModel" | "totals" | "oneHourCacheCreationTokens" | "speed" | "reportedCostUsd"
>;

export interface PricedUsage {
Expand All @@ -234,11 +244,15 @@ export interface PricedUsage {
readonly speedPremiumUsd: number;
}

function costByCategory(totals: UsageTokenTotals, rates: TokenRates): UsageCategoryCost {
function costByCategory(record: PricedRecord, rates: TokenRates): UsageCategoryCost {
const { totals } = record;
const oneHourWrites = record.oneHourCacheCreationTokens ?? 0;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
return {
input: totals.uncachedInputTokens * rates.inputCostPerToken,
cacheRead: totals.cachedInputTokens * rates.cacheReadCostPerToken,
cacheWrite: totals.cacheCreationTokens * rates.cacheCreationCostPerToken,
cacheWrite:
(totals.cacheCreationTokens - oneHourWrites) * rates.cacheCreationCostPerToken +
oneHourWrites * rates.oneHourCacheCreationCostPerToken,
output: totals.outputTokens * rates.outputCostPerToken,
};
}
Expand All @@ -256,7 +270,7 @@ export function priceUsage(
record: PricedRecord,
overrides?: RateTable,
): PricedUsage {
const { model, totals, reportedCostUsd } = record;
const { model, reportedCostUsd } = record;
const override = overrides?.get(model.trim());
const reported =
override === undefined && reportedCostUsd !== null && Number.isFinite(reportedCostUsd)
Expand All @@ -273,11 +287,11 @@ export function priceUsage(
return reported === null ? unsplit(0, "unpriced") : unsplit(reported, "providerReported");
}

const listCost = costByCategory(totals, ratesAt(rate, record.speed));
const listCost = costByCategory(record, ratesAt(rate, record.speed));
const listCostUsd = sumCategories(listCost);
if (reported !== null && listCostUsd <= 0) return unsplit(reported, "providerReported");
const premiumUsd =
record.speed === "standard" ? 0 : listCostUsd - sumCategories(costByCategory(totals, rate));
record.speed === "standard" ? 0 : listCostUsd - sumCategories(costByCategory(record, rate));
const scale = reported === null ? 1 : reported / listCostUsd;
return {
costUsd: reported ?? listCostUsd,
Expand Down
26 changes: 21 additions & 5 deletions apps/server/src/usage/usageScanCache.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,15 @@ describe("scan cache round trip", () => {
[
"/a.jsonl",
100,
[record(), record({ dedupeKey: "msg_2:", model: "claude-opus-5-5", speed: "fast" })],
[
record(),
record({
dedupeKey: "msg_2:",
model: "claude-opus-5-5",
speed: "fast",
oneHourCacheCreationTokens: 6,
}),
],
],
[
"/b.jsonl",
Expand Down Expand Up @@ -162,7 +170,7 @@ describe("scan cache round trip", () => {
expect(restored).toEqual(original);
});

it.each([5, 6, 7])(
it.each([5, 6, 7, 8])(
"retains v%s history and compatible positions but invalidates obsolete state",
(version) => {
const original = cacheWith([["/gone.jsonl", 100, [record({ speed: "fast" })]]]);
Expand Down Expand Up @@ -213,12 +221,20 @@ describe("scan cache round trip", () => {
};

const restored = decodeScanCache(JSON.parse(JSON.stringify(legacy)), TEST_FORMAT_MAP);
expect(restored.get("/gone.jsonl")).toEqual(original.get("/gone.jsonl"));
expect(restored.get("/gone-codex.jsonl")).toEqual({
...original.get("/gone-codex.jsonl"),
expect(restored.get("/gone.jsonl")).toEqual({
...original.get("/gone.jsonl"),
size: -1,
position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null },
});
expect(restored.get("/gone-codex.jsonl")).toEqual(
version >= 8
? original.get("/gone-codex.jsonl")
: {
...original.get("/gone-codex.jsonl"),
size: -1,
position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null },
},
);
expect(restored.get("/gone-pi.jsonl")).toEqual(
version >= 6
? original.get("/gone-pi.jsonl")
Expand Down
31 changes: 21 additions & 10 deletions apps/server/src/usage/usageScanCache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,17 @@ import * as Schema from "effect/Schema";
import { CLAUDE_SYNTHETIC_MODEL } from "../provider/Drivers/claudeUsage.ts";
import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts";

const USAGE_SCAN_CACHE_VERSION = 8 as const;
const USAGE_SCAN_CACHE_VERSION = 9 as const;
Comment thread
n3a7-7 marked this conversation as resolved.
Resolved
const SPEED_COMPATIBLE_SINCE_VERSION = 4;

/**
* An older server sharing that directory cannot read a newer cache and would
* replace it, dropping saved usage for deleted transcripts. Separate files keep
* both.
*/
export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v8.json";
export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v9.json";
export const LEGACY_SCAN_CACHE_FILE_NAMES = [
"usage-scan-cache-v8.json",
"usage-scan-cache-v7.json",
"usage-scan-cache-v6.json",
"usage-scan-cache-v5.json",
Expand Down Expand Up @@ -58,7 +59,8 @@ type SerializedRecord = readonly [
reportedCostUsd: number | null,
speed: number,
rateModelIndex?: number | null,
unresolvedServiceTierTurnId?: string,
unresolvedServiceTierTurnId?: string | null,
oneHourCacheCreationTokens?: number,
];

interface SerializedFile {
Expand Down Expand Up @@ -117,9 +119,12 @@ function serializeFile(entry: CachedFile, tables: InternTables): SerializedFile
record.rateModel === undefined
? null
: intern(tables.models, tables.modelIndex, record.rateModel),
...(record.unresolvedServiceTierTurnId === undefined
? ([] as const)
: ([record.unresolvedServiceTierTurnId] as const)),
// Optional trailing slots are written only when set, so most rows stay short.
...(record.oneHourCacheCreationTokens !== undefined
? ([record.unresolvedServiceTierTurnId ?? null, record.oneHourCacheCreationTokens] as const)
: record.unresolvedServiceTierTurnId !== undefined
? ([record.unresolvedServiceTierTurnId] as const)
: ([] as const)),
];
return {
s: entry.size,
Expand Down Expand Up @@ -232,6 +237,7 @@ export function decodeScanCache(
speedIndex,
rateModelIndex,
unresolvedServiceTierTurnId,
oneHourCacheCreation,
] = row as SerializedRecord;
const speed = typeof speedIndex === "number" ? SPEEDS[speedIndex] : undefined;

Expand All @@ -241,8 +247,8 @@ export function decodeScanCache(
typeof timestampMs !== "number" ||
!Number.isFinite(timestampMs) ||
model === undefined ||
(unresolvedServiceTierTurnId !== undefined &&
typeof unresolvedServiceTierTurnId !== "string") ||
(unresolvedServiceTierTurnId != null && typeof unresolvedServiceTierTurnId !== "string") ||
(oneHourCacheCreation !== undefined && !Number.isFinite(oneHourCacheCreation)) ||
(rateModelIndex != null && rateModel === undefined) ||
!Number.isFinite(uncached) ||
!Number.isFinite(cached) ||
Expand Down Expand Up @@ -273,9 +279,12 @@ export function decodeScanCache(
},
reportedCostUsd: typeof reportedCostUsd === "number" ? reportedCostUsd : null,
speed,
...(version >= 7 && unresolvedServiceTierTurnId !== undefined
...(version >= 7 && typeof unresolvedServiceTierTurnId === "string"
? { unresolvedServiceTierTurnId }
: {}),
...(oneHourCacheCreation !== undefined && oneHourCacheCreation > 0
? { oneHourCacheCreationTokens: oneHourCacheCreation }
: {}),
dedupeKey: typeof dedupeKey === "string" ? dedupeKey : null,
});
}
Expand Down Expand Up @@ -312,8 +321,10 @@ export function decodeScanCache(
}
// Keep history for deleted rollouts, but re-parse incompatible reducer state.
// Older Codex entries lack provenance state (v7) or misread compaction and resume counters (v8).
// Older Claude entries lack the one-hour cache-write split (v9).
const legacy =
format.state !== undefined && (version < 6 || (entry.p === "codex" && version < 8));
(format.state !== undefined && (version < 6 || (entry.p === "codex" && version < 8))) ||
(entry.p === "claude" && version < 9);
// A corrupt state disqualifies the entry: resuming with it would attach
// appended usage to the wrong model or replay fork-copied history.
if (!legacy && !isValidState.get(entry.p)?.(entry.cs)) continue;
Expand Down
2 changes: 2 additions & 0 deletions packages/provider-core/src/server/usage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ export interface UsageRecord {
readonly rateModel?: string;
readonly sessionId: string;
readonly totals: UsageTokenTotals;
/** Part of `totals.cacheCreationTokens` written with Anthropic's one-hour TTL, billed higher. */
readonly oneHourCacheCreationTokens?: number;
readonly reportedCostUsd: number | null;
readonly speed: UsageSpeed;
readonly unresolvedServiceTierTurnId?: string;
Expand Down
Loading