Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/server/src/usage/UsageService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3207,7 +3207,7 @@ describe("UsageService", () => {
60 + 20 * (version === 4 ? 1 : 2),
);
yield* upgraded.awaitPersisted;
assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v16.json");
assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v17.json");
assert.deepStrictEqual(
(yield* Effect.promise(() => NodeFSP.readdir(stateDir)))
.filter((name) => name.startsWith("usage-scan-cache"))
Expand Down
26 changes: 26 additions & 0 deletions apps/server/src/usage/usageAggregation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,32 @@ describe("UsageAggregator", () => {
}
});

it("keeps a live Grok turn over a saved pre-v17 maximum in either source order", () => {
const turn = (timestampMs: number, outputTokens: number) =>
record({
provider: "grok",
model: "grok-4.5",
timestampMs,
dedupeKey: `s:p:${timestampMs}:grok-4.5`,
reportedCostUsd: outputTokens / 10,
totals: { ...record().totals, uncachedInputTokens: 0, outputTokens },
});
const first = turn(Date.parse("2026-08-08T10:00:00Z"), 10);
const resumed = turn(Date.parse("2026-08-08T11:00:00Z"), 20);
const savedMaximum = { ...first, totals: resumed.totals, reportedCostUsd: 2 };

const expected = aggregate([first, resumed]).buckets;

for (const order of [
[first, resumed, savedMaximum],
[savedMaximum, first, resumed],
]) {
const result = aggregate(order);
expect(result.duplicatesDropped).toBe(1);
expect(result.buckets).toEqual(expected);
}
});

it("does not count a key whose first copy fell outside the window", () => {
const result = aggregate([
record({ dedupeKey: "msg_1:", timestampMs: Date.parse("2026-07-01T12:00:00Z") }),
Expand Down
31 changes: 30 additions & 1 deletion apps/server/src/usage/usageAggregation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,30 @@ export function mergeUsageCopies(held: UsageRecord, copy: UsageRecord): UsageRec
};
}

/**
* Grok reports each turn once, complete, so real copies are equal. A larger one is a deleted
* transcript's pre-v17 maximum over turns that shared a prompt id, whichever source it came from.
*/
function smallestGrokCopy(held: UsageRecord, copy: UsageRecord): UsageRecord {
const a = held.totals;
const b = copy.totals;
const reportedCostUsd =
held.reportedCostUsd === null || copy.reportedCostUsd === null
? (held.reportedCostUsd ?? copy.reportedCostUsd)
: Math.min(held.reportedCostUsd, copy.reportedCostUsd);
return {
...held,
totals: {
uncachedInputTokens: Math.min(a.uncachedInputTokens, b.uncachedInputTokens),
cachedInputTokens: Math.min(a.cachedInputTokens, b.cachedInputTokens),
cacheCreationTokens: Math.min(a.cacheCreationTokens, b.cacheCreationTokens),
outputTokens: Math.min(a.outputTokens, b.outputTokens),
reasoningTokens: Math.min(a.reasoningTokens, b.reasoningTokens),
},
reportedCostUsd,
};
}

interface Contribution {
record: UsageRecord;
/** Null when the first copy fell outside the window. */
Expand Down Expand Up @@ -217,7 +241,12 @@ export class UsageAggregator {
const held = this.#seen.get(record.dedupeKey);
if (held !== undefined) {
this.#duplicatesDropped += 1;
if (held.bucket !== null) held.record = mergeUsageCopies(held.record, record);
if (held.bucket !== null) {
held.record =
record.provider === "grok"
? smallestGrokCopy(held.record, record)
: mergeUsageCopies(held.record, record);
}
return false;
}
}
Expand Down
23 changes: 23 additions & 0 deletions apps/server/src/usage/usageScanCache.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,29 @@ describe("scan cache round trip", () => {
});
});

it.each(["s:p:grok-4.5", "s:p:1786000000000:grok-4.5"])(
"re-parses v16 Grok transcripts with key %s",
(dedupeKey) => {
const original = cacheWith([
["/grok/updates.jsonl", 100, [record({ provider: "grok", model: "grok-4.5", dedupeKey })]],
]);
original.set("/grok/updates.jsonl", {
...original.get("/grok/updates.jsonl")!,
provider: "grok",
});
const encoded = parse(makeScanCacheWriter()(original, {}));
const restored = decodeScanCache({ ...encoded, version: 16 }, TEST_FORMAT_MAP);
expect(restored.get("/grok/updates.jsonl")).toEqual({
...original.get("/grok/updates.jsonl"),
size: -1,
position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null },
records: [
record({ provider: "grok", model: "grok-4.5", dedupeKey: "s:p:1786000000000:grok-4.5" }),
],
});
},
);

it("rewrites only changed entries and still restores the whole cache", () => {
const write = makeScanCacheWriter();
const cache = cacheWith([
Expand Down
18 changes: 15 additions & 3 deletions apps/server/src/usage/usageScanCache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,17 @@ import { CLAUDE_SYNTHETIC_MODEL } from "../provider/Drivers/claudeUsage.ts";
import { mergeUsageCopies } from "./usageAggregation.ts";
import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts";

const USAGE_SCAN_CACHE_VERSION = 16 as const;
const USAGE_SCAN_CACHE_VERSION = 17 as const;
const SPEED_COMPATIBLE_SINCE_VERSION = 4;

/**
* An older server sharing that directory cannot read a newer cache and would
* replace it, dropping saved usage for deleted transcripts. Separate files keep
* the newer one; older files are removed once it is saved.
*/
export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v16.json";
export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v17.json";
export const LEGACY_SCAN_CACHE_FILE_NAMES = [
"usage-scan-cache-v16.json",
"usage-scan-cache-v15.json",
"usage-scan-cache-v14.json",
"usage-scan-cache-v13.json",
Expand Down Expand Up @@ -264,7 +265,16 @@ export function decodeScanCache(
...(version >= 7 && unresolvedServiceTierTurnId != null
? { unresolvedServiceTierTurnId }
: {}),
dedupeKey: typeof dedupeKey === "string" ? dedupeKey : null,
dedupeKey:
typeof dedupeKey !== "string"
? null
: // Saved copies of moved transcripts must still match the re-parsed v17 key.
provider === "grok" &&
version < 17 &&
dedupeKey.endsWith(`:${model}`) &&
!dedupeKey.endsWith(`:${timestampMs}:${model}`)
? `${dedupeKey.slice(0, -model.length)}${timestampMs}:${model}`

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟠 High usage/usageScanCache.ts:276

A moved Grok transcript can inflate usage because this migration assigns a previously merged legacy record to the first turn’s new key. If the cached record already combines two turns at $2, aggregation merges it with the live first turn at $1 and then counts the live second turn at $2, reporting $4 instead of $3. Reconcile reduced legacy records against the reparsed transcript rather than treating the migrated key as an exact copy.

🤖 Copy this AI Prompt to have your agent fix this:
In file @apps/server/src/usage/usageScanCache.ts around line 276:

A moved Grok transcript can inflate usage because this migration assigns a previously merged legacy record to the first turn’s new key. If the cached record already combines two turns at $2, aggregation merges it with the live first turn at $1 and then counts the live second turn at $2, reporting $4 instead of $3. Reconcile reduced legacy records against the reparsed transcript rather than treating the migrated key as an exact copy.

: dedupeKey,
});
}
return records;
Expand Down Expand Up @@ -301,8 +311,10 @@ export function decodeScanCache(
// Keep history for deleted rollouts, but re-parse incompatible reducer state.
// v13 adds Claude 1-hour cache writes; v14 uses Codex request ledgers.
// v15 repairs skipped Codex requests whose counters rewound; clean v14 entries stay served.
// v17 keys Grok turns by time, restoring turns previously dropped as repeats.
const legacy =
(entry.p === "claude" && version < 13) ||
(entry.p === "grok" && version < 17) ||
(format.state !== undefined &&
(version < 6 ||
(entry.p === "codex" && (version < 14 || (version < 15 && malformedRecords > 0)))));
Expand Down
2 changes: 1 addition & 1 deletion apps/server/src/usage/usageTranscriptStreaming.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ describe("large usage records", () => {
outputTokens: 99,
reasoningTokens: 10,
},
dedupeKey: "s1:p1:grok-4.5-build",
dedupeKey: "s1:p1:1785578400123:grok-4.5-build",
reportedCostUsd: 0.25,
});
});
Expand Down
19 changes: 17 additions & 2 deletions packages/provider-grok/src/server/usage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,9 @@ describe("parseGrokLine", () => {
reasoningTokens: 180,
});
expect(record?.reportedCostUsd).toBeCloseTo(230_272_000 / GROK_COST_USD_TICKS_PER_DOLLAR, 12);
expect(record?.dedupeKey).toBe("019fec1a-12f7-72f2-9b1f-7778a00aea3c:prompt-1:grok-4.5-build");
expect(record?.dedupeKey).toBe(
"019fec1a-12f7-72f2-9b1f-7778a00aea3c:prompt-1:1786372566485:grok-4.5-build",
);
});

it("emits one record per model when modelUsage has several entries", () => {
Expand Down Expand Up @@ -142,7 +144,9 @@ describe("parseGrokLine", () => {
reasoningTokens: 180,
});
expect(record?.reportedCostUsd).toBeCloseTo(230_272_000 / GROK_COST_USD_TICKS_PER_DOLLAR, 12);
expect(record?.dedupeKey).toBe("019fec1a-12f7-72f2-9b1f-7778a00aea3c:prompt-1:grok");
expect(record?.dedupeKey).toBe(
"019fec1a-12f7-72f2-9b1f-7778a00aea3c:prompt-1:1786372566485:grok",
);
});

it("pro-rates top-level cost ticks across multi-model turns without per-model ticks", () => {
Expand Down Expand Up @@ -230,6 +234,17 @@ describe("parseGrokLine", () => {
expect(parseGrokLine(line)[0]?.dedupeKey).toBeNull();
});

it("keeps later turns that reuse a prompt id after the session resumes in a new process", () => {
const first = parseGrokLine(turnCompleted({ promptId: "t3-xai-prompt-1" }))[0];
const resumed = parseGrokLine(
turnCompleted({ promptId: "t3-xai-prompt-1", agentTimestampMs: 1_786_459_000_000 }),
)[0];
const copied = parseGrokLine(turnCompleted({ promptId: "t3-xai-prompt-1" }))[0];

expect(resumed?.dedupeKey).not.toBe(first?.dedupeKey);
expect(copied?.dedupeKey).toBe(first?.dedupeKey);
});

it("ignores non-turn lines and empty usage", () => {
expect(parseGrokLine(JSON.stringify({ method: "session/update", params: {} }))).toEqual([]);
expect(parseGrokLine("not json")).toEqual([]);
Expand Down
6 changes: 4 additions & 2 deletions packages/provider-grok/src/server/usage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ function parseGrokRecord(parsed: unknown): readonly UsageRecord[] {
const usageRecord = usage as Record<string, unknown>;

const sessionId = typeof paramsRecord["sessionId"] === "string" ? paramsRecord["sessionId"] : "";
// T3's prompt ids restart with each Grok process, so a resumed session reuses them.
const promptId = typeof updateRecord["prompt_id"] === "string" ? updateRecord["prompt_id"] : null;

const meta = paramsRecord["_meta"];
Expand Down Expand Up @@ -136,7 +137,7 @@ function parseGrokRecord(parsed: unknown): readonly UsageRecord[] {
reportedCostUsd: grokCostTicksToUsd(topLevel.costUsdTicks),
speed: "standard",
// No prompt id means we cannot tell two same-second updates apart.
dedupeKey: promptId === null ? null : `${sessionId}:${promptId}:grok`,
dedupeKey: promptId === null ? null : `${sessionId}:${promptId}:${timestampMs}:grok`,
},
];
}
Expand Down Expand Up @@ -174,7 +175,8 @@ function parseGrokRecord(parsed: unknown): readonly UsageRecord[] {
totals,
reportedCostUsd,
speed: "standard",
dedupeKey: promptId === null ? null : `${sessionId}:${promptId}:${entry.model}`,
dedupeKey:
promptId === null ? null : `${sessionId}:${promptId}:${timestampMs}:${entry.model}`,
});
}
return results;
Expand Down
Loading