From 658688aceff4fb2b3b470040519e8de6e8bbb6f9 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Mon, 9 Nov 2026 19:01:49 -0600 Subject: [PATCH] perf(cursor): improve account usage scanning - Limit concurrent account refreshes and prune expired cached records - Avoid hashing disjoint pages while preserving overlap deduplication --- apps/server/src/usage/UsageService.test.ts | 37 ++++++-- .../src/server/CursorUsageAccounts.ts | 4 +- .../src/server/accountCache.test.ts | 10 +++ .../src/server/accountCache.ts | 18 ++-- .../src/server/accountUsage.test.ts | 27 +++++- .../src/server/accountUsage.ts | 90 +++++++++++-------- 6 files changed, 133 insertions(+), 53 deletions(-) diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index 27137e4fd615..a3775c6aebe1 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -1580,6 +1580,8 @@ describe("UsageService", () => { Effect.gen(function* () { const { settings, home } = yield* setup; const cursor = makeFakeCursor(); + let inFlight = 0; + let mostInFlight = 0; const started = yield* Deferred.make(); const gate = yield* Deferred.make(); yield* Effect.gen(function* () { @@ -1587,17 +1589,28 @@ describe("UsageService", () => { layerCursorUsageAccounts( Layer.succeed(CursorAccountReader.CursorAccountReader, { read: (credential, sinceMs, untilMs) => - (credential as { apiKey?: string }).apiKey === "failing-key" - ? Effect.succeed({ + Effect.gen(function* () { + inFlight++; + mostInFlight = Math.max(mostInFlight, inFlight); + if (credential.apiKey === "failing-key") { + yield* Deferred.await(started); + return { accountKey: null, records: [], missing: false, error: "Cursor account usage could not be read.", - }) - : Deferred.succeed(started, undefined).pipe( - Effect.andThen(Deferred.await(gate)), - Effect.andThen(cursor.read(credential, sinceMs, untilMs)), - ), + }; + } + yield* Deferred.succeed(started, undefined); + yield* Deferred.await(gate); + return yield* cursor.read(credential, sinceMs, untilMs); + }).pipe( + Effect.ensuring( + Effect.sync(() => { + inFlight--; + }), + ), + ), }), ), ); @@ -1613,6 +1626,14 @@ describe("UsageService", () => { instanceId: ProviderInstanceId.make("cursor-work"), environment: { CURSOR_API_KEY: "new-key" }, }, + { + instanceId: ProviderInstanceId.make("cursor-third"), + environment: { CURSOR_API_KEY: "third-key" }, + }, + { + instanceId: ProviderInstanceId.make("cursor-fourth"), + environment: { CURSOR_API_KEY: "fourth-key" }, + }, ], windowStartMs: 0, retentionCutoffMs: 0, @@ -1624,6 +1645,8 @@ describe("UsageService", () => { assert.isTrue(sources.some((source) => source.refreshing === true)); yield* Deferred.succeed(gate, undefined); yield* Fiber.join(first); + assert.strictEqual(mostInFlight, 2); + assert.strictEqual(inFlight, 0); }).pipe( Effect.provide( layerService({ diff --git a/packages/provider-cursor/src/server/CursorUsageAccounts.ts b/packages/provider-cursor/src/server/CursorUsageAccounts.ts index 21ab0e62e894..8c36c107a409 100644 --- a/packages/provider-cursor/src/server/CursorUsageAccounts.ts +++ b/packages/provider-cursor/src/server/CursorUsageAccounts.ts @@ -104,6 +104,7 @@ const make = Effect.gen(function* () { ); yield* Effect.addFinalizer(() => awaitPersisted); + const refreshLock = yield* Semaphore.make(2); const refreshAccount = Effect.fn("CursorUsageAccounts.refreshAccount")(function* ( credential: CursorCredentialSource, credentialKey: string, @@ -149,7 +150,7 @@ const make = Effect.gen(function* () { failures.delete(credentialKey); if (merged.changed) cacheDirty = true; yield* schedulePersist; - }); + }, refreshLock.withPermit); const startRefresh = ( credential: CursorCredentialSource, @@ -256,6 +257,7 @@ const make = Effect.gen(function* () { retentionCutoffMs, awaitRefresh, ), + { concurrency: 2 }, ); // Keys for one account share its source, and a repeated source would count sessions twice. // A failed key can hold older records, so a current read wins, then a refresh the client diff --git a/packages/provider-cursor/src/server/accountCache.test.ts b/packages/provider-cursor/src/server/accountCache.test.ts index 55b6ba5df3c6..939bc6fc22f0 100644 --- a/packages/provider-cursor/src/server/accountCache.test.ts +++ b/packages/provider-cursor/src/server/accountCache.test.ts @@ -65,6 +65,16 @@ describe("cursorAccountCache", () => { assert.isFalse( mergeCursorFetch(cache, "a", range, [record(10 * DAY_MS)], 11 * DAY_MS, 0).changed, ); + const pruned = mergeCursorFetch( + cache, + "a", + range, + [record(10 * DAY_MS)], + 11 * DAY_MS, + 2 * DAY_MS, + ); + assert.isTrue(pruned.changed); + assert.deepStrictEqual(pruned.cache.records, [record(10 * DAY_MS)]); assert.isTrue( mergeCursorFetch( cache, diff --git a/packages/provider-cursor/src/server/accountCache.ts b/packages/provider-cursor/src/server/accountCache.ts index ffbfcc640053..637d2b96d1ec 100644 --- a/packages/provider-cursor/src/server/accountCache.ts +++ b/packages/provider-cursor/src/server/accountCache.ts @@ -53,11 +53,22 @@ export function mergeCursorFetch( record.timestampMs >= range.sinceMs && record.timestampMs <= range.untilMs; const previous = cache?.records ?? []; const added = fetched.filter(inRange); - const replacedKeys = new Set(previous.filter(inRange).map((record) => record.dedupeKey)); + const replacedKeys = new Set(); + let pruned = false; + for (const record of previous) { + if (inRange(record)) replacedKeys.add(record.dedupeKey); + else if (record.timestampMs < retentionStartMs) pruned = true; + } const changed = cache === undefined || + pruned || replacedKeys.size !== added.length || added.some((record) => !replacedKeys.has(record.dedupeKey)); + const records = !changed + ? previous + : previous + .filter((record) => !inRange(record) && record.timestampMs >= retentionStartMs) + .concat(added); return { changed, cache: { @@ -65,10 +76,7 @@ export function mergeCursorFetch( sinceMs: retentionStartMs, untilMs: range.untilMs, fetchedAtMs: nowMs, - records: [ - ...previous.filter((record) => !inRange(record) && record.timestampMs >= retentionStartMs), - ...added, - ], + records, } satisfies CursorAccountCache, }; } diff --git a/packages/provider-cursor/src/server/accountUsage.test.ts b/packages/provider-cursor/src/server/accountUsage.test.ts index d907c46f9ac3..94c65d546b95 100644 --- a/packages/provider-cursor/src/server/accountUsage.test.ts +++ b/packages/provider-cursor/src/server/accountUsage.test.ts @@ -115,6 +115,10 @@ describe("Cursor account history", () => { reasoningTokens: 0, }); assert.strictEqual(result.records[0]?.reportedCostUsd, 0.25); + assert.strictEqual( + result.records[0]?.dedupeKey, + "cursor-account:2188c26da759c4fae3aa0252fc6d725eff0922ef0b022551e6f0987d02bfab33:3149aaf8419ad6b66ef7330853d8fe85e6f2e9876e2ad5e85f88bee8c8d6d133:0", + ); assert.isFalse(result.accountKey?.includes("demo") ?? true); }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); @@ -209,7 +213,8 @@ describe("Cursor account history", () => { model: "gpt-5", tokenUsage: { inputTokens: 10, outputTokens: 5, totalCents: 1 }, }); - for (const total of [2000, 2001]) { + for (const total of [2000, 2001, undefined]) { + const expectedCount = total ?? 2001; let requests = 0; const result = yield* readAccountUsage(credential, 0, 1781000000000, async () => { requests++; @@ -219,19 +224,33 @@ describe("Cursor account history", () => { requests === 1 ? Array.from({ length: 1000 }, (_, index) => event(index)) : requests === 2 - ? Array.from({ length: 1000 }, (_, index) => event(999 + index)) + ? Array.from({ length: 1000 }, (_, index) => { + const row = event(999 + index); + return { + tokenUsage: { + totalCents: row.tokenUsage.totalCents, + outputTokens: row.tokenUsage.outputTokens, + inputTokens: row.tokenUsage.inputTokens, + }, + model: row.model, + timestamp: row.timestamp, + }; + }) : [event(1999)], }); }); assert.isNull(result.error); - assert.strictEqual(result.records.length, total); + assert.strictEqual(result.records.length, expectedCount); assert.strictEqual(requests, 3); assert.strictEqual(result.records.at(-1)?.timestampMs, 1780000001999); assert.strictEqual( result.records.filter((record) => record.timestampMs === 1780000000999).length, total === 2000 ? 1 : 2, ); - assert.strictEqual(new Set(result.records.map((record) => record.dedupeKey)).size, total); + assert.strictEqual( + new Set(result.records.map((record) => record.dedupeKey)).size, + expectedCount, + ); } let requests = 0; const inconsistent = yield* readAccountUsage(credential, 0, 1781000000000, async () => { diff --git a/packages/provider-cursor/src/server/accountUsage.ts b/packages/provider-cursor/src/server/accountUsage.ts index 6d28410f7c83..dff8f3d877e2 100644 --- a/packages/provider-cursor/src/server/accountUsage.ts +++ b/packages/provider-cursor/src/server/accountUsage.ts @@ -3,7 +3,6 @@ import * as Digest from "effect/crypto/Digest"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Base64Url from "effect/encoding/Base64Url"; -import * as Hex from "effect/encoding/Hex"; import * as Fiber from "effect/Fiber"; import * as FetchHttpClient from "effect/http/FetchHttpClient"; import * as HttpClient from "effect/http/HttpClient"; @@ -55,12 +54,6 @@ const billedEventIdentity = (...fields: readonly unknown[]) => JSON.stringify(fi const textEncoder = new TextEncoder(); -// Untraced: each usage scan hashes every billed event. -const accountHash = Effect.fnUntraced(function* (value: string) { - const digest = yield* Digest.Digest; - return Hex.encode(yield* digest.digest(Digest.sha256, textEncoder.encode(value))); -}); - function canonicalJson(value: unknown): string { if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; if (value !== null && typeof value === "object") { @@ -93,6 +86,9 @@ export const readCursorAccountUsage = Effect.fn("readCursorAccountUsage")(functi endDate: number, ): Effect.fn.Return { const httpClient = yield* HttpClient.HttpClient; + const digest = yield* Digest.Digest; + // Per-event hashes share the scan span and use the native hex encoder. + const accountHash = (value: string) => digest.digestHex(Digest.sha256, textEncoder.encode(value)); const credential = yield* Effect.result(exchangeCursorApiKey(credentialSource.apiKey)); if (Result.isFailure(credential)) { return { @@ -124,8 +120,9 @@ export const readCursorAccountUsage = Effect.fn("readCursorAccountUsage")(functi ); const records: UsageRecord[] = []; const occurrences = new Map(); - const pages: unknown[][] = []; - let completed = false; + const normalized: Array<(Omit & { dedupeKey: string }) | null> = []; + const pages: Array<{ start: number; end: number; overlap: number }> = []; + let previousEvents: unknown[] = []; const pageSize = 1000; let total: number | undefined; const readPage = Effect.fn("CursorAccountUsage.readPage")( @@ -210,30 +207,23 @@ export const readCursorAccountUsage = Effect.fn("readCursorAccountUsage")(functi const next = ahead.shift(); const events = yield* next === undefined ? readPage(page) : Fiber.join(next); if (!Array.isArray(events)) return events; - pages.push(events); - if (events.length < pageSize) { - completed = true; - break; - } - } - if (!completed) return yield* invalid("Account usage page limit exceeded"); - const rawCount = pages.reduce((sum, page) => sum + page.length, 0); - if (total !== undefined && rawCount < total) - return yield* invalid("Incomplete account usage pages"); - let removalsRemaining = total === undefined ? 0 : rawCount - total; - let previousKeys: string[] = []; - for (const events of pages) { - const eventKeys = - removalsRemaining > 0 - ? yield* Effect.forEach(events, (event) => accountHash(canonicalJson(event))) - : []; - const removalCount = Math.min(removalsRemaining, boundaryOverlap(previousKeys, eventKeys)); - removalsRemaining -= removalCount; - previousKeys = eventKeys; - for (const raw of events.slice(removalCount)) { + const start = normalized.length; + const firstTimestamp = object(events[0]).timestamp; + // Equal raw events must have equal timestamps; avoid canonicalizing disjoint pages. + const mayOverlap = + (typeof firstTimestamp !== "string" && typeof firstTimestamp !== "number") || + previousEvents.some((event) => object(event).timestamp === firstTimestamp); + const overlap = mayOverlap + ? boundaryOverlap(previousEvents.map(canonicalJson), events.map(canonicalJson)) + : 0; + previousEvents = events; + for (const raw of events) { const event = object(raw); const usage = object(event.tokenUsage); - if (event.tokenUsage === undefined || event.tokenUsage === null) continue; + if (event.tokenUsage === undefined || event.tokenUsage === null) { + normalized.push(null); + continue; + } for (const key of [ "inputTokens", "outputTokens", @@ -260,7 +250,10 @@ export const readCursorAccountUsage = Effect.fn("readCursorAccountUsage")(functi !event.model ) return yield* invalid("Invalid account usage event"); - if (timestampMs < sinceMs || timestampMs > endDate) continue; + if (timestampMs < sinceMs || timestampMs > endDate) { + normalized.push(null); + continue; + } const totals = { uncachedInputTokens: tokens(usage.inputTokens), cachedInputTokens: tokens(usage.cacheReadTokens), @@ -275,9 +268,7 @@ export const readCursorAccountUsage = Effect.fn("readCursorAccountUsage")(functi const key = yield* accountHash( billedEventIdentity(timestampMs, event.model, sessionId, totals, reportedCostUsd), ); - const occurrence = occurrences.get(key) ?? 0; - occurrences.set(key, occurrence + 1); - records.push({ + normalized.push({ provider: "cursor", timestampMs, model: event.model, @@ -286,12 +277,39 @@ export const readCursorAccountUsage = Effect.fn("readCursorAccountUsage")(functi totals, reportedCostUsd, speed: "standard", - dedupeKey: `cursor-account:${accountKey}:${key}:${occurrence}`, + dedupeKey: key, }); } + pages.push({ start, end: normalized.length, overlap }); + if (events.length < pageSize) break; + yield* Effect.yieldNow; + } + const rawCount = normalized.length; + if (total !== undefined && rawCount < total) + return yield* invalid("Incomplete account usage pages"); + let removalsRemaining = total === undefined ? 0 : rawCount - total; + // The final count proves which boundary copies to remove, including non-token events. + for (const page of pages) { + const removalCount = Math.min(removalsRemaining, page.overlap); + removalsRemaining -= removalCount; + for (let index = page.start; index < page.end; index++) { + const record = normalized[index]; + normalized[index] = null; + if (index < page.start + removalCount || record == null) continue; + const key = record.dedupeKey; + const occurrence = occurrences.get(key) ?? 0; + occurrences.set(key, occurrence + 1); + record.dedupeKey = `cursor-account:${accountKey}:${key}:${occurrence}`; + records.push(record); + } yield* Effect.yieldNow; } if (removalsRemaining !== 0) return yield* invalid("Inconsistent account usage boundaries"); + yield* Effect.annotateCurrentSpan({ + "cursor.usage.pages": pages.length, + "cursor.usage.events": rawCount, + "cursor.usage.records": records.length, + }); return { accountKey, records,