diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index dc3f7ac927147..0ebb1228e0114 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -3508,6 +3508,96 @@ describe("UsageService", () => { }), ); + it.effect( + "reuses old live transcripts across all-time refreshes without retaining deleted history", + () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-12-01T00:00:00Z")); + const { transcript, settings, home } = yield* setup; + const lastWrite = Date.parse("2026-08-01T10:00:00Z") / 1000; + yield* Effect.promise(async () => { + await NodeFSP.writeFile(transcript, claudeLine(1, 5)); + await NodeFSP.utimes(transcript, lastWrite, lastWrite); + }); + yield* Effect.gen(function* () { + const service = yield* UsageService.make; + const allTime = { + ...WINDOW, + sinceDay: UsageDay.make("1970-01-01"), + untilDay: UsageDay.make("2026-12-01"), + }; + const open = vi.spyOn(NodeFSP, "open"); + yield* Effect.addFinalizer(() => Effect.sync(() => open.mockRestore())); + open.mockClear(); + assert.strictEqual(totalOutputTokens(yield* service.readSummary(allTime)), 5); + assert.isTrue(open.mock.calls.some(([path]) => path === transcript)); + open.mockClear(); + for (const sinceDay of ["2026-11-02", "2026-09-03"]) { + const recent = { ...allTime, sinceDay: UsageDay.make(sinceDay) }; + assert.deepStrictEqual((yield* service.readSummary(recent)).buckets, []); + } + assert.strictEqual( + totalOutputTokens(yield* service.readSummary({ ...allTime, awaitRefresh: true })), + 5, + ); + assert.isFalse(open.mock.calls.some(([path]) => path === transcript)); + + yield* Effect.promise(async () => { + await NodeFSP.appendFile(transcript, claudeLine(2, 7)); + await NodeFSP.utimes(transcript, lastWrite + 86400, lastWrite + 86400); + }); + assert.strictEqual(totalOutputTokens(yield* service.readSummary(allTime)), 12); + yield* Effect.promise(async () => { + await NodeFSP.writeFile(transcript, claudeLine(3, 9)); + await NodeFSP.utimes(transcript, lastWrite + 172800, lastWrite + 172800); + }); + assert.strictEqual(totalOutputTokens(yield* service.readSummary(allTime)), 9); + + yield* Effect.promise(() => NodeFSP.rm(transcript)); + assert.deepStrictEqual((yield* service.readSummary(allTime)).buckets, []); + yield* service.awaitPersisted; + const restarted = yield* UsageService.make; + assert.deepStrictEqual((yield* restarted.readSummary(allTime)).buckets, []); + }).pipe( + Effect.provide(layerService({ prefix: "usage-service-all-time-test", home, settings })), + ); + }).pipe(Effect.scoped), + ); + + it.effect("answers a provisional all-time read with old and recent transcripts", () => + Effect.gen(function* () { + yield* TestClock.setTime(Date.parse("2026-12-01T00:00:00Z")); + const { transcript, settings, home } = yield* setup; + const recent = NodePath.join(NodePath.dirname(transcript), "recent.jsonl"); + yield* Effect.promise(async () => { + const oldWrite = Date.parse("2026-08-01T10:00:00Z") / 1000; + const recentWrite = Date.parse("2026-11-30T10:00:00Z") / 1000; + await NodeFSP.writeFile(transcript, claudeLine(1, 5)); + await NodeFSP.utimes(transcript, oldWrite, oldWrite); + await NodeFSP.writeFile(recent, claudeLine(2, 3)); + await NodeFSP.utimes(recent, recentWrite, recentWrite); + }); + yield* Effect.gen(function* () { + const service = yield* UsageService.make; + const allTime = { + ...WINDOW, + sinceDay: UsageDay.make("1970-01-01"), + untilDay: UsageDay.make("2026-12-01"), + }; + assert.strictEqual(totalOutputTokens(yield* service.readSummary(allTime)), 8); + // The recent file keeps the source refreshing, so this read cannot fall back to an exact scan. + assert.strictEqual( + totalOutputTokens(yield* service.readSummary({ ...allTime, provisional: true })), + 8, + ); + }).pipe( + Effect.provide( + layerService({ prefix: "usage-service-all-time-provisional", home, settings }), + ), + ); + }).pipe(Effect.scoped), + ); + it.effect("reports saved usage of a removed directory only for windows it reaches", () => Effect.gen(function* () { yield* TestClock.setTime(Date.parse("2026-08-12T00:00:00Z")); diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index a55532913c3c6..c297474ef4e3b 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -162,6 +162,8 @@ export const make = Effect.gen(function* () { const readerContext = yield* Effect.context(); const fileCache: ScanCache = new Map(); + // Old live files stay warm without enlarging persisted history or normal-window cache walks. + const historicalFileCache: ScanCache = new Map(); // Reads from concurrent windows can still hold the original record after it is corrected. const resolvedRecords = new WeakMap(); const correctedRecord = (record: UsageRecord) => resolvedRecords.get(record) ?? record; @@ -518,7 +520,7 @@ export const make = Effect.gen(function* () { readonly failed?: true; readonly update?: { readonly entry: CachedFile; readonly replaces: CachedFile | undefined }; }> { - const cached = fileCache.get(filePath); + const cached = fileCache.get(filePath) ?? historicalFileCache.get(filePath); // Provider is part of the identity: if both providers were ever pointed // at one directory, a hit parsed by the other parser must not be reused. if ( @@ -647,6 +649,7 @@ export const make = Effect.gen(function* () { const scanTranscriptDir = Effect.fn("UsageService.scanTranscriptDir")(function* ( source: TranscriptSource, windowStartMs: number, + retentionCutoffMs: number, fromMemory: boolean, ) { const { provider, format, dir, volumeId, fileName } = source; @@ -662,6 +665,17 @@ export const make = Effect.gen(function* () { const { files, walked, failedPaths } = yield* Effect.promise(() => listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }), ); + if (windowStartMs < retentionCutoffMs && failedPaths === 0) { + const livePaths = new Set(files.map((file) => file.path)); + for (const [filePath, entry] of historicalFileCache) { + if ( + entry.mtimeMs >= windowStartMs && + isWithinDirectory(filePath, dir) && + !livePaths.has(filePath) + ) + historicalFileCache.delete(filePath); + } + } const read = yield* Effect.forEach( files, (file) => @@ -675,18 +689,21 @@ export const make = Effect.gen(function* () { // A scan of another window may have cached its own read of this file // meanwhile. Then keep whichever read saw the later file, so a slower // scan never replaces newer usage with older. - const current = fileCache.get(path); + const current = fileCache.get(path) ?? historicalFileCache.get(path); if ( current === update.replaces || current === undefined || isLaterRead(update.entry, current) ) { - fileCache.set(path, { + const cache = update.entry.mtimeMs < retentionCutoffMs ? historicalFileCache : fileCache; + if (cache === fileCache) historicalFileCache.delete(path); + else if (fileCache.delete(path)) cacheDirty = true; + cache.set(path, { ...update.entry, records: correctedBatch(update.entry.records), tailRecords: correctedBatch(update.entry.tailRecords), }); - cacheDirty = true; + if (cache === fileCache) cacheDirty = true; } return { path, records, malformedRecords }; }); @@ -755,7 +772,12 @@ export const make = Effect.gen(function* () { // Scan independent sources together, directories sequentially to bound open files; keep order, since the first copy decides a duplicate's bucket. const [transcripts, scanDirs] = yield* Effect.all( - [Effect.forEach(dirs, (dir) => scanTranscriptDir(dir, windowStartMs, fromMemory)), scans], + [ + Effect.forEach(dirs, (dir) => + scanTranscriptDir(dir, windowStartMs, retentionCutoffMs, fromMemory), + ), + scans, + ], { concurrency: "unbounded" }, ); const scanned: readonly ScannedDir[] = [...transcripts, ...pending, ...scanDirs.flat()]; @@ -820,8 +842,10 @@ export const make = Effect.gen(function* () { ); // A window no complete scan has read would show partial totals, so every directory must cover it. // Saved files are keyed by path, so two providers reading one directory keep only one parse. + // Windows reaching past retention need the in-memory historical files, which a saved snapshot lacks. const fromMemory = provisional && + windowStartMs >= retentionCutoffMs && resolved.dirs.length > 0 && new Set(resolved.dirs.map((source) => source.dir)).size === resolved.dirs.length && resolved.dirs.every( diff --git a/apps/web/src/components/usage/UsagePage.tsx b/apps/web/src/components/usage/UsagePage.tsx index 8bc8a93a43d69..c542a5631bfbc 100644 --- a/apps/web/src/components/usage/UsagePage.tsx +++ b/apps/web/src/components/usage/UsagePage.tsx @@ -51,7 +51,6 @@ import { formatTokens, formatUsageContractMismatch, formatUsd, - makeWindow, usageSourceMessages, } from "@t3tools/shared/usageFormat"; import { Button, InlineButton } from "../ui/button"; @@ -80,6 +79,7 @@ import { WorkspacePageContainer } from "../WorkspacePageContainer"; import { UsageLimitsSection } from "./UsageLimits"; import { UsagePriceOverrides } from "./UsagePriceOverrides"; import { UsageProviderChart } from "./UsageProviderChart"; +import { makeUsageWindow, trimUsageWindow } from "./usageWindow"; import { SpeedPremium, UsageModelDialog } from "./UsageModelDialog"; import { UsageShareBar } from "./UsageShareBar"; import { @@ -188,11 +188,7 @@ export function UsagePage() { }; const [windowSelection, setWindowSelection] = useState(() => ({ days: preferences.windowDays, - window: makeWindow( - preferences.windowDays, - undefined, - preferences.windowDays === 1 ? "hour" : "day", - ), + window: makeUsageWindow(preferences.windowDays), })); const metric = preferences.metric; const showingLimits = metric === "limits"; @@ -204,7 +200,6 @@ export function UsagePage() { useState | null>(null); const hiddenProviders = new Set(preferences.hiddenProviders); const { days: windowDays, window } = windowSelection; - const isPast24Hours = windowDays === 1; const { merged: answeredUsage, environments, @@ -214,7 +209,7 @@ export function UsagePage() { refresh, } = useUsage(window, selectedEnvironmentIds, hiddenProviders); const merged = shown?.merged ?? answeredUsage; - const shownWindow = shown?.window ?? window; + const shownWindow = trimUsageWindow(shown?.window ?? window, merged.daily); const shownHourly = shownWindow.resolution === "hour"; const refreshingUsage = isRefreshing && !showingLimits; const showingKept = shown !== null && shown.window !== window; @@ -240,7 +235,7 @@ export function UsagePage() { updatePreferences({ windowDays: days }); setWindowSelection({ days, - window: makeWindow(days, undefined, days === 1 ? "hour" : "day"), + window: makeUsageWindow(days), }); }; const selectMetric = (nextMetric: UsageMetric) => { @@ -305,7 +300,7 @@ export function UsagePage() { }); return; } - const nextWindow = makeWindow(windowDays, undefined, isPast24Hours ? "hour" : "day"); + const nextWindow = makeUsageWindow(windowDays); if ( nextWindow.sinceDay !== window.sinceDay || nextWindow.untilDay !== window.untilDay || @@ -357,7 +352,7 @@ export function UsagePage() { ? `${formatDateTimeShort(shownWindow.sinceTime, shownWindow.timeZone, hour12)} to ${formatDateTimeShort(shownWindow.untilTime, shownWindow.timeZone, hour12)}` : `${formatDayShort(shownWindow.sinceDay)} to ${formatDayShort(shownWindow.untilDay)}`; const topbarContent = ( -
+

Usage

@@ -390,7 +385,7 @@ export function UsagePage() { {windowLabel} ) : null} -
+
{/* Limits are always current, so they have no period. The period sits before the metric tabs so dropping it never moves them. */} {showingLimits ? null : ( @@ -436,7 +431,7 @@ export function UsagePage() {
-
+
{showingLimits ? null : (