diff --git a/apps/server/src/project/AgentSessionScanner.test.ts b/apps/server/src/project/AgentSessionScanner.test.ts index 29a55ec22ba1..f2bfb656a525 100644 --- a/apps/server/src/project/AgentSessionScanner.test.ts +++ b/apps/server/src/project/AgentSessionScanner.test.ts @@ -1,5 +1,6 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import * as NodeOS from "node:os"; +import * as NodeZlib from "node:zlib"; import { describe, expect, it } from "@effect/vitest"; import { AgentSessionImportSource, @@ -10,6 +11,7 @@ import { type ServerSettings as ContractServerSettings, } from "@t3tools/contracts"; import { symlinksSupported } from "@t3tools/shared/testing/symlinks"; +import * as ByteSize from "effect/ByteSize"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -106,17 +108,44 @@ const makeTempDir = Effect.fn("AgentSessionScanner.test.makeTempDir")(function* const writeTranscript = Effect.fn("AgentSessionScanner.test.writeTranscript")(function* (input: { readonly filePath: string; - readonly contents: string; + readonly contents: string | Uint8Array; readonly mtimeMs: number; }) { const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; yield* fileSystem.makeDirectory(path.dirname(input.filePath), { recursive: true }); - yield* fileSystem.writeFileString(input.filePath, input.contents); + yield* typeof input.contents === "string" + ? fileSystem.writeFileString(input.filePath, input.contents) + : fileSystem.writeFile(input.filePath, input.contents); const seconds = input.mtimeMs / 1000; yield* fileSystem.utimes(input.filePath, seconds, seconds); }); +const ZSTD_BLOCK_BYTES = 128 * 1024; + +/** + * A single zstd frame holding `text` followed by `paddingBlocks` run-length blocks of spaces, so + * gibibytes of decompressed padding cost tens of kibibytes on disk. The padding extends the last + * record. It must share the frame because Node's decompressor stops after the first one. + */ +const zstdRolloutWithPadding = (text: string, paddingBlocks: number) => { + const raw = new TextEncoder().encode(text); + const blockHeader = (size: number, type: 0 | 1, isLast: boolean) => { + const header = (size << 3) | (type << 1) | (isLast ? 1 : 0); + return [header & 0xff, (header >> 8) & 0xff, (header >> 16) & 0xff]; + }; + const frame = new Uint8Array(6 + 3 + raw.byteLength + paddingBlocks * 4); + // Magic number, a frame header without optional fields, and a 128 KiB window. + frame.set([0x28, 0xb5, 0x2f, 0xfd, 0x00, 0x38]); + frame.set(blockHeader(raw.byteLength, 0, paddingBlocks === 0), 6); + frame.set(raw, 9); + for (let index = 0; index < paddingBlocks; index += 1) { + const header = blockHeader(ZSTD_BLOCK_BYTES, 1, index === paddingBlocks - 1); + frame.set([...header, 0x20], 9 + raw.byteLength + index * 4); + } + return frame; +}; + const claudeSessionLine = (cwd: string) => `${JSON.stringify({ type: "user", cwd, sessionId: "s1" })}\n${JSON.stringify({ type: "assistant" })}\n`; @@ -284,6 +313,64 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }), ); + it.effect("finds the cwd of zstd-compressed Codex rollouts beside plain ones", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + const otherWorkspace = yield* makeTempDir("t3code-workspace-other-"); + + const rollout = (name: string) => + path.join(codexHomePath, "sessions", "2026", "02", "09", name); + + // The leading record is larger than one metadata read, so the cwd only + // appears after several decompressed chunks. + yield* writeTranscript({ + filePath: rollout("rollout-2026-02-09T10-00-00-aaa.jsonl.zst"), + contents: NodeZlib.zstdCompressSync( + encodeTranscriptRecord({ type: "turn_context", payload: { pad: "x".repeat(40_000) } }) + + "\n" + + codexRolloutLine(workspace), + ), + mtimeMs: Date.parse("2026-02-09T10:00:00.000Z"), + }); + yield* writeTranscript({ + filePath: rollout("rollout-2026-02-09T11-00-00-bbb.jsonl"), + contents: codexRolloutLine(otherWorkspace), + mtimeMs: Date.parse("2026-02-09T11:00:00.000Z"), + }); + yield* writeTranscript({ + filePath: rollout("rollout-2026-02-09T12-00-00-corrupt.jsonl.zst"), + contents: new TextEncoder().encode("not a zstd frame"), + mtimeMs: Date.parse("2026-02-09T12:00:00.000Z"), + }); + + const result = yield* runScan({ claudeHomePath, codexHomePath }); + + expect(result.candidates).toEqual([ + { + path: otherWorkspace, + title: path.basename(otherWorkspace), + sources: ["codex"], + threadCount: 1, + lastActiveAt: "2026-02-09T11:00:00.000Z", + alreadyImported: false, + git: null, + }, + { + path: workspace, + title: path.basename(workspace), + sources: ["codex"], + threadCount: 1, + lastActiveAt: "2026-02-09T10:00:00.000Z", + alreadyImported: false, + git: null, + }, + ]); + }), + ); + it.effect("merges the same cwd seen by both agents and flags imported projects", () => Effect.gen(function* () { const path = yield* Path.Path; @@ -1068,6 +1155,184 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }), ); + it.effect("imports zstd-compressed and plain Codex rollouts together", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + + const rollout = (sessionId: string, prompt: string) => + [ + encodeTranscriptRecord({ + type: "session_meta", + payload: { id: sessionId, cwd: workspace }, + }), + // Compresses to a few bytes but decompresses past several read chunks. + encodeTranscriptRecord({ type: "turn_context", payload: { pad: "x".repeat(100_000) } }), + encodeTranscriptRecord({ + type: "event_msg", + timestamp: "2026-08-24T10:00:00.000Z", + payload: { type: "user_message", message: prompt }, + }), + encodeTranscriptRecord({ + type: "response_item", + timestamp: "2026-08-24T10:01:00.000Z", + payload: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: `Answer to ${prompt}` }], + }, + }), + ].join("\n"); + const sessionsDir = path.join(codexHomePath, "sessions", "2026", "08", "24"); + + yield* writeTranscript({ + filePath: path.join(sessionsDir, "rollout-compressed.jsonl.zst"), + contents: NodeZlib.zstdCompressSync(rollout("codex-compressed", "Compressed prompt")), + mtimeMs: nowMs - 60 * 60 * 1000, + }); + yield* writeTranscript({ + filePath: path.join(sessionsDir, "rollout-plain.jsonl"), + contents: rollout("codex-plain", "Plain prompt"), + mtimeMs: nowMs - 2 * 60 * 60 * 1000, + }); + + const threads = yield* runRecentThreads({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }); + + expect(threads.map((thread) => thread.providerSessionId)).toEqual([ + "codex-compressed", + "codex-plain", + ]); + expect(threads.map((thread) => thread.messages.map((message) => message.text))).toEqual([ + ["Compressed prompt", "Answer to Compressed prompt"], + ["Plain prompt", "Answer to Plain prompt"], + ]); + }), + ); + + it.effect("charges decompressed zstd bytes against the aggregate import budget", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + + const rollout = (sessionId: string) => + [ + encodeTranscriptRecord({ + type: "session_meta", + payload: { id: sessionId, cwd: workspace }, + }), + encodeTranscriptRecord({ + type: "event_msg", + payload: { type: "user_message", message: "Imported prompt" }, + }), + ].join("\n"); + const gibibyteBlocks = (1024 * 1024 * 1024) / ZSTD_BLOCK_BYTES; + const sessionsDir = path.join(codexHomePath, "sessions", "2026", "08", "24"); + // The two large rollouts take 2.5 GiB each of the 4 GiB import budget once decompressed, + // yet only a few tens of KiB on disk. The third is tiny and would fit if the budget + // were charged by size on disk. + for (const [index, { sessionId, paddingBlocks }] of [ + { sessionId: "large-first", paddingBlocks: 2.5 * gibibyteBlocks }, + { sessionId: "large-second", paddingBlocks: 2.5 * gibibyteBlocks }, + { sessionId: "small-last", paddingBlocks: 0 }, + ].entries()) { + yield* writeTranscript({ + filePath: path.join(sessionsDir, `rollout-${sessionId}.jsonl.zst`), + contents: zstdRolloutWithPadding(rollout(sessionId), paddingBlocks), + mtimeMs: nowMs - index * 1_000, + }); + } + + const outcomes = yield* runRecentThreadOutcomes({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }); + + expect( + outcomes.map((outcome) => + outcome._tag === "Importable" ? outcome.thread.providerSessionId : outcome._tag, + ), + ).toEqual(["large-first", "Skipped", "Skipped"]); + }), + ); + + it.effect("imports a zstd rollout that decompresses to exactly the remaining budget", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const fileSystem = yield* FileSystem.FileSystem; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + + const rollout = [ + encodeTranscriptRecord({ + type: "session_meta", + payload: { id: "exact-limit", cwd: workspace }, + }), + encodeTranscriptRecord({ + type: "event_msg", + payload: { type: "user_message", message: "Imported prompt" }, + }), + ].join("\n"); + const decompressedBytes = + new TextEncoder().encode(rollout).byteLength + 3 * ZSTD_BLOCK_BYTES; + const sessionsDir = path.join(codexHomePath, "sessions", "2026", "08", "24"); + const reservingPath = path.join(sessionsDir, "rollout-reserving.jsonl"); + yield* writeTranscript({ + filePath: reservingPath, + contents: codexRolloutLine(workspace), + mtimeMs: nowMs, + }); + yield* writeTranscript({ + filePath: path.join(sessionsDir, "rollout-exact-limit.jsonl.zst"), + contents: zstdRolloutWithPadding(rollout, 3), + mtimeMs: nowMs - 1_000, + }); + + // A plain file's size is reserved from the 4 GiB import budget before it is read. Reporting + // all but `decompressedBytes` of it leaves the zstd rollout a read limit of exactly its + // decompressed length; the mismatch with the real file then ends the read at once. + const simulatedFileSystem = FileSystem.FileSystem.of({ + ...fileSystem, + stat: (filePath) => + fileSystem + .stat(filePath) + .pipe( + Effect.map((info) => + filePath === reservingPath + ? { ...info, size: ByteSize.bytes(4 * 1024 * 1024 * 1024 - decompressedBytes) } + : info, + ), + ), + }); + const outcomes = yield* runRecentThreadOutcomes({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }).pipe(Effect.provideService(FileSystem.FileSystem, simulatedFileSystem)); + + expect( + outcomes.map((outcome) => + outcome._tag === "Importable" ? outcome.thread.providerSessionId : outcome._tag, + ), + ).toEqual(["Skipped", "exact-limit"]); + }), + ); + it.effect("keeps the provider instance that owns a custom session home", () => Effect.gen(function* () { const path = yield* Path.Path; diff --git a/apps/server/src/project/AgentSessionScanner.ts b/apps/server/src/project/AgentSessionScanner.ts index e516ce0e53b9..156aaac4cc17 100644 --- a/apps/server/src/project/AgentSessionScanner.ts +++ b/apps/server/src/project/AgentSessionScanner.ts @@ -1,4 +1,6 @@ +import * as NodeStream from "@effect/platform-node/NodeStream"; import * as NodeOS from "node:os"; +import * as NodeZlib from "node:zlib"; import { AgentSessionScanError, @@ -13,6 +15,7 @@ import { type AgentSessionScanResult, type ProviderInstanceConfig, } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -20,6 +23,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Pull from "effect/Pull"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; @@ -118,6 +122,9 @@ const decodeCodexTurnMetadata = Schema.decodeUnknownOption(CodexTurnMetadata); type DecodedTranscriptRecord = typeof TranscriptRecord.Type; +const TRANSCRIPT_EXTENSION = /\.jsonl(?:\.zst)?$/; +const isZstdTranscript = (filePath: string) => filePath.endsWith(".jsonl.zst"); + interface AgentSessionTranscriptMetadata { readonly source: AgentSessionSource; readonly providerInstanceId: ProviderInstanceId; @@ -184,6 +191,10 @@ interface TranscriptCandidate { readonly size: number; } +interface ImportByteBudget { + bytesRemaining: number; +} + interface MetadataReadBudget { bytesRemaining: number; operationsRemaining: number; @@ -731,6 +742,50 @@ export const make = Effect.gen(function* () { } as const; }); + /** + * Codex can store rollouts as `.jsonl.zst`. A compressed file is decompressed from the same + * descriptor that the identity checks stat, so a replaced file is still detected, and + * `readAlloc` keeps its plain-file contract over the decompressed bytes. + */ + const openTranscript = Effect.fn("AgentSessionScanner.openTranscript")(function* ( + filePath: string, + ) { + const file = yield* fileSystem.open(filePath, { flag: "r" }); + if (!isZstdTranscript(filePath)) return file; + + const compressed = Stream.fromPull( + Effect.succeed( + file.readAlloc(TRANSCRIPT_PREFIX_BYTES).pipe( + Effect.flatMap( + Option.match({ + onNone: () => Cause.done(), + onSome: (chunk) => Effect.succeed([chunk] as const), + }), + ), + ), + ), + ); + const pull = yield* Stream.toPull( + NodeStream.pipeThroughSimple(compressed, () => NodeZlib.createZstdDecompress()), + ); + const queued: Array = []; + return { + stat: file.stat, + // Runs once per decompressed chunk; avoid a span for each read. + readAlloc: Effect.fnUntraced(function* (size: number) { + let head = queued.shift(); + while (head === undefined) { + const chunks = yield* pull.pipe(Pull.catchDone(() => Effect.succeed(null))); + if (chunks === null) return Option.none(); + queued.push(...chunks); + head = queued.shift(); + } + if (head.byteLength > size) queued.unshift(head.subarray(size)); + return Option.some(head.subarray(0, size)); + }), + }; + }); + const readCwd = Effect.fn("AgentSessionScanner.readCwd")(function* ( source: AgentSessionSource, transcript: TranscriptCandidate, @@ -746,15 +801,19 @@ export const make = Effect.gen(function* () { return null; } budget.operationsRemaining -= 1; + // The size on disk of a compressed file says nothing about its decompressed length. + const transcriptSize = isZstdTranscript(transcript.filePath) + ? Number.POSITIVE_INFINITY + : transcript.size; return yield* Effect.scoped( - fileSystem.open(transcript.filePath, { flag: "r" }).pipe( + openTranscript(transcript.filePath).pipe( Effect.flatMap((file) => Effect.gen(function* () { const decoder = new TextDecoder(); let remaining = ""; let bytesRead = 0; let recordsRead = 0; - const maxBytes = Math.min(MAX_TRANSCRIPT_SCAN_BYTES, transcript.size); + const maxBytes = Math.min(MAX_TRANSCRIPT_SCAN_BYTES, transcriptSize); const reserveRecord = () => { if ( recordsRead === MAX_METADATA_RECORDS_PER_TRANSCRIPT || @@ -803,7 +862,7 @@ export const make = Effect.gen(function* () { } } - if (bytesRead < transcript.size) { + if (bytesRead < transcriptSize) { budget.truncated = true; return null; } @@ -819,11 +878,20 @@ export const make = Effect.gen(function* () { expected: ReturnType, recordLimit: number, source: AgentSessionSource, + budget: ImportByteBudget, ) { if (expected.size > MAX_IMPORTED_TRANSCRIPT_BYTES) return null; + const compressed = isZstdTranscript(filePath); + // A plain file's size is reserved before it is read. A compressed file's size on disk says + // nothing about its decompressed length, so it is read to its end under whatever is left of + // the aggregate budget and charged for each decompressed byte, even if the read then fails. + const readLimit = compressed + ? Math.min(MAX_IMPORTED_TRANSCRIPT_BYTES, budget.bytesRemaining) + : expected.size; + return yield* Effect.scoped( - fileSystem.open(filePath, { flag: "r" }).pipe( + openTranscript(filePath).pipe( Effect.flatMap((file) => Effect.gen(function* () { if (!sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat))) { @@ -870,15 +938,19 @@ export const make = Effect.gen(function* () { return true; }; - while (bytesRead < expected.size) { + let reachedEnd = false; + while (bytesRead < readLimit) { const next = yield* file.readAlloc( - Math.min(TRANSCRIPT_PREFIX_BYTES, expected.size - bytesRead), + Math.min(TRANSCRIPT_PREFIX_BYTES, readLimit - bytesRead), ); if (Option.isNone(next)) { - return null; + if (!compressed) return null; + reachedEnd = true; + break; } bytesRead += next.value.byteLength; + if (compressed) budget.bytesRemaining -= next.value.byteLength; const withinBudget = yield* Effect.try(() => { let start = 0; while (start < next.value.byteLength) { @@ -895,6 +967,9 @@ export const make = Effect.gen(function* () { if (!withinBudget) return null; } + // Filling the limit exactly is not EOF; only an empty read shows the transcript fits. + // A byte found past the limit is left uncharged so the budget never goes negative. + if (compressed && !reachedEnd && Option.isSome(yield* file.readAlloc(1))) return null; if (recordStarted && !(yield* Effect.try(finishRecord))) return null; return sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat)) ? { records, recordCount, cwd, claudeFields } @@ -1011,7 +1086,7 @@ export const make = Effect.gen(function* () { } const directory = path.join(sessionsDir, year, month, day); for (const entry of (yield* readDirectory(directory)).toSorted().toReversed()) { - if (!entry.startsWith("rollout-") || !entry.endsWith(".jsonl")) continue; + if (!entry.startsWith("rollout-") || !TRANSCRIPT_EXTENSION.test(entry)) continue; if (operationsRemaining <= 0) { truncated = true; break; @@ -1364,7 +1439,7 @@ export const make = Effect.gen(function* () { (source) => `${source.providerInstanceId}\0${source.filePath}`, ); const importedSessions = new Set(); - let bytesRemaining = MAX_IMPORT_BYTES; + const importBudget: ImportByteBudget = { bytesRemaining: MAX_IMPORT_BYTES }; let transcriptsRemaining = MAX_IMPORT_TRANSCRIPTS; let recordsRemaining = MAX_IMPORT_RECORDS; return Stream.fromIteratorSucceed(eligibleTranscripts.values(), 1).pipe( @@ -1375,7 +1450,9 @@ export const make = Effect.gen(function* () { ); if ( completed === undefined && - (transcriptsRemaining === 0 || bytesRemaining === 0 || recordsRemaining === 0) + (transcriptsRemaining === 0 || + importBudget.bytesRemaining === 0 || + recordsRemaining === 0) ) { return Option.some({ _tag: "Skipped" }); } @@ -1401,17 +1478,18 @@ export const make = Effect.gen(function* () { transcriptsRemaining === 0 || recordsRemaining === 0 || identity.size > MAX_IMPORTED_TRANSCRIPT_BYTES || - identity.size > bytesRemaining + identity.size > importBudget.bytesRemaining ) { return Option.some({ _tag: "Skipped" }); } transcriptsRemaining -= 1; - bytesRemaining -= identity.size; + if (!isZstdTranscript(transcript.filePath)) importBudget.bytesRemaining -= identity.size; const snapshot = yield* readTranscript( transcript.filePath, identity, recordsRemaining, candidate.source, + importBudget, ); if (snapshot === null) { return Option.some({ _tag: "Skipped" }); @@ -1433,7 +1511,9 @@ export const make = Effect.gen(function* () { { source: candidate.source, providerInstanceId: candidate.providerInstanceId, - fallbackSessionId: path.basename(transcript.filePath, ".jsonl"), + fallbackSessionId: path + .basename(transcript.filePath) + .replace(TRANSCRIPT_EXTENSION, ""), lastActiveAtMs: transcript.mtimeMs, }, snapshot.records,