diff --git a/apps/server/src/project/AgentSessionScanner.test.ts b/apps/server/src/project/AgentSessionScanner.test.ts index 956a65976369f..fb2fe71265f51 100644 --- a/apps/server/src/project/AgentSessionScanner.test.ts +++ b/apps/server/src/project/AgentSessionScanner.test.ts @@ -5,6 +5,7 @@ import * as NodeChildProcess from "node:child_process"; import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodeSqlite from "node:sqlite"; +import * as NodeZlib from "node:zlib"; import { describe, expect, it } from "@effect/vitest"; import { AgentSessionImportSource, @@ -147,6 +148,21 @@ const writeTranscript = Effect.fn("AgentSessionScanner.test.writeTranscript")(fu yield* fileSystem.utimes(input.filePath, seconds, seconds); }); +const writeCompressedTranscript = Effect.fn("AgentSessionScanner.test.writeCompressedTranscript")( + function* (input: { + readonly filePath: string; + readonly contents: string; + readonly mtimeMs: number; + }) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fileSystem.makeDirectory(path.dirname(input.filePath), { recursive: true }); + yield* fileSystem.writeFile(input.filePath, NodeZlib.zstdCompressSync(input.contents)); + const seconds = input.mtimeMs / 1000; + yield* fileSystem.utimes(input.filePath, seconds, seconds); + }, +); + const claudeSessionLine = (cwd: string) => `${JSON.stringify({ type: "user", cwd, sessionId: "s1" })}\n${JSON.stringify({ type: "assistant" })}\n`; @@ -396,6 +412,42 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }), ); + it.effect("discovers zstd-compressed Codex rollouts", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-compressed-"); + + yield* writeCompressedTranscript({ + filePath: path.join( + codexHomePath, + "sessions", + "2026", + "04", + "01", + "rollout-2026-04-01T09-00-00-aaa.jsonl.zst", + ), + contents: codexRolloutLine(workspace), + mtimeMs: Date.parse("2026-04-01T09:00:00.000Z"), + }); + + const result = yield* runScan({ claudeHomePath, codexHomePath }); + + expect(result.candidates).toEqual([ + { + path: workspace, + title: path.basename(workspace), + sources: ["codex"], + threadCount: 1, + lastActiveAt: "2026-04-01T09:00:00.000Z", + alreadyImported: false, + git: null, + }, + ]); + }), + ); + it.effect("returns the imported project ID through a realpath alias", () => Effect.gen(function* () { const path = yield* Path.Path; @@ -1512,89 +1564,95 @@ it.layer(NodeServices.layer)("AgentSessionScanner", (it) => { }), ); - it.effect("imports a rewound Codex thread with the history its new rollout inherits", () => - Effect.gen(function* () { - const path = yield* Path.Path; - const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); - yield* TestClock.setTime(nowMs); - const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); - const codexHomePath = yield* makeTempDir("t3code-codex-home-"); - const workspace = yield* makeTempDir("t3code-workspace-"); - const threadId = "019a0000-0000-7000-8000-000000000001"; - const rolloutId = "019a0000-0000-7000-8000-000000000002"; - const sessionsDir = path.join(codexHomePath, "sessions", "2026", "08", "24"); - const prompt = (message: string) => - encodeTranscriptRecord({ type: "event_msg", payload: { type: "user_message", message } }); - const reply = (text: string) => - encodeTranscriptRecord({ - type: "response_item", - payload: { - type: "message", - role: "assistant", - content: [{ type: "output_text", text }], - }, - }); - const keptHistory = [ - encodeTranscriptRecord({ - type: "session_meta", - payload: { id: threadId, cwd: workspace, history_mode: "paginated" }, - }), - prompt("Keep the first request"), - reply("First reply"), - ] - .map((record) => `${record}\n`) - .join(""); - - yield* writeTranscript({ - filePath: path.join(sessionsDir, `rollout-2026-08-24T10-00-00-${threadId}.jsonl`), - contents: `${keptHistory}${prompt("Rewound request")}\n${reply("Rewound reply")}\n`, - mtimeMs: nowMs - 60_000, - }); - yield* writeTranscript({ - filePath: path.join( - sessionsDir, - `rollout-2026-08-24T11-00-00-${threadId}_${rolloutId}.jsonl`, - ), - contents: [ + it.effect.each([".jsonl", ".jsonl.zst"] as const)( + "imports a rewound Codex thread with the history its new %s rollout inherits", + (extension) => + Effect.gen(function* () { + const write = extension === ".jsonl" ? writeTranscript : writeCompressedTranscript; + const path = yield* Path.Path; + const nowMs = Date.parse("2026-08-24T12:00:00.000Z"); + yield* TestClock.setTime(nowMs); + const claudeHomePath = yield* makeTempDir("t3code-claude-home-"); + const codexHomePath = yield* makeTempDir("t3code-codex-home-"); + const workspace = yield* makeTempDir("t3code-workspace-"); + const threadId = "019a0000-0000-7000-8000-000000000001"; + const rolloutId = "019a0000-0000-7000-8000-000000000002"; + const sessionsDir = path.join(codexHomePath, "sessions", "2026", "08", "24"); + const prompt = (message: string) => encodeTranscriptRecord({ - type: "session_meta", + type: "event_msg", + payload: { type: "user_message", message }, + }); + const reply = (text: string) => + encodeTranscriptRecord({ + type: "response_item", payload: { - id: threadId, - cwd: workspace, - history_mode: "paginated", - history_base: { - thread_id: threadId, - end_ordinal_exclusive: 3, - end_byte_offset: Buffer.byteLength(keptHistory), - }, + type: "message", + role: "assistant", + content: [{ type: "output_text", text }], }, + }); + const keptHistory = [ + encodeTranscriptRecord({ + type: "session_meta", + payload: { id: threadId, cwd: workspace, history_mode: "paginated" }, }), - prompt("Ask again"), - reply("Final reply"), - ].join("\n"), - mtimeMs: nowMs, - }); + prompt("Keep the first request"), + reply("First reply"), + ] + .map((record) => `${record}\n`) + .join(""); - const threads = yield* runRecentThreads({ - claudeHomePath, - codexHomePath, - workspaceRoot: workspace, - }); + yield* write({ + filePath: path.join(sessionsDir, `rollout-2026-08-24T10-00-00-${threadId}${extension}`), + contents: `${keptHistory}${prompt("Rewound request")}\n${reply("Rewound reply")}\n`, + mtimeMs: nowMs - 60_000, + }); + yield* write({ + filePath: path.join( + sessionsDir, + `rollout-2026-08-24T11-00-00-${threadId}_${rolloutId}${extension}`, + ), + contents: [ + encodeTranscriptRecord({ + type: "session_meta", + payload: { + id: threadId, + cwd: workspace, + history_mode: "paginated", + history_base: { + thread_id: threadId, + end_ordinal_exclusive: 3, + end_byte_offset: Buffer.byteLength(keptHistory), + }, + }, + }), + prompt("Ask again"), + reply("Final reply"), + ].join("\n"), + mtimeMs: nowMs, + }); - expect( - threads.map((thread) => ({ - id: thread.providerSessionId, - title: thread.title, - messages: thread.messages.map((message) => message.text), - })), - ).toEqual([ - { - id: threadId, - title: "Keep the first request", - messages: ["Keep the first request", "First reply", "Ask again", "Final reply"], - }, - ]); - }), + const threads = yield* runRecentThreads({ + claudeHomePath, + codexHomePath, + workspaceRoot: workspace, + }); + + expect( + threads.map((thread) => ({ + id: thread.providerSessionId, + title: thread.title, + messages: thread.messages.map((message) => message.text), + })), + ).toEqual([ + { + id: threadId, + title: "Keep the first request", + messages: ["Keep the first request", "First reply", "Ask again", "Final reply"], + }, + ]); + }), ); it.effect("excludes Codex subagents from discovery and the recent-thread budget", () => diff --git a/apps/server/src/project/AgentSessionScanner.ts b/apps/server/src/project/AgentSessionScanner.ts index 6c8410241a3f5..4d655de4b939e 100644 --- a/apps/server/src/project/AgentSessionScanner.ts +++ b/apps/server/src/project/AgentSessionScanner.ts @@ -1,5 +1,8 @@ import * as NodeOS from "node:os"; import * as NodeSqlite from "node:sqlite"; +import * as NodeZlib from "node:zlib"; + +import * as NodeStream from "@effect/platform-node/NodeStream"; import { AgentSessionScanError, @@ -14,6 +17,7 @@ import { type AgentSessionScanResult, type ProviderInstanceConfig, } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; @@ -22,6 +26,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Pull from "effect/Pull"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; @@ -170,7 +175,7 @@ const decodeCodexHistoryBase = Schema.decodeUnknownOption( ); // Original rollouts end in their thread ID, rewound ones in `_`. const CODEX_ROLLOUT_ID_PATTERN = - /-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))?\.jsonl$/i; + /-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:_([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))?\.jsonl(?:\.zst)?$/i; // Codex records a chat's project only here; moving a chat leaves its rollout cwd unchanged. const CODEX_STATE_DATABASE = "state_5.sqlite"; const decodeCodexSessionIndexEntry = Schema.decodeUnknownOption( @@ -250,6 +255,10 @@ interface TranscriptCandidate { readonly projectRoots?: ReadonlyArray | undefined; } +interface ImportByteBudget { + bytesRemaining: number; +} + interface MetadataReadBudget { bytesRemaining: number; operationsRemaining: number; @@ -786,6 +795,39 @@ type TranscriptIdentity = Pick< const fileIdKey = (inode: TranscriptIdentity["inode"]) => (inode === null ? null : String(inode)); +const isCompressedTranscript = (filePath: string) => filePath.endsWith(".zst"); + +/** + * Reads a transcript in chunks of at most `size` bytes. Codex may store rollouts + * as `.jsonl.zst`; those yield decompressed bytes so every budget counts what is + * actually parsed. The decompressor honors backpressure, so memory stays bounded. + */ +const openTranscriptChunks = Effect.fnUntraced(function* (file: FileSystem.File, filePath: string) { + if (!isCompressedTranscript(filePath)) return (size: number) => file.readAlloc(size); + const pull = yield* Stream.fromEffectRepeat( + file + .readAlloc(TRANSCRIPT_PREFIX_BYTES) + .pipe(Effect.flatMap(Option.match({ onNone: () => Cause.done(), onSome: Effect.succeed }))), + ).pipe( + NodeStream.pipeThroughDuplex({ evaluate: () => NodeZlib.createZstdDecompress() }), + Stream.toPull, + ); + const pending: Array = []; + return (size: number) => + Effect.gen(function* () { + while (pending.length === 0) { + for (const chunk of yield* pull) if (chunk.byteLength > 0) pending.push(chunk); + } + const chunk = pending[0]!; + if (chunk.byteLength <= size) { + pending.shift(); + return Option.some(chunk); + } + pending[0] = chunk.subarray(size); + return Option.some(chunk.subarray(0, size)); + }).pipe(Pull.catchDone(() => Effect.succeedNone)); +}); + function sameTranscriptIdentity(left: TranscriptIdentity, right: TranscriptIdentity): boolean { // Unsafe legacy numbers may already have lost file ID precision. if ( @@ -1001,11 +1043,16 @@ export const make = Effect.gen(function* () { fileSystem.open(transcript.filePath, { flag: "r" }).pipe( Effect.flatMap((file) => Effect.gen(function* () { + const compressed = isCompressedTranscript(transcript.filePath); + const readChunk = yield* openTranscriptChunks(file, transcript.filePath); const decoder = new TextDecoder(); let remaining = ""; let bytesRead = 0; let recordsRead = 0; - const maxBytes = Math.min(MAX_TRANSCRIPT_SCAN_BYTES, transcript.size); + // The decompressed size of a compressed rollout is unknown until its end. + const maxBytes = compressed + ? MAX_TRANSCRIPT_SCAN_BYTES + : Math.min(MAX_TRANSCRIPT_SCAN_BYTES, transcript.size); const reserveRecord = () => { if ( recordsRead === MAX_METADATA_RECORDS_PER_TRANSCRIPT || @@ -1037,7 +1084,7 @@ export const make = Effect.gen(function* () { ); budget.operationsRemaining -= 1; budget.bytesRemaining -= readSize; - const next = yield* file.readAlloc(readSize); + const next = yield* readChunk(readSize); if (Option.isNone(next)) { return readLastRecord(); } @@ -1054,7 +1101,7 @@ export const make = Effect.gen(function* () { } } - if (bytesRead < transcript.size) { + if (compressed || bytesRead < transcript.size) { budget.truncated = true; return null; } @@ -1069,10 +1116,19 @@ export const make = Effect.gen(function* () { filePath: string, expected: ReturnType, source: AgentSessionSource, + budget: ImportByteBudget, prefix?: { readonly endByteOffset: number; readonly historyBytes: number }, ) { - const readBytes = prefix?.endByteOffset ?? expected.size; - if (readBytes > expected.size || readBytes > MAX_IMPORTED_TRANSCRIPT_BYTES) return null; + const compressed = isCompressedTranscript(filePath); + // A full read of a compressed rollout has no known length, so it is charged as it + // decompresses; callers charge every other read up front. + const chargeAsRead = compressed && prefix === undefined; + const readBytes = chargeAsRead + ? Math.min(MAX_IMPORTED_TRANSCRIPT_BYTES, budget.bytesRemaining) + : (prefix?.endByteOffset ?? expected.size); + if ((!compressed && readBytes > expected.size) || readBytes > MAX_IMPORTED_TRANSCRIPT_BYTES) { + return null; + } return yield* Effect.scoped( fileSystem.open(filePath, { flag: "r" }).pipe( @@ -1081,6 +1137,7 @@ export const make = Effect.gen(function* () { if (!sameTranscriptIdentity(expected, transcriptIdentity(filePath, yield* file.stat))) { return null; } + const readChunk = yield* openTranscriptChunks(file, filePath); const records: Array = []; const claudeHistory = createClaudeHistoryWindow(); let cwd = Option.none(); @@ -1141,14 +1198,16 @@ export const make = Effect.gen(function* () { }; while (bytesRead < readBytes) { - const next = yield* file.readAlloc( + const next = yield* readChunk( Math.min(TRANSCRIPT_PREFIX_BYTES, readBytes - bytesRead), ); if (Option.isNone(next)) { + if (chargeAsRead) break; return null; } bytesRead += next.value.byteLength; + if (chargeAsRead) budget.bytesRemaining -= next.value.byteLength; const withinBudget = yield* Effect.try({ try: () => { let start = 0; @@ -1167,6 +1226,9 @@ export const make = Effect.gen(function* () { }); if (!withinBudget) return null; } + if (chargeAsRead && bytesRead === readBytes && Option.isSome(yield* readChunk(1))) { + return null; + } if ( recordStarted && @@ -1362,7 +1424,12 @@ export const make = Effect.gen(function* () { } const directory = path.join(sessionsDir, year, month, day); for (const entry of (yield* readDirectory(directory)).toSorted().toReversed()) { - if (!entry.startsWith("rollout-") || !entry.endsWith(".jsonl")) continue; + if ( + !entry.startsWith("rollout-") || + !(entry.endsWith(".jsonl") || entry.endsWith(".jsonl.zst")) + ) { + continue; + } if (operationsRemaining <= 0) { truncated = true; break; @@ -1809,7 +1876,7 @@ export const make = Effect.gen(function* () { ); const importedSessions = new Set(); const codexThreadNamesByHome = new Map>(); - let bytesRemaining = MAX_IMPORT_BYTES; + const byteBudget: ImportByteBudget = { bytesRemaining: MAX_IMPORT_BYTES }; let transcriptsRemaining = MAX_IMPORT_TRANSCRIPTS; const codexRolloutPaths = new Map>(); const codexRolloutPath = (providerInstanceId: ProviderInstanceId, rolloutId: string) => { @@ -1843,15 +1910,18 @@ export const make = Effect.gen(function* () { const filePath = visited.has(base.thread_id) ? undefined : codexRolloutPath(providerInstanceId, base.thread_id); - if (filePath === undefined || base.end_byte_offset > bytesRemaining) return null; + if (filePath === undefined || base.end_byte_offset > byteBudget.bytesRemaining) { + return null; + } visited.add(base.thread_id); const stats = yield* statOption(filePath); if (Option.isNone(stats) || stats.value.type !== "File") return null; - bytesRemaining -= base.end_byte_offset; + byteBudget.bytesRemaining -= base.end_byte_offset; const inherited = yield* readTranscript( filePath, transcriptIdentity(filePath, stats.value), "codex", + byteBudget, { endByteOffset: base.end_byte_offset, historyBytes }, ); if (inherited === null) return null; @@ -1868,7 +1938,10 @@ export const make = Effect.gen(function* () { const completed = completedByFile.get( `${candidate.providerInstanceId}\0${transcript.filePath}`, ); - if (completed === undefined && (transcriptsRemaining === 0 || bytesRemaining === 0)) { + if ( + completed === undefined && + (transcriptsRemaining === 0 || byteBudget.bytesRemaining === 0) + ) { return Option.some({ _tag: "Skipped" }); } const stats = yield* statOption(transcript.filePath); @@ -1892,13 +1965,20 @@ export const make = Effect.gen(function* () { if ( transcriptsRemaining === 0 || identity.size > MAX_IMPORTED_TRANSCRIPT_BYTES || - identity.size > bytesRemaining + identity.size > byteBudget.bytesRemaining ) { return Option.some({ _tag: "Skipped" }); } transcriptsRemaining -= 1; - bytesRemaining -= identity.size; - const snapshot = yield* readTranscript(transcript.filePath, identity, candidate.source); + if (!isCompressedTranscript(transcript.filePath)) { + byteBudget.bytesRemaining -= identity.size; + } + const snapshot = yield* readTranscript( + transcript.filePath, + identity, + candidate.source, + byteBudget, + ); if (snapshot === null) { return Option.some({ _tag: "Skipped" }); } @@ -1929,7 +2009,10 @@ export const make = Effect.gen(function* () { { source: candidate.source, providerInstanceId: candidate.providerInstanceId, - fallbackSessionId: path.basename(transcript.filePath, ".jsonl"), + fallbackSessionId: path.basename( + path.basename(transcript.filePath, ".zst"), + ".jsonl", + ), lastActiveAtMs: transcript.mtimeMs, }, inheritedRecords === null