From bbe299712a4f5d290784e15c15cca85072dda9fa Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 19 Oct 2026 19:29:59 -0700 Subject: [PATCH 1/3] fix(session): commit an in-place evidence index append --- src/session/optimized-context-store.test.ts | 82 +++++++++++++++++++++ src/session/optimized-context-store.ts | 67 ++++++++++++++++- 2 files changed, 147 insertions(+), 2 deletions(-) diff --git a/src/session/optimized-context-store.test.ts b/src/session/optimized-context-store.test.ts index c722f9eab..c15e945e2 100644 --- a/src/session/optimized-context-store.test.ts +++ b/src/session/optimized-context-store.test.ts @@ -843,6 +843,88 @@ describe("createOptimizedContextStore checkpoint", () => { expect(new TextDecoder().decode(blob)).toBe("rewritten-in-place"); }); + test("an in-place append of evidence-archive/index.jsonl is in the next commit", async () => { + const dir = tempDir(); + const store = await createOptimizedContextStore(dir); + const indexDir = path.join(dir, "evidence-archive"); + fs.mkdirSync(indexDir); + const indexPath = path.join(indexDir, "index.jsonl"); + fs.writeFileSync(indexPath, '{"id":"seed"}\n'); + await store.writeTurns([turn("seed")]); + await store.writeMetadata(EMPTY_CHECKPOINT_METADATA); + const first = await store.commit({ message: "seed evidence index" }); + + const before = fs.statSync(indexPath); + const dirBefore = fs.statSync(indexDir); + fs.appendFileSync(indexPath, '{"id":"appended"}\n'); + const after = fs.statSync(indexPath); + const dirAfter = fs.statSync(indexDir); + expect(after.ino).toBe(before.ino); + expect(dirAfter.mtimeMs).toBe(dirBefore.mtimeMs); + expect(dirAfter.ino).toBe(dirBefore.ino); + + const second = await store.commit({ message: "publish evidence index append" }); + expect(second.hash).not.toBe(first.hash); + const { blob } = await git.readBlob({ + fs, + dir, + oid: second.hash, + filepath: "evidence-archive/index.jsonl", + }); + expect(new TextDecoder().decode(blob)).toContain('{"id":"appended"}'); + }); + + test("an unchanged evidence index is not re-read and a no-op does not walk tool-output", async () => { + const dir = tempDir(); + const store = await createOptimizedContextStore(dir); + const blobCount = 24; + for (let index = 0; index < blobCount; index++) { + await store.writeBlob( + `blob-${index}`, + new TextEncoder().encode(`body-${index}`), + "text/plain", + ); + } + const indexDir = path.join(dir, "evidence-archive"); + fs.mkdirSync(indexDir); + fs.writeFileSync(path.join(indexDir, "index.jsonl"), '{"id":"seed"}\n'); + await store.writeTurns([turn("seed")]); + await store.writeMetadata(EMPTY_CHECKPOINT_METADATA); + await store.commit({ message: "seed evidence and blobs" }); + + const walkedToolOutput: string[][] = []; + const originalStatus = git.statusMatrix.bind(git); + git.statusMatrix = (async (args) => { + const filepaths = args.filepaths ?? []; + if ( + filepaths.some( + (filepath) => + filepath === "tool-output" || filepath.startsWith("tool-output/"), + ) + ) { + walkedToolOutput.push([...filepaths]); + } + return originalStatus(args); + }) as typeof git.statusMatrix; + let reads = 0; + let lstats = 0; + try { + reads = await countPathOps("readFile", "index.jsonl", () => + store.commit({ message: "no writes" }), + ); + lstats = await countPathOps( + "lstat", + `${path.sep}tool-output${path.sep}`, + () => store.commit({ message: "still no writes" }), + ); + } finally { + git.statusMatrix = originalStatus; + } + expect(reads).toBe(0); + expect(walkedToolOutput).toEqual([]); + expect(lstats).toBe(blobCount); + }); + test("a turn commit does not lstat every seeded tool-output blob", async () => { const dir = tempDir(); const store = await createOptimizedContextStore(dir); diff --git a/src/session/optimized-context-store.ts b/src/session/optimized-context-store.ts index 3fc3aefd9..6e11f859c 100644 --- a/src/session/optimized-context-store.ts +++ b/src/session/optimized-context-store.ts @@ -46,6 +46,7 @@ const METADATA_FILE = "metadata.json"; const TOKEN_USAGE_SIDECAR_FILE = "token-usage.jsonl"; const TOOL_OUTPUT_DIR = "tool-output"; const EVIDENCE_ARCHIVE_DIR = "evidence-archive"; +const EVIDENCE_INDEX_FILE = `${EVIDENCE_ARCHIVE_DIR}/index.jsonl`; const OWNER_MARKER_FILE = "session-owner.json"; const log = getLogger([LOG_NAMESPACE_ROOT, "session", "context-store"]); @@ -1489,9 +1490,13 @@ export async function createSessionStores( // Fingerprints from the last publish. Segment stats avoid re-reading an // unchanged segment. Tool-output blobs are remembered per file: a directory // mtime does not move when an existing blob is rewritten in place, and - // statusMatrix is not asked to walk that directory on a no-op. + // statusMatrix is not asked to walk that directory on a no-op. The evidence + // index is one file: an in-place append does not move the archive directory, + // so it is fingerprinted on its own and is not a reason to walk tool-output. let publishedSegmentStats = new Map(); let publishedToolBlobs = new Map(); + let publishedEvidenceIndex: SegmentStat | undefined; + let evidenceIndexObserved = false; let blobDirsObserved = false; let toolOutputFingerprint: DirectoryFingerprint | null = null; let evidenceFingerprint: DirectoryFingerprint | null = null; @@ -1575,6 +1580,53 @@ export async function createSessionStores( return drifted; } + async function statEvidenceIndex(): Promise { + try { + const fileStats = await fs.promises.lstat( + path.join(dir, EVIDENCE_INDEX_FILE), + ); + if (!fileStats.isFile()) return undefined; + return { + bytes: fileStats.size, + mtimeMs: fileStats.mtimeMs, + ino: fileStats.ino, + }; + } catch (cause) { + if ( + cause instanceof Error && + "code" in cause && + cause.code === "ENOENT" + ) { + return undefined; + } + throw cause; + } + } + + // In-place appends do not move the archive directory. Compare the one index + // file saved at the last publish. Absent before and after is not drift, and + // a match is not read. A store that has not published yet leaves this to the + // directory walk. + async function driftedPublishedEvidenceIndex(): Promise<{ + filepath: string; + present: boolean; + } | null> { + if (!evidenceIndexObserved) return null; + const observed = await statEvidenceIndex(); + if (observed === undefined) { + return publishedEvidenceIndex === undefined + ? null + : { filepath: EVIDENCE_INDEX_FILE, present: false }; + } + if ( + publishedEvidenceIndex === undefined || + segmentStatDrifted(observed, publishedEvidenceIndex) + ) { + return { filepath: EVIDENCE_INDEX_FILE, present: true }; + } + return null; + } + async function blobDirectoriesNeedWalk(): Promise { if (!blobDirsObserved) return true; const tool = await directoryFingerprint(dir, TOOL_OUTPUT_DIR); @@ -1612,6 +1664,8 @@ export async function createSessionStores( toolOutputFingerprint = tool; evidenceFingerprint = evidence; blobDirsObserved = true; + publishedEvidenceIndex = await statEvidenceIndex(); + evidenceIndexObserved = true; } const ownerToken = opts?.ownerId ?? globalThis.crypto.randomUUID(); const explicitOwnerId = opts?.ownerId; @@ -2466,7 +2520,10 @@ export async function createSessionStores( // mtime or inode, so the directory walk stays off. Stat only the // blobs recorded at the last publish, and only when nothing else // is already forcing a commit (that path git.adds the directory). + // The evidence index is the same kind of in-place append, and it + // is one file: stat it, do not walk tool-output to notice it. let toolBlobsDrifted = false; + let evidenceIndexDrifted = false; if (!rootsDiffer) { for (const blob of await driftedPublishedToolBlobs()) { toolBlobsDrifted = true; @@ -2475,8 +2532,14 @@ export async function createSessionStores( restatBlobs.add(blob.filepath); } else toRemove.push(blob.filepath); } + const index = await driftedPublishedEvidenceIndex(); + if (index !== null) { + evidenceIndexDrifted = true; + if (index.present) toAdd.push(index.filepath); + else toRemove.push(index.filepath); + } } - if (!rootsDiffer && !toolBlobsDrifted) { + if (!rootsDiffer && !toolBlobsDrifted && !evidenceIndexDrifted) { const [head] = await base.log(1); if (head !== undefined) { pendingBlobFilepaths.clear(); From c9ecb62db2e10e3ce6e1728d7368be8e7f8c8d56 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 19 Oct 2026 19:34:44 -0700 Subject: [PATCH 2/3] fix(session): keep the evidence index fingerprint on a no-op --- src/session/optimized-context-store.test.ts | 51 +++++++++++++++++++++ src/session/optimized-context-store.ts | 9 +++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/session/optimized-context-store.test.ts b/src/session/optimized-context-store.test.ts index c15e945e2..f031c4369 100644 --- a/src/session/optimized-context-store.test.ts +++ b/src/session/optimized-context-store.test.ts @@ -874,6 +874,57 @@ describe("createOptimizedContextStore checkpoint", () => { expect(new TextDecoder().decode(blob)).toContain('{"id":"appended"}'); }); + test("an append during a no-op commit is not remembered as already published", async () => { + const dir = tempDir(); + const store = await createOptimizedContextStore(dir); + const indexPath = path.join(dir, "evidence-archive", "index.jsonl"); + fs.mkdirSync(path.dirname(indexPath)); + fs.writeFileSync(indexPath, '{"id":"seed"}\n'); + await store.writeTurns([turn("seed")]); + await store.writeMetadata(EMPTY_CHECKPOINT_METADATA); + const first = await store.commit({ message: "seed evidence index" }); + + const evidenceDir = path.join(dir, "evidence-archive"); + let indexStats = 0; + let appended = false; + const original = fs.promises.lstat; + fs.promises.lstat = (async (target: fs.PathLike, opts?: fs.StatOptions) => { + if (typeof target === "string" && target === indexPath) indexStats += 1; + // The drift check stats the index. The later directory stat is + // rememberPublishedTree, which used to restat the index and store a + // fingerprint the no-op never committed. + if ( + typeof target === "string" && + target === evidenceDir && + indexStats >= 1 && + !appended + ) { + appended = true; + fs.appendFileSync(indexPath, '{"id":"during-noop"}\n'); + } + return original.call(fs.promises, target, opts); + }) as typeof fs.promises.lstat; + let noopHash = ""; + try { + const noop = await store.commit({ message: "no writes" }); + noopHash = noop.hash; + } finally { + fs.promises.lstat = original; + } + expect(appended).toBe(true); + expect(noopHash).toBe(first.hash); + + const third = await store.commit({ message: "publish append from no-op window" }); + expect(third.hash).not.toBe(first.hash); + const { blob } = await git.readBlob({ + fs, + dir, + oid: third.hash, + filepath: "evidence-archive/index.jsonl", + }); + expect(new TextDecoder().decode(blob)).toContain('{"id":"during-noop"}'); + }); + test("an unchanged evidence index is not re-read and a no-op does not walk tool-output", async () => { const dir = tempDir(); const store = await createOptimizedContextStore(dir); diff --git a/src/session/optimized-context-store.ts b/src/session/optimized-context-store.ts index 6e11f859c..ee4cdba23 100644 --- a/src/session/optimized-context-store.ts +++ b/src/session/optimized-context-store.ts @@ -1664,8 +1664,13 @@ export async function createSessionStores( toolOutputFingerprint = tool; evidenceFingerprint = evidence; blobDirsObserved = true; - publishedEvidenceIndex = await statEvidenceIndex(); - evidenceIndexObserved = true; + // A no-op already compared the index and found it unchanged. Restatting + // here would store a fingerprint for bytes appended after that check and + // before this return, and the next commit would treat them as published. + if (refreshBlobs || !evidenceIndexObserved) { + publishedEvidenceIndex = await statEvidenceIndex(); + evidenceIndexObserved = true; + } } const ownerToken = opts?.ownerId ?? globalThis.crypto.randomUUID(); const explicitOwnerId = opts?.ownerId; From c4699bb9429c730e01866cdf3894cd985a02db64 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 19 Oct 2026 19:37:31 -0700 Subject: [PATCH 3/3] ci(session): record the evidence-index lstat in the perf baseline --- scripts/results/session-store-perf-baseline.json | 6 +++--- src/session/optimized-context-store.test.ts | 8 ++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/results/session-store-perf-baseline.json b/scripts/results/session-store-perf-baseline.json index 673cbd8ec..aa5971f74 100644 --- a/scripts/results/session-store-perf-baseline.json +++ b/scripts/results/session-store-perf-baseline.json @@ -12,14 +12,14 @@ "turnCount": 200, "wallMs": 814, "loadedTurns": 250, - "totalFsOps": 15736, + "totalFsOps": 15936, "gitPathFsOps": 7717, "fsOps": { "readFile": 7878, "mkdir": 246, "writeFile": 1274, "stat": 2580, - "lstat": 2948, + "lstat": 3148, "readdir": 611, "access": 1, "open": 198 @@ -33,7 +33,7 @@ }, "perTurn": { "wallMs": 4.07, - "fsOps": 78.68 + "fsOps": 79.68 } }, "turnPersistenceCensus": { diff --git a/src/session/optimized-context-store.test.ts b/src/session/optimized-context-store.test.ts index f031c4369..2a7cb629d 100644 --- a/src/session/optimized-context-store.test.ts +++ b/src/session/optimized-context-store.test.ts @@ -863,7 +863,9 @@ describe("createOptimizedContextStore checkpoint", () => { expect(dirAfter.mtimeMs).toBe(dirBefore.mtimeMs); expect(dirAfter.ino).toBe(dirBefore.ino); - const second = await store.commit({ message: "publish evidence index append" }); + const second = await store.commit({ + message: "publish evidence index append", + }); expect(second.hash).not.toBe(first.hash); const { blob } = await git.readBlob({ fs, @@ -914,7 +916,9 @@ describe("createOptimizedContextStore checkpoint", () => { expect(appended).toBe(true); expect(noopHash).toBe(first.hash); - const third = await store.commit({ message: "publish append from no-op window" }); + const third = await store.commit({ + message: "publish append from no-op window", + }); expect(third.hash).not.toBe(first.hash); const { blob } = await git.readBlob({ fs,