fix(session): commit an in-place evidence index append

scripts/results/session-store-perf-baseline.json

@@ -12,14 +12,14 @@
     "turnCount": 200,
     "wallMs": 814,
     "loadedTurns": 250,
-    "totalFsOps": 15736,
+    "totalFsOps": 15936,
     "gitPathFsOps": 7717,
     "fsOps": {
       "readFile": 7878,
       "mkdir": 246,
       "writeFile": 1274,
       "stat": 2580,
-      "lstat": 2948,
+      "lstat": 3148,
       "readdir": 611,
       "access": 1,
       "open": 198
@@ -33,7 +33,7 @@
     },
     "perTurn": {
       "wallMs": 4.07,
-      "fsOps": 78.68
+      "fsOps": 79.68
     }
   },
   "turnPersistenceCensus": {

src/session/optimized-context-store.test.ts

@@ -843,6 +843,143 @@ describe("createOptimizedContextStore checkpoint", () => {
     expect(new TextDecoder().decode(blob)).toBe("rewritten-in-place");
   });
 
+  test("an in-place append of evidence-archive/index.jsonl is in the next commit", async () => {
+    const dir = tempDir();
+    const store = await createOptimizedContextStore(dir);
+    const indexDir = path.join(dir, "evidence-archive");
+    fs.mkdirSync(indexDir);
+    const indexPath = path.join(indexDir, "index.jsonl");
+    fs.writeFileSync(indexPath, '{"id":"seed"}\n');
+    await store.writeTurns([turn("seed")]);
+    await store.writeMetadata(EMPTY_CHECKPOINT_METADATA);
+    const first = await store.commit({ message: "seed evidence index" });
+
+    const before = fs.statSync(indexPath);
+    const dirBefore = fs.statSync(indexDir);
+    fs.appendFileSync(indexPath, '{"id":"appended"}\n');
+    const after = fs.statSync(indexPath);
+    const dirAfter = fs.statSync(indexDir);
+    expect(after.ino).toBe(before.ino);
+    expect(dirAfter.mtimeMs).toBe(dirBefore.mtimeMs);
+    expect(dirAfter.ino).toBe(dirBefore.ino);
+
+    const second = await store.commit({
+      message: "publish evidence index append",
+    });
+    expect(second.hash).not.toBe(first.hash);
+    const { blob } = await git.readBlob({
+      fs,
+      dir,
+      oid: second.hash,
+      filepath: "evidence-archive/index.jsonl",
+    });
+    expect(new TextDecoder().decode(blob)).toContain('{"id":"appended"}');
+  });
+
+  test("an append during a no-op commit is not remembered as already published", async () => {
+    const dir = tempDir();
+    const store = await createOptimizedContextStore(dir);
+    const indexPath = path.join(dir, "evidence-archive", "index.jsonl");
+    fs.mkdirSync(path.dirname(indexPath));
+    fs.writeFileSync(indexPath, '{"id":"seed"}\n');
+    await store.writeTurns([turn("seed")]);
+    await store.writeMetadata(EMPTY_CHECKPOINT_METADATA);
+    const first = await store.commit({ message: "seed evidence index" });
+
+    const evidenceDir = path.join(dir, "evidence-archive");
+    let indexStats = 0;
+    let appended = false;
+    const original = fs.promises.lstat;
+    fs.promises.lstat = (async (target: fs.PathLike, opts?: fs.StatOptions) => {
+      if (typeof target === "string" && target === indexPath) indexStats += 1;
+      // The drift check stats the index. The later directory stat is
+      // rememberPublishedTree, which used to restat the index and store a
+      // fingerprint the no-op never committed.
+      if (
+        typeof target === "string" &&
+        target === evidenceDir &&
+        indexStats >= 1 &&
+        !appended
+      ) {
+        appended = true;
+        fs.appendFileSync(indexPath, '{"id":"during-noop"}\n');
+      }
+      return original.call(fs.promises, target, opts);
+    }) as typeof fs.promises.lstat;
+    let noopHash = "";
+    try {
+      const noop = await store.commit({ message: "no writes" });
+      noopHash = noop.hash;
+    } finally {
+      fs.promises.lstat = original;
+    }
+    expect(appended).toBe(true);
+    expect(noopHash).toBe(first.hash);
+
+    const third = await store.commit({
+      message: "publish append from no-op window",
+    });
+    expect(third.hash).not.toBe(first.hash);
+    const { blob } = await git.readBlob({
+      fs,
+      dir,
+      oid: third.hash,
+      filepath: "evidence-archive/index.jsonl",
+    });
+    expect(new TextDecoder().decode(blob)).toContain('{"id":"during-noop"}');
+  });
+
+  test("an unchanged evidence index is not re-read and a no-op does not walk tool-output", async () => {
+    const dir = tempDir();
+    const store = await createOptimizedContextStore(dir);
+    const blobCount = 24;
+    for (let index = 0; index < blobCount; index++) {
+      await store.writeBlob(
+        `blob-${index}`,
+        new TextEncoder().encode(`body-${index}`),
+        "text/plain",
+      );
+    }
+    const indexDir = path.join(dir, "evidence-archive");
+    fs.mkdirSync(indexDir);
+    fs.writeFileSync(path.join(indexDir, "index.jsonl"), '{"id":"seed"}\n');
+    await store.writeTurns([turn("seed")]);
+    await store.writeMetadata(EMPTY_CHECKPOINT_METADATA);
+    await store.commit({ message: "seed evidence and blobs" });
+
+    const walkedToolOutput: string[][] = [];
+    const originalStatus = git.statusMatrix.bind(git);
+    git.statusMatrix = (async (args) => {
+      const filepaths = args.filepaths ?? [];
+      if (
+        filepaths.some(
+          (filepath) =>
+            filepath === "tool-output" || filepath.startsWith("tool-output/"),
+        )
+      ) {
+        walkedToolOutput.push([...filepaths]);
+      }
+      return originalStatus(args);
+    }) as typeof git.statusMatrix;
+    let reads = 0;
+    let lstats = 0;
+    try {
+      reads = await countPathOps("readFile", "index.jsonl", () =>
+        store.commit({ message: "no writes" }),
+      );
+      lstats = await countPathOps(
+        "lstat",
+        `${path.sep}tool-output${path.sep}`,
+        () => store.commit({ message: "still no writes" }),
+      );
+    } finally {
+      git.statusMatrix = originalStatus;
+    }
+    expect(reads).toBe(0);
+    expect(walkedToolOutput).toEqual([]);
+    expect(lstats).toBe(blobCount);
+  });
+
   test("a turn commit does not lstat every seeded tool-output blob", async () => {
     const dir = tempDir();
     const store = await createOptimizedContextStore(dir);

src/session/optimized-context-store.ts

@@ -46,6 +46,7 @@ const METADATA_FILE = "metadata.json";
 const TOKEN_USAGE_SIDECAR_FILE = "token-usage.jsonl";
 const TOOL_OUTPUT_DIR = "tool-output";
 const EVIDENCE_ARCHIVE_DIR = "evidence-archive";
+const EVIDENCE_INDEX_FILE = `${EVIDENCE_ARCHIVE_DIR}/index.jsonl`;
 const OWNER_MARKER_FILE = "session-owner.json";
 
 const log = getLogger([LOG_NAMESPACE_ROOT, "session", "context-store"]);
@@ -1489,9 +1490,13 @@ export async function createSessionStores(
   // Fingerprints from the last publish. Segment stats avoid re-reading an
   // unchanged segment. Tool-output blobs are remembered per file: a directory
   // mtime does not move when an existing blob is rewritten in place, and
-  // statusMatrix is not asked to walk that directory on a no-op.
+  // statusMatrix is not asked to walk that directory on a no-op. The evidence
+  // index is one file: an in-place append does not move the archive directory,
+  // so it is fingerprinted on its own and is not a reason to walk tool-output.
   let publishedSegmentStats = new Map<string, SegmentStat>();
   let publishedToolBlobs = new Map<string, SegmentStat>();
+  let publishedEvidenceIndex: SegmentStat | undefined;
+  let evidenceIndexObserved = false;
   let blobDirsObserved = false;
   let toolOutputFingerprint: DirectoryFingerprint | null = null;
   let evidenceFingerprint: DirectoryFingerprint | null = null;
@@ -1575,6 +1580,53 @@ export async function createSessionStores(
     return drifted;
   }
 
+  async function statEvidenceIndex(): Promise<SegmentStat | undefined> {
+    try {
+      const fileStats = await fs.promises.lstat(
+        path.join(dir, EVIDENCE_INDEX_FILE),
+      );
+      if (!fileStats.isFile()) return undefined;
+      return {
+        bytes: fileStats.size,
+        mtimeMs: fileStats.mtimeMs,
+        ino: fileStats.ino,
+      };
+    } catch (cause) {
+      if (
+        cause instanceof Error &&
+        "code" in cause &&
+        cause.code === "ENOENT"
+      ) {
+        return undefined;
+      }
+      throw cause;
+    }
+  }
+
+  // In-place appends do not move the archive directory. Compare the one index
+  // file saved at the last publish. Absent before and after is not drift, and
+  // a match is not read. A store that has not published yet leaves this to the
+  // directory walk.
+  async function driftedPublishedEvidenceIndex(): Promise<{
+    filepath: string;
+    present: boolean;
+  } | null> {
+    if (!evidenceIndexObserved) return null;
+    const observed = await statEvidenceIndex();
+    if (observed === undefined) {
+      return publishedEvidenceIndex === undefined
+        ? null
+        : { filepath: EVIDENCE_INDEX_FILE, present: false };
+    }
+    if (
+      publishedEvidenceIndex === undefined ||
+      segmentStatDrifted(observed, publishedEvidenceIndex)
+    ) {
+      return { filepath: EVIDENCE_INDEX_FILE, present: true };
+    }
+    return null;
+  }
+
   async function blobDirectoriesNeedWalk(): Promise<boolean> {
     if (!blobDirsObserved) return true;
     const tool = await directoryFingerprint(dir, TOOL_OUTPUT_DIR);
@@ -1612,6 +1664,13 @@ export async function createSessionStores(
     toolOutputFingerprint = tool;
     evidenceFingerprint = evidence;
     blobDirsObserved = true;
+    // A no-op already compared the index and found it unchanged. Restatting
+    // here would store a fingerprint for bytes appended after that check and
+    // before this return, and the next commit would treat them as published.
+    if (refreshBlobs || !evidenceIndexObserved) {
+      publishedEvidenceIndex = await statEvidenceIndex();
+      evidenceIndexObserved = true;
+    }
   }
   const ownerToken = opts?.ownerId ?? globalThis.crypto.randomUUID();
   const explicitOwnerId = opts?.ownerId;
@@ -2466,7 +2525,10 @@ export async function createSessionStores(
             // mtime or inode, so the directory walk stays off. Stat only the
             // blobs recorded at the last publish, and only when nothing else
             // is already forcing a commit (that path git.adds the directory).
+            // The evidence index is the same kind of in-place append, and it
+            // is one file: stat it, do not walk tool-output to notice it.
             let toolBlobsDrifted = false;
+            let evidenceIndexDrifted = false;
             if (!rootsDiffer) {
               for (const blob of await driftedPublishedToolBlobs()) {
                 toolBlobsDrifted = true;
@@ -2475,8 +2537,14 @@ export async function createSessionStores(
                   restatBlobs.add(blob.filepath);
                 } else toRemove.push(blob.filepath);
               }
+              const index = await driftedPublishedEvidenceIndex();
+              if (index !== null) {
+                evidenceIndexDrifted = true;
+                if (index.present) toAdd.push(index.filepath);
+                else toRemove.push(index.filepath);
+              }
             }
-            if (!rootsDiffer && !toolBlobsDrifted) {
+            if (!rootsDiffer && !toolBlobsDrifted && !evidenceIndexDrifted) {
               const [head] = await base.log(1);
               if (head !== undefined) {
                 pendingBlobFilepaths.clear();