fix(server): finished runs rescan the file index off the event loop

apps/server/src/workspace/WorkspaceEntries.test.ts

@@ -756,10 +756,13 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => {
           if (created.ok) {
             const finder = created.value;
             finders += 1;
-            const scanFiles = finder.scanFiles.bind(finder);
             finder.scanFiles = () => {
+              throw new Error("refresh must not rescan on the event loop");
+            };
+            const scanFilesAsync = finder.scanFilesAsync.bind(finder);
+            finder.scanFilesAsync = () => {
               rescans += 1;
-              return scanFiles();
+              return scanFilesAsync();
             };
             const waitForIndexReady = finder.waitForIndexReady.bind(finder);
             finder.waitForIndexReady = (timeoutMs) =>
@@ -789,6 +792,50 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => {
         expect({ finders, rescans }).toEqual({ finders: 1, rescans: 1 });
       }),
     );
+
+    it.effect("shares one native progress read between concurrent refreshes", () =>
+      Effect.gen(function* () {
+        const cwd = yield* makeTempDir({ prefix: "t3code-workspace-content-refresh-pool-" });
+        yield* writeTextFile(cwd, "src/notes.ts", "const needle = 1;\n");
+        // Each read holds a libuv pool thread while it waits on fff's picker lock.
+        const readsInFlight = new Set<Promise<unknown>>();
+        let maxReadsInFlight = 0;
+        const createFinder = FileFinder.create.bind(FileFinder);
+        vi.spyOn(FileFinder, "create").mockImplementation((options) => {
+          const created = createFinder(options);
+          if (created.ok) {
+            const finder = created.value;
+            const getScanProgressAsync = finder.getScanProgressAsync.bind(finder);
+            finder.getScanProgressAsync = () => {
+              const read = getScanProgressAsync();
+              if (!readsInFlight.has(read)) {
+                readsInFlight.add(read);
+                maxReadsInFlight = Math.max(maxReadsInFlight, readsInFlight.size);
+                void read.finally(() => readsInFlight.delete(read));
+              }
+              return read;
+            };
+          }
+          return created;
+        });
+        const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
+        yield* workspaceEntries.searchContents({
+          cwd,
+          query: "needle",
+          limit: 100,
+          caseSensitive: true,
+          wholeWord: false,
+          useRegex: false,
+        });
+
+        yield* Effect.all(
+          Array.from({ length: 8 }, () => workspaceEntries.refresh(cwd)),
+          { concurrency: "unbounded" },
+        );
+
+        expect(maxReadsInFlight).toBe(1);
+      }),
+    );
   });
 
   describe("browse", () => {

apps/server/src/workspace/WorkspaceSearchIndex.ts

@@ -401,12 +401,14 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
   const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn(
     "WorkspaceSearchIndex.refresh",
   )(function* () {
-    const result = yield* Effect.try({
-      try: () => finder.scanFiles(),
+    // The rescan recovers events the watcher dropped. It waits on fff's picker lock, so it must
+    // not run on the event loop: a contended lock froze the server for tens of seconds.
+    const result = yield* Effect.tryPromise({
+      try: () => finder.scanFilesAsync(),
       catch: (cause) =>
         new WorkspaceSearchIndexRefreshFailed({
           cwd,
-          reason: "FileFinder.scanFiles threw unexpectedly.",
+          reason: "FileFinder.scanFilesAsync rejected unexpectedly.",
           cause,
         }),
     });

patches/@ff-labs__fff-node@0.11.0.patch

@@ -1,16 +1,5 @@
-diff --git a/package.json b/package.json
---- a/package.json
-+++ b/package.json
-@@ -11,7 +11,7 @@
-     ".": {
-       "types": "./dist/index.d.ts",
-       "import": "./dist/index.js",
--      "require": "./dist/index.cjs",
-+      "require": "./dist/index.js",
-       "default": "./dist/index.js"
-     },
-     "./package.json": "./package.json"
 diff --git a/dist/fff-api.d.ts b/dist/fff-api.d.ts
+index 0f90eedca20655ccd30b8f28e433b2094341fbdf..c2ef86927d6e3ab0e8584330fa0ac3fdacc06cf5 100644
 --- a/dist/fff-api.d.ts
 +++ b/dist/fff-api.d.ts
 @@ -199,7 +199,7 @@ export type Location = {
@@ -40,7 +29,36 @@ diff --git a/dist/fff-api.d.ts b/dist/fff-api.d.ts
      scores: Score[];
      /** Total number of items (files + dirs) that matched */
      totalMatched: number;
+diff --git a/dist/finder.d.ts b/dist/finder.d.ts
+index 452102d720cc19b0560cfc9e2f1a320687047c92..092162277e7a906f65a559d874fbe19571e62133 100644
+--- a/dist/finder.d.ts
++++ b/dist/finder.d.ts
+@@ -242,6 +242,11 @@ export declare class FileFinder implements FileFinderApi {
+      * background watcher might have missed.
+      */
+     scanFiles(): Result<void>;
++    /**
++     * Trigger a rescan on a native thread so a contended picker lock never blocks the JS thread.
++     * Patched in T3 Code.
++     */
++    scanFilesAsync(): Promise<Result<void>>;
+     /**
+      * Check if a scan is currently in progress.
+      */
+@@ -254,6 +259,11 @@ export declare class FileFinder implements FileFinderApi {
+      * Get the current scan progress.
+      */
+     getScanProgress(): Result<ScanProgress>;
++    /**
++     * Read scan progress on a native thread. Concurrent callers share one call.
++     * Patched in T3 Code.
++     */
++    getScanProgressAsync(): Promise<Result<ScanProgress>>;
+     /**
+      * Wait for the initial file scan to complete.
+      * Non-blocking: polls `isScanning` and yields to the event loop between checks.
 diff --git a/dist/index.js b/dist/index.js
+index 7c1d722eabc8d8eb98998eb48c5fe9d70b749d10..9643e1614ff128bd0cef71368b531314dfdee678 100644
 --- a/dist/index.js
 +++ b/dist/index.js
 @@ -1,7 +1,7 @@
@@ -263,3 +281,175 @@ diff --git a/dist/index.js b/dist/index.js
    }
    load({
      library: LIBRARY_KEY,
+@@ -1167,6 +1183,29 @@ function ffiMultiGrep(handle, patternsJoined, constraints, maxFileSize, maxMatch
+ function ffiScanFiles(handle) {
+   return callVoidResult("fff_scan_files", [DataType.External], [handle]);
+ }
++async function ffiScanFilesAsync(handle) {
++  loadLibrary();
++  const rawPtr = await load({
++    library: LIBRARY_KEY,
++    funcName: "fff_scan_files",
++    retType: DataType.External,
++    paramsType: [DataType.External],
++    paramsValue: [handle],
++    freeResultMemory: false,
++    runInNewThread: true
++  });
++  const [structData] = restorePointer({
++    retType: [FFF_RESULT_STRUCT],
++    paramsValue: wrapPointer([rawPtr])
++  });
++  if (structData.success === 0) {
++    const errorStr = readCString(structData.error);
++    freeResult(rawPtr);
++    return err(errorStr || "Unknown error");
++  }
++  freeResult(rawPtr);
++  return { ok: true, value: void 0 };
++}
+ function ffiIsScanning(handle) {
+   loadLibrary();
+   return load({
+@@ -1212,6 +1251,48 @@ function ffiGetScanProgress(handle) {
+   });
+   return { ok: true, value: result };
+ }
++async function ffiGetScanProgressAsync(handle) {
++  loadLibrary();
++  const rawPtr = await load({
++    library: LIBRARY_KEY,
++    funcName: "fff_get_scan_progress",
++    retType: DataType.External,
++    paramsType: [DataType.External],
++    paramsValue: [handle],
++    freeResultMemory: false,
++    runInNewThread: true
++  });
++  const [structData] = restorePointer({
++    retType: [FFF_RESULT_STRUCT],
++    paramsValue: wrapPointer([rawPtr])
++  });
++  if (structData.success === 0) {
++    const errorStr = readCString(structData.error);
++    freeResult(rawPtr);
++    return err(errorStr || "Unknown error");
++  }
++  const handlePtr = structData.handle;
++  freeResult(rawPtr);
++  if (isNullPointer(handlePtr)) return err("scan progress returned null");
++  const [sp] = restorePointer({
++    retType: [FFF_SCAN_PROGRESS_STRUCT],
++    paramsValue: wrapPointer([handlePtr])
++  });
++  const result = {
++    scannedFilesCount: Number(sp.scanned_files_count),
++    isScanning: sp.is_scanning !== 0,
++    isWatcherReady: sp.is_watcher_ready !== 0,
++    isWarmupComplete: sp.is_warmup_complete !== 0
++  };
++  load({
++    library: LIBRARY_KEY,
++    funcName: "fff_free_scan_progress",
++    retType: DataType.Void,
++    paramsType: [DataType.External],
++    paramsValue: [handlePtr]
++  });
++  return { ok: true, value: result };
++}
+ function ffiWaitForScan(handle, timeoutMs) {
+   return callBoolResult(
+     "fff_wait_for_scan",
+@@ -1429,6 +1510,9 @@ var FileFinder = class _FileFinder {
+   handle;
+   /** Native ids of this instance's active watch subscriptions. */
+   watchers = /* @__PURE__ */ new Set();
++  pendingScan = null;
++  pendingScanProgress = null;
++  pendingNativeCalls = /* @__PURE__ */ new Set();
+   constructor(handle) {
+     this.handle = handle;
+   }
+@@ -1485,6 +1569,16 @@ var FileFinder = class _FileFinder {
+   destroy() {
+     if (this.handle !== null) {
+       const handle = this.handle;
++      if (this.pendingNativeCalls.size > 0) {
++        this.handle = null;
++        const watchers = this.watchers;
++        this.watchers = /* @__PURE__ */ new Set();
++        Promise.allSettled(this.pendingNativeCalls).then(() => {
++          ffiDestroy(handle);
++          ffiWatchCleanupAfterDestroy(handle, watchers);
++        });
++        return;
++      }
+       ffiDestroy(handle);
+       this.handle = null;
+       ffiWatchCleanupAfterDestroy(handle, this.watchers);
+@@ -1757,6 +1851,42 @@ var FileFinder = class _FileFinder {
+     if (!guard.ok) return guard;
+     return ffiScanFiles(guard.value);
+   }
++  /**
++   * Trigger a rescan without blocking the JS thread. Patched in T3 Code:
++   * `fff_scan_files` can wait on the picker lock, so it runs on a native thread.
++   */
++  scanFilesAsync() {
++    const guard = this.ensureAlive();
++    if (!guard.ok) return Promise.resolve(guard);
++    if (this.pendingScan) return this.pendingScan;
++    const scan = this.runOnNativeThread(ffiScanFilesAsync(guard.value)).finally(() => {
++      if (this.pendingScan === scan) this.pendingScan = null;
++    });
++    this.pendingScan = scan;
++    return scan;
++  }
++  /**
++   * Read scan progress without blocking the JS thread. Patched in T3 Code: concurrent readers
++   * share one call, because each call holds a libuv pool thread while it waits on the picker lock.
++   */
++  getScanProgressAsync() {
++    const guard = this.ensureAlive();
++    if (!guard.ok) return Promise.resolve(guard);
++    if (this.pendingScanProgress) return this.pendingScanProgress;
++    const read = this.runOnNativeThread(ffiGetScanProgressAsync(guard.value)).finally(() => {
++      if (this.pendingScanProgress === read) this.pendingScanProgress = null;
++    });
++    this.pendingScanProgress = read;
++    return read;
++  }
++  /** Keeps the handle alive until a native-thread call settles. Patched in T3 Code. */
++  runOnNativeThread(call) {
++    const tracked = call.catch((cause) => err(String(cause))).finally(() => {
++      this.pendingNativeCalls.delete(tracked);
++    });
++    this.pendingNativeCalls.add(tracked);
++    return tracked;
++  }
+   /**
+    * Check if a scan is currently in progress.
+    */
+@@ -1838,7 +1968,8 @@ var FileFinder = class _FileFinder {
+     if (!guard.ok) return guard;
+     const deadline = Date.now() + timeoutMs;
+     while (true) {
+-      const progress = this.getScanProgress();
++      // Patched in T3 Code: the progress read takes the picker lock, so it runs on a native thread.
++      const progress = await this.getScanProgressAsync();
+       if (!progress.ok) return progress;
+       if (!progress.value.isScanning && progress.value.isWarmupComplete) {
+         return { ok: true, value: true };
+diff --git a/package.json b/package.json
+index e000a78b3f79826200979ebb3fe297b599bcea67..84325461c2d963cb42cf72267079d36a419c1398 100644
+--- a/package.json
++++ b/package.json
+@@ -11,7 +11,7 @@
+     ".": {
+       "types": "./dist/index.d.ts",
+       "import": "./dist/index.js",
+-      "require": "./dist/index.cjs",
++      "require": "./dist/index.js",
+       "default": "./dist/index.js"
+     },
+     "./package.json": "./package.json"