diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index b0633dac04cc7..624be9dc1bd55 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -756,10 +756,13 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { if (created.ok) { const finder = created.value; finders += 1; - const scanFiles = finder.scanFiles.bind(finder); finder.scanFiles = () => { + throw new Error("refresh must not rescan on the event loop"); + }; + const scanFilesAsync = finder.scanFilesAsync.bind(finder); + finder.scanFilesAsync = () => { rescans += 1; - return scanFiles(); + return scanFilesAsync(); }; const waitForIndexReady = finder.waitForIndexReady.bind(finder); finder.waitForIndexReady = (timeoutMs) => @@ -789,6 +792,50 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { expect({ finders, rescans }).toEqual({ finders: 1, rescans: 1 }); }), ); + + it.effect("shares one native progress read between concurrent refreshes", () => + Effect.gen(function* () { + const cwd = yield* makeTempDir({ prefix: "t3code-workspace-content-refresh-pool-" }); + yield* writeTextFile(cwd, "src/notes.ts", "const needle = 1;\n"); + // Each read holds a libuv pool thread while it waits on fff's picker lock. + const readsInFlight = new Set>(); + let maxReadsInFlight = 0; + const createFinder = FileFinder.create.bind(FileFinder); + vi.spyOn(FileFinder, "create").mockImplementation((options) => { + const created = createFinder(options); + if (created.ok) { + const finder = created.value; + const getScanProgressAsync = finder.getScanProgressAsync.bind(finder); + finder.getScanProgressAsync = () => { + const read = getScanProgressAsync(); + if (!readsInFlight.has(read)) { + readsInFlight.add(read); + maxReadsInFlight = Math.max(maxReadsInFlight, readsInFlight.size); + void read.finally(() => readsInFlight.delete(read)); + } + return read; + }; + } + return created; + }); + const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + yield* workspaceEntries.searchContents({ + cwd, + query: "needle", + limit: 100, + caseSensitive: true, + wholeWord: false, + useRegex: false, + }); + + yield* Effect.all( + Array.from({ length: 8 }, () => workspaceEntries.refresh(cwd)), + { concurrency: "unbounded" }, + ); + + expect(maxReadsInFlight).toBe(1); + }), + ); }); describe("browse", () => { diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index ecb8c6ba87a73..5bffa3a2d9c0e 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -401,12 +401,14 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn( "WorkspaceSearchIndex.refresh", )(function* () { - const result = yield* Effect.try({ - try: () => finder.scanFiles(), + // The rescan recovers events the watcher dropped. It waits on fff's picker lock, so it must + // not run on the event loop: a contended lock froze the server for tens of seconds. + const result = yield* Effect.tryPromise({ + try: () => finder.scanFilesAsync(), catch: (cause) => new WorkspaceSearchIndexRefreshFailed({ cwd, - reason: "FileFinder.scanFiles threw unexpectedly.", + reason: "FileFinder.scanFilesAsync rejected unexpectedly.", cause, }), }); diff --git a/patches/@ff-labs__fff-node@0.11.0.patch b/patches/@ff-labs__fff-node@0.11.0.patch index 76e45e152a51e..d2063031855b7 100644 --- a/patches/@ff-labs__fff-node@0.11.0.patch +++ b/patches/@ff-labs__fff-node@0.11.0.patch @@ -1,16 +1,5 @@ -diff --git a/package.json b/package.json ---- a/package.json -+++ b/package.json -@@ -11,7 +11,7 @@ - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", -- "require": "./dist/index.cjs", -+ "require": "./dist/index.js", - "default": "./dist/index.js" - }, - "./package.json": "./package.json" diff --git a/dist/fff-api.d.ts b/dist/fff-api.d.ts +index 0f90eedca20655ccd30b8f28e433b2094341fbdf..c2ef86927d6e3ab0e8584330fa0ac3fdacc06cf5 100644 --- a/dist/fff-api.d.ts +++ b/dist/fff-api.d.ts @@ -199,7 +199,7 @@ export type Location = { @@ -40,7 +29,36 @@ diff --git a/dist/fff-api.d.ts b/dist/fff-api.d.ts scores: Score[]; /** Total number of items (files + dirs) that matched */ totalMatched: number; +diff --git a/dist/finder.d.ts b/dist/finder.d.ts +index 452102d720cc19b0560cfc9e2f1a320687047c92..092162277e7a906f65a559d874fbe19571e62133 100644 +--- a/dist/finder.d.ts ++++ b/dist/finder.d.ts +@@ -242,6 +242,11 @@ export declare class FileFinder implements FileFinderApi { + * background watcher might have missed. + */ + scanFiles(): Result; ++ /** ++ * Trigger a rescan on a native thread so a contended picker lock never blocks the JS thread. ++ * Patched in T3 Code. ++ */ ++ scanFilesAsync(): Promise>; + /** + * Check if a scan is currently in progress. + */ +@@ -254,6 +259,11 @@ export declare class FileFinder implements FileFinderApi { + * Get the current scan progress. + */ + getScanProgress(): Result; ++ /** ++ * Read scan progress on a native thread. Concurrent callers share one call. ++ * Patched in T3 Code. ++ */ ++ getScanProgressAsync(): Promise>; + /** + * Wait for the initial file scan to complete. + * Non-blocking: polls `isScanning` and yields to the event loop between checks. diff --git a/dist/index.js b/dist/index.js +index 7c1d722eabc8d8eb98998eb48c5fe9d70b749d10..9643e1614ff128bd0cef71368b531314dfdee678 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1,7 +1,7 @@ @@ -263,3 +281,175 @@ diff --git a/dist/index.js b/dist/index.js } load({ library: LIBRARY_KEY, +@@ -1167,6 +1183,29 @@ function ffiMultiGrep(handle, patternsJoined, constraints, maxFileSize, maxMatch + function ffiScanFiles(handle) { + return callVoidResult("fff_scan_files", [DataType.External], [handle]); + } ++async function ffiScanFilesAsync(handle) { ++ loadLibrary(); ++ const rawPtr = await load({ ++ library: LIBRARY_KEY, ++ funcName: "fff_scan_files", ++ retType: DataType.External, ++ paramsType: [DataType.External], ++ paramsValue: [handle], ++ freeResultMemory: false, ++ runInNewThread: true ++ }); ++ const [structData] = restorePointer({ ++ retType: [FFF_RESULT_STRUCT], ++ paramsValue: wrapPointer([rawPtr]) ++ }); ++ if (structData.success === 0) { ++ const errorStr = readCString(structData.error); ++ freeResult(rawPtr); ++ return err(errorStr || "Unknown error"); ++ } ++ freeResult(rawPtr); ++ return { ok: true, value: void 0 }; ++} + function ffiIsScanning(handle) { + loadLibrary(); + return load({ +@@ -1212,6 +1251,48 @@ function ffiGetScanProgress(handle) { + }); + return { ok: true, value: result }; + } ++async function ffiGetScanProgressAsync(handle) { ++ loadLibrary(); ++ const rawPtr = await load({ ++ library: LIBRARY_KEY, ++ funcName: "fff_get_scan_progress", ++ retType: DataType.External, ++ paramsType: [DataType.External], ++ paramsValue: [handle], ++ freeResultMemory: false, ++ runInNewThread: true ++ }); ++ const [structData] = restorePointer({ ++ retType: [FFF_RESULT_STRUCT], ++ paramsValue: wrapPointer([rawPtr]) ++ }); ++ if (structData.success === 0) { ++ const errorStr = readCString(structData.error); ++ freeResult(rawPtr); ++ return err(errorStr || "Unknown error"); ++ } ++ const handlePtr = structData.handle; ++ freeResult(rawPtr); ++ if (isNullPointer(handlePtr)) return err("scan progress returned null"); ++ const [sp] = restorePointer({ ++ retType: [FFF_SCAN_PROGRESS_STRUCT], ++ paramsValue: wrapPointer([handlePtr]) ++ }); ++ const result = { ++ scannedFilesCount: Number(sp.scanned_files_count), ++ isScanning: sp.is_scanning !== 0, ++ isWatcherReady: sp.is_watcher_ready !== 0, ++ isWarmupComplete: sp.is_warmup_complete !== 0 ++ }; ++ load({ ++ library: LIBRARY_KEY, ++ funcName: "fff_free_scan_progress", ++ retType: DataType.Void, ++ paramsType: [DataType.External], ++ paramsValue: [handlePtr] ++ }); ++ return { ok: true, value: result }; ++} + function ffiWaitForScan(handle, timeoutMs) { + return callBoolResult( + "fff_wait_for_scan", +@@ -1429,6 +1510,9 @@ var FileFinder = class _FileFinder { + handle; + /** Native ids of this instance's active watch subscriptions. */ + watchers = /* @__PURE__ */ new Set(); ++ pendingScan = null; ++ pendingScanProgress = null; ++ pendingNativeCalls = /* @__PURE__ */ new Set(); + constructor(handle) { + this.handle = handle; + } +@@ -1485,6 +1569,16 @@ var FileFinder = class _FileFinder { + destroy() { + if (this.handle !== null) { + const handle = this.handle; ++ if (this.pendingNativeCalls.size > 0) { ++ this.handle = null; ++ const watchers = this.watchers; ++ this.watchers = /* @__PURE__ */ new Set(); ++ Promise.allSettled(this.pendingNativeCalls).then(() => { ++ ffiDestroy(handle); ++ ffiWatchCleanupAfterDestroy(handle, watchers); ++ }); ++ return; ++ } + ffiDestroy(handle); + this.handle = null; + ffiWatchCleanupAfterDestroy(handle, this.watchers); +@@ -1757,6 +1851,42 @@ var FileFinder = class _FileFinder { + if (!guard.ok) return guard; + return ffiScanFiles(guard.value); + } ++ /** ++ * Trigger a rescan without blocking the JS thread. Patched in T3 Code: ++ * `fff_scan_files` can wait on the picker lock, so it runs on a native thread. ++ */ ++ scanFilesAsync() { ++ const guard = this.ensureAlive(); ++ if (!guard.ok) return Promise.resolve(guard); ++ if (this.pendingScan) return this.pendingScan; ++ const scan = this.runOnNativeThread(ffiScanFilesAsync(guard.value)).finally(() => { ++ if (this.pendingScan === scan) this.pendingScan = null; ++ }); ++ this.pendingScan = scan; ++ return scan; ++ } ++ /** ++ * Read scan progress without blocking the JS thread. Patched in T3 Code: concurrent readers ++ * share one call, because each call holds a libuv pool thread while it waits on the picker lock. ++ */ ++ getScanProgressAsync() { ++ const guard = this.ensureAlive(); ++ if (!guard.ok) return Promise.resolve(guard); ++ if (this.pendingScanProgress) return this.pendingScanProgress; ++ const read = this.runOnNativeThread(ffiGetScanProgressAsync(guard.value)).finally(() => { ++ if (this.pendingScanProgress === read) this.pendingScanProgress = null; ++ }); ++ this.pendingScanProgress = read; ++ return read; ++ } ++ /** Keeps the handle alive until a native-thread call settles. Patched in T3 Code. */ ++ runOnNativeThread(call) { ++ const tracked = call.catch((cause) => err(String(cause))).finally(() => { ++ this.pendingNativeCalls.delete(tracked); ++ }); ++ this.pendingNativeCalls.add(tracked); ++ return tracked; ++ } + /** + * Check if a scan is currently in progress. + */ +@@ -1838,7 +1968,8 @@ var FileFinder = class _FileFinder { + if (!guard.ok) return guard; + const deadline = Date.now() + timeoutMs; + while (true) { +- const progress = this.getScanProgress(); ++ // Patched in T3 Code: the progress read takes the picker lock, so it runs on a native thread. ++ const progress = await this.getScanProgressAsync(); + if (!progress.ok) return progress; + if (!progress.value.isScanning && progress.value.isWarmupComplete) { + return { ok: true, value: true }; +diff --git a/package.json b/package.json +index e000a78b3f79826200979ebb3fe297b599bcea67..84325461c2d963cb42cf72267079d36a419c1398 100644 +--- a/package.json ++++ b/package.json +@@ -11,7 +11,7 @@ + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", +- "require": "./dist/index.cjs", ++ "require": "./dist/index.js", + "default": "./dist/index.js" + }, + "./package.json": "./package.json" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6451f3f97cb29..3b75d43d34955 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -358,7 +358,7 @@ patchedDependencies: '@alchemy.run/test@2.0.0-beta.85': 29c0539ea71d9cc372f0b16558640599beb9bc609605feb2a493da268d72713e '@clerk/expo-native-components@0.4.4': 7a808b92da8460750b85818552e13be3fb73d111691c7db76965970351e124e4 '@effect/vitest@4.1.2': 299510b71c36537a832a67e9d160a5f112a5f326a29e95c41cb45fc8234e47c7 - '@ff-labs/fff-node@0.11.0': cc71fcec015f4d39bc96ae2e2bf6b0ca678124c722c66a7841efce9d34b5983c + '@ff-labs/fff-node@0.11.0': 34dd25f8217a0a0e9c7287bde40c6bb46e42b96375817815e77f1df0741b1692 '@legendapp/list@3.7.2': 3c68607b8bb7a531873bcbe0b197d163ef1fdb4d4cf52a4ddf9d2b3804671658 '@muse-code/sdk@1.4.6': 7b495b607fc5d76b444c7c4d58821e839726a30e6f3c53564f0d59f2d5b42101 '@pierre/diffs@1.7.0-beta.3': aea143f4a2e59744371219c4e7ae675ec739c9ba16f1732e2f11748b496ea05e @@ -832,7 +832,7 @@ importers: version: 4.1.2(bufferutil@4.1.0)(effect@4.1.2)(utf-8-validate@6.0.6) '@ff-labs/fff-node': specifier: 0.11.0 - version: 0.11.0(patch_hash=cc71fcec015f4d39bc96ae2e2bf6b0ca678124c722c66a7841efce9d34b5983c) + version: 0.11.0(patch_hash=34dd25f8217a0a0e9c7287bde40c6bb46e42b96375817815e77f1df0741b1692) '@muse-code/sdk': specifier: 1.4.6 version: 1.4.6(patch_hash=7b495b607fc5d76b444c7c4d58821e839726a30e6f3c53564f0d59f2d5b42101)(@types/node@24.12.4) @@ -14717,7 +14717,7 @@ snapshots: '@ff-labs/fff-bin-win32-x64@0.11.0': optional: true - '@ff-labs/fff-node@0.11.0(patch_hash=cc71fcec015f4d39bc96ae2e2bf6b0ca678124c722c66a7841efce9d34b5983c)': + '@ff-labs/fff-node@0.11.0(patch_hash=34dd25f8217a0a0e9c7287bde40c6bb46e42b96375817815e77f1df0741b1692)': dependencies: ffi-rs: 1.3.2 optionalDependencies: