From 555711a17859c95ec78e843c5e4ad607d4ba2853 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 12 Nov 2026 17:42:02 +0000 Subject: [PATCH] fix(server): file index refreshes no longer block the server on fff's lock Run finalization refreshes the workspace search index after every run. The refresh called fff's scanFiles() and polled getScanProgress() synchronously on the JS thread, and both wait on fff's picker lock. While fff's watcher held that lock, for example while indexing a newly created folder, the whole server stopped answering. The fff-node patch adds scanFilesAsync() and getScanProgressAsync(), which run those calls on a libuv pool thread through ffi-rs. Concurrent callers share one call per finder, waitForIndexReady polls through the async read, and destroy() frees the native handle only after pending calls settle. The refresh now uses scanFilesAsync(). Fixes pingdotgg/t3code#22586 AI-assisted; verified with the tests and tools listed above. --- .../src/workspace/WorkspaceEntries.test.ts | 8 +- .../src/workspace/WorkspaceSearchIndex.ts | 7 +- patches/@ff-labs__fff-node@0.11.0.patch | 207 ++++++++++++++++-- pnpm-lock.yaml | 6 +- 4 files changed, 196 insertions(+), 32 deletions(-) diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index 87cab4a2c88c3..610568078e31a 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -813,10 +813,14 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { if (created.ok) { const finder = created.value; finders += 1; - const scanFiles = finder.scanFiles.bind(finder); + // The synchronous rescan waits on fff's picker lock on the event loop. finder.scanFiles = () => { + throw new Error("refresh must not rescan on the JS thread"); + }; + const scanFilesAsync = finder.scanFilesAsync.bind(finder); + finder.scanFilesAsync = () => { rescans += 1; - return scanFiles(); + return scanFilesAsync(); }; const waitForIndexReady = finder.waitForIndexReady.bind(finder); finder.waitForIndexReady = (timeoutMs) => diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index 23d17759c9fc1..f276bfd9654ee 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -399,12 +399,13 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn( "WorkspaceSearchIndex.refresh", )(function* () { - const result = yield* Effect.try({ - try: () => finder.scanFiles(), + // scanFiles waits on fff's picker lock, which its watcher can hold for seconds. + const result = yield* Effect.tryPromise({ + try: () => finder.scanFilesAsync(), catch: (cause) => new WorkspaceSearchIndexRefreshFailed({ cwd, - reason: "FileFinder.scanFiles threw unexpectedly.", + reason: "FileFinder.scanFilesAsync rejected unexpectedly.", cause, }), }); diff --git a/patches/@ff-labs__fff-node@0.11.0.patch b/patches/@ff-labs__fff-node@0.11.0.patch index 76e45e152a51e..8468783d2a6a5 100644 --- a/patches/@ff-labs__fff-node@0.11.0.patch +++ b/patches/@ff-labs__fff-node@0.11.0.patch @@ -1,16 +1,5 @@ -diff --git a/package.json b/package.json ---- a/package.json -+++ b/package.json -@@ -11,7 +11,7 @@ - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", -- "require": "./dist/index.cjs", -+ "require": "./dist/index.js", - "default": "./dist/index.js" - }, - "./package.json": "./package.json" diff --git a/dist/fff-api.d.ts b/dist/fff-api.d.ts +index 0f90eedca20655ccd30b8f28e433b2094341fbdf..c2ef86927d6e3ab0e8584330fa0ac3fdacc06cf5 100644 --- a/dist/fff-api.d.ts +++ b/dist/fff-api.d.ts @@ -199,7 +199,7 @@ export type Location = { @@ -40,7 +29,34 @@ diff --git a/dist/fff-api.d.ts b/dist/fff-api.d.ts scores: Score[]; /** Total number of items (files + dirs) that matched */ totalMatched: number; +diff --git a/dist/finder.d.ts b/dist/finder.d.ts +index 452102d720cc19b0560cfc9e2f1a320687047c92..3d3211c70a7de4b97792edb32cc9466c8ba89547 100644 +--- a/dist/finder.d.ts ++++ b/dist/finder.d.ts +@@ -242,6 +242,10 @@ export declare class FileFinder implements FileFinderApi { + * background watcher might have missed. + */ + scanFiles(): Result; ++ /** ++ * Trigger a rescan without blocking the JS thread. Patched in T3 Code. ++ */ ++ scanFilesAsync(): Promise>; + /** + * Check if a scan is currently in progress. + */ +@@ -254,6 +258,10 @@ export declare class FileFinder implements FileFinderApi { + * Get the current scan progress. + */ + getScanProgress(): Result; ++ /** ++ * Read scan progress without blocking the JS thread. Patched in T3 Code. ++ */ ++ getScanProgressAsync(): Promise>; + /** + * Wait for the initial file scan to complete. + * Non-blocking: polls `isScanning` and yields to the event loop between checks. diff --git a/dist/index.js b/dist/index.js +index 7c1d722eabc8d8eb98998eb48c5fe9d70b749d10..138e590bdcb1bc4fd33f7d195b8d588ac7f86cc9 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1,7 +1,7 @@ @@ -86,7 +102,45 @@ diff --git a/dist/index.js b/dist/index.js if (isNullPointer(ptr)) return null; try { const [str] = restorePointer({ -@@ -426,9 +437,9 @@ function ffiDestroy(handle) { +@@ -303,6 +314,26 @@ function freeResult(resultPtr) { + function readResultEnvelope(funcName, paramsType, paramsValue) { + loadLibrary(); + const { rawPtr, struct: structData } = callRaw(funcName, paramsType, paramsValue); ++ return envelopeFromRaw(rawPtr, structData); ++} ++async function readResultEnvelopeOnNativeThread(funcName, handle) { ++ loadLibrary(); ++ const rawPtr = await load({ ++ library: LIBRARY_KEY, ++ funcName, ++ retType: DataType.External, ++ paramsType: [DataType.External], ++ paramsValue: [handle], ++ freeResultMemory: false, ++ runInNewThread: true ++ }); ++ const [structData] = restorePointer({ ++ retType: [FFF_RESULT_STRUCT], ++ paramsValue: wrapPointer([rawPtr]) ++ }); ++ return envelopeFromRaw(rawPtr, structData); ++} ++function envelopeFromRaw(rawPtr, structData) { + if (structData.success === 0) { + const errorStr = readCString(structData.error); + freeResult(rawPtr); +@@ -311,7 +342,9 @@ function readResultEnvelope(funcName, paramsType, paramsValue) { + return { rawPtr, struct: structData }; + } + function callVoidResult(funcName, paramsType, paramsValue) { +- const res = readResultEnvelope(funcName, paramsType, paramsValue); ++ return voidResultFromEnvelope(readResultEnvelope(funcName, paramsType, paramsValue)); ++} ++function voidResultFromEnvelope(res) { + if ("ok" in res) return res; + freeResult(res.rawPtr); + return { ok: true, value: void 0 }; +@@ -426,9 +459,9 @@ function ffiDestroy(handle) { }); } var FFF_FILE_ITEM_STRUCT = { @@ -99,7 +153,7 @@ diff --git a/dist/index.js b/dist/index.js size: DataType.U64, modified: DataType.U64, access_frecency_score: DataType.I64, -@@ -445,8 +456,9 @@ var FFF_SCORE_STRUCT = { +@@ -445,8 +478,9 @@ var FFF_SCORE_STRUCT = { distance_penalty: DataType.I32, current_file_penalty: DataType.I32, combo_match_boost: DataType.I32, @@ -110,7 +164,7 @@ diff --git a/dist/index.js b/dist/index.js }; var FFF_SEARCH_RESULT_STRUCT = { items: DataType.External, -@@ -462,8 +474,8 @@ var FFF_SEARCH_RESULT_STRUCT = { +@@ -462,8 +496,8 @@ var FFF_SEARCH_RESULT_STRUCT = { location_end_col: DataType.I32 }; var FFF_DIR_ITEM_STRUCT = { @@ -121,7 +175,7 @@ diff --git a/dist/index.js b/dist/index.js max_access_frecency: DataType.I32 }; var FFF_DIR_SEARCH_RESULT_STRUCT = { -@@ -475,9 +487,9 @@ var FFF_DIR_SEARCH_RESULT_STRUCT = { +@@ -475,9 +509,9 @@ var FFF_DIR_SEARCH_RESULT_STRUCT = { }; var FFF_MIXED_ITEM_STRUCT = { item_type: DataType.U8, @@ -134,7 +188,7 @@ diff --git a/dist/index.js b/dist/index.js size: DataType.U64, modified: DataType.U64, access_frecency_score: DataType.I64, -@@ -500,10 +512,10 @@ var FFF_MIXED_SEARCH_RESULT_STRUCT = { +@@ -500,10 +534,10 @@ var FFF_MIXED_SEARCH_RESULT_STRUCT = { location_end_col: DataType.I32 }; var FFF_GREP_MATCH_STRUCT = { @@ -149,7 +203,7 @@ diff --git a/dist/index.js b/dist/index.js match_ranges: DataType.External, context_before: DataType.External, context_after: DataType.External, -@@ -598,18 +610,13 @@ function readMixedItemFromRaw(raw) { +@@ -598,18 +632,13 @@ function readMixedItemFromRaw(raw) { } function callAccessor(funcName, resultPtr, index, structDef) { loadLibrary(); @@ -170,7 +224,7 @@ diff --git a/dist/index.js b/dist/index.js } function ptrOffset(base, bytes) { return load({ -@@ -766,6 +773,7 @@ function parseSearchResult(rawPtr) { +@@ -766,6 +795,7 @@ function parseSearchResult(rawPtr) { } const items = []; const scores = []; @@ -178,7 +232,7 @@ diff --git a/dist/index.js b/dist/index.js for (let i = 0; i < count; i++) { const rawItem = callAccessor( "fff_search_result_get_item", -@@ -774,13 +782,15 @@ function parseSearchResult(rawPtr) { +@@ -774,13 +804,15 @@ function parseSearchResult(rawPtr) { FFF_FILE_ITEM_STRUCT ); items.push(readFileItemFromRaw(rawItem)); @@ -201,7 +255,7 @@ diff --git a/dist/index.js b/dist/index.js } load({ library: LIBRARY_KEY, -@@ -824,6 +834,7 @@ function parseDirSearchResult(rawPtr) { +@@ -824,6 +856,7 @@ function parseDirSearchResult(rawPtr) { const count = sr.count; const items = []; const scores = []; @@ -209,7 +263,7 @@ diff --git a/dist/index.js b/dist/index.js for (let i = 0; i < count; i++) { const rawItem = callAccessor( "fff_dir_search_result_get_item", -@@ -832,13 +843,15 @@ function parseDirSearchResult(rawPtr) { +@@ -832,13 +865,15 @@ function parseDirSearchResult(rawPtr) { FFF_DIR_ITEM_STRUCT ); items.push(readDirItemFromRaw(rawItem)); @@ -232,7 +286,7 @@ diff --git a/dist/index.js b/dist/index.js } load({ library: LIBRARY_KEY, -@@ -897,6 +910,7 @@ function parseMixedSearchResult(rawPtr) { +@@ -897,6 +932,7 @@ function parseMixedSearchResult(rawPtr) { } const items = []; const scores = []; @@ -240,7 +294,7 @@ diff --git a/dist/index.js b/dist/index.js for (let i = 0; i < count; i++) { const rawItem = callAccessor( "fff_mixed_search_result_get_item", -@@ -905,13 +919,15 @@ function parseMixedSearchResult(rawPtr) { +@@ -905,13 +941,15 @@ function parseMixedSearchResult(rawPtr) { FFF_MIXED_ITEM_STRUCT ); items.push(readMixedItemFromRaw(rawItem)); @@ -263,3 +317,108 @@ diff --git a/dist/index.js b/dist/index.js } load({ library: LIBRARY_KEY, +@@ -1188,7 +1226,11 @@ var FFF_SCAN_PROGRESS_STRUCT = { + }; + function ffiGetScanProgress(handle) { + loadLibrary(); +- const res = readResultEnvelope("fff_get_scan_progress", [DataType.External], [handle]); ++ return scanProgressFromEnvelope( ++ readResultEnvelope("fff_get_scan_progress", [DataType.External], [handle]) ++ ); ++} ++function scanProgressFromEnvelope(res) { + if ("ok" in res) return res; + const handlePtr = res.struct.handle; + freeResult(res.rawPtr); +@@ -1429,6 +1471,8 @@ var FileFinder = class _FileFinder { + handle; + /** Native ids of this instance's active watch subscriptions. */ + watchers = /* @__PURE__ */ new Set(); ++ /** Patched in T3 Code: in-flight native-thread calls by function name. */ ++ nativeThreadCalls = /* @__PURE__ */ new Map(); + constructor(handle) { + this.handle = handle; + } +@@ -1485,12 +1529,37 @@ var FileFinder = class _FileFinder { + destroy() { + if (this.handle !== null) { + const handle = this.handle; +- ffiDestroy(handle); ++ const watchers = this.watchers; + this.handle = null; +- ffiWatchCleanupAfterDestroy(handle, this.watchers); +- this.watchers.clear(); ++ this.watchers = /* @__PURE__ */ new Set(); ++ const release = () => { ++ ffiDestroy(handle); ++ ffiWatchCleanupAfterDestroy(handle, watchers); ++ }; ++ // Patched in T3 Code: a native-thread call may still be using the handle. ++ if (this.nativeThreadCalls.size > 0) { ++ Promise.allSettled(this.nativeThreadCalls.values()).then(release); ++ } else { ++ release(); ++ } + } + } ++ /** ++ * Patched in T3 Code: runs a handle-only call on a libuv pool thread so waiting on fff's ++ * picker lock never blocks the JS thread. Concurrent callers share one call, so one finder ++ * holds at most one pool thread per function. ++ */ ++ callOnNativeThread(funcName, fromEnvelope) { ++ const guard = this.ensureAlive(); ++ if (!guard.ok) return Promise.resolve(guard); ++ const pending = this.nativeThreadCalls.get(funcName); ++ if (pending) return pending; ++ const call = readResultEnvelopeOnNativeThread(funcName, guard.value).then(fromEnvelope, (cause) => err(String(cause))).finally(() => { ++ this.nativeThreadCalls.delete(funcName); ++ }); ++ this.nativeThreadCalls.set(funcName, call); ++ return call; ++ } + /** + * Check if this instance has been destroyed. + */ +@@ -1757,6 +1826,18 @@ var FileFinder = class _FileFinder { + if (!guard.ok) return guard; + return ffiScanFiles(guard.value); + } ++ /** ++ * Trigger a rescan without blocking the JS thread. Patched in T3 Code. ++ */ ++ scanFilesAsync() { ++ return this.callOnNativeThread("fff_scan_files", voidResultFromEnvelope); ++ } ++ /** ++ * Read scan progress without blocking the JS thread. Patched in T3 Code. ++ */ ++ getScanProgressAsync() { ++ return this.callOnNativeThread("fff_get_scan_progress", scanProgressFromEnvelope); ++ } + /** + * Check if a scan is currently in progress. + */ +@@ -1838,7 +1919,8 @@ var FileFinder = class _FileFinder { + if (!guard.ok) return guard; + const deadline = Date.now() + timeoutMs; + while (true) { +- const progress = this.getScanProgress(); ++ // Patched in T3 Code: the progress read takes the picker lock. ++ const progress = await this.getScanProgressAsync(); + if (!progress.ok) return progress; + if (!progress.value.isScanning && progress.value.isWarmupComplete) { + return { ok: true, value: true }; +diff --git a/package.json b/package.json +index e000a78b3f79826200979ebb3fe297b599bcea67..84325461c2d963cb42cf72267079d36a419c1398 100644 +--- a/package.json ++++ b/package.json +@@ -11,7 +11,7 @@ + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", +- "require": "./dist/index.cjs", ++ "require": "./dist/index.js", + "default": "./dist/index.js" + }, + "./package.json": "./package.json" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0a9aa3d0f4c16..71a14ccacb0c5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -363,7 +363,7 @@ patchedDependencies: '@clerk/expo-native-components@0.4.4': 7a808b92da8460750b85818552e13be3fb73d111691c7db76965970351e124e4 '@cursor/sdk@1.2.0': 57f502128bf6accd976963b126e227c86a45d815a53a8dc450461beae3bed4a4 '@effect/vitest@4.1.2': 299510b71c36537a832a67e9d160a5f112a5f326a29e95c41cb45fc8234e47c7 - '@ff-labs/fff-node@0.11.0': cc71fcec015f4d39bc96ae2e2bf6b0ca678124c722c66a7841efce9d34b5983c + '@ff-labs/fff-node@0.11.0': a71dc3a5544fc47afacf0fe9da04dccec3018e625a76c96087ebfdf35e293768 '@legendapp/list@3.7.2': 3c68607b8bb7a531873bcbe0b197d163ef1fdb4d4cf52a4ddf9d2b3804671658 '@muse-code/sdk@1.4.6': 7b495b607fc5d76b444c7c4d58821e839726a30e6f3c53564f0d59f2d5b42101 '@pierre/diffs@1.7.0-beta.3': 4a64f63a0def94c5add9d86d70560d26d6857e1f5150133862e42e1fd77bc4e8 @@ -856,7 +856,7 @@ importers: version: 4.1.2(bufferutil@4.1.0)(effect@4.1.2)(utf-8-validate@6.0.6) '@ff-labs/fff-node': specifier: 0.11.0 - version: 0.11.0(patch_hash=cc71fcec015f4d39bc96ae2e2bf6b0ca678124c722c66a7841efce9d34b5983c) + version: 0.11.0(patch_hash=a71dc3a5544fc47afacf0fe9da04dccec3018e625a76c96087ebfdf35e293768) '@muse-code/sdk': specifier: 1.4.6 version: 1.4.6(patch_hash=7b495b607fc5d76b444c7c4d58821e839726a30e6f3c53564f0d59f2d5b42101)(@types/node@24.12.4) @@ -15144,7 +15144,7 @@ snapshots: '@ff-labs/fff-bin-win32-x64@0.11.0': optional: true - '@ff-labs/fff-node@0.11.0(patch_hash=cc71fcec015f4d39bc96ae2e2bf6b0ca678124c722c66a7841efce9d34b5983c)': + '@ff-labs/fff-node@0.11.0(patch_hash=a71dc3a5544fc47afacf0fe9da04dccec3018e625a76c96087ebfdf35e293768)': dependencies: ffi-rs: 1.3.2 optionalDependencies: