From a0258142981bbf3188a57dee3c2e4da55179076d Mon Sep 17 00:00:00 2001 From: Lakshmi Tanmay Date: Mon, 2 Nov 2026 15:33:48 +0530 Subject: [PATCH 1/6] perf(server): keep workspace content searches responsive --- .../workspace/WorkspaceSearchIndex.test.ts | 77 +++++++++++++++++++ .../src/workspace/WorkspaceSearchIndex.ts | 18 +++-- 2 files changed, 87 insertions(+), 8 deletions(-) diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index 2f9ac05bf9e3c..fae7a3d03a314 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -144,6 +144,83 @@ it.effect("continues whole-word searches after a filtered grep page", () => ), ); +it.effect.each([false, true])( + "resumes empty budget pages without losing results (hasMatch: %s)", + (hasMatch) => + Effect.scoped( + Effect.gen(function* () { + const clock = vi.spyOn(performance, "now").mockReturnValue(0); + const cursors = [1, 2].map( + (offset) => + ({ + __brand: "GrepCursor", + _offset: offset, + }) as GrepCursor, + ); + let eventLoopServiced = false; + let eventLoopTask: NodeJS.Immediate | undefined; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + if (eventLoopTask) clearImmediate(eventLoopTask); + }), + ); + const grep = vi.fn((_query: string, options?: GrepOptions) => { + const page = options?.cursor ? cursors.indexOf(options.cursor) + 1 : 0; + if (page === 0) { + clock.mockReturnValue(1_000); + eventLoopTask = setImmediate(() => { + eventLoopServiced = true; + }); + } else { + expect(eventLoopServiced).toBe(true); + } + expect(options?.enforceTimeBudget).toBe(true); + const value: GrepResult = { + items: + page === 2 && hasMatch + ? [ + { + ...fileItem("src/late.ts"), + isBinary: false, + lineNumber: 1, + col: 0, + byteOffset: 0, + lineContent: "needle", + matchRanges: [[0, 6]], + }, + ] + : [], + totalMatched: page === 2 && hasMatch ? 1 : 0, + totalFilesSearched: 1, + totalFiles: 3, + filteredFileCount: 3, + nextCursor: cursors[page] ?? null, + }; + return { ok: true as const, value }; + }); + const finder = { + destroy: vi.fn(), + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), + grep, + } as unknown as FileFinder; + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); + + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project", "content"); + const result = yield* searchIndex.searchContents({ + query: "needle", + limit: 10, + caseSensitive: true, + wholeWord: false, + useRegex: false, + }); + + expect(result.matches.map((match) => match.path)).toEqual(hasMatch ? ["src/late.ts"] : []); + expect(result.truncated).toBe(false); + expect(grep).toHaveBeenCalledTimes(3); + }), + ), +); + // fff reads HOME on Linux and macOS but the user profile on Windows, so a temp HOME cannot stand in there. it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( "refuses to index a workspace rooted at the home directory", diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index f276bfd9654ee..b408bb45bff89 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -1,4 +1,6 @@ import * as NodeModule from "node:module"; +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Yield to Node I/O, not just the Effect scheduler. +import * as NodeTimersPromises from "node:timers/promises"; import type { DirItem, @@ -36,7 +38,7 @@ const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2; const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds"; const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000; const WORKSPACE_INDEX_IDLE_TTL = "15 minutes"; -const CONTENT_SEARCH_TIME_BUDGET_MS = 250; +const CONTENT_SEARCH_TIME_BUDGET_MS = 25; const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100; export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedError()( @@ -492,7 +494,6 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( "WorkspaceSearchIndex.searchContents", )(function* (input) { let { searchQuery, regexMode } = buildContentSearchQuery(input); - const deadline = performance.now() + CONTENT_SEARCH_TIME_BUDGET_MS; // Grep cursors advance per file; whole-word filtering needs enough raw candidates before advancing. const rawPageSize = input.wholeWord ? Math.max(input.limit, CONTENT_SEARCH_MAX_MATCHES_PER_FILE) @@ -503,7 +504,6 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( let retryLiterally = false; do { - const remainingTimeBudgetMs = Math.max(1, Math.ceil(deadline - performance.now())); const result = yield* runSearch(input.query, input.limit, "grep", () => finder.grep(searchQuery, { mode: regexMode ? "regex" : "plain", @@ -511,7 +511,8 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( maxMatchesPerFile: Math.min(CONTENT_SEARCH_MAX_MATCHES_PER_FILE, rawPageSize), pageSize: rawPageSize, cursor: nextCursor, - timeBudgetMs: remainingTimeBudgetMs, + timeBudgetMs: CONTENT_SEARCH_TIME_BUDGET_MS, + enforceTimeBudget: true, }), ); // fff searches an invalid pattern as a literal, which would keep the `(?i)` prefix. @@ -537,10 +538,11 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( } nextCursor = result.nextCursor; regexFallbackError ??= result.regexFallbackError; - } while ( - retryLiterally || - (matches.length < input.limit && nextCursor !== null && performance.now() < deadline) - ); + // The RPC has no resume cursor: finish paging internally, letting Node service I/O between pages. + if (matches.length < input.limit && nextCursor !== null) { + yield* Effect.promise((signal) => NodeTimersPromises.setImmediate(undefined, { signal })); + } + } while (retryLiterally || (matches.length < input.limit && nextCursor !== null)); return { matches: matches.slice(0, input.limit), From a3189132eef76b452cda2dd41c3a54ec6e86a1c4 Mon Sep 17 00:00:00 2001 From: Lakshmi Tanmay Date: Mon, 2 Nov 2026 16:32:37 +0530 Subject: [PATCH 2/6] perf(server): yield between search pages without diagnostic suppression --- apps/server/src/workspace/WorkspaceSearchIndex.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index b408bb45bff89..d5d620f2e37f9 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -1,6 +1,4 @@ import * as NodeModule from "node:module"; -// @effect-diagnostics-next-line nodeBuiltinImport:off -- Yield to Node I/O, not just the Effect scheduler. -import * as NodeTimersPromises from "node:timers/promises"; import type { DirItem, @@ -540,7 +538,10 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( regexFallbackError ??= result.regexFallbackError; // The RPC has no resume cursor: finish paging internally, letting Node service I/O between pages. if (matches.length < input.limit && nextCursor !== null) { - yield* Effect.promise((signal) => NodeTimersPromises.setImmediate(undefined, { signal })); + yield* Effect.callback((resume) => { + const immediate = setImmediate(() => resume(Effect.void)); + return Effect.sync(() => clearImmediate(immediate)); + }); } } while (retryLiterally || (matches.length < input.limit && nextCursor !== null)); From 76317b36c3a5628f7dea331376c815077b6e3d2f Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Fri, 6 Nov 2026 08:38:23 +0000 Subject: [PATCH 3/6] fix(server): rerun content search pages that lose native fallback results --- .../workspace/WorkspaceSearchIndex.test.ts | 48 +++++++++++++++++++ .../src/workspace/WorkspaceSearchIndex.ts | 13 +++-- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index fae7a3d03a314..c630ca55d3151 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -221,6 +221,54 @@ it.effect.each([false, true])( ), ); +it.effect("reruns an enforced page whose path constraint filtered every file", () => + Effect.scoped( + Effect.gen(function* () { + const grep = vi.fn((_query: string, options?: GrepOptions) => { + const fellBack = options?.enforceTimeBudget === false; + const value: GrepResult = { + items: fellBack + ? [ + { + ...fileItem("arch/x86/kernel/head64.c"), + isBinary: false, + lineNumber: 1, + col: 0, + byteOffset: 0, + lineContent: "asmlinkage __visible void start_kernel(void)", + matchRanges: [[0, 44]], + }, + ] + : [], + totalMatched: fellBack ? 1 : 0, + totalFilesSearched: fellBack ? 3 : 0, + totalFiles: 3, + filteredFileCount: fellBack ? 3 : 0, + nextCursor: null, + }; + return { ok: true as const, value }; + }); + const finder = { + destroy: vi.fn(), + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), + grep, + } as unknown as FileFinder; + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); + + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project", "content"); + const result = yield* searchIndex.searchContents({ + query: "[a-z]{90}[-+*/]{5}|asmlinkage __visible.*start_kernel", + limit: 10, + caseSensitive: true, + wholeWord: false, + useRegex: true, + }); + + expect(result.matches.map((match) => match.path)).toEqual(["arch/x86/kernel/head64.c"]); + }), + ), +); + // fff reads HOME on Linux and macOS but the user profile on Windows, so a temp HOME cannot stand in there. it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( "refuses to index a workspace rooted at the home directory", diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index d5d620f2e37f9..427f394a8761f 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -501,8 +501,8 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( let regexFallbackError: string | undefined; let retryLiterally = false; - do { - const result = yield* runSearch(input.query, input.limit, "grep", () => + const grepPage = (enforceTimeBudget: boolean) => + runSearch(input.query, input.limit, "grep", () => finder.grep(searchQuery, { mode: regexMode ? "regex" : "plain", smartCase: !input.caseSensitive && !regexMode, @@ -510,9 +510,16 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( pageSize: rawPageSize, cursor: nextCursor, timeBudgetMs: CONTENT_SEARCH_TIME_BUDGET_MS, - enforceTimeBudget: true, + enforceTimeBudget, }), ); + + do { + let result = yield* grepPage(true); + // fff 0.11 drops an empty enforced raw-query fallback page's cursor when a parsed path constraint matches no files. + if (result.filteredFileCount === 0 && result.totalFiles > 0) { + result = yield* grepPage(false); + } // fff searches an invalid pattern as a literal, which would keep the `(?i)` prefix. retryLiterally = regexMode && !input.caseSensitive && result.regexFallbackError !== undefined; if (retryLiterally) { From b2c3f2381fd673249066c612252ce5466baee9a4 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 14 Nov 2026 12:07:04 +0000 Subject: [PATCH 4/6] fix(server): rerun content search fallback pages whose constraint matched files fff drops the cursor of an enforced raw-query fallback page that times out, whether or not the parsed path constraint matched any files. Rerun any empty enforced page that reports the scan ended without enforcement, instead of only those whose constraint matched nothing. Co-Authored-By: Claude Opus 5.5 --- .../workspace/WorkspaceSearchIndex.test.ts | 62 +++++++++---------- .../src/workspace/WorkspaceSearchIndex.ts | 5 +- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index f7b04a1e8965e..bbfa535250458 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -549,38 +549,38 @@ it.effect("pages past empty time-budget pages to a late match", () => ).pipe(Effect.provide(testLayer)), ); -it.effect("reruns an enforced page whose path constraint filtered every file", () => - Effect.scoped( - Effect.gen(function* () { - const grep = vi.fn(async (_query: string, options?: GrepOptions) => { - const fellBack = options?.enforceTimeBudget === false; - const items = fellBack - ? [grepMatch("arch/x86/kernel/head64.c", "asmlinkage __visible void start_kernel(void)")] - : []; - const value: GrepResult = { - items, - totalMatched: items.length, - totalFilesSearched: fellBack ? 3 : 0, - totalFiles: 3, - filteredFileCount: fellBack ? 3 : 0, - nextCursor: null, - }; - return { ok: true as const, value }; - }); - contentFinder(grep); - - const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project", "content"); - const result = yield* searchIndex.searchContents({ - query: "[a-z]{90}[-+*/]{5}|asmlinkage __visible.*start_kernel", - limit: 10, - caseSensitive: true, - wholeWord: false, - useRegex: true, - }); +it.effect.each([0, 1])( + "reruns an empty enforced fallback page whose constraint matched %i files", + (constrainedFileCount) => + Effect.scoped( + Effect.gen(function* () { + const grep = vi.fn(async (_query: string, options?: GrepOptions) => { + const fellBack = options?.enforceTimeBudget === false; + const items = fellBack ? [grepMatch("zzzz.txt", "*.ts needle")] : []; + const value: GrepResult = { + items, + totalMatched: items.length, + totalFilesSearched: fellBack ? 3 : constrainedFileCount, + totalFiles: 3, + filteredFileCount: fellBack ? 3 : constrainedFileCount, + nextCursor: null, + }; + return { ok: true as const, value }; + }); + contentFinder(grep); + + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project", "content"); + const result = yield* searchIndex.searchContents({ + query: "*.ts needle", + limit: 10, + caseSensitive: true, + wholeWord: false, + useRegex: false, + }); - expect(result.matches.map((match) => match.path)).toEqual(["arch/x86/kernel/head64.c"]); - }), - ).pipe(Effect.provide(testLayer)), + expect(result.matches.map((match) => match.path)).toEqual(["zzzz.txt"]); + }), + ).pipe(Effect.provide(testLayer)), ); // fff reads HOME on Linux and macOS but the user profile on Windows, so a temp HOME cannot stand in there. diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index afa49dfed4ca1..a2fd9337ffe69 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -1021,8 +1021,9 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( do { let result = yield* grepPage(true); - // fff 0.11 drops an empty enforced raw-query fallback page's cursor when a parsed path constraint matches no files. - if (result.filteredFileCount === 0 && result.totalFiles > 0) { + // fff 0.11 drops the cursor when an enforced raw-query fallback page (after a constraint like `*.ts` finds nothing) + // times out, so an empty enforced page that claims the scan ended is rerun without enforcement. + if (result.items.length === 0 && result.nextCursor === null) { result = yield* grepPage(false); } // fff searches an invalid pattern as a literal, which would keep the `(?i)` prefix. From 37bc27a77a458b099d52715ab9d9e965b6a440d5 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 14 Nov 2026 13:18:21 +0000 Subject: [PATCH 5/6] perf(server): filter whole-word content searches in fff Whole-word searches sent fff the bare query and dropped substring hits in JS, so a repo full of needleSuffix lines pushed hundreds of thousands of candidates through fff before the five real matches arrived (12 s on 4,000 files). Plain ASCII whole-word queries now go to fff as a boundary regex, and JS still computes the exact ranges and applies the existing word rule. Co-Authored-By: Claude Opus 5.5 --- .../workspace/WorkspaceSearchIndex.test.ts | 1 + .../src/workspace/WorkspaceSearchIndex.ts | 52 +++++++++++++++++-- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index bbfa535250458..04860117154d1 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -234,6 +234,7 @@ it.effect("continues whole-word searches after a filtered grep page", () => truncated: false, }); expect(grep).toHaveBeenCalledTimes(2); + expect(grep.mock.calls[0]?.[0]).toBe(String.raw`(?:^|\W)needle(?:\W|$)`); expect(grep.mock.calls[1]?.[1]?.cursor).toBe(nextCursor); }), ).pipe(Effect.provide(testLayer)), diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index a2fd9337ffe69..fbf288307a899 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -251,9 +251,33 @@ function codePointBefore(line: string, index: number): string | undefined { return codePointAt(line, previousIndex); } +function escapeRegex(text: string): string { + return text.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&"); +} + function buildContentSearchQuery(input: Omit): { readonly searchQuery: string; readonly regexMode: boolean; + readonly wholeWordPrefilter: boolean; +} { + // Without a native prefilter every substring hit crosses into JS, which takes seconds on large repos. + // fff compiles regexes bytewise, so `\W` also accepts non-ASCII and the JS check below stays authoritative; + // non-ASCII queries skip this because bytewise `(?i)` may not fold them. + if (input.wholeWord && !input.useRegex && /^[\x20-\x7e]+$/.test(input.query)) { + const left = /\w/.test(input.query[0] ?? "") ? String.raw`(?:^|\W)` : ""; + const right = /\w/.test(input.query.at(-1) ?? "") ? String.raw`(?:\W|$)` : ""; + return { + searchQuery: `${input.caseSensitive ? "" : "(?i)"}${left}${escapeRegex(input.query)}${right}`, + regexMode: true, + wholeWordPrefilter: true, + }; + } + return { ...buildPlainOrRegexQuery(input), wholeWordPrefilter: false }; +} + +function buildPlainOrRegexQuery(input: Omit): { + readonly searchQuery: string; + readonly regexMode: boolean; } { if (input.caseSensitive) { return { searchQuery: input.query, regexMode: input.useRegex }; @@ -275,6 +299,18 @@ function mapContentMatchRanges( })); } +function findLiteralRanges( + line: string, + query: string, + caseSensitive: boolean, +): Array<{ readonly start: number; readonly end: number }> { + const pattern = new RegExp(escapeRegex(query), caseSensitive ? "gu" : "giu"); + return Array.from(line.matchAll(pattern), (match) => ({ + start: match.index, + end: match.index + match[0].length, + })); +} + // Post-filter whole words: consuming boundary regexes swallow adjacent separators and widen ranges; \b misses punctuation edges. function isWholeWordRange( line: string, @@ -996,7 +1032,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( const searchContents: WorkspaceSearchIndex["Service"]["searchContents"] = Effect.fn( "WorkspaceSearchIndex.searchContents", )(function* (input) { - let { searchQuery, regexMode } = buildContentSearchQuery(input); + let { searchQuery, regexMode, wholeWordPrefilter } = buildContentSearchQuery(input); // Grep cursors advance per file; whole-word filtering needs enough raw candidates before advancing. const rawPageSize = input.wholeWord ? Math.max(input.limit, CONTENT_SEARCH_MAX_MATCHES_PER_FILE) @@ -1030,15 +1066,21 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( retryLiterally = regexMode && !input.caseSensitive && result.regexFallbackError !== undefined; if (retryLiterally) { regexFallbackError = result.regexFallbackError; - ({ searchQuery, regexMode } = buildContentSearchQuery({ ...input, useRegex: false })); + ({ searchQuery, regexMode, wholeWordPrefilter } = buildContentSearchQuery({ + ...input, + useRegex: false, + wholeWord: false, + })); continue; } for (const match of result.items) { if (isIgnoredPath(ignoredPaths, match.relativePath, hostIsWindows)) continue; - const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter( - (range) => !input.wholeWord || isWholeWordRange(match.lineContent, range), - ); + const matchRanges = ( + wholeWordPrefilter + ? findLiteralRanges(match.lineContent, input.query, input.caseSensitive) + : mapContentMatchRanges(match.lineContent, match.matchRanges) + ).filter((range) => !input.wholeWord || isWholeWordRange(match.lineContent, range)); // fff reports hits past its 512-char lineContent cap with no ranges, so whole words can't be verified there. if (input.wholeWord && matchRanges.length === 0) continue; matches.push({ From 4d0a558969b7c6443fa23d88d3d2ab678330af19 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sat, 14 Nov 2026 13:34:15 +0000 Subject: [PATCH 6/6] fix(server): keep fff query parsing for whole-word searches fff rewrites glob and path syntax out of a query even in regex mode, so the escaped whole-word prefilter turned '*.ts needle' into an invalid regex and returned nothing. Only queries made of ASCII word characters, which fff leaves alone and need no escaping, now use the prefilter; everything else takes the previous path. Co-Authored-By: Claude Opus 5.5 --- .../workspace/WorkspaceSearchIndex.test.ts | 34 +++++++++++++++++++ .../src/workspace/WorkspaceSearchIndex.ts | 21 +++++------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index 04860117154d1..d1f9d28882807 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -614,3 +614,37 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( }), ).pipe(Effect.provide(testLayer)), ); + +// Real fff: it parses glob and path syntax out of the query, which a native whole-word prefilter must not break. +it.effect.each([ + { query: "*.ts needle", expected: ["words.ts:2"] }, + { query: "*.ts", expected: ["notes.md:1"] }, + { query: "needle", expected: ["notes.md:1", "words.ts:2"] }, +])("matches whole words in a real workspace for $query", ({ query, expected }) => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-whole-word-" }); + yield* fileSystem.writeFileString( + path.join(cwd, "words.ts"), + "needleSuffix\nneedle\ntrailer\n", + ); + yield* fileSystem.writeFileString(path.join(cwd, "notes.md"), "needle *.ts\n"); + + const searchIndex = yield* WorkspaceSearchIndex.make(cwd, "content"); + const result = yield* searchIndex.searchContents({ + query, + limit: 500, + caseSensitive: true, + wholeWord: true, + useRegex: false, + }); + + expect(result.regexFallbackError).toBeUndefined(); + expect(result.matches.map((match) => `${match.path}:${match.lineNumber}`).toSorted()).toEqual( + expected, + ); + }), + ).pipe(Effect.provide(testLayer)), +); diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index fbf288307a899..d45f20117abae 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -237,6 +237,7 @@ function mapMixedSearchResult( } const WORD_CHARACTER = /[\p{Letter}\p{Mark}\p{Number}_]/u; +const ASCII_WORD = /^\w+$/; function codePointAt(line: string, index: number): string | undefined { const codePoint = line.codePointAt(index); @@ -251,23 +252,17 @@ function codePointBefore(line: string, index: number): string | undefined { return codePointAt(line, previousIndex); } -function escapeRegex(text: string): string { - return text.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&"); -} - function buildContentSearchQuery(input: Omit): { readonly searchQuery: string; readonly regexMode: boolean; readonly wholeWordPrefilter: boolean; } { // Without a native prefilter every substring hit crosses into JS, which takes seconds on large repos. - // fff compiles regexes bytewise, so `\W` also accepts non-ASCII and the JS check below stays authoritative; - // non-ASCII queries skip this because bytewise `(?i)` may not fold them. - if (input.wholeWord && !input.useRegex && /^[\x20-\x7e]+$/.test(input.query)) { - const left = /\w/.test(input.query[0] ?? "") ? String.raw`(?:^|\W)` : ""; - const right = /\w/.test(input.query.at(-1) ?? "") ? String.raw`(?:\W|$)` : ""; + // Only plain ASCII words qualify: fff rewrites queries with glob or path syntax even in regex mode, and its + // bytewise `\W` also accepts non-ASCII, so the JS whole-word check below stays authoritative. + if (input.wholeWord && !input.useRegex && ASCII_WORD.test(input.query)) { return { - searchQuery: `${input.caseSensitive ? "" : "(?i)"}${left}${escapeRegex(input.query)}${right}`, + searchQuery: `${input.caseSensitive ? "" : "(?i)"}(?:^|\\W)${input.query}(?:\\W|$)`, regexMode: true, wholeWordPrefilter: true, }; @@ -299,12 +294,12 @@ function mapContentMatchRanges( })); } -function findLiteralRanges( +function findWordQueryRanges( line: string, query: string, caseSensitive: boolean, ): Array<{ readonly start: number; readonly end: number }> { - const pattern = new RegExp(escapeRegex(query), caseSensitive ? "gu" : "giu"); + const pattern = new RegExp(query, caseSensitive ? "gu" : "giu"); return Array.from(line.matchAll(pattern), (match) => ({ start: match.index, end: match.index + match[0].length, @@ -1078,7 +1073,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( if (isIgnoredPath(ignoredPaths, match.relativePath, hostIsWindows)) continue; const matchRanges = ( wholeWordPrefilter - ? findLiteralRanges(match.lineContent, input.query, input.caseSensitive) + ? findWordQueryRanges(match.lineContent, input.query, input.caseSensitive) : mapContentMatchRanges(match.lineContent, match.matchRanges) ).filter((range) => !input.wholeWord || isWholeWordRange(match.lineContent, range)); // fff reports hits past its 512-char lineContent cap with no ranges, so whole words can't be verified there.