diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index 45102e4c1cfea..d1f9d28882807 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -234,6 +234,7 @@ it.effect("continues whole-word searches after a filtered grep page", () => truncated: false, }); expect(grep).toHaveBeenCalledTimes(2); + expect(grep.mock.calls[0]?.[0]).toBe(String.raw`(?:^|\W)needle(?:\W|$)`); expect(grep.mock.calls[1]?.[1]?.cursor).toBe(nextCursor); }), ).pipe(Effect.provide(testLayer)), @@ -482,6 +483,107 @@ it.effect("finds symlinks in folders past the first page of the index", () => ).pipe(Effect.provide(testLayer)), ); +function grepMatch(relativePath: string, lineContent: string): GrepResult["items"][number] { + return { + ...fileItem(relativePath), + isBinary: false, + lineNumber: 1, + col: 0, + byteOffset: 0, + lineContent, + matchRanges: [[0, lineContent.length]], + }; +} + +function contentFinder(grepAsync: (query: string, options?: GrepOptions) => Promise) { + const finder = { + destroy: vi.fn(), + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), + getScanProgressAsync: async () => ({ ok: true as const, value: { isScanning: false } }), + watch: () => ({ ok: true as const, value: () => undefined }), + grepAsync, + } as unknown as FileFinder; + vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); +} + +it.effect("pages past empty time-budget pages to a late match", () => + Effect.scoped( + Effect.gen(function* () { + const clock = vi.spyOn(performance, "now").mockReturnValue(0); + const cursors = [1, 2].map( + (offset) => ({ __brand: "GrepCursor", _offset: offset }) as GrepCursor, + ); + const grep = vi.fn(async (_query: string, options?: GrepOptions) => { + const page = options?.cursor ? cursors.indexOf(options.cursor) + 1 : 0; + // Each page outlasts any overall deadline; paging must still reach the end. + clock.mockReturnValue((page + 1) * 1_000); + const items = page === 2 ? [grepMatch("src/late.ts", "needle")] : []; + const value: GrepResult = { + items, + totalMatched: items.length, + totalFilesSearched: 1, + totalFiles: 3, + filteredFileCount: 3, + nextCursor: cursors[page] ?? null, + }; + return { ok: true as const, value }; + }); + contentFinder(grep); + + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project", "content"); + const result = yield* searchIndex.searchContents({ + query: "needle", + limit: 10, + caseSensitive: true, + wholeWord: false, + useRegex: false, + }); + + expect(result.matches.map((match) => match.path)).toEqual(["src/late.ts"]); + expect(result.truncated).toBe(false); + expect(grep.mock.calls.map(([, options]) => options?.enforceTimeBudget)).toEqual([ + true, + true, + true, + ]); + }), + ).pipe(Effect.provide(testLayer)), +); + +it.effect.each([0, 1])( + "reruns an empty enforced fallback page whose constraint matched %i files", + (constrainedFileCount) => + Effect.scoped( + Effect.gen(function* () { + const grep = vi.fn(async (_query: string, options?: GrepOptions) => { + const fellBack = options?.enforceTimeBudget === false; + const items = fellBack ? [grepMatch("zzzz.txt", "*.ts needle")] : []; + const value: GrepResult = { + items, + totalMatched: items.length, + totalFilesSearched: fellBack ? 3 : constrainedFileCount, + totalFiles: 3, + filteredFileCount: fellBack ? 3 : constrainedFileCount, + nextCursor: null, + }; + return { ok: true as const, value }; + }); + contentFinder(grep); + + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project", "content"); + const result = yield* searchIndex.searchContents({ + query: "*.ts needle", + limit: 10, + caseSensitive: true, + wholeWord: false, + useRegex: false, + }); + + expect(result.matches.map((match) => match.path)).toEqual(["zzzz.txt"]); + }), + ).pipe(Effect.provide(testLayer)), +); + // fff reads HOME on Linux and macOS but the user profile on Windows, so a temp HOME cannot stand in there. it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( "refuses to index a workspace rooted at the home directory", @@ -512,3 +614,37 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( }), ).pipe(Effect.provide(testLayer)), ); + +// Real fff: it parses glob and path syntax out of the query, which a native whole-word prefilter must not break. +it.effect.each([ + { query: "*.ts needle", expected: ["words.ts:2"] }, + { query: "*.ts", expected: ["notes.md:1"] }, + { query: "needle", expected: ["notes.md:1", "words.ts:2"] }, +])("matches whole words in a real workspace for $query", ({ query, expected }) => + Effect.scoped( + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-whole-word-" }); + yield* fileSystem.writeFileString( + path.join(cwd, "words.ts"), + "needleSuffix\nneedle\ntrailer\n", + ); + yield* fileSystem.writeFileString(path.join(cwd, "notes.md"), "needle *.ts\n"); + + const searchIndex = yield* WorkspaceSearchIndex.make(cwd, "content"); + const result = yield* searchIndex.searchContents({ + query, + limit: 500, + caseSensitive: true, + wholeWord: true, + useRegex: false, + }); + + expect(result.regexFallbackError).toBeUndefined(); + expect(result.matches.map((match) => `${match.path}:${match.lineNumber}`).toSorted()).toEqual( + expected, + ); + }), + ).pipe(Effect.provide(testLayer)), +); diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index 22e307c182616..d45f20117abae 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -47,7 +47,7 @@ const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2; const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds"; const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000; const WORKSPACE_INDEX_IDLE_TTL = "15 minutes"; -const CONTENT_SEARCH_TIME_BUDGET_MS = 250; +const CONTENT_SEARCH_TIME_BUDGET_MS = 25; const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100; const SYMLINK_SCAN_CONCURRENCY = 16; // A scan no watcher can invalidate is reused this long, so typing does not rescan every folder. @@ -237,6 +237,7 @@ function mapMixedSearchResult( } const WORD_CHARACTER = /[\p{Letter}\p{Mark}\p{Number}_]/u; +const ASCII_WORD = /^\w+$/; function codePointAt(line: string, index: number): string | undefined { const codePoint = line.codePointAt(index); @@ -254,6 +255,24 @@ function codePointBefore(line: string, index: number): string | undefined { function buildContentSearchQuery(input: Omit): { readonly searchQuery: string; readonly regexMode: boolean; + readonly wholeWordPrefilter: boolean; +} { + // Without a native prefilter every substring hit crosses into JS, which takes seconds on large repos. + // Only plain ASCII words qualify: fff rewrites queries with glob or path syntax even in regex mode, and its + // bytewise `\W` also accepts non-ASCII, so the JS whole-word check below stays authoritative. + if (input.wholeWord && !input.useRegex && ASCII_WORD.test(input.query)) { + return { + searchQuery: `${input.caseSensitive ? "" : "(?i)"}(?:^|\\W)${input.query}(?:\\W|$)`, + regexMode: true, + wholeWordPrefilter: true, + }; + } + return { ...buildPlainOrRegexQuery(input), wholeWordPrefilter: false }; +} + +function buildPlainOrRegexQuery(input: Omit): { + readonly searchQuery: string; + readonly regexMode: boolean; } { if (input.caseSensitive) { return { searchQuery: input.query, regexMode: input.useRegex }; @@ -275,6 +294,18 @@ function mapContentMatchRanges( })); } +function findWordQueryRanges( + line: string, + query: string, + caseSensitive: boolean, +): Array<{ readonly start: number; readonly end: number }> { + const pattern = new RegExp(query, caseSensitive ? "gu" : "giu"); + return Array.from(line.matchAll(pattern), (match) => ({ + start: match.index, + end: match.index + match[0].length, + })); +} + // Post-filter whole words: consuming boundary regexes swallow adjacent separators and widen ranges; \b misses punctuation edges. function isWholeWordRange( line: string, @@ -996,8 +1027,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( const searchContents: WorkspaceSearchIndex["Service"]["searchContents"] = Effect.fn( "WorkspaceSearchIndex.searchContents", )(function* (input) { - let { searchQuery, regexMode } = buildContentSearchQuery(input); - const deadline = performance.now() + CONTENT_SEARCH_TIME_BUDGET_MS; + let { searchQuery, regexMode, wholeWordPrefilter } = buildContentSearchQuery(input); // Grep cursors advance per file; whole-word filtering needs enough raw candidates before advancing. const rawPageSize = input.wholeWord ? Math.max(input.limit, CONTENT_SEARCH_MAX_MATCHES_PER_FILE) @@ -1007,31 +1037,45 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( let regexFallbackError: string | undefined; let retryLiterally = false; - do { - const remainingTimeBudgetMs = Math.max(1, Math.ceil(deadline - performance.now())); - const result = yield* runSearch(input.query, input.limit, "grep", () => + const grepPage = (enforceTimeBudget: boolean) => + runSearch(input.query, input.limit, "grep", () => finder.grepAsync(searchQuery, { mode: regexMode ? "regex" : "plain", smartCase: !input.caseSensitive && !regexMode, maxMatchesPerFile: Math.min(CONTENT_SEARCH_MAX_MATCHES_PER_FILE, rawPageSize), pageSize: rawPageSize, cursor: nextCursor, - timeBudgetMs: remainingTimeBudgetMs, + timeBudgetMs: CONTENT_SEARCH_TIME_BUDGET_MS, + enforceTimeBudget, }), ); + + do { + let result = yield* grepPage(true); + // fff 0.11 drops the cursor when an enforced raw-query fallback page (after a constraint like `*.ts` finds nothing) + // times out, so an empty enforced page that claims the scan ended is rerun without enforcement. + if (result.items.length === 0 && result.nextCursor === null) { + result = yield* grepPage(false); + } // fff searches an invalid pattern as a literal, which would keep the `(?i)` prefix. retryLiterally = regexMode && !input.caseSensitive && result.regexFallbackError !== undefined; if (retryLiterally) { regexFallbackError = result.regexFallbackError; - ({ searchQuery, regexMode } = buildContentSearchQuery({ ...input, useRegex: false })); + ({ searchQuery, regexMode, wholeWordPrefilter } = buildContentSearchQuery({ + ...input, + useRegex: false, + wholeWord: false, + })); continue; } for (const match of result.items) { if (isIgnoredPath(ignoredPaths, match.relativePath, hostIsWindows)) continue; - const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter( - (range) => !input.wholeWord || isWholeWordRange(match.lineContent, range), - ); + const matchRanges = ( + wholeWordPrefilter + ? findWordQueryRanges(match.lineContent, input.query, input.caseSensitive) + : mapContentMatchRanges(match.lineContent, match.matchRanges) + ).filter((range) => !input.wholeWord || isWholeWordRange(match.lineContent, range)); // fff reports hits past its 512-char lineContent cap with no ranges, so whole words can't be verified there. if (input.wholeWord && matchRanges.length === 0) continue; matches.push({ @@ -1043,10 +1087,8 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( } nextCursor = result.nextCursor; regexFallbackError ??= result.regexFallbackError; - } while ( - retryLiterally || - (matches.length < input.limit && nextCursor !== null && performance.now() < deadline) - ); + // The RPC has no resume cursor, so page to the limit; short enforced pages keep each native call brief. + } while (retryLiterally || (matches.length < input.limit && nextCursor !== null)); return { matches: matches.slice(0, input.limit),