diff --git a/apps/server/src/binMain.ts b/apps/server/src/binMain.ts index 0dabd73873440..fc700f824a7ef 100644 --- a/apps/server/src/binMain.ts +++ b/apps/server/src/binMain.ts @@ -19,6 +19,9 @@ export async function runCliCommand() { // Exit 2 identifies document rejections; OS and process failures remain retryable. process.exitCode = 2; } + } else if (command === "__workspace-search") { + const { runWorkspaceSearchWorker } = await import("./workspace/workspaceSearchWorker.ts"); + runWorkspaceSearchWorker(); } else if (command === "__managed-chatgpt-token") { const { runManagedChatGptTokenCli } = await import("./cli/managedChatGptToken.ts"); await runManagedChatGptTokenCli(process.argv.slice(3)); diff --git a/apps/server/src/workspace/WorkspaceEntries.test.ts b/apps/server/src/workspace/WorkspaceEntries.test.ts index 348b5f877ac24..34472879b58e8 100644 --- a/apps/server/src/workspace/WorkspaceEntries.test.ts +++ b/apps/server/src/workspace/WorkspaceEntries.test.ts @@ -15,6 +15,7 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as WorkspaceEntries from "./WorkspaceEntries.ts"; import * as WorkspacePaths from "./WorkspacePaths.ts"; +import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts"; vi.mock("node:fs/promises", async (importOriginal) => { const actual = await importOriginal(); @@ -33,6 +34,30 @@ const layerTest = Layer.empty.pipe( Layer.provideMerge(NodeServices.layer), ); +// Keeps finders in this process so a test can intercept the native calls. +const layerInProcessFinders = Layer.effect( + WorkspaceEntries.WorkspaceEntries, + WorkspaceEntries.make, +).pipe( + Layer.provide(Layer.fresh(WorkspaceSearchIndex.WorkspaceSearchIndexMap.layerNoDeps)), + Layer.provide( + Layer.succeed(WorkspaceSearchIndex.WorkspaceSearchWorker, { + create: async (options) => { + const created = FileFinder.create(options); + if (!created.ok) return created; + const finder = new Proxy(created.value, { + get: (target, method: keyof FileFinder) => + method === ("alive" as string) + ? true + : async (...args: Array) => + (target[method] as (...args: Array) => unknown)(...args), + }); + return { ok: true, value: finder as unknown as WorkspaceSearchIndex.RemoteFileFinder }; + }, + }), + ), +); + const makeTempDir = Effect.fn(function* (opts?: { prefix?: string; git?: boolean }) { const fileSystem = yield* FileSystem.FileSystem; const dir = yield* fileSystem.makeTempDirectoryScoped({ @@ -857,7 +882,7 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => { expect(result.matches.map((match) => match.path)).toEqual(["src/notes.ts"]); // The second search ran on the rescanned live finder, not a rebuild. expect({ finders, rescans }).toEqual({ finders: 1, rescans: 1 }); - }), + }).pipe(Effect.provide(layerInProcessFinders)), ); }); diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index c01600842f2f6..6a15df93e3b88 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -1,11 +1,8 @@ -import { - FileFinder, - type FileItem, - type GrepCursor, - type GrepOptions, - type GrepResult, -} from "@ff-labs/fff-node"; +// @effect-diagnostics nodeBuiltinImport:off -- observes the real search process. +import * as NodeChildProcess from "node:child_process"; +import * as NodeEvents from "node:events"; import * as NodeServices from "@effect/platform-node/NodeServices"; +import type { FileItem, GrepCursor, GrepOptions, GrepResult } from "@ff-labs/fff-node"; import { afterEach, expect, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; @@ -17,10 +14,49 @@ import { vi } from "vite-plus/test"; import * as VcsProcess from "../vcs/VcsProcess.ts"; import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts"; -const testLayer = VcsProcess.layer.pipe(Layer.provideMerge(NodeServices.layer)); +vi.mock("node:child_process", { spy: true }); + +const testLayer = Layer.mergeAll(VcsProcess.layer, WorkspaceSearchIndex.layerWorker).pipe( + Layer.provideMerge(NodeServices.layer), +); afterEach(() => { vi.restoreAllMocks(); + vi.mocked(NodeChildProcess.spawn).mockClear(); +}); + +const withFinder = (finder: Partial) => + Effect.provideService(WorkspaceSearchIndex.WorkspaceSearchWorker, { + create: async () => ({ + ok: true, + value: { + alive: true, + destroy: vi.fn(async () => {}), + waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), + ...finder, + } as WorkspaceSearchIndex.RemoteFileFinder, + }), + }); + +const spawnedWorkers = () => + vi + .mocked(NodeChildProcess.spawn) + .mock.results.map((result) => result.value as NodeChildProcess.ChildProcess); + +const searchNeedle = (searchIndex: WorkspaceSearchIndex.WorkspaceSearchIndex["Service"]) => + searchIndex.searchContents({ + query: "needle", + limit: 10, + caseSensitive: true, + wholeWord: false, + useRegex: false, + }); + +const makeNeedleWorkspace = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-search-worker-" }); + yield* fileSystem.writeFileString(`${cwd}/notes.ts`, "const needle = 1;\n"); + return cwd; }); function fileItem(relativePath: string): FileItem { @@ -43,7 +79,7 @@ it.effect("filters image searches before applying the result limit", () => ...Array.from({ length: 200 }, (_, index) => fileItem(`src/file-${index}.ts`)), fileItem("public/icon.svg"), ]; - const fileSearch = vi.fn(() => ({ + const fileSearch = vi.fn(async () => ({ ok: true as const, value: { items, @@ -52,14 +88,9 @@ it.effect("filters image searches before applying the result limit", () => totalFiles: items.length, }, })); - const finder = { - destroy: vi.fn(), - waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - fileSearch, - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); - - const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project"); + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project").pipe( + withFinder({ fileSearch }), + ); const resultWithoutKind = yield* searchIndex.search("", 200, undefined, true); const resultWithDirectoryKind = yield* searchIndex.search("", 200, "directory", true); @@ -89,7 +120,7 @@ it.effect("fetches past a first page of Git-ignored results", () => fileItem("hidden/c.ts"), fileItem("src/a.ts"), ]; - const fileSearch = vi.fn((_query: string, options?: { pageSize?: number }) => ({ + const fileSearch = vi.fn(async (_query: string, options?: { pageSize?: number }) => ({ ok: true as const, value: { items: items.slice(0, options?.pageSize), @@ -98,14 +129,8 @@ it.effect("fetches past a first page of Git-ignored results", () => totalFiles: items.length, }, })); - const finder = { - destroy: vi.fn(), - waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - fileSearch, - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); - const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + const searchIndex = yield* WorkspaceSearchIndex.make(cwd).pipe(withFinder({ fileSearch })); const result = yield* searchIndex.search("a", 1, "file", false); expect(result).toEqual({ entries: [{ kind: "file", path: "src/a.ts" }], truncated: false }); @@ -120,17 +145,14 @@ it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( Effect.scoped( Effect.gen(function* () { const items = [fileItem("dir\\one/report.txt")]; - const finder = { - destroy: vi.fn(), - waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - fileSearch: vi.fn(() => ({ - ok: true as const, - value: { items, scores: [], totalMatched: 1, totalFiles: 1 }, - })), - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); - - const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project"); + const fileSearch = vi.fn(async () => ({ + ok: true as const, + value: { items, scores: [], totalMatched: 1, totalFiles: 1 }, + })); + + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project").pipe( + withFinder({ fileSearch }), + ); const result = yield* searchIndex.search("report", 50, "file"); expect(result.entries).toEqual([{ kind: "file", path: "dir\\one/report.txt" }]); @@ -174,7 +196,7 @@ it.effect("continues whole-word searches after a filtered grep page", () => filteredFileCount: 1, nextCursor: cursor, }); - const grep = vi.fn((_query: string, options?: GrepOptions) => + const grep = vi.fn(async (_query: string, options?: GrepOptions) => options?.cursor ? { ok: true as const, value: grepResult("needle", [[0, 6]], null) } : { @@ -182,14 +204,9 @@ it.effect("continues whole-word searches after a filtered grep page", () => value: grepResult("needleSuffix", [[0, 6]], nextCursor), }, ); - const finder = { - destroy: vi.fn(), - waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - grep, - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); - - const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project", "content"); + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project", "content").pipe( + withFinder({ grep }), + ); const result = yield* searchIndex.searchContents({ query: "needle", limit: 1, @@ -215,6 +232,92 @@ it.effect("continues whole-word searches after a filtered grep page", () => ).pipe(Effect.provide(testLayer)), ); +it.effect("reads an incomplete scan from the finder that answers the search", () => + Effect.scoped( + Effect.gen(function* () { + let firstExited = false; + const timedOutScan = vi.fn(async () => ({ ok: true as const, value: false })); + const fileSearch = vi.fn(async () => ({ + ok: true as const, + value: { items: [fileItem("src/a.ts")], scores: [], totalMatched: 1, totalFiles: 1 }, + })); + const first = { + get alive() { + return !firstExited; + }, + destroy: vi.fn(async () => {}), + waitForIndexReady: timedOutScan, + // The scan finished, then the process exited right after replying. + isScanning: vi.fn(async () => { + firstExited = true; + return false; + }), + fileSearch: vi.fn(async () => { + throw new Error("The workspace search worker exited."); + }), + } as unknown as WorkspaceSearchIndex.RemoteFileFinder; + const rebuilt = { + alive: true, + destroy: vi.fn(async () => {}), + waitForIndexReady: timedOutScan, + isScanning: vi.fn(async () => true), + fileSearch, + } as unknown as WorkspaceSearchIndex.RemoteFileFinder; + const finders = [first, rebuilt]; + const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project").pipe( + Effect.provideService(WorkspaceSearchIndex.WorkspaceSearchWorker, { + create: async () => ({ ok: true, value: finders.shift()! }), + }), + ); + + const duringExit = yield* searchIndex.search("", 10, "file").pipe( + Effect.map(({ truncated }) => truncated), + Effect.catchTags({ + WorkspaceSearchIndexSearchFailed: (error) => Effect.succeed(error._tag), + }), + ); + const afterRebuild = yield* searchIndex.search("", 10, "file"); + + expect(duringExit).toBe("WorkspaceSearchIndexSearchFailed"); + expect(afterRebuild).toEqual({ + entries: [{ kind: "file", path: "src/a.ts" }], + truncated: true, + }); + }), + ).pipe(Effect.provide(testLayer)), +); + +it.live("rebuilds the index in a new search process after the worker exits", () => + Effect.gen(function* () { + const cwd = yield* makeNeedleWorkspace; + const searchIndex = yield* WorkspaceSearchIndex.make(cwd, "content"); + const before = yield* searchNeedle(searchIndex); + const [crashed] = spawnedWorkers(); + yield* Effect.promise(() => { + const exited = NodeEvents.EventEmitter.once(crashed!, "exit"); + crashed!.kill(); + return exited; + }); + + expect(yield* searchNeedle(searchIndex)).toEqual(before); + expect(before.matches.map((match) => match.path)).toEqual(["notes.ts"]); + expect(spawnedWorkers()).toHaveLength(2); + }).pipe(Effect.scoped, Effect.provide(testLayer)), +); + +it.live("ends the search process once its last index closes", () => + Effect.gen(function* () { + const cwd = yield* makeNeedleWorkspace; + yield* Effect.scoped(Effect.flatMap(WorkspaceSearchIndex.make(cwd, "content"), searchNeedle)); + const [worker] = spawnedWorkers(); + + const code = + worker!.exitCode ?? + (yield* Effect.promise(() => NodeEvents.EventEmitter.once(worker!, "exit")))[0]; + expect(code).toBe(0); + }).pipe(Effect.scoped, Effect.provide(testLayer)), +); + // fff reads HOME on Linux and macOS but the user profile on Windows, so a temp HOME cannot stand in there. it.effect.skipIf(HostProcess.Platform.defaultValue() === "win32")( "refuses to index a workspace rooted at the home directory", diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index 5d5d39bf55612..60e3b3ee805db 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -1,18 +1,24 @@ -import * as NodeModule from "node:module"; +// @effect-diagnostics nodeBuiltinImport:off -- the search process is an external runtime boundary. +import * as NodeChildProcess from "node:child_process"; +import * as NodeURL from "node:url"; import type { DirItem, FileItem, FileFinder as FileFinderType, GrepCursor, + InitOptions, MixedItem, Result, } from "@ff-labs/fff-node"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as LayerMap from "effect/LayerMap"; import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; import type { ProjectEntry, @@ -27,10 +33,6 @@ import { isProjectFaviconPath } from "@t3tools/shared/projectFavicon"; import * as VcsProcess from "../vcs/VcsProcess.ts"; -// fff-node's native library stays external; single-executables require disk packages instead of importing them. -const requireForFff = NodeModule.createRequire(import.meta.url); -const { FileFinder } = requireForFff("@ff-labs/fff-node") as typeof import("@ff-labs/fff-node"); - const WORKSPACE_INDEX_MAX_ENTRIES = 25_000; const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2; const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds"; @@ -290,6 +292,151 @@ function withDirectoryAncestors(entries: ReadonlyArray): ProjectEn return [...entryByPath.values()]; } +type RemoteFinderMethod = + | "destroy" + | "directorySearch" + | "fileSearch" + | "grep" + | "isScanning" + | "mixedSearch" + | "scanFilesAsync" + | "waitForIndexReady"; + +export type RemoteFileFinder = { + readonly [Method in RemoteFinderMethod]: ( + ...args: Parameters + ) => Promise>>; +} & { + // False once the process holding the native index has exited. + readonly alive: boolean; +}; + +// One process owns every finder. Native calls already ran one at a time on the server thread, so a +// pool would add runtimes and index copies without making any single search faster. +export class WorkspaceSearchWorker extends Context.Service< + WorkspaceSearchWorker, + { + readonly create: (options: InitOptions) => Promise>; + } +>()("t3/workspace/WorkspaceSearchIndex/WorkspaceSearchWorker") {} + +const makeWorker = Effect.gen(function* () { + const executable = yield* HostProcess.ExecutablePath; + const isExecutable = yield* HostProcess.IsExecutable; + const environment = yield* HostProcess.Environment; + // In a bundle import.meta.url is a chunk beside bin.mjs; development and tests use this source module. + const entry = NodeURL.fileURLToPath( + new URL(import.meta.url.endsWith(".ts") ? "../bin.ts" : "./bin.mjs", import.meta.url), + ); + let nextId = 0; + let current: ReturnType | undefined; + + function start() { + const child = NodeChildProcess.spawn( + executable, + [...(isExecutable ? [] : [entry]), "__workspace-search"], + { + stdio: ["ignore", "ignore", "inherit", "ipc"], + serialization: "advanced", + windowsHide: true, + env: { ...environment, ELECTRON_RUN_AS_NODE: "1" }, + }, + ); + const pending = new Map & { readonly method: string }>(); + const worker = { + child, + finders: 0, + exited: false, + request: (finderId: number, method: string, args: ReadonlyArray) => { + if (worker.exited) return Promise.reject(new Error("The workspace search worker exited.")); + const id = ++nextId; + const request = { ...Promise.withResolvers(), method }; + pending.set(id, request); + child.send({ id, finderId, method, args }, (error) => { + if (error) child.kill(); + }); + return request.promise; + }, + release: () => { + if (--worker.finders > 0 || worker !== current) return; + // The last index expired; the next search starts a fresh process. + current = undefined; + child.disconnect(); + }, + }; + const exit = () => { + worker.exited = true; + if (current === worker) current = undefined; + for (const request of pending.values()) { + // The native index went with the process, so there is nothing left to destroy. + if (request.method === "destroy") request.resolve(undefined); + else request.reject(new Error("The workspace search worker exited.")); + } + pending.clear(); + }; + child.on("message", (reply: { id: number; value?: unknown; error?: unknown }) => { + const request = pending.get(reply.id); + pending.delete(reply.id); + if ("error" in reply) request?.reject(reply.error); + else request?.resolve(reply.value); + }); + child.once("exit", exit); + child.once("error", exit); + return worker; + } + + yield* Effect.addFinalizer(() => Effect.sync(() => current?.child.kill())); + + return WorkspaceSearchWorker.of({ + create: async (options) => { + const worker = (current ??= start()); + const finderId = ++nextId; + worker.finders++; + const created = (await worker + .request(finderId, "create", [options]) + .catch((error: unknown) => { + worker.release(); + throw error; + })) as Result; + if (!created.ok) { + worker.release(); + return created; + } + const call = + (method: Method) => + (...args: Parameters) => + worker.request(finderId, method, args) as Promise< + Awaited> + >; + return { + ok: true, + value: { + get alive() { + return !worker.exited; + }, + destroy: async () => { + if (worker.exited) return; + try { + await worker.request(finderId, "destroy", []); + } finally { + worker.release(); + } + }, + directorySearch: call("directorySearch"), + fileSearch: call("fileSearch"), + grep: call("grep"), + isScanning: call("isScanning"), + mixedSearch: call("mixedSearch"), + scanFilesAsync: call("scanFilesAsync"), + waitForIndexReady: call("waitForIndexReady"), + }, + }; + }, + }); +}); + +export const layerWorker = Layer.effect(WorkspaceSearchWorker, makeWorker); + // fff applies .gitignore files only; Git also ignores paths from .git/info/exclude and the global excludes file. const listGitIgnoredPaths = Effect.fn("WorkspaceSearchIndex.listGitIgnoredPaths")(function* ( vcsProcess: VcsProcess.VcsProcess["Service"], @@ -325,6 +472,13 @@ const listGitIgnoredPaths = Effect.fn("WorkspaceSearchIndex.listGitIgnoredPaths" ); }); +function withIncompleteScan( + result: T, + incompleteBeforeQuery: boolean, +): T { + return incompleteBeforeQuery ? { ...result, truncated: true } : result; +} + function mixedItemPath(item: MixedItem): string { return item.item.relativePath; } @@ -346,12 +500,13 @@ function isIgnoredPath( } const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* ( + worker: WorkspaceSearchWorker["Service"], cwd: string, variant: WorkspaceSearchIndexVariant, ) { - const result = yield* Effect.try({ + const result = yield* Effect.tryPromise({ try: () => - FileFinder.create({ + worker.create({ basePath: cwd, disableMmapCache: true, // Content indexing costs CPU/memory; path-only consumers use the lightweight index. @@ -374,7 +529,7 @@ const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* ( const waitForIndexReady = Effect.fn("WorkspaceSearchIndex.waitForIndexReady")(function* ( cwd: string, - finder: FileFinderType, + finder: RemoteFileFinder, onFailure: (input: { readonly reason: string; readonly cause?: unknown }) => E, ): Effect.fn.Return { const result = yield* Effect.tryPromise({ @@ -400,46 +555,85 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( cwd: string, variant: WorkspaceSearchIndexVariant = "paths", ) { + const worker = yield* WorkspaceSearchWorker; const hostIsWindows = yield* HostProcess.isWindows; - const finder = yield* Effect.acquireRelease(createFinder(cwd, variant), (finder) => - Effect.try({ - try: () => finder.destroy(), - catch: (cause) => new WorkspaceSearchIndexDestroyFailed({ cwd, cause }), - }).pipe(Effect.orDie), - ); - let initialScanTimedOut = false; const vcsProcess = yield* VcsProcess.VcsProcess; - let ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd, hostIsWindows); - yield* waitForIndexReady( - cwd, - finder, - ({ reason, cause }) => - new WorkspaceSearchIndexCreateFailed({ + let ignoredPaths: ReadonlySet = new Set(); + const indexScope = yield* Effect.scope; + const reopenLock = yield* Semaphore.make(1); + let initialScanTimedOut = false; + const open = Effect.fn("WorkspaceSearchIndex.open")(function* () { + const scope = yield* Scope.fork(indexScope); + return yield* Effect.gen(function* () { + const finder = yield* Effect.acquireRelease(createFinder(worker, cwd, variant), (finder) => + Effect.tryPromise({ + try: () => finder.destroy(), + catch: (cause) => new WorkspaceSearchIndexDestroyFailed({ cwd, cause }), + }).pipe(Effect.orDie), + ); + initialScanTimedOut = false; + ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd, hostIsWindows); + yield* waitForIndexReady( cwd, - reason, - cause, - }), - ).pipe( - Effect.catchTags({ - WorkspaceSearchIndexScanTimedOut: (error) => - variant === "paths" - ? Effect.sync(() => { - initialScanTimedOut = true; - }) - : Effect.fail(error), - }), + finder, + ({ reason, cause }) => + new WorkspaceSearchIndexCreateFailed({ + cwd, + reason, + cause, + }), + ).pipe( + Effect.catchTags({ + WorkspaceSearchIndexScanTimedOut: (error) => + variant === "paths" + ? Effect.sync(() => { + initialScanTimedOut = true; + }) + : Effect.fail(error), + }), + ); + return finder; + }).pipe( + Scope.provide(scope), + Effect.onError((cause) => Scope.close(scope, Exit.failCause(cause))), + ); + }); + let finder = yield* open(); + // A search worker that exited took the native index with it; the next use rebuilds it. + const currentFinder = reopenLock.withPermits(1)( + Effect.suspend(() => + finder.alive + ? Effect.succeed(finder) + : open().pipe(Effect.map((opened) => (finder = opened))), + ), ); - const hasIncompleteInitialScan = () => initialScanTimedOut && finder.isScanning(); - const runSearch = Effect.fn("WorkspaceSearchIndex.runSearch")(function* ( query: string, pageSize: number, operation: "directorySearch" | "fileSearch" | "grep" | "mixedSearch", - execute: () => Result, - ): Effect.fn.Return { - const result = yield* Effect.try({ - try: execute, + execute: (finder: RemoteFileFinder) => Promise>, + ): Effect.fn.Return< + { readonly value: A; readonly incompleteBeforeQuery: boolean }, + WorkspaceSearchIndexSearchFailed + > { + const finder = yield* currentFinder.pipe( + Effect.mapError( + (cause) => + new WorkspaceSearchIndexSearchFailed({ + cwd, + queryLength: query.length, + pageSize, + reason: "The workspace search index could not be reopened.", + cause, + }), + ), + ); + // Read from the finder that answers, before it answers: a scan finishing mid-query still leaves a partial result. + const incompleteBeforeQuery = + initialScanTimedOut && (yield* Effect.promise(() => finder.isScanning().catch(() => true))); + const result = yield* Effect.tryPromise({ + try: () => execute(finder), catch: (cause) => new WorkspaceSearchIndexSearchFailed({ cwd, @@ -457,7 +651,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( reason: result.error, }); } - return result.value; + return { value: result.value, incompleteBeforeQuery }; }); // Grows the first page until hidden entries can no longer crowd visible ones out of it. @@ -465,14 +659,24 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( query: string, pageSize: number, operation: "directorySearch" | "fileSearch" | "mixedSearch", - execute: (pageSize: number) => Result>, + execute: (finder: RemoteFileFinder, pageSize: number) => Promise>>, pathOf: (item: I) => string, - ): Effect.fn.Return, WorkspaceSearchIndexSearchFailed> { + ): Effect.fn.Return< + { readonly value: SearchPage; readonly incompleteBeforeQuery: boolean }, + WorkspaceSearchIndexSearchFailed + > { let size = pageSize; + let incomplete = false; while (true) { const fetchSize = size; - const result = yield* runSearch(query, fetchSize, operation, () => execute(fetchSize)); - if (ignoredPaths.size === 0) return result; + const { value: result, incompleteBeforeQuery } = yield* runSearch( + query, + fetchSize, + operation, + (finder) => execute(finder, fetchSize), + ); + incomplete ||= incompleteBeforeQuery; + if (ignoredPaths.size === 0) return { value: result, incompleteBeforeQuery: incomplete }; const items = result.items.filter( (item) => !isIgnoredPath(ignoredPaths, pathOf(item), hostIsWindows), ); @@ -482,8 +686,11 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( fetchSize >= WORKSPACE_INDEX_PAGE_SIZE ) { return { - items, - totalMatched: result.totalMatched - (result.items.length - items.length), + value: { + items, + totalMatched: result.totalMatched - (result.items.length - items.length), + }, + incompleteBeforeQuery: incomplete, }; } size = Math.min(WORKSPACE_INDEX_PAGE_SIZE, fetchSize * 4); @@ -493,6 +700,16 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn( "WorkspaceSearchIndex.refresh", )(function* () { + const finder = yield* currentFinder.pipe( + Effect.mapError( + (cause) => + new WorkspaceSearchIndexRefreshFailed({ + cwd, + reason: "The workspace search index could not be reopened.", + cause, + }), + ), + ); // scanFiles waits on fff's picker lock, which its watcher can hold for seconds. const result = yield* Effect.tryPromise({ try: () => finder.scanFilesAsync(), @@ -526,12 +743,11 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")( function* () { - const incompleteBeforeQuery = hasIncompleteInitialScan(); - const result = yield* searchVisible( + const { value: result, incompleteBeforeQuery } = yield* searchVisible( "", WORKSPACE_INDEX_PAGE_SIZE, "mixedSearch", - (pageSize) => finder.mixedSearch("", { pageSize }), + (finder, pageSize) => finder.mixedSearch("", { pageSize }), mixedItemPath, ); const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES, hostIsWindows); @@ -555,33 +771,42 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( ) { const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1); if (kind === "file" || imageOnly) { - const result = yield* searchVisible( + const { value: result, incompleteBeforeQuery } = yield* searchVisible( query, pageSize, "fileSearch", - (size) => finder.fileSearch(query, { pageSize: size }), + (finder, size) => finder.fileSearch(query, { pageSize: size }), (item) => item.relativePath, ); - return mapFileSearchResult(result, limit, hostIsWindows, imageOnly); + return withIncompleteScan( + mapFileSearchResult(result, limit, hostIsWindows, imageOnly), + incompleteBeforeQuery, + ); } if (kind === "directory") { - const result = yield* searchVisible( + const { value: result, incompleteBeforeQuery } = yield* searchVisible( query, pageSize, "directorySearch", - (size) => finder.directorySearch(query, { pageSize: size }), + (finder, size) => finder.directorySearch(query, { pageSize: size }), (item) => item.relativePath, ); - return mapDirectorySearchResult(result, limit, hostIsWindows); + return withIncompleteScan( + mapDirectorySearchResult(result, limit, hostIsWindows), + incompleteBeforeQuery, + ); } - const result = yield* searchVisible( + const { value: result, incompleteBeforeQuery } = yield* searchVisible( query, pageSize, "mixedSearch", - (size) => finder.mixedSearch(query, { pageSize: size }), + (finder, size) => finder.mixedSearch(query, { pageSize: size }), mixedItemPath, ); - return mapMixedSearchResult(result, limit, hostIsWindows); + return withIncompleteScan( + mapMixedSearchResult(result, limit, hostIsWindows), + incompleteBeforeQuery, + ); }); const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn( @@ -591,12 +816,11 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( // matching is subsequence-based, so dropping them still matches "[id]" and "a{b}" paths. // "{" is dropped only when the query finds nothing as a glob, so "*.{ts,tsx}" keeps working. const query = rawQuery.replaceAll("[", ""); - const incompleteBeforeQuery = hasIncompleteInitialScan(); let mapped = yield* searchNames(query, limit, kind, imageOnly); if (mapped.entries.length === 0 && query.includes("{")) { mapped = yield* searchNames(query.replaceAll("{", ""), limit, kind, imageOnly); } - return { ...mapped, truncated: incompleteBeforeQuery || mapped.truncated }; + return mapped; }); const searchContents: WorkspaceSearchIndex["Service"]["searchContents"] = Effect.fn( @@ -615,7 +839,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( do { const remainingTimeBudgetMs = Math.max(1, Math.ceil(deadline - performance.now())); - const result = yield* runSearch(input.query, input.limit, "grep", () => + const { value: result } = yield* runSearch(input.query, input.limit, "grep", (finder) => finder.grep(searchQuery, { mode: regexMode ? "regex" : "plain", smartCase: !input.caseSensitive && !regexMode, @@ -692,5 +916,6 @@ export class WorkspaceSearchIndexMap extends LayerMap.Service(); + // The finders die with this process; the server closing the channel ends it. + process.on("disconnect", () => process.exit(0)); + process.on( + "message", + async (message: { + readonly id: number; + readonly finderId: number; + readonly method: "create" | "destroy" | Exclude; + readonly args: Array; + }) => { + try { + if (message.method === "create") { + const created = FileFinder.create(message.args[0] as InitOptions); + if (created.ok) finders.set(message.finderId, created.value); + process.send!({ id: message.id, value: created.ok ? { ok: true } : created }); + return; + } + const finder = finders.get(message.finderId)!; + if (message.method === "destroy") finders.delete(message.finderId); + const method = finder[message.method] as (...args: Array) => unknown; + process.send!({ id: message.id, value: await method.apply(finder, message.args) }); + } catch (error) { + process.send!({ id: message.id, error }); + } + }, + ); +}