From 12ec207b35067b8fe34f9addbc84604b1f93280d Mon Sep 17 00:00:00 2001 From: Lakshmi Tanmay Date: Fri, 6 Nov 2026 21:51:03 +0530 Subject: [PATCH 1/3] perf(server): skip non-image results in project icon searches --- .../workspace/WorkspaceSearchIndex.test.ts | 172 ++++++++++++++---- .../src/workspace/WorkspaceSearchIndex.ts | 65 +++++-- 2 files changed, 195 insertions(+), 42 deletions(-) diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index c01600842f2f6..2cc985edb32cf 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -8,6 +8,8 @@ import { import * as NodeServices from "@effect/platform-node/NodeServices"; import { afterEach, expect, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { PROJECT_FAVICON_EXTENSIONS, isProjectFaviconPath } from "@t3tools/shared/projectFavicon"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -36,39 +38,147 @@ function fileItem(relativePath: string): FileItem { }; } -it.effect("filters image searches before applying the result limit", () => - Effect.scoped( - Effect.gen(function* () { - const items = [ - ...Array.from({ length: 200 }, (_, index) => fileItem(`src/file-${index}.ts`)), - fileItem("public/icon.svg"), - ]; - const fileSearch = vi.fn(() => ({ - ok: true as const, - value: { - items, - scores: [], - totalMatched: items.length, - totalFiles: items.length, - }, - })); - const finder = { - destroy: vi.fn(), - waitForIndexReady: vi.fn(async () => ({ ok: true as const, value: true })), - fileSearch, - } as unknown as FileFinder; - vi.spyOn(FileFinder, "create").mockReturnValueOnce({ ok: true, value: finder }); +it.live("bounds native image results without losing matches, ranking, or query constraints", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-image-search-" }); + yield* fileSystem.makeDirectory(path.join(cwd, "assets")); + yield* fileSystem.makeDirectory(path.join(cwd, "other")); + yield* fileSystem.makeDirectory(path.join(cwd, "assetss")); + yield* fileSystem.makeDirectory(path.join(cwd, "assets/images")); + const imagePaths = PROJECT_FAVICON_EXTENSIONS.flatMap((extension) => [ + `assets/logo${extension}`, + `other/logo${extension.toUpperCase()}`, + ]); + for (const relativePath of [ + ...Array.from({ length: 220 }, (_, index) => `assets/source-${index}.ts`), + "assets/logo.ts", + "assets/logotype.png", + "assetss/logotype.png", + "assets/images/logo.PNG", + ...imagePaths, + ]) { + yield* fileSystem.writeFileString(path.join(cwd, relativePath), ""); + yield* fileSystem.utimes(path.join(cwd, relativePath), 0, 0); + } - const searchIndex = yield* WorkspaceSearchIndex.make("/workspace/project"); - const resultWithoutKind = yield* searchIndex.search("", 200, undefined, true); - const resultWithDirectoryKind = yield* searchIndex.search("", 200, "directory", true); + const createFinder = vi.spyOn(FileFinder, "create"); + const fileSearch = vi.spyOn(FileFinder.prototype, "fileSearch"); + const searchIndex = yield* WorkspaceSearchIndex.make(cwd); + const limited = yield* searchIndex.search("", 200, undefined, true); + expect(limited.entries).toHaveLength(imagePaths.length + 3); + expect(limited.truncated).toBe(false); + const nativeResult = fileSearch.mock.results[0]?.value; + expect(nativeResult.ok).toBe(true); + expect(nativeResult.value.items.length).toBeLessThanOrEqual(201); + const allImages = yield* searchIndex.search("", 1000, "file", true); + expect(allImages.entries.map((entry) => entry.path).toSorted()).toEqual( + [ + ...imagePaths, + "assets/logotype.png", + "assetss/logotype.png", + "assets/images/logo.PNG", + ].toSorted(), + ); + expect(allImages.truncated).toBe(false); - expect(resultWithoutKind.entries).toEqual([{ kind: "file", path: "public/icon.svg" }]); - expect(resultWithDirectoryKind.entries).toEqual([{ kind: "file", path: "public/icon.svg" }]); - expect(fileSearch).toHaveBeenCalledTimes(2); - expect(fileSearch).toHaveBeenCalledWith("", { pageSize: 25_002 }); - }), - ).pipe(Effect.provide(testLayer)), + for (const query of [ + "", + "logo", + "lgo", + "logo.png", + "*.png", + "*.ts", + "!*.png", + "assets/**", + "**/", + "!assets/", + "assets*/", + "/assets", + "assets/", + "logo:12", + "/assets/logotype.png:12", + "/assets/logotype.png", + "/assets/images", + "assets/images/", + "logo *.svg !other", + ]) { + const ordinary = yield* searchIndex.search(query, 1000, "file"); + const expected = ordinary.entries.filter((entry) => isProjectFaviconPath(entry.path)); + for (const limit of [1, expected.length || 1, 200]) { + const images = yield* searchIndex.search(query, limit, "directory", true); + expect(images, `query=${query} limit=${limit}`).toEqual({ + entries: expected.slice(0, limit), + truncated: expected.length > limit, + }); + } + } + yield* fileSystem.makeDirectory(path.join(cwd, "public")); + const tiedImagePaths = Array.from({ length: 220 }, (_, index) => `public/icon-${index}.png`); + for (const relativePath of [ + ...tiedImagePaths, + ...Array.from({ length: 220 }, (_, index) => `public/source-${index}.ts`), + ]) { + yield* fileSystem.writeFileString(path.join(cwd, relativePath), ""); + yield* fileSystem.utimes(path.join(cwd, relativePath), 0, 0); + } + yield* searchIndex.refresh(); + const tiedMatches = yield* searchIndex.search("public/", 1000, "file"); + const expectedTiedImages = tiedMatches.entries.filter((entry) => + isProjectFaviconPath(entry.path), + ); + expect(expectedTiedImages.map((entry) => entry.path).toSorted()).toEqual( + tiedImagePaths.toSorted(), + ); + for (const limit of [200, 220]) { + const images = yield* searchIndex.search("public/", limit, "file", true); + expect(images).toEqual({ + entries: expectedTiedImages.slice(0, limit), + truncated: limit < tiedImagePaths.length, + }); + expect(new Set(images.entries.map((entry) => entry.path)).size).toBe(images.entries.length); + } + const largeCwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-image-rank-" }); + yield* fileSystem.makeDirectory(path.join(largeCwd, "x/logo"), { recursive: true }); + yield* fileSystem.makeDirectory(path.join(largeCwd, "a")); + yield* fileSystem.makeDirectory(path.join(largeCwd, "b")); + for (let index = 0; index < 14_998; index += 1) { + yield* fileSystem.writeFileString(path.join(largeCwd, `b/logo-${index}.png`), ""); + } + yield* fileSystem.writeFileString(path.join(largeCwd, "x/logo/source.ts"), ""); + const pairPaths = ["a/xxlogo.png", "x/logo/logo.png"]; + for (const relativePath of pairPaths) { + yield* fileSystem.writeFileString(path.join(largeCwd, relativePath), ""); + } + const largeIndex = yield* WorkspaceSearchIndex.make(largeCwd); + const created: ReturnType | undefined = + createFinder.mock.results.at(-1)?.value; + if (!created?.ok) return yield* Effect.die("Missing native finder"); + const indexed = yield* Deferred.make(); + const newImagePath = path.join(largeCwd, "b/watched-logo.png"); + const subscription = created.value.watch("b/watched-logo.png", (events) => { + if (events.some((event) => event.path === newImagePath)) { + Deferred.doneUnsafe(indexed, Effect.void); + } + }); + if (!subscription.ok) return yield* Effect.die(subscription.error); + yield* Effect.addFinalizer(() => Effect.sync(subscription.value)); + for (const watched of [false, true]) { + if (watched) { + yield* fileSystem.writeFileString(newImagePath, ""); + yield* Deferred.await(indexed); + } + const ordinary = yield* largeIndex.search("logo", 20_000, "file"); + const images = yield* largeIndex.search("logo", 20_000, "file", true); + expect(images.entries).toHaveLength(watched ? 15_001 : 15_000); + expect(images.truncated).toBe(false); + expect(images.entries.filter((entry) => pairPaths.includes(entry.path))).toEqual( + ordinary.entries.filter((entry) => pairPaths.includes(entry.path)), + ); + } + expect((yield* searchIndex.search("", 1, "directory")).entries[0]?.kind).toBe("directory"); + }).pipe(Effect.scoped, Effect.provide(testLayer)), ); it.effect("fetches past a first page of Git-ignored results", () => diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.ts b/apps/server/src/workspace/WorkspaceSearchIndex.ts index 5d5d39bf55612..62277b7a54403 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.ts @@ -23,7 +23,7 @@ import type { ProjectSearchEntriesResult, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; -import { isProjectFaviconPath } from "@t3tools/shared/projectFavicon"; +import { PROJECT_FAVICON_EXTENSIONS, isProjectFaviconPath } from "@t3tools/shared/projectFavicon"; import * as VcsProcess from "../vcs/VcsProcess.ts"; @@ -31,6 +31,14 @@ import * as VcsProcess from "../vcs/VcsProcess.ts"; const requireForFff = NodeModule.createRequire(import.meta.url); const { FileFinder } = requireForFff("@ff-labs/fff-node") as typeof import("@ff-labs/fff-node"); +// A glob intersects existing extension constraints; fff ORs bare `*.ext` constraints together. +const IMAGE_SEARCH_CONSTRAINT = `**/*.{${PROJECT_FAVICON_EXTENSIONS.map((extension) => + extension.slice(1).replace(/[a-z]/g, (letter) => `[${letter}${letter.toUpperCase()}]`), +).join(",")}}`; + +// fff 0.11 disables filename fallback above this match count per base/overflow segment. +const FFF_FILENAME_FALLBACK_MAX_MATCHES = 15_000; + const WORKSPACE_INDEX_MAX_ENTRIES = 25_000; const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2; const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds"; @@ -157,6 +165,14 @@ function toDirectoryEntry(item: DirItem, hostIsWindows: boolean): ProjectEntry | return normalizedPath ? { path: normalizedPath, kind: "directory" } : null; } +function isLoneFuzzyPath(query: string): boolean { + const trimmed = query.trim(); + return ( + !/\s/u.test(trimmed) && + (trimmed.startsWith("/") || (trimmed.endsWith("/") && !/^[*!\\]/u.test(trimmed))) + ); +} + type SearchPage = { readonly items: ReadonlyArray; readonly totalMatched: number }; function mapFileSearchResult( @@ -461,13 +477,13 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( }); // Grows the first page until hidden entries can no longer crowd visible ones out of it. - const searchVisible = Effect.fn("WorkspaceSearchIndex.searchVisible")(function* ( + const searchVisible = Effect.fn("WorkspaceSearchIndex.searchVisible")(function* ( query: string, pageSize: number, operation: "directorySearch" | "fileSearch" | "mixedSearch", - execute: (pageSize: number) => Result>, + execute: (pageSize: number) => Result & Metadata>, pathOf: (item: I) => string, - ): Effect.fn.Return, WorkspaceSearchIndexSearchFailed> { + ): Effect.fn.Return & Metadata, WorkspaceSearchIndexSearchFailed> { let size = pageSize; while (true) { const fetchSize = size; @@ -482,6 +498,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( fetchSize >= WORKSPACE_INDEX_PAGE_SIZE ) { return { + ...result, items, totalMatched: result.totalMatched - (result.items.length - items.length), }; @@ -553,16 +570,42 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( kind: ProjectEntryKind | undefined, imageOnly: boolean | undefined, ) { - const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1); + const pageSize = Math.max(1, limit + 1); if (kind === "file" || imageOnly) { - const result = yield* searchVisible( + // Adding a constraint changes fff's lone-path parse; offset pages can skip tied matches. + let postFilterImages = imageOnly && isLoneFuzzyPath(query); + const nativePageSize = postFilterImages ? WORKSPACE_INDEX_PAGE_SIZE : pageSize; + const nativeQuery = + imageOnly && !postFilterImages ? `${query} ${IMAGE_SEARCH_CONSTRAINT}` : query; + let result = yield* searchVisible( query, - pageSize, + nativePageSize, "fileSearch", - (size) => finder.fileSearch(query, { pageSize: size }), - (item) => item.relativePath, + (size) => finder.fileSearch(nativeQuery, { pageSize: size }), + (item: FileItem) => item.relativePath, ); - return mapFileSearchResult(result, limit, hostIsWindows, imageOnly); + if ( + imageOnly && + !postFilterImages && + query.trim() && + result.totalFiles > FFF_FILENAME_FALLBACK_MAX_MATCHES && + result.totalMatched > 0 + ) { + const unfiltered = yield* runSearch(query, 1, "fileSearch", () => + finder.fileSearch(query, { pageSize: 1 }), + ); + if (unfiltered.totalMatched > FFF_FILENAME_FALLBACK_MAX_MATCHES) { + postFilterImages = true; + result = yield* searchVisible( + query, + WORKSPACE_INDEX_PAGE_SIZE, + "fileSearch", + (size) => finder.fileSearch(query, { pageSize: size }), + (item: FileItem) => item.relativePath, + ); + } + } + return mapFileSearchResult(result, limit, hostIsWindows, postFilterImages); } if (kind === "directory") { const result = yield* searchVisible( @@ -570,7 +613,7 @@ export const make = Effect.fn("WorkspaceSearchIndex.make")(function* ( pageSize, "directorySearch", (size) => finder.directorySearch(query, { pageSize: size }), - (item) => item.relativePath, + (item: DirItem) => item.relativePath, ); return mapDirectorySearchResult(result, limit, hostIsWindows); } From 29f2ff957a60e2f504412c2557617e97cb208a32 Mon Sep 17 00:00:00 2001 From: Lakshmi Tanmay Date: Fri, 6 Nov 2026 22:18:13 +0530 Subject: [PATCH 2/3] fix(server): bound image search compatibility results --- .../workspace/WorkspaceSearchIndex.test.ts | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index 2cc985edb32cf..0bd1ea5cd4d50 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -47,10 +47,19 @@ it.live("bounds native image results without losing matches, ranking, or query c yield* fileSystem.makeDirectory(path.join(cwd, "other")); yield* fileSystem.makeDirectory(path.join(cwd, "assetss")); yield* fileSystem.makeDirectory(path.join(cwd, "assets/images")); - const imagePaths = PROJECT_FAVICON_EXTENSIONS.flatMap((extension) => [ - `assets/logo${extension}`, - `other/logo${extension.toUpperCase()}`, - ]); + const vcsProcess = yield* VcsProcess.VcsProcess; + yield* vcsProcess.run({ operation: "test.gitInit", command: "git", args: ["init"], cwd }); + yield* fileSystem.writeFileString(path.join(cwd, ".git/info/exclude"), "hidden/\n"); + yield* fileSystem.makeDirectory(path.join(cwd, "hidden")); + yield* fileSystem.writeFileString(path.join(cwd, "hidden/logo.png"), ""); + const imagePaths = [ + ...PROJECT_FAVICON_EXTENSIONS.flatMap((extension) => [ + `assets/logo${extension}`, + `other/logo${extension.toUpperCase()}`, + ]), + "assets/foo{bar}.png", + "assets/[id].png", + ]; for (const relativePath of [ ...Array.from({ length: 220 }, (_, index) => `assets/source-${index}.ts`), "assets/logo.ts", @@ -103,6 +112,11 @@ it.live("bounds native image results without losing matches, ranking, or query c "/assets/images", "assets/images/", "logo *.svg !other", + "foo{bar}.png", + "[id].png", + "*.{png,svg}", + "hidden/", + "/hidden/", ]) { const ordinary = yield* searchIndex.search(query, 1000, "file"); const expected = ordinary.entries.filter((entry) => isProjectFaviconPath(entry.path)); @@ -141,6 +155,14 @@ it.live("bounds native image results without losing matches, ranking, or query c } const largeCwd = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-image-rank-" }); yield* fileSystem.makeDirectory(path.join(largeCwd, "x/logo"), { recursive: true }); + yield* vcsProcess.run({ + operation: "test.gitInit", + command: "git", + args: ["init"], + cwd: largeCwd, + }); + yield* fileSystem.writeFileString(path.join(largeCwd, ".git/info/exclude"), "hidden.txt\n"); + yield* fileSystem.writeFileString(path.join(largeCwd, "hidden.txt"), ""); yield* fileSystem.makeDirectory(path.join(largeCwd, "a")); yield* fileSystem.makeDirectory(path.join(largeCwd, "b")); for (let index = 0; index < 14_998; index += 1) { @@ -178,6 +200,28 @@ it.live("bounds native image results without losing matches, ranking, or query c ); } expect((yield* searchIndex.search("", 1, "directory")).entries[0]?.kind).toBe("directory"); + let materialized = 0; + fileSearch.mockImplementation((_query, options) => { + const count = Math.min(options?.pageSize ?? 100, 30_000); + materialized += count; + return { + ok: true, + value: { + ...nativeResult.value, + totalFiles: 30_000, + totalMatched: 30_000, + items: Array.from({ length: count }, (_, index) => ({ + ...nativeResult.value.items[0], + relativePath: index === 29_999 ? "assets/late.png" : `assets/source-${index}.ts`, + })), + }, + }; + }); + expect(yield* searchIndex.search("assets/", 200, "file", true)).toEqual({ + entries: [], + truncated: true, + }); + expect(materialized).toBeLessThanOrEqual(25_002); }).pipe(Effect.scoped, Effect.provide(testLayer)), ); From 780960d4b9599625ce9c0ea9d6600974e7d28be2 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 15 Nov 2026 03:41:50 +0000 Subject: [PATCH 3/3] test(server): type the native image search result Co-Authored-By: Claude Opus 5.5 --- apps/server/src/workspace/WorkspaceSearchIndex.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts index 418aff4447b32..6c4e873f12b47 100644 --- a/apps/server/src/workspace/WorkspaceSearchIndex.test.ts +++ b/apps/server/src/workspace/WorkspaceSearchIndex.test.ts @@ -77,7 +77,10 @@ it.live("image searches match the image subset of a file search without fetching const fileSearch = vi.spyOn(FileFinder.prototype, "fileSearchAsync"); const searchIndex = yield* WorkspaceSearchIndex.make(cwd); yield* searchIndex.search("", 200, undefined, true); - const nativeResult = yield* Effect.promise(() => fileSearch.mock.results[0]?.value); + const nativeResult = yield* Effect.promise( + (): ReturnType => fileSearch.mock.results[0]?.value, + ); + if (!nativeResult.ok) return yield* Effect.die(nativeResult.error); expect(nativeResult.value.items.length).toBeLessThanOrEqual(201); // The image glob must not change fff's parse of fuzzy paths, globs, negations or locations.