fix(web): find ignored files beyond the clone picker limit

apps/server/src/environment/ServerEnvironment.test.ts

@@ -113,6 +113,8 @@ it.layer(NodeServices.layer)("ServerEnvironmentLive", (it) => {
           mode === "web" ? undefined : "desktop-managed",
         );
         expect(descriptor.capabilities).toMatchObject({
+          ignoredFileListing: true,
+          ignoredFileBrowsing: true,
           settledWorktreeCleanup: true,
           storageCleanupRun: true,
           projectWorktreeCleanup: true,

apps/server/src/environment/ServerEnvironment.ts

@@ -258,6 +258,7 @@ export const make = Effect.gen(function* () {
       environmentIcon: true,
       projectCloneTracking: true,
       ignoredFileListing: true,
+      ignoredFileBrowsing: true,
       ...(serverSelfUpdate === null ? {} : { serverSelfUpdate }),
       ...(serverInstallation === null ? {} : { serverInstallation }),
       ...(serverSelfUpdate === "boot-service" || desktopAppUpdate

apps/server/src/workspace/WorkspaceEntries.test.ts

@@ -6,13 +6,20 @@ import * as FileSystem from "effect/FileSystem";
 import * as Layer from "effect/Layer";
 import * as Path from "effect/Path";
 import * as PlatformError from "effect/PlatformError";
+import { ChildProcessSpawner } from "effect/process";
 import { vi } from "vite-plus/test";
 
 import * as ServerConfig from "../config.ts";
 import * as HostProcess from "@t3tools/shared/HostProcess";
+import {
+  type ProjectListIgnoredFilesResult,
+  VcsProcessSpawnError,
+  VcsProcessTimeoutError,
+} from "@t3tools/contracts";
 import * as VcsProcess from "../vcs/VcsProcess.ts";
 import * as WorkspaceEntries from "./WorkspaceEntries.ts";
 import * as WorkspacePaths from "./WorkspacePaths.ts";
+import * as WorkspaceSearchIndex from "./WorkspaceSearchIndex.ts";
 
 vi.mock("node:fs/promises", async (importOriginal) => {
   const actual = await importOriginal<typeof import("node:fs/promises")>();
@@ -285,6 +292,199 @@ it.layer(layerTest, { excludeTestServices: true })("WorkspaceEntries", (it) => {
   });
 
   describe("listIgnoredFiles", () => {
+    it.effect("browses every captured ignored path in bounded pages without duplicates", () =>
+      Effect.gen(function* () {
+        const cwd = yield* makeTempDir({ git: true });
+        const fileSystem = yield* FileSystem.FileSystem;
+        yield* writeTextFile(cwd, ".gitignore", "*.log\n.env*\nnode_modules/\n");
+        yield* writeTextFile(cwd, ".agents/keep.txt", "tracked");
+        yield* writeTextFile(cwd, "services/api/index.ts", "tracked");
+        yield* git(cwd, ["add", ".agents/keep.txt", "services/api/index.ts"]);
+        const logs = Array.from(
+          { length: 605 },
+          (_, i) => `.agents/run-${String(i).padStart(4, "0")}.log`,
+        );
+        yield* Effect.forEach(logs, (name) => writeTextFile(cwd, name, "x"));
+        yield* writeTextFile(cwd, "services/api/.env.local", "A=1");
+        yield* writeTextFile(cwd, "node_modules/excluded.log", "excluded");
+
+        const entries = yield* WorkspaceEntries.WorkspaceEntries;
+        const seen: string[] = [];
+        let after: string | undefined;
+        let pageCount = 0;
+        do {
+          const page: ProjectListIgnoredFilesResult = yield* entries.listIgnoredFiles({
+            cwd,
+            ...(after ? { after } : {}),
+          });
+          expect(page.files.length).toBeLessThanOrEqual(200);
+          expect(page.truncated).toBe(page.nextCursor !== undefined);
+          expect(page.outputTruncated).not.toBe(true);
+          if (pageCount === 0) {
+            expect(page.files[0]).toEqual({ path: "services/api/.env.local", byteLength: 3 });
+            expect(page.nextCursor).toBe(".agents/run-0198.log");
+            yield* fileSystem.remove(`${cwd}/.agents/run-0198.log`);
+          }
+          seen.push(...page.files.map((file) => file.path));
+          after = page.nextCursor;
+          pageCount++;
+          expect(pageCount).toBeLessThanOrEqual(4);
+        } while (after !== undefined);
+        expect(pageCount).toBe(4);
+        expect(new Set(seen).size).toBe(seen.length);
+        expect(seen.toSorted()).toEqual(["services/api/.env.local", ...logs].toSorted());
+
+        const matches: string[] = [];
+        after = undefined;
+        let filteredPages = 0;
+        do {
+          const page: ProjectListIgnoredFilesResult = yield* entries.listIgnoredFiles({
+            cwd,
+            query: "  .AGENTS/RUN-0  ",
+            ...(after ? { after } : {}),
+          });
+          expect(page.files.length).toBeLessThanOrEqual(200);
+          matches.push(...page.files.map((file) => file.path));
+          after = page.nextCursor;
+          filteredPages++;
+          expect(filteredPages).toBeLessThanOrEqual(4);
+        } while (after !== undefined);
+        expect(filteredPages).toBe(4);
+        expect(new Set(matches).size).toBe(matches.length);
+        expect(matches).toEqual(logs.filter((name) => name !== ".agents/run-0198.log"));
+      }),
+    );
+
+    it.effect("advances past an empty candidate page and a removed or symlink cursor", () =>
+      Effect.gen(function* () {
+        const cwd = yield* makeTempDir();
+        const fileSystem = yield* FileSystem.FileSystem;
+        const missing = Array.from({ length: 199 }, (_, i) => `a-${String(i).padStart(3, "0")}`);
+        yield* writeTextFile(cwd, "z-file", "last");
+        yield* fileSystem.symlink(`${cwd}/z-file`, `${cwd}/a-link`);
+        const entries = yield* WorkspaceEntries.make.pipe(
+          Effect.provideService(
+            VcsProcess.VcsProcess,
+            VcsProcess.VcsProcess.of({
+              run: () =>
+                Effect.succeed({
+                  exitCode: ChildProcessSpawner.ExitCode(0),
+                  stdout: [...missing, "a-link", "z-file"].join("\0") + "\0",
+                  stderr: "",
+                  stdoutTruncated: false,
+                  stderrTruncated: false,
+                }),
+            }),
+          ),
+          Effect.provide(WorkspaceSearchIndex.WorkspaceSearchIndexMap.layer),
+        );
+        const first = yield* entries.listIgnoredFiles({ cwd });
+        expect(first).toEqual({ files: [], truncated: true, nextCursor: "a-link" });
+        yield* fileSystem.remove(`${cwd}/a-link`);
+        expect(yield* entries.listIgnoredFiles({ cwd, after: first.nextCursor })).toEqual({
+          files: [{ path: "z-file", byteLength: 4 }],
+          truncated: false,
+        });
+      }),
+    );
+
+    it.effect("drops only the incomplete trailing record when Git output is truncated", () =>
+      Effect.gen(function* () {
+        const cwd = yield* makeTempDir();
+        yield* writeTextFile(cwd, ".env", "A=1");
+        yield* writeTextFile(cwd, "partial", "must not be listed");
+        for (const stdout of [".env\0partial", ".env\0"]) {
+          const entries = yield* WorkspaceEntries.make.pipe(
+            Effect.provideService(
+              VcsProcess.VcsProcess,
+              VcsProcess.VcsProcess.of({
+                run: () =>
+                  Effect.succeed({
+                    exitCode: ChildProcessSpawner.ExitCode(0),
+                    stdout,
+                    stderr: "",
+                    stdoutTruncated: true,
+                    stderrTruncated: false,
+                  }),
+              }),
+            ),
+            Effect.provide(WorkspaceSearchIndex.WorkspaceSearchIndexMap.layer),
+          );
+          expect(yield* entries.listIgnoredFiles({ cwd })).toEqual({
+            files: [{ path: ".env", byteLength: 3 }],
+            truncated: false,
+            outputTruncated: true,
+          });
+        }
+      }),
+    );
+
+    it.effect("preserves Git timeout and spawn failures as typed listing errors", () =>
+      Effect.gen(function* () {
+        const cwd = yield* makeTempDir();
+        const context = { cwd, command: "git", operation: "WorkspaceEntries.listIgnoredFiles" };
+        for (const cause of [
+          new VcsProcessSpawnError({ ...context, cause: new Error("git unavailable") }),
+          new VcsProcessTimeoutError({ ...context, timeoutMs: 10_000 }),
+        ]) {
+          const entries = yield* WorkspaceEntries.make.pipe(
+            Effect.provideService(
+              VcsProcess.VcsProcess,
+              VcsProcess.VcsProcess.of({
+                run: () => Effect.fail(cause),
+              }),
+            ),
+            Effect.provide(WorkspaceSearchIndex.WorkspaceSearchIndexMap.layer),
+          );
+          const error = yield* entries.listIgnoredFiles({ cwd }).pipe(Effect.flip);
+          expect(error).toMatchObject({ _tag: "WorkspaceEntriesListIgnoredFilesError", cause });
+        }
+      }),
+    );
+
+    it.effect("returns a typed failure when Git exits unsuccessfully", () =>
+      Effect.gen(function* () {
+        const cwd = yield* makeTempDir();
+        const entries = yield* WorkspaceEntries.WorkspaceEntries;
+        const error = yield* entries.listIgnoredFiles({ cwd }).pipe(Effect.flip);
+        expect(error._tag).toBe("WorkspaceEntriesListIgnoredFilesError");
+        if (error._tag === "WorkspaceEntriesListIgnoredFilesError") {
+          expect(error.cause).toMatchObject({ _tag: "VcsProcessExitError" });
+        }
+      }),
+    );
+
+    it.effect("searches all ignored paths before limiting results", () =>
+      Effect.gen(function* () {
+        const cwd = yield* makeTempDir({ git: true });
+        yield* writeTextFile(cwd, ".gitignore", "*.log\nnode_modules/\n");
+        yield* writeTextFile(cwd, "data/keep.txt", "tracked");
+        yield* git(cwd, ["add", "data/keep.txt"]);
+        yield* Effect.forEach(
+          Array.from({ length: 205 }, (_, i) => i),
+          (i) => writeTextFile(cwd, `data/generated-${String(i).padStart(3, "0")}.log`, "x"),
+        );
+        yield* writeTextFile(cwd, "data/z-Target.log", "found");
+        yield* writeTextFile(cwd, "node_modules/z-target.log", "excluded");
+
+        const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries;
+        expect(
+          yield* workspaceEntries.listIgnoredFiles({ cwd, query: "  DaTa/Z-TaRgEt.LoG  " }),
+        ).toEqual({
+          files: [{ path: "data/z-Target.log", byteLength: 5 }],
+          truncated: false,
+        });
+        const unfiltered = yield* workspaceEntries.listIgnoredFiles({ cwd });
+        expect(unfiltered.files.map((file) => file.path)).not.toContain("data/z-Target.log");
+        expect(yield* workspaceEntries.listIgnoredFiles({ cwd, query: "" })).toEqual(unfiltered);
+        expect(yield* workspaceEntries.listIgnoredFiles({ cwd, query: "   " })).toEqual(unfiltered);
+        expect(yield* workspaceEntries.listIgnoredFiles({ cwd, query: "node_modules" })).toEqual({
+          files: [],
+          truncated: false,
+        });
+      }),
+    );
+
     it.effect("lists ignored files with sizes and skips wholly ignored folders", () =>
       Effect.gen(function* () {
         const cwd = yield* makeTempDir({ git: true });

apps/server/src/workspace/WorkspaceEntries.ts

@@ -8,18 +8,20 @@ import * as Path from "effect/Path";
 import * as RcMap from "effect/RcMap";
 import * as Schema from "effect/Schema";
 
-import type {
-  FilesystemBrowseInput,
-  FilesystemBrowseResult,
-  ProjectEntry,
-  ProjectListEntriesInput,
-  ProjectListEntriesResult,
-  ProjectListIgnoredFilesInput,
-  ProjectListIgnoredFilesResult,
-  ProjectSearchContentsInput,
-  ProjectSearchContentsResult,
-  ProjectSearchEntriesInput,
-  ProjectSearchEntriesResult,
+import {
+  PROJECT_IGNORED_FILES_LIMIT,
+  isDefaultCopiedFile,
+  type FilesystemBrowseInput,
+  type FilesystemBrowseResult,
+  type ProjectEntry,
+  type ProjectListEntriesInput,
+  type ProjectListEntriesResult,
+  type ProjectListIgnoredFilesInput,
+  type ProjectListIgnoredFilesResult,
+  type ProjectSearchContentsInput,
+  type ProjectSearchContentsResult,
+  type ProjectSearchEntriesInput,
+  type ProjectSearchEntriesResult,
 } from "@t3tools/contracts";
 import * as HostProcess from "@t3tools/shared/HostProcess";
 import { isExplicitRelativePath, isWindowsAbsolutePath } from "@t3tools/shared/path";
@@ -70,6 +72,18 @@ export class WorkspaceEntriesReadDirectoryError extends Schema.TaggedError<Works
   }
 }
 
+export class WorkspaceEntriesListIgnoredFilesError extends Schema.TaggedError<WorkspaceEntriesListIgnoredFilesError>()(
+  "WorkspaceEntriesListIgnoredFilesError",
+  {
+    cwd: Schema.String,
+    cause: Schema.Defect(),
+  },
+) {
+  override get message(): string {
+    return "Failed to list ignored files.";
+  }
+}
+
 export const WorkspaceEntriesBrowseError = Schema.Union([
   WorkspaceEntriesWindowsPathUnsupportedError,
   WorkspaceEntriesCurrentProjectRequiredError,
@@ -78,6 +92,7 @@ export const WorkspaceEntriesBrowseError = Schema.Union([
 export type WorkspaceEntriesBrowseError = typeof WorkspaceEntriesBrowseError.Type;
 
 export const WorkspaceEntriesError = Schema.Union([
+  WorkspaceEntriesListIgnoredFilesError,
   WorkspaceEntriesReadDirectoryError,
   WorkspacePaths.WorkspaceRootNotExistsError,
   WorkspacePaths.WorkspaceRootCreateFailedError,
@@ -111,9 +126,6 @@ export class WorkspaceEntries extends Context.Service<
   }
 >()("t3/workspace/WorkspaceEntries") {}
 
-/** Ignored files offered for copying; more than this is a build folder, not config. */
-const IGNORED_FILES_LIMIT = 200;
-
 /** Paths sent to one `git check-ignore` call when labelling a folder's children. */
 const IGNORE_CHECK_CHUNK_SIZE = 1000;
 
@@ -412,19 +424,39 @@ export const make = Effect.gen(function* () {
           "-z",
         ],
         cwd: normalizedCwd,
-        allowNonZeroExit: true,
         timeoutMs: 10_000,
         maxOutputBytes: 4 * 1024 * 1024,
       })
-      .pipe(Effect.orElseSucceed(() => undefined));
-    if (!result || result.exitCode !== 0) return { files: [], truncated: false };
-    const paths = result.stdout
+      .pipe(
+        Effect.mapError(
+          (cause) => new WorkspaceEntriesListIgnoredFilesError({ cwd: normalizedCwd, cause }),
+        ),
+      );
+    const stdout = result.stdoutTruncated
+      ? result.stdout.slice(0, result.stdout.lastIndexOf("\0") + 1)
+      : result.stdout;
+    const query = input.query?.trim().toLowerCase() ?? "";
+    const paths = stdout
       .split("\0")
-      .filter((entry) => entry.length > 0 && !entry.endsWith("/"))
+      .filter(
+        (entry) => entry.length > 0 && !entry.endsWith("/") && entry.toLowerCase().includes(query),
+      )
       .toSorted();
+    const after = input.after;
+    const afterIsDefault = after !== undefined && isDefaultCopiedFile(after);
+    const prioritizedPaths = [
+      ...paths.filter(isDefaultCopiedFile),
+      ...paths.filter((entry) => !isDefaultCopiedFile(entry)),
+    ].filter((entry) => {
+      if (after === undefined) return true;
+      const isDefault = isDefaultCopiedFile(entry);
+      return isDefault === afterIsDefault ? entry > after : afterIsDefault;
+    });
+    const candidates = prioritizedPaths.slice(0, PROJECT_IGNORED_FILES_LIMIT);
+    const truncated = prioritizedPaths.length > PROJECT_IGNORED_FILES_LIMIT;
     const files = yield* Effect.promise(() =>
       Promise.all(
-        paths.slice(0, IGNORED_FILES_LIMIT).map(async (relativePath) => {
+        candidates.map(async (relativePath) => {
           // lstat: a symlink usually points outside the workspace, where reads refuse to go.
           const stat = await NodeFSP.lstat(path.join(normalizedCwd, relativePath)).catch(
             () => null,
@@ -435,7 +467,9 @@ export const make = Effect.gen(function* () {
     );
     return {
       files: files.filter((file) => file !== null),
-      truncated: paths.length > IGNORED_FILES_LIMIT,
+      truncated,
+      ...(truncated ? { nextCursor: candidates.at(-1) } : {}),
+      ...(result.stdoutTruncated ? { outputTruncated: true } : {}),
     };
   });