From 931572feb094f99634c196aa95d0dd9e46293936 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Sat, 14 Nov 2026 23:12:28 +0200 Subject: [PATCH] fix(web): opening a very large folder no longer freezes the Files panel Scient's lazy Files tree applied each listing to the tree as one batch, and the tree reworks a folder's children for every row added or removed: opening a 50,000-entry folder blocked the panel for seconds, and a refresh after most of it was deleted for about 15 s. When the batch would cost more than a rebuild (T3's measured break-even, as in its #29996), the tree is rebuilt from the known entries and then given one real change as a small batch, so it still clears a drag, remaps rename, focus and selection, and refreshes an open search as a batch does. When entries changed kind, a first rebuild without either form ends a rename or selection on them, so neither moves to the replacement. Rows the tree shows before the disk has them, such as a new entry being named, are tracked and kept unless their folder is removed. Removed entries are forgotten in one pass instead of one scan each. With a search open, a large listing that removes matching rows can show the old matches for one update, which may scroll the results to the top; opening a folder only adds rows and keeps the scroll. Co-Authored-By: Claude Opus 5.5 --- .../files/FileBrowserPanel.browser.test.tsx | 53 +++- .../files/LazyWorkspaceTreeController.test.ts | 229 ++++++++++++++++++ .../files/LazyWorkspaceTreeController.ts | 102 +++++++- .../src/scient/files/useScientFileActions.tsx | 2 +- 4 files changed, 375 insertions(+), 11 deletions(-) diff --git a/apps/web/src/components/files/FileBrowserPanel.browser.test.tsx b/apps/web/src/components/files/FileBrowserPanel.browser.test.tsx index 9d8bac44b8..1bd9017aea 100644 --- a/apps/web/src/components/files/FileBrowserPanel.browser.test.tsx +++ b/apps/web/src/components/files/FileBrowserPanel.browser.test.tsx @@ -120,13 +120,15 @@ describe("Files panel search", () => { workspaceMutationId: null, }; - async function renderPanel() { + async function renderPanel({ cwd = props.cwd }: { readonly cwd?: string } = {}) { const container = document.createElement("div"); // The tree virtualizes its rows, so it needs a real viewport to render any. container.style.cssText = "display: flex; flex-direction: column; width: 320px; height: 480px;"; await using stack = new AsyncDisposableStack(); - const screen = stack.use(await render(, { container })); - const rerender = () => screen.rerender(); + const screen = stack.use( + await render(, { container }), + ); + const rerender = () => screen.rerender(); function searchInput() { const input = container!.querySelector('input[type="search"]'); expect(input).not.toBeNull(); @@ -353,6 +355,51 @@ describe("Files panel search", () => { await chooseFromFilesMenu(screen.container, "Collapse all folders"); await expect.poll(screen.visiblePaths).toEqual(["src/", "package.json"]); }); + describe("a very large folder", () => { + const LARGE_FOLDER_SIZE = 50_000; + // Out of sort order, because inserting in order hides the cost of each insert. + const largeFolder = () => + Array.from({ length: LARGE_FOLDER_SIZE }, (_, index) => + entry(`big/f${String((index * 7_919) % LARGE_FOLDER_SIZE).padStart(5, "0")}.txt`), + ); + const treeRow = (container: HTMLElement, path: string) => + container + .querySelector("file-tree-container") + ?.shadowRoot?.querySelector(`[data-item-path="${CSS.escape(path)}"]`) ?? null; + + it("opens without freezing and keeps the folder that was open", async () => { + setup(); + files.directories[""] = [entry("big", "directory"), ...files.directories[""]!]; + files.directories.big = largeFolder(); + // Its own folder, so the folders it opens are not remembered for the other tests. + await using screen = await renderPanel({ cwd: "/large-folder-open" }); + await expect.poll(screen.visiblePaths).toEqual(["big/", "src/", "package.json"]); + treeRow(screen.container, "src/")!.click(); + await expect.poll(screen.visiblePaths).toContain("src/app.ts"); + + treeRow(screen.container, "big/")!.click(); + // Batching every row blocked the panel past this wait; the rebuild takes about 170 ms. + await expect.poll(screen.visiblePaths).toContain("big/f00000.txt"); + expect(treeRow(screen.container, "big/")!.getAttribute("aria-expanded")).toBe("true"); + const src = treeRow(screen.container, "src/"); + if (src) expect(src.getAttribute("aria-expanded")).toBe("true"); + }); + + it("keeps an open search accurate when the folder's listing arrives", async () => { + setup(); + files.directories[""] = [entry("big", "directory"), ...files.directories[""]!]; + files.directories.big = largeFolder(); + await using screen = await renderPanel({ cwd: "/large-folder-search" }); + await expect.poll(screen.visiblePaths).toContain("big/"); + files.searchEntries = [{ path: "big/f49999.txt", kind: "file" }]; + await screen.search("f49999"); + await expect.poll(screen.visiblePaths).toEqual(["big/", "big/f49999.txt"]); + expect(screen.searchInput().value).toBe("f49999"); + await screen.search(""); + await expect.poll(screen.visiblePaths).toEqual(["big/", "src/", "package.json"]); + }); + }); + it("copies with the pointer and returns to the row after keyboard dismissal", async () => { setup(); files.directories = { "": [entry("README.md")] }; diff --git a/apps/web/src/scient/files/LazyWorkspaceTreeController.test.ts b/apps/web/src/scient/files/LazyWorkspaceTreeController.test.ts index dfad64c09e..913c633724 100644 --- a/apps/web/src/scient/files/LazyWorkspaceTreeController.test.ts +++ b/apps/web/src/scient/files/LazyWorkspaceTreeController.test.ts @@ -531,4 +531,233 @@ describe("LazyWorkspaceTreeController", () => { expect(harness.snapshot()?.rootError).toContain("incomplete directory listing"); harness.destroy(); }); + + describe("very large folders", () => { + const bigFolder = (count: number) => + Array.from({ length: count }, (_, index) => + file(`big/f${String(index).padStart(5, "0")}.txt`), + ); + const expandedPaths = (model: FileTree, paths: readonly string[]) => + paths.filter((path) => { + const item = model.getItem(path); + return item !== null && "isExpanded" in item && item.isExpanded(); + }); + const expand = (model: FileTree, path: string) => { + const item = model.getItem(path); + if (item && "expand" in item) item.expand(); + }; + + it("rebuilds the tree for a large listing, keeping what it shows open, selected and new", async () => { + let bigEntries = bigFolder(3_000); + const harness = makeController(async (relativeDirectory) => { + switch (relativeDirectory) { + case "": + return complete( + directory("big"), + directory("docs"), + directory("src"), + file("README.md"), + ); + case "src": + return complete(directory("src/lib"), file("src/a.ts")); + case "docs": + return complete(file("docs/guide.md")); + default: + return complete(...bigEntries); + } + }); + await harness.controller.start(); + for (const path of ["src", "docs"]) { + expand(harness.model, path); + await harness.controller.load(path); + } + harness.model.getItem("src/a.ts")?.select(); + // New rows the disk does not have yet, one of them hidden in a closed folder. + harness.controller.addProvisional("src/untitled.md"); + harness.controller.addProvisional("docs/untitled folder/"); + const docs = harness.model.getItem("docs/"); + if (docs && "collapse" in docs) docs.collapse(); + const resetPaths = vi.spyOn(harness.model, "resetPaths"); + const batch = vi.spyOn(harness.model, "batch"); + + expand(harness.model, "big"); + await harness.controller.load("big"); + + expect(resetPaths).toHaveBeenCalledTimes(1); + // One real change follows the rebuild, so the tree updates what depends on changes. + expect(batch).toHaveBeenCalledTimes(1); + expect(batch.mock.calls[0]?.[0]).toHaveLength(1); + expect(harness.model.getItem("big/f02999.txt")).not.toBeNull(); + expect(harness.model.getItem("src/untitled.md")).not.toBeNull(); + expect(harness.model.getItem("docs/untitled folder/")).not.toBeNull(); + expect(harness.model.getItem("src/lib/")).not.toBeNull(); + expect(expandedPaths(harness.model, ["big/", "docs/", "src/", "src/lib/"])).toEqual([ + "big/", + "src/", + ]); + expect(harness.model.getSelectedPaths()).toEqual(["src/a.ts"]); + // big, its files, docs, src with its three rows, and the README. + expect(harness.model.getVisibleCount()).toBe(1 + 3_000 + 1 + 4 + 1); + expect(harness.snapshot()?.entries.size).toBe(4 + 2 + 1 + 3_000); + + // Most of the folder is deleted: the refresh rebuilds again and drops the rows. + bigEntries = bigEntries.slice(0, 10); + await harness.controller.refresh(); + expect(resetPaths).toHaveBeenCalledTimes(2); + expect(harness.model.getItem("big/f00009.txt")).not.toBeNull(); + expect(harness.model.getItem("big/f00010.txt")).toBeNull(); + expect(harness.model.getItem("src/untitled.md")).not.toBeNull(); + expect(harness.model.getVisibleCount()).toBe(1 + 10 + 1 + 4 + 1); + expect(harness.snapshot()?.entries.size).toBe(4 + 2 + 1 + 10); + harness.destroy(); + }); + + it("drops the selection of an entry that changed kind but keeps its focus, as a batch does", async () => { + let bigEntries = bigFolder(3_000); + const harness = makeController(async (relativeDirectory) => + relativeDirectory === "" ? complete(directory("big")) : complete(...bigEntries), + ); + await harness.controller.start(); + expand(harness.model, "big"); + await harness.controller.load("big"); + harness.model.getItem("big/f00001.txt")?.select(); + harness.model.focusPath("big/f00001.txt"); + const resetPaths = vi.spyOn(harness.model, "resetPaths"); + + // f00001.txt is now a folder, and the rest of the folder changed with it. + bigEntries = [ + directory("big/f00001.txt"), + ...bigFolder(3_000) + .slice(2) + .map((entry) => file(entry.relativePath.replace("/f", "/g"))), + ]; + await harness.controller.refresh(); + + // A first rebuild without either form of the changed entry ends its selection. + expect(resetPaths).toHaveBeenCalledTimes(2); + expect(harness.model.getItem("big/f00001.txt/")).not.toBeNull(); + expect(harness.model.getItem("big/g02999.txt")).not.toBeNull(); + expect(harness.model.getItem("big/f00002.txt")).toBeNull(); + expect(harness.model.getSelectedPaths()).toEqual([]); + expect(harness.model.getFocusedPath()).toBe("big/f00001.txt/"); + harness.destroy(); + }); + + it("keeps an open search, and what it restores, through a large folder's rebuild", async () => { + const harness = makeController(async (relativeDirectory) => + relativeDirectory === "" + ? complete(directory("big"), file("README.md")) + : complete(...bigFolder(3_000)), + ); + await harness.controller.start(); + const resetPaths = vi.spyOn(harness.model, "resetPaths"); + harness.model.setSearch("f0299"); + + await harness.controller.primePaths(["big/f02999.txt"]); + + expect(resetPaths).toHaveBeenCalledTimes(1); + expect(harness.model.isSearchOpen()).toBe(true); + expect(harness.model.getSearchMatchingPaths()).toEqual( + Array.from({ length: 10 }, (_, index) => `big/f0299${index}.txt`), + ); + harness.model.closeSearch(); + expect(expandedPaths(harness.model, ["big/"])).toEqual([]); + harness.destroy(); + }); + + it("drops a new row with the folder a large listing removes", async () => { + let bigEntries = [directory("big/gone"), ...bigFolder(3_000)]; + const harness = makeController(async (relativeDirectory) => { + if (relativeDirectory === "") return complete(directory("big")); + if (relativeDirectory === "big/gone") return complete(file("big/gone/kept.txt")); + return complete(...bigEntries); + }); + await harness.controller.start(); + expand(harness.model, "big"); + await harness.controller.load("big"); + expand(harness.model, "big/gone"); + await harness.controller.load("big/gone"); + harness.controller.addProvisional("big/gone/untitled.txt"); + const resetPaths = vi.spyOn(harness.model, "resetPaths"); + + bigEntries = bigFolder(3_000).map((entry) => file(entry.relativePath.replace("/f", "/g"))); + await harness.controller.refresh(); + + expect(resetPaths).toHaveBeenCalledTimes(1); + expect(harness.model.getItem("big/gone/")).toBeNull(); + expect(harness.model.getItem("big/gone/untitled.txt")).toBeNull(); + expect(harness.model.getItem("big/g00000.txt")).not.toBeNull(); + harness.destroy(); + }); + + it("never shrinks an open search's results while a large listing adds to them", async () => { + const rootFiles = Array.from({ length: 50 }, (_, index) => + file(`f0r${String(index).padStart(2, "0")}.txt`), + ); + const harness = makeController(async (relativeDirectory) => + relativeDirectory === "" + ? complete(directory("big"), ...rootFiles) + : complete(...bigFolder(3_000)), + ); + await harness.controller.start(); + harness.model.setSearch("f0"); + const counts: number[] = []; + const stop = harness.model.subscribe(() => counts.push(harness.model.getVisibleCount())); + counts.length = 0; + + await harness.controller.primePaths(["big/f02999.txt"]); + stop(); + + expect(Math.min(...counts)).toBeGreaterThanOrEqual(50); + expect(counts.at(-1)).toBe(50 + 1 + 3_000); + harness.destroy(); + }); + + it("ends the selection of a thousand entries that changed kind without batching them", async () => { + let bigEntries = bigFolder(3_000); + const harness = makeController(async (relativeDirectory) => + relativeDirectory === "" ? complete(directory("big")) : complete(...bigEntries), + ); + await harness.controller.start(); + expand(harness.model, "big"); + await harness.controller.load("big"); + for (const entry of bigEntries.slice(0, 1_000)) { + harness.model.getItem(entry.relativePath)?.select(); + } + expect(harness.model.getSelectedPaths()).toHaveLength(1_000); + const resetPaths = vi.spyOn(harness.model, "resetPaths"); + const batch = vi.spyOn(harness.model, "batch"); + + bigEntries = bigFolder(3_000).map((entry, index) => + index < 1_000 ? directory(entry.relativePath) : entry, + ); + await harness.controller.refresh(); + + expect(resetPaths).toHaveBeenCalledTimes(2); + expect(batch.mock.calls.map(([operations]) => operations.length)).toEqual([1]); + expect(harness.model.getItem("big/f00999.txt/")).not.toBeNull(); + expect(harness.model.getItem("big/f01000.txt")).not.toBeNull(); + expect(harness.model.getSelectedPaths()).toEqual([]); + harness.destroy(); + }); + + it("keeps batching a small listing into a large tree", async () => { + const harness = makeController(async (relativeDirectory) => { + if (relativeDirectory === "") return complete(directory("big"), directory("src")); + if (relativeDirectory === "src") return complete(file("src/a.ts"), file("src/b.ts")); + return complete(...bigFolder(3_000)); + }); + await harness.controller.start(); + await harness.controller.load("big"); + const resetPaths = vi.spyOn(harness.model, "resetPaths"); + const batch = vi.spyOn(harness.model, "batch"); + + await harness.controller.load("src"); + + expect(batch).toHaveBeenCalledTimes(1); + expect(resetPaths).not.toHaveBeenCalled(); + expect(harness.model.getItem("src/b.ts")).not.toBeNull(); + harness.destroy(); + }); + }); }); diff --git a/apps/web/src/scient/files/LazyWorkspaceTreeController.ts b/apps/web/src/scient/files/LazyWorkspaceTreeController.ts index 97007116ad..5ed82d39ac 100644 --- a/apps/web/src/scient/files/LazyWorkspaceTreeController.ts +++ b/apps/web/src/scient/files/LazyWorkspaceTreeController.ts @@ -34,6 +34,9 @@ interface LazyWorkspaceTreeControllerOptions { const EMPTY_CHILDREN: ReadonlySet = new Set(); const MAX_CONCURRENT_DIRECTORY_LOADS = 4; +// A batched add or remove reworks its folder's siblings, a rebuild every path once. T3 measured +// the break-even between 200 and 500 operations on flat folders of 50,000 and 332,000 entries. +const REBUILD_COST_IN_FOLDER_OPERATIONS = 300; function treePath(entry: ProjectDirectoryEntry): string { return entry.kind === "directory" ? `${entry.relativePath}/` : entry.relativePath; @@ -61,6 +64,7 @@ export class LazyWorkspaceTreeController { readonly #requestVersions = new Map(); readonly #inFlight = new Map>(); readonly #unloadedDirectories = new Set(); + readonly #provisionalTreePaths = new Set(); readonly #directoryLoadWaiters: Array<() => void> = []; #view: ProjectDirectoryView; @@ -187,17 +191,25 @@ export class LazyWorkspaceTreeController { }); } else if (moved) { // Into a folder not listed yet: its listing brings the item. - this.#forgetSubtree(to); + this.#forgetSubtrees(new Set([to])); this.#model.remove(treePath({ ...moved, relativePath: to }), { recursive: true }); } this.#emit(); } + /** Shows a row before the disk has it, such as a new entry while it is named. */ + addProvisional(treePath: string): void { + this.#provisionalTreePaths.add(treePath); + this.#model.add(treePath); + } + /** * Follows an entry the tree already shows before the disk has it (a new * file or folder named in place), so the next listing finds it known. */ noteAdded(relativePath: string, kind: ProjectDirectoryEntry["kind"]): void { + this.#provisionalTreePaths.delete(relativePath); + this.#provisionalTreePaths.delete(`${relativePath}/`); if (this.#destroyed || this.#entries.has(relativePath)) return; // Any folder the tree knows, loading included: a listing still on its way then // reconciles against what the tree shows instead of adding the row twice. @@ -385,15 +397,18 @@ export class LazyWorkspaceTreeController { } const previousChildren = this.#branches.get(relativeDirectory)?.children ?? EMPTY_CHILDREN; + const previousPathCount = this.#entries.size; const operations: FileTreeBatchOperation[] = []; + const removedChildren = new Set(); for (const previousPath of previousChildren) { const previousEntry = this.#entries.get(previousPath); const nextEntry = byPath.get(previousPath); if (previousEntry && (!nextEntry || previousEntry.kind !== nextEntry.kind)) { operations.push({ type: "remove", path: treePath(previousEntry), recursive: true }); - this.#forgetSubtree(previousPath); + removedChildren.add(previousPath); } } + this.#forgetSubtrees(removedChildren); for (const entry of byPath.values()) { const previousEntry = this.#entries.get(entry.relativePath); if (!previousEntry || previousEntry.kind !== entry.kind) { @@ -413,19 +428,92 @@ export class LazyWorkspaceTreeController { hasLoadedOnce: true, }); this.#emit(); - if (operations.length > 0) this.#model.batch(operations); + if (operations.length > 0) { + // Every operation here lands among this folder's children. + const batchWork = operations.length * Math.max(previousChildren.size, byPath.size); + const rebuildWork = Math.max(previousPathCount, this.#entries.size); + if (batchWork > REBUILD_COST_IN_FOLDER_OPERATIONS * rebuildWork) this.#rebuild(operations); + else this.#model.batch(operations); + } this.#loadExpandedBranches(); } - #forgetSubtree(relativePath: string): void { - const prefix = `${relativePath}/`; + /** + * Rebuilds the tree from the known entries, then applies part of the change as an ordinary + * batch: Pierre clears a drag, remaps rename, focus and selection and refreshes an open search + * only on a real change, which a rebuild is not. + */ + #rebuild(operations: readonly FileTreeBatchOperation[]): void { + const model = this.#model; + const removed = new Set(); + const added = new Set(); + for (const op of operations) { + if (op.type === "remove") removed.add(op.path); + else if (op.type === "add") added.add(op.path); + } + const isRemoved = (path: string) => { + for (let end = path.indexOf("/"); end !== -1; end = path.indexOf("/", end + 1)) { + if (end < path.length - 1 && removed.has(path.slice(0, end + 1))) return true; + } + return removed.has(path); + }; + const otherKind = (path: string) => (path.endsWith("/") ? path.slice(0, -1) : `${path}/`); + const kindChanges = operations.flatMap((op) => + op.type === "remove" && added.has(otherKind(op.path)) ? [op.path] : [], + ); + const carried = operations.find((op) => op.type === "add") ?? operations[0]!; + + const paths = new Set(); + const expandedPaths: string[] = []; + for (const entry of this.#entries.values()) { + const path = treePath(entry); + paths.add(path); + if (entry.kind !== "directory") continue; + const item = model.getItem(path); + if (item && "isExpanded" in item && item.isExpanded()) expandedPaths.push(path); + } + for (const path of this.#provisionalTreePaths) { + if (model.getItem(path) === null || isRemoved(path)) this.#provisionalTreePaths.delete(path); + else paths.add(path); + } + // A row the tree moved before telling the controller is still a visible one. + for (const row of model.getVisibleRows(0, model.getVisibleCount() - 1)) { + if (!isRemoved(row.path)) paths.add(row.path); + } + const focusedPath = model.getFocusedPath(); + if (kindChanges.length > 0) { + // A rebuild would move a rename or selection to the entry of the other kind, so a first + // one without either form ends them. + const withoutReplaced = new Set(paths); + for (const path of kindChanges) withoutReplaced.delete(otherKind(path)); + model.resetPaths([...withoutReplaced], { initialExpandedPaths: expandedPaths }); + } + if (carried.type === "add") paths.delete(carried.path); + else if (carried.type === "remove") paths.add(carried.path); + model.resetPaths([...paths], { initialExpandedPaths: expandedPaths }); + model.batch([carried]); + // As in a batch, focus stays on an entry that changed kind. + if (focusedPath !== null && kindChanges.includes(focusedPath)) { + model.focusPath(otherKind(focusedPath)); + } + } + + /** Forgets these entries and everything under them, in one pass however many there are. */ + #forgetSubtrees(relativePaths: ReadonlySet): void { + if (relativePaths.size === 0) return; + const isForgotten = (path: string) => { + for (let current = path; current.length > 0; current = parentDirectory(current)) { + if (relativePaths.has(current)) return true; + } + return false; + }; for (const path of this.#entries.keys()) { - if (path !== relativePath && !path.startsWith(prefix)) continue; + if (!isForgotten(path)) continue; this.#entries.delete(path); this.#unloadedDirectories.delete(path); } for (const path of this.#branches.keys()) { - if (path !== relativePath && !path.startsWith(prefix)) continue; + if (!isForgotten(path)) continue; this.#branches.delete(path); this.#requestVersions.set(path, (this.#requestVersions.get(path) ?? 0) + 1); this.#inFlight.delete(path); diff --git a/apps/web/src/scient/files/useScientFileActions.tsx b/apps/web/src/scient/files/useScientFileActions.tsx index cf72cb09ea..2446b829bb 100644 --- a/apps/web/src/scient/files/useScientFileActions.tsx +++ b/apps/web/src/scient/files/useScientFileActions.tsx @@ -543,7 +543,7 @@ export function useScientFileActions(input: { await windowFocused(); // Clicked elsewhere meanwhile (the chat box, say): that is where the typing goes. if (early.typed.abandoned) return; - model.add(treePath); + controller.addProvisional(treePath); pendingCreateRef.current = { relativePath, kind }; input.quietSelection(); model.startRenaming(treePath, { removeIfCanceled: true });