diff --git a/apps/web/src/components/files/FileBrowserPanel.browser.test.tsx b/apps/web/src/components/files/FileBrowserPanel.browser.test.tsx
index 9d8bac44b8..1bd9017aea 100644
--- a/apps/web/src/components/files/FileBrowserPanel.browser.test.tsx
+++ b/apps/web/src/components/files/FileBrowserPanel.browser.test.tsx
@@ -120,13 +120,15 @@ describe("Files panel search", () => {
workspaceMutationId: null,
};
- async function renderPanel() {
+ async function renderPanel({ cwd = props.cwd }: { readonly cwd?: string } = {}) {
const container = document.createElement("div");
// The tree virtualizes its rows, so it needs a real viewport to render any.
container.style.cssText = "display: flex; flex-direction: column; width: 320px; height: 480px;";
await using stack = new AsyncDisposableStack();
- const screen = stack.use(await render(, { container }));
- const rerender = () => screen.rerender();
+ const screen = stack.use(
+ await render(, { container }),
+ );
+ const rerender = () => screen.rerender();
function searchInput() {
const input = container!.querySelector('input[type="search"]');
expect(input).not.toBeNull();
@@ -353,6 +355,51 @@ describe("Files panel search", () => {
await chooseFromFilesMenu(screen.container, "Collapse all folders");
await expect.poll(screen.visiblePaths).toEqual(["src/", "package.json"]);
});
+ describe("a very large folder", () => {
+ const LARGE_FOLDER_SIZE = 50_000;
+ // Out of sort order, because inserting in order hides the cost of each insert.
+ const largeFolder = () =>
+ Array.from({ length: LARGE_FOLDER_SIZE }, (_, index) =>
+ entry(`big/f${String((index * 7_919) % LARGE_FOLDER_SIZE).padStart(5, "0")}.txt`),
+ );
+ const treeRow = (container: HTMLElement, path: string) =>
+ container
+ .querySelector("file-tree-container")
+ ?.shadowRoot?.querySelector(`[data-item-path="${CSS.escape(path)}"]`) ?? null;
+
+ it("opens without freezing and keeps the folder that was open", async () => {
+ setup();
+ files.directories[""] = [entry("big", "directory"), ...files.directories[""]!];
+ files.directories.big = largeFolder();
+ // Its own folder, so the folders it opens are not remembered for the other tests.
+ await using screen = await renderPanel({ cwd: "/large-folder-open" });
+ await expect.poll(screen.visiblePaths).toEqual(["big/", "src/", "package.json"]);
+ treeRow(screen.container, "src/")!.click();
+ await expect.poll(screen.visiblePaths).toContain("src/app.ts");
+
+ treeRow(screen.container, "big/")!.click();
+ // Batching every row blocked the panel past this wait; the rebuild takes about 170 ms.
+ await expect.poll(screen.visiblePaths).toContain("big/f00000.txt");
+ expect(treeRow(screen.container, "big/")!.getAttribute("aria-expanded")).toBe("true");
+ const src = treeRow(screen.container, "src/");
+ if (src) expect(src.getAttribute("aria-expanded")).toBe("true");
+ });
+
+ it("keeps an open search accurate when the folder's listing arrives", async () => {
+ setup();
+ files.directories[""] = [entry("big", "directory"), ...files.directories[""]!];
+ files.directories.big = largeFolder();
+ await using screen = await renderPanel({ cwd: "/large-folder-search" });
+ await expect.poll(screen.visiblePaths).toContain("big/");
+ files.searchEntries = [{ path: "big/f49999.txt", kind: "file" }];
+ await screen.search("f49999");
+ await expect.poll(screen.visiblePaths).toEqual(["big/", "big/f49999.txt"]);
+ expect(screen.searchInput().value).toBe("f49999");
+ await screen.search("");
+ await expect.poll(screen.visiblePaths).toEqual(["big/", "src/", "package.json"]);
+ });
+ });
+
it("copies with the pointer and returns to the row after keyboard dismissal", async () => {
setup();
files.directories = { "": [entry("README.md")] };
diff --git a/apps/web/src/scient/files/LazyWorkspaceTreeController.test.ts b/apps/web/src/scient/files/LazyWorkspaceTreeController.test.ts
index dfad64c09e..913c633724 100644
--- a/apps/web/src/scient/files/LazyWorkspaceTreeController.test.ts
+++ b/apps/web/src/scient/files/LazyWorkspaceTreeController.test.ts
@@ -531,4 +531,233 @@ describe("LazyWorkspaceTreeController", () => {
expect(harness.snapshot()?.rootError).toContain("incomplete directory listing");
harness.destroy();
});
+
+ describe("very large folders", () => {
+ const bigFolder = (count: number) =>
+ Array.from({ length: count }, (_, index) =>
+ file(`big/f${String(index).padStart(5, "0")}.txt`),
+ );
+ const expandedPaths = (model: FileTree, paths: readonly string[]) =>
+ paths.filter((path) => {
+ const item = model.getItem(path);
+ return item !== null && "isExpanded" in item && item.isExpanded();
+ });
+ const expand = (model: FileTree, path: string) => {
+ const item = model.getItem(path);
+ if (item && "expand" in item) item.expand();
+ };
+
+ it("rebuilds the tree for a large listing, keeping what it shows open, selected and new", async () => {
+ let bigEntries = bigFolder(3_000);
+ const harness = makeController(async (relativeDirectory) => {
+ switch (relativeDirectory) {
+ case "":
+ return complete(
+ directory("big"),
+ directory("docs"),
+ directory("src"),
+ file("README.md"),
+ );
+ case "src":
+ return complete(directory("src/lib"), file("src/a.ts"));
+ case "docs":
+ return complete(file("docs/guide.md"));
+ default:
+ return complete(...bigEntries);
+ }
+ });
+ await harness.controller.start();
+ for (const path of ["src", "docs"]) {
+ expand(harness.model, path);
+ await harness.controller.load(path);
+ }
+ harness.model.getItem("src/a.ts")?.select();
+ // New rows the disk does not have yet, one of them hidden in a closed folder.
+ harness.controller.addProvisional("src/untitled.md");
+ harness.controller.addProvisional("docs/untitled folder/");
+ const docs = harness.model.getItem("docs/");
+ if (docs && "collapse" in docs) docs.collapse();
+ const resetPaths = vi.spyOn(harness.model, "resetPaths");
+ const batch = vi.spyOn(harness.model, "batch");
+
+ expand(harness.model, "big");
+ await harness.controller.load("big");
+
+ expect(resetPaths).toHaveBeenCalledTimes(1);
+ // One real change follows the rebuild, so the tree updates what depends on changes.
+ expect(batch).toHaveBeenCalledTimes(1);
+ expect(batch.mock.calls[0]?.[0]).toHaveLength(1);
+ expect(harness.model.getItem("big/f02999.txt")).not.toBeNull();
+ expect(harness.model.getItem("src/untitled.md")).not.toBeNull();
+ expect(harness.model.getItem("docs/untitled folder/")).not.toBeNull();
+ expect(harness.model.getItem("src/lib/")).not.toBeNull();
+ expect(expandedPaths(harness.model, ["big/", "docs/", "src/", "src/lib/"])).toEqual([
+ "big/",
+ "src/",
+ ]);
+ expect(harness.model.getSelectedPaths()).toEqual(["src/a.ts"]);
+ // big, its files, docs, src with its three rows, and the README.
+ expect(harness.model.getVisibleCount()).toBe(1 + 3_000 + 1 + 4 + 1);
+ expect(harness.snapshot()?.entries.size).toBe(4 + 2 + 1 + 3_000);
+
+ // Most of the folder is deleted: the refresh rebuilds again and drops the rows.
+ bigEntries = bigEntries.slice(0, 10);
+ await harness.controller.refresh();
+ expect(resetPaths).toHaveBeenCalledTimes(2);
+ expect(harness.model.getItem("big/f00009.txt")).not.toBeNull();
+ expect(harness.model.getItem("big/f00010.txt")).toBeNull();
+ expect(harness.model.getItem("src/untitled.md")).not.toBeNull();
+ expect(harness.model.getVisibleCount()).toBe(1 + 10 + 1 + 4 + 1);
+ expect(harness.snapshot()?.entries.size).toBe(4 + 2 + 1 + 10);
+ harness.destroy();
+ });
+
+ it("drops the selection of an entry that changed kind but keeps its focus, as a batch does", async () => {
+ let bigEntries = bigFolder(3_000);
+ const harness = makeController(async (relativeDirectory) =>
+ relativeDirectory === "" ? complete(directory("big")) : complete(...bigEntries),
+ );
+ await harness.controller.start();
+ expand(harness.model, "big");
+ await harness.controller.load("big");
+ harness.model.getItem("big/f00001.txt")?.select();
+ harness.model.focusPath("big/f00001.txt");
+ const resetPaths = vi.spyOn(harness.model, "resetPaths");
+
+ // f00001.txt is now a folder, and the rest of the folder changed with it.
+ bigEntries = [
+ directory("big/f00001.txt"),
+ ...bigFolder(3_000)
+ .slice(2)
+ .map((entry) => file(entry.relativePath.replace("/f", "/g"))),
+ ];
+ await harness.controller.refresh();
+
+ // A first rebuild without either form of the changed entry ends its selection.
+ expect(resetPaths).toHaveBeenCalledTimes(2);
+ expect(harness.model.getItem("big/f00001.txt/")).not.toBeNull();
+ expect(harness.model.getItem("big/g02999.txt")).not.toBeNull();
+ expect(harness.model.getItem("big/f00002.txt")).toBeNull();
+ expect(harness.model.getSelectedPaths()).toEqual([]);
+ expect(harness.model.getFocusedPath()).toBe("big/f00001.txt/");
+ harness.destroy();
+ });
+
+ it("keeps an open search, and what it restores, through a large folder's rebuild", async () => {
+ const harness = makeController(async (relativeDirectory) =>
+ relativeDirectory === ""
+ ? complete(directory("big"), file("README.md"))
+ : complete(...bigFolder(3_000)),
+ );
+ await harness.controller.start();
+ const resetPaths = vi.spyOn(harness.model, "resetPaths");
+ harness.model.setSearch("f0299");
+
+ await harness.controller.primePaths(["big/f02999.txt"]);
+
+ expect(resetPaths).toHaveBeenCalledTimes(1);
+ expect(harness.model.isSearchOpen()).toBe(true);
+ expect(harness.model.getSearchMatchingPaths()).toEqual(
+ Array.from({ length: 10 }, (_, index) => `big/f0299${index}.txt`),
+ );
+ harness.model.closeSearch();
+ expect(expandedPaths(harness.model, ["big/"])).toEqual([]);
+ harness.destroy();
+ });
+
+ it("drops a new row with the folder a large listing removes", async () => {
+ let bigEntries = [directory("big/gone"), ...bigFolder(3_000)];
+ const harness = makeController(async (relativeDirectory) => {
+ if (relativeDirectory === "") return complete(directory("big"));
+ if (relativeDirectory === "big/gone") return complete(file("big/gone/kept.txt"));
+ return complete(...bigEntries);
+ });
+ await harness.controller.start();
+ expand(harness.model, "big");
+ await harness.controller.load("big");
+ expand(harness.model, "big/gone");
+ await harness.controller.load("big/gone");
+ harness.controller.addProvisional("big/gone/untitled.txt");
+ const resetPaths = vi.spyOn(harness.model, "resetPaths");
+
+ bigEntries = bigFolder(3_000).map((entry) => file(entry.relativePath.replace("/f", "/g")));
+ await harness.controller.refresh();
+
+ expect(resetPaths).toHaveBeenCalledTimes(1);
+ expect(harness.model.getItem("big/gone/")).toBeNull();
+ expect(harness.model.getItem("big/gone/untitled.txt")).toBeNull();
+ expect(harness.model.getItem("big/g00000.txt")).not.toBeNull();
+ harness.destroy();
+ });
+
+ it("never shrinks an open search's results while a large listing adds to them", async () => {
+ const rootFiles = Array.from({ length: 50 }, (_, index) =>
+ file(`f0r${String(index).padStart(2, "0")}.txt`),
+ );
+ const harness = makeController(async (relativeDirectory) =>
+ relativeDirectory === ""
+ ? complete(directory("big"), ...rootFiles)
+ : complete(...bigFolder(3_000)),
+ );
+ await harness.controller.start();
+ harness.model.setSearch("f0");
+ const counts: number[] = [];
+ const stop = harness.model.subscribe(() => counts.push(harness.model.getVisibleCount()));
+ counts.length = 0;
+
+ await harness.controller.primePaths(["big/f02999.txt"]);
+ stop();
+
+ expect(Math.min(...counts)).toBeGreaterThanOrEqual(50);
+ expect(counts.at(-1)).toBe(50 + 1 + 3_000);
+ harness.destroy();
+ });
+
+ it("ends the selection of a thousand entries that changed kind without batching them", async () => {
+ let bigEntries = bigFolder(3_000);
+ const harness = makeController(async (relativeDirectory) =>
+ relativeDirectory === "" ? complete(directory("big")) : complete(...bigEntries),
+ );
+ await harness.controller.start();
+ expand(harness.model, "big");
+ await harness.controller.load("big");
+ for (const entry of bigEntries.slice(0, 1_000)) {
+ harness.model.getItem(entry.relativePath)?.select();
+ }
+ expect(harness.model.getSelectedPaths()).toHaveLength(1_000);
+ const resetPaths = vi.spyOn(harness.model, "resetPaths");
+ const batch = vi.spyOn(harness.model, "batch");
+
+ bigEntries = bigFolder(3_000).map((entry, index) =>
+ index < 1_000 ? directory(entry.relativePath) : entry,
+ );
+ await harness.controller.refresh();
+
+ expect(resetPaths).toHaveBeenCalledTimes(2);
+ expect(batch.mock.calls.map(([operations]) => operations.length)).toEqual([1]);
+ expect(harness.model.getItem("big/f00999.txt/")).not.toBeNull();
+ expect(harness.model.getItem("big/f01000.txt")).not.toBeNull();
+ expect(harness.model.getSelectedPaths()).toEqual([]);
+ harness.destroy();
+ });
+
+ it("keeps batching a small listing into a large tree", async () => {
+ const harness = makeController(async (relativeDirectory) => {
+ if (relativeDirectory === "") return complete(directory("big"), directory("src"));
+ if (relativeDirectory === "src") return complete(file("src/a.ts"), file("src/b.ts"));
+ return complete(...bigFolder(3_000));
+ });
+ await harness.controller.start();
+ await harness.controller.load("big");
+ const resetPaths = vi.spyOn(harness.model, "resetPaths");
+ const batch = vi.spyOn(harness.model, "batch");
+
+ await harness.controller.load("src");
+
+ expect(batch).toHaveBeenCalledTimes(1);
+ expect(resetPaths).not.toHaveBeenCalled();
+ expect(harness.model.getItem("src/b.ts")).not.toBeNull();
+ harness.destroy();
+ });
+ });
});
diff --git a/apps/web/src/scient/files/LazyWorkspaceTreeController.ts b/apps/web/src/scient/files/LazyWorkspaceTreeController.ts
index 97007116ad..5ed82d39ac 100644
--- a/apps/web/src/scient/files/LazyWorkspaceTreeController.ts
+++ b/apps/web/src/scient/files/LazyWorkspaceTreeController.ts
@@ -34,6 +34,9 @@ interface LazyWorkspaceTreeControllerOptions {
const EMPTY_CHILDREN: ReadonlySet = new Set();
const MAX_CONCURRENT_DIRECTORY_LOADS = 4;
+// A batched add or remove reworks its folder's siblings, a rebuild every path once. T3 measured
+// the break-even between 200 and 500 operations on flat folders of 50,000 and 332,000 entries.
+const REBUILD_COST_IN_FOLDER_OPERATIONS = 300;
function treePath(entry: ProjectDirectoryEntry): string {
return entry.kind === "directory" ? `${entry.relativePath}/` : entry.relativePath;
@@ -61,6 +64,7 @@ export class LazyWorkspaceTreeController {
readonly #requestVersions = new Map();
readonly #inFlight = new Map>();
readonly #unloadedDirectories = new Set();
+ readonly #provisionalTreePaths = new Set();
readonly #directoryLoadWaiters: Array<() => void> = [];
#view: ProjectDirectoryView;
@@ -187,17 +191,25 @@ export class LazyWorkspaceTreeController {
});
} else if (moved) {
// Into a folder not listed yet: its listing brings the item.
- this.#forgetSubtree(to);
+ this.#forgetSubtrees(new Set([to]));
this.#model.remove(treePath({ ...moved, relativePath: to }), { recursive: true });
}
this.#emit();
}
+ /** Shows a row before the disk has it, such as a new entry while it is named. */
+ addProvisional(treePath: string): void {
+ this.#provisionalTreePaths.add(treePath);
+ this.#model.add(treePath);
+ }
+
/**
* Follows an entry the tree already shows before the disk has it (a new
* file or folder named in place), so the next listing finds it known.
*/
noteAdded(relativePath: string, kind: ProjectDirectoryEntry["kind"]): void {
+ this.#provisionalTreePaths.delete(relativePath);
+ this.#provisionalTreePaths.delete(`${relativePath}/`);
if (this.#destroyed || this.#entries.has(relativePath)) return;
// Any folder the tree knows, loading included: a listing still on its way then
// reconciles against what the tree shows instead of adding the row twice.
@@ -385,15 +397,18 @@ export class LazyWorkspaceTreeController {
}
const previousChildren = this.#branches.get(relativeDirectory)?.children ?? EMPTY_CHILDREN;
+ const previousPathCount = this.#entries.size;
const operations: FileTreeBatchOperation[] = [];
+ const removedChildren = new Set();
for (const previousPath of previousChildren) {
const previousEntry = this.#entries.get(previousPath);
const nextEntry = byPath.get(previousPath);
if (previousEntry && (!nextEntry || previousEntry.kind !== nextEntry.kind)) {
operations.push({ type: "remove", path: treePath(previousEntry), recursive: true });
- this.#forgetSubtree(previousPath);
+ removedChildren.add(previousPath);
}
}
+ this.#forgetSubtrees(removedChildren);
for (const entry of byPath.values()) {
const previousEntry = this.#entries.get(entry.relativePath);
if (!previousEntry || previousEntry.kind !== entry.kind) {
@@ -413,19 +428,92 @@ export class LazyWorkspaceTreeController {
hasLoadedOnce: true,
});
this.#emit();
- if (operations.length > 0) this.#model.batch(operations);
+ if (operations.length > 0) {
+ // Every operation here lands among this folder's children.
+ const batchWork = operations.length * Math.max(previousChildren.size, byPath.size);
+ const rebuildWork = Math.max(previousPathCount, this.#entries.size);
+ if (batchWork > REBUILD_COST_IN_FOLDER_OPERATIONS * rebuildWork) this.#rebuild(operations);
+ else this.#model.batch(operations);
+ }
this.#loadExpandedBranches();
}
- #forgetSubtree(relativePath: string): void {
- const prefix = `${relativePath}/`;
+ /**
+ * Rebuilds the tree from the known entries, then applies part of the change as an ordinary
+ * batch: Pierre clears a drag, remaps rename, focus and selection and refreshes an open search
+ * only on a real change, which a rebuild is not.
+ */
+ #rebuild(operations: readonly FileTreeBatchOperation[]): void {
+ const model = this.#model;
+ const removed = new Set();
+ const added = new Set();
+ for (const op of operations) {
+ if (op.type === "remove") removed.add(op.path);
+ else if (op.type === "add") added.add(op.path);
+ }
+ const isRemoved = (path: string) => {
+ for (let end = path.indexOf("/"); end !== -1; end = path.indexOf("/", end + 1)) {
+ if (end < path.length - 1 && removed.has(path.slice(0, end + 1))) return true;
+ }
+ return removed.has(path);
+ };
+ const otherKind = (path: string) => (path.endsWith("/") ? path.slice(0, -1) : `${path}/`);
+ const kindChanges = operations.flatMap((op) =>
+ op.type === "remove" && added.has(otherKind(op.path)) ? [op.path] : [],
+ );
+ const carried = operations.find((op) => op.type === "add") ?? operations[0]!;
+
+ const paths = new Set();
+ const expandedPaths: string[] = [];
+ for (const entry of this.#entries.values()) {
+ const path = treePath(entry);
+ paths.add(path);
+ if (entry.kind !== "directory") continue;
+ const item = model.getItem(path);
+ if (item && "isExpanded" in item && item.isExpanded()) expandedPaths.push(path);
+ }
+ for (const path of this.#provisionalTreePaths) {
+ if (model.getItem(path) === null || isRemoved(path)) this.#provisionalTreePaths.delete(path);
+ else paths.add(path);
+ }
+ // A row the tree moved before telling the controller is still a visible one.
+ for (const row of model.getVisibleRows(0, model.getVisibleCount() - 1)) {
+ if (!isRemoved(row.path)) paths.add(row.path);
+ }
+ const focusedPath = model.getFocusedPath();
+ if (kindChanges.length > 0) {
+ // A rebuild would move a rename or selection to the entry of the other kind, so a first
+ // one without either form ends them.
+ const withoutReplaced = new Set(paths);
+ for (const path of kindChanges) withoutReplaced.delete(otherKind(path));
+ model.resetPaths([...withoutReplaced], { initialExpandedPaths: expandedPaths });
+ }
+ if (carried.type === "add") paths.delete(carried.path);
+ else if (carried.type === "remove") paths.add(carried.path);
+ model.resetPaths([...paths], { initialExpandedPaths: expandedPaths });
+ model.batch([carried]);
+ // As in a batch, focus stays on an entry that changed kind.
+ if (focusedPath !== null && kindChanges.includes(focusedPath)) {
+ model.focusPath(otherKind(focusedPath));
+ }
+ }
+
+ /** Forgets these entries and everything under them, in one pass however many there are. */
+ #forgetSubtrees(relativePaths: ReadonlySet): void {
+ if (relativePaths.size === 0) return;
+ const isForgotten = (path: string) => {
+ for (let current = path; current.length > 0; current = parentDirectory(current)) {
+ if (relativePaths.has(current)) return true;
+ }
+ return false;
+ };
for (const path of this.#entries.keys()) {
- if (path !== relativePath && !path.startsWith(prefix)) continue;
+ if (!isForgotten(path)) continue;
this.#entries.delete(path);
this.#unloadedDirectories.delete(path);
}
for (const path of this.#branches.keys()) {
- if (path !== relativePath && !path.startsWith(prefix)) continue;
+ if (!isForgotten(path)) continue;
this.#branches.delete(path);
this.#requestVersions.set(path, (this.#requestVersions.get(path) ?? 0) + 1);
this.#inFlight.delete(path);
diff --git a/apps/web/src/scient/files/useScientFileActions.tsx b/apps/web/src/scient/files/useScientFileActions.tsx
index cf72cb09ea..2446b829bb 100644
--- a/apps/web/src/scient/files/useScientFileActions.tsx
+++ b/apps/web/src/scient/files/useScientFileActions.tsx
@@ -543,7 +543,7 @@ export function useScientFileActions(input: {
await windowFocused();
// Clicked elsewhere meanwhile (the chat box, say): that is where the typing goes.
if (early.typed.abandoned) return;
- model.add(treePath);
+ controller.addProvisional(treePath);
pendingCreateRef.current = { relativePath, kind };
input.quietSelection();
model.startRenaming(treePath, { removeIfCanceled: true });