From 9b6943aadf5dd816f79e8cdc7b200090f7f4829e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ademir=20Jos=C3=A9=20Ferreira=20J=C3=BAnior?= Date: Tue, 22 Sep 2026 13:34:14 -0300 Subject: [PATCH 1/2] Support isolated parallel desktop E2E workers --- .github/workflows/ci.yml | 4 +- CONTRIBUTING.md | 16 +- docs/architecture.md | 4 +- e2e/desktop/run.ts | 473 ++++++++++++------ e2e/desktop/specs/diagnostics.spec.ts | 4 +- e2e/desktop/support/artifacts.ts | 8 +- e2e/desktop/support/runContext.ts | 1 + e2e/desktop/support/scenarioSelection.test.ts | 64 ++- e2e/desktop/support/scenarioSelection.ts | 70 ++- e2e/desktop/support/suite.ts | 17 +- e2e/desktop/support/workerPool.test.ts | 69 +++ e2e/desktop/support/workerPool.ts | 41 ++ package.json | 2 +- src-tauri/src/lib.rs | 15 +- 14 files changed, 595 insertions(+), 193 deletions(-) create mode 100644 e2e/desktop/support/workerPool.test.ts create mode 100644 e2e/desktop/support/workerPool.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8467d78..34a4f0cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,7 +125,7 @@ jobs: run: pnpm build:e2e:desktop - name: Run desktop E2E suite - run: pnpm test:e2e:desktop:run + run: pnpm test:e2e:desktop:run -- --workers 1 - name: Upload failure evidence if: failure() @@ -144,7 +144,7 @@ jobs: Start-Sleep -Seconds 10 $orphans = Get-Process -Name leafdown-e2e, msedgedriver -ErrorAction SilentlyContinue - $listeners = Get-NetTCPConnection -LocalPort 4445 -State Listen -ErrorAction SilentlyContinue + $listeners = Get-NetTCPConnection -LocalPort (4445..4448) -State Listen -ErrorAction SilentlyContinue if ($orphans -or $listeners) { $orphans | Format-Table -AutoSize Name, Id, StartTime diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0dd29829..3a5a047d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,7 +80,15 @@ Run the assembled desktop E2E suite, which requires Windows, with: pnpm test:e2e:desktop ``` -This explicit suite is not part of `pnpm check`; CI runs it as its own job on every pull request and every push to `main`, so it is enforced without changing what you run locally. It builds an isolated debug binary with test-only WebDriver capabilities, then runs one embedded WebDriver worker at a time on port 4445 across fresh application sessions. The embedded provider does not require an external WebDriver. Keep port 4445 available while it runs. The test identifier, persisted store, and target directory are separate from ordinary Leafdown builds. +This explicit suite is not part of `pnpm check`; CI runs it as its own job on every pull request and every push to `main`, so it is enforced without changing what you run locally. It builds one isolated debug binary with test-only WebDriver capabilities, then runs fresh application sessions through the embedded provider without requiring an external WebDriver. The runner defaults to one worker on port 4445. CI requests that deterministic single-worker mode explicitly until comparable hosted-run timings support a change. + +Run the full suite with two workers against an already built binary: + +```powershell +pnpm test:e2e:desktop:run -- --workers 2 +``` + +Worker counts from 1 through 4 are accepted. Workers use consecutive ports beginning at 4445, so keep that range available for the requested count. Each worker receives a distinct runtime application identifier, persisted-data and WebView2 roots, fixture tree, context file, and artifact directory. Independent scenario groups may overlap; the `persistence` target always keeps its write and fresh-process restart sequence ordered on one worker. One local Windows sample on 2026-09-22 ran the already-built full suite in about 123 seconds with one worker and 74 seconds with two. That sample supports opt-in local concurrency but is not treated as hosted-CI evidence. While iterating on an already built E2E binary, run one target without rebuilding it: @@ -88,14 +96,14 @@ While iterating on an already built E2E binary, run one target without rebuildin pnpm test:e2e:desktop:run -- --scenario folder-watcher ``` -Valid targets are `block-selection`, `diagnostics`, `document-lifecycle`, `folder-watcher`, `rendered-images`, `rendered-html`, `missing-document-error`, `persistence`, and `window-lifecycle`. The `persistence` target runs its write and fresh-process restart scenarios in order. Focused runs create the same fixtures and isolated state, preserve the same failure evidence and cleanup, and start a fresh packaged-app process; they do not check whether the binary is current. Run `pnpm build:e2e:desktop` first whenever E2E binary inputs change. The default `pnpm test:e2e:desktop` command and CI still run the full suite. +Valid targets are `block-selection`, `diagnostics`, `document-lifecycle`, `folder-watcher`, `rendered-images`, `rendered-html`, `missing-document-error`, `persistence`, and `window-lifecycle`. Focused runs create the same fixtures and isolated state, preserve the same failure evidence and cleanup, and start a fresh packaged-app process; they do not check whether the binary is current. Run `pnpm build:e2e:desktop` first whenever E2E binary inputs change. The default `pnpm test:e2e:desktop` command and CI still run the full suite. -The runner resets only the isolated E2E persisted store, leaving the application to write its own defaults, creates temporary filesystem fixtures, and removes both after the suite. Each run writes ignored runner, frontend, backend, and focused diagnostic evidence under `e2e/desktop/artifacts///`. A failed test also captures a screenshot, the real diagnostics summary, the test error, and a semantic UI snapshot that excludes editor content. A failed run additionally writes `fixture-manifest.json` under `e2e/desktop/artifacts//`, recording each temporary fixture's path, expected and actual hash and size, and modification time before cleanup removes it. Local artifacts are retained until manually deleted. Treat them as potentially sensitive because diagnostics and errors may contain local paths. CI uploads the same evidence only when the job fails, retained for seven days; those artifacts contain runner paths rather than a contributor's. +The runner resets only each worker's isolated E2E persisted store, leaving the application to write its own defaults, creates temporary filesystem fixtures, and removes the worker-owned state after the suite. Each run writes ignored runner, frontend, backend, and focused diagnostic evidence under `e2e/desktop/artifacts//worker-//`. A failed test also captures a screenshot, the real diagnostics summary, the test error, and a semantic UI snapshot that excludes editor content. A failed worker additionally writes `fixture-manifest.json` under its worker directory, recording its scenario and port plus each temporary fixture's path, expected and actual hash and size, and modification time before cleanup removes it. Other workers finish their queued groups and clean up independently. Local artifacts are retained until manually deleted. Treat them as potentially sensitive because diagnostics and errors may contain local paths. CI uploads the same evidence only when the job fails, retained for seven days; those artifacts contain runner paths rather than a contributor's. To verify the failure-evidence path, run the suite with the forced-failure flag: ```powershell -$env:LEAFDOWN_E2E_FORCE_FAILURE=1; pnpm test:e2e:desktop; $env:LEAFDOWN_E2E_FORCE_FAILURE=$null +$env:LEAFDOWN_E2E_FORCE_FAILURE=1; pnpm test:e2e:desktop:run -- --workers 2; $env:LEAFDOWN_E2E_FORCE_FAILURE=$null ``` The Diagnostics scenario should fail, retain its evidence, clean its fixture and store state, and return a nonzero exit code. diff --git a/docs/architecture.md b/docs/architecture.md index 48b9582e..de5604ff 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -190,7 +190,9 @@ Automated tests focus on: - Safe live HTML and literal fallback, attribute and namespace rejection, script/resource-load prevention, exact HTML round trips, and source-projection history and save finalization. Verify parsing and block-atom layout in the desktop WebView as well as the DOM test environment. - Context popup layout and caret-based marker visibility. -The assembled desktop E2E suite complements those component and boundary tests without replacing them. It requires Windows and stays outside `pnpm check`, running through an explicit command locally and as its own CI job on every pull request and push to `main`. It runs one embedded WebDriver worker at a time against an isolated debug binary and starts fresh application processes for independent scenarios. Developers can run an already built target with `pnpm test:e2e:desktop:run -- --scenario `; the runner validates the target before setup and preserves the selected scenario's fixture, isolated-state, artifact, fresh-process, port-release, and cleanup guarantees. The `persistence` target always runs its write and restart scenarios in order. This focused interface never checks binary freshness, so developers rebuild explicitly when binary inputs change; the default command and CI run every scenario. The suite retains the Help → Diagnostics smoke path, then adds narrow assembled-boundary assertions for the document lifecycle, real folder-watcher refresh, typed backend error propagation, persisted settings across restart, injected frame controls, and the clean window-close handshake. +The assembled desktop E2E suite complements those component and boundary tests without replacing them. It requires Windows and stays outside `pnpm check`, running through an explicit command locally and as its own CI job on every pull request and push to `main`. One isolated debug binary serves every worker. The runner schedules independent scenario groups with configurable bounded concurrency, while each worker receives its own WebDriver port, runtime application identifier, persisted-data and WebView2 roots, fixture tree, context file, and artifact namespace. A worker starts a fresh application process per scenario; the persistence write and restart scenarios remain one ordered, stateful group. Failure evidence and cleanup stay worker-owned so one failed worker cannot remove another worker's state, and healthy workers finish already queued coverage. + +Developers can run an already built target with `pnpm test:e2e:desktop:run -- --scenario ` or the full suite with `--workers <1-4>`; the runner validates both options before setup and preserves the fixture, isolated-state, artifact, fresh-process, port-release, and cleanup guarantees. This interface never checks binary freshness, so developers rebuild explicitly when binary inputs change. The default command and CI run every scenario with one worker; local concurrency is opt-in until hosted-CI timing and reliability evidence justify changing that deterministic default. The suite retains the Help → Diagnostics smoke path, then adds narrow assembled-boundary assertions for the document lifecycle, real folder-watcher refresh, typed backend error propagation, persisted settings across restart, injected frame controls, and the clean window-close handshake. User-visible acceptance paths use semantic UI interactions. Scenarios decide that an operation happened from state that outlives it — on-disk contents, menu item state, editor contents, or diagnostic records — rather than from an affordance that dismisses on a timer. Where the notification is itself the reported outcome, the desktop E2E build disables toast auto-dismissal so the assertion reads a settled affordance instead of racing it. Direct bridge execution is limited to corroborating diagnostic state, while Node-side filesystem, persisted-store, log, and process access provides deterministic setup or evidence around the native boundary. WebDriver plugins, permissions, and frontend integration remain limited to the dedicated desktop E2E build and are excluded from ordinary application builds. diff --git a/e2e/desktop/run.ts b/e2e/desktop/run.ts index e8d46913..66674bc7 100644 --- a/e2e/desktop/run.ts +++ b/e2e/desktop/run.ts @@ -8,8 +8,9 @@ import { setTimeout as delay } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import type { DesktopE2ERunContext } from "./support/runContext.js"; -import { selectScenarioNames } from "./support/scenarioSelection.js"; -import { RUN_LABEL, WEBDRIVER_PORT } from "./support/suite.js"; +import { selectDesktopE2ERun } from "./support/scenarioSelection.js"; +import { RUN_LABEL, WEBDRIVER_BASE_PORT } from "./support/suite.js"; +import { runWorkerPool } from "./support/workerPool.js"; interface Scenario { name: string; @@ -18,62 +19,46 @@ interface Scenario { recentFolders?: string[]; } -const selectedScenarioNames = selectScenarioNames(process.argv.slice(2)); +interface WorkerContext { + appDataDirectory: string; + artifactsRoot: string; + context: DesktopE2ERunContext; + contextPath: string; + label: string; + localDataDirectory: string; + port: number; + scenarios: Map; + storeDirectory: string; + temporaryRoot: string; +} +const selection = selectDesktopE2ERun(process.argv.slice(2)); const repositoryRoot = fileURLToPath(new URL("../..", import.meta.url)); const artifactsRoot = path.join(repositoryRoot, "e2e", "desktop", "artifacts", RUN_LABEL); -const contextPath = path.join(artifactsRoot, "run-context.json"); -const e2eAppDataDirectory = path.join(process.env.APPDATA ?? "", "com.azganoth.leafdown.e2e"); -const e2eStoreDirectory = path.join(e2eAppDataDirectory, "tauri-plugin-zustand"); -const recentItemsPath = path.join(e2eStoreDirectory, "recent-items.dev.json"); -const settingsPath = path.join(e2eStoreDirectory, "settings.dev.json"); const writeJson = (filePath: string, value: unknown) => writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`); -const resetPersistedState = async (recentFiles: string[] = [], recentFolders: string[] = []) => { - await rm(e2eStoreDirectory, { force: true, recursive: true }); - await mkdir(e2eStoreDirectory, { recursive: true }); - await writeJson(recentItemsPath, { recentFiles, recentFolders, version: 1 }); -}; +const rejectedReasons = (results: PromiseSettledResult[]): unknown[] => + results.flatMap((result) => (result.status === "rejected" ? [result.reason as unknown] : [])); -const runWdio = (scenario: Scenario) => - new Promise((resolve, reject) => { - const wdioExecutable = path.join( - repositoryRoot, - "node_modules", - "@wdio", - "cli", - "bin", - "wdio.js", - ); +const settleTasks = async (description: string, tasks: Promise[]) => { + const failures = rejectedReasons(await Promise.allSettled(tasks)); - const child = spawn(process.execPath, [wdioExecutable, "run", "e2e/desktop/wdio.conf.ts"], { - cwd: repositoryRoot, - env: { - ...process.env, - LEAFDOWN_E2E_ARTIFACT_RUN: RUN_LABEL, - LEAFDOWN_E2E_CONTEXT_PATH: contextPath, - LEAFDOWN_E2E_SCENARIO: scenario.name, - LEAFDOWN_E2E_SPEC: `e2e/desktop/specs/${scenario.name}.spec.ts`, - }, - stdio: "inherit", - }); + if (failures.length > 0) { + throw new AggregateError(failures, description); + } +}; - child.once("error", reject); - child.once("exit", (code, signal) => { - if (code === 0) { - resolve(); - return; - } +const removeDirectories = async (directories: string[]) => { + const results = await Promise.allSettled( + directories.map((directory) => + rm(directory, { force: true, maxRetries: 20, recursive: true, retryDelay: 250 }), + ), + ); - reject( - new Error( - `Desktop E2E scenario ${scenario.name} failed (code=${String(code)}, signal=${String(signal)}).`, - ), - ); - }); - }); + return rejectedReasons(results); +}; const sha256 = (contents: Buffer | string) => createHash("sha256").update(contents).digest("hex"); @@ -102,20 +87,19 @@ const fileEvidence = async (filePath: string, expectedContents?: string) => { } }; -const isPortFree = () => +const isPortFree = (port: number) => new Promise((resolve) => { const server = createServer(); server.once("error", () => resolve(false)); server.once("listening", () => server.close(() => resolve(true))); - server.listen(WEBDRIVER_PORT, "127.0.0.1"); + server.listen(port, "127.0.0.1"); }); -// The embedded driver releases the port asynchronously as it shuts down. -const waitForPortRelease = async (timeoutMs = 10_000) => { +const waitForPortRelease = async (port: number, timeoutMs = 10_000) => { const deadline = Date.now() + timeoutMs; - while (!(await isPortFree())) { + while (!(await isPortFree(port))) { if (Date.now() > deadline) { return false; } @@ -126,28 +110,32 @@ const waitForPortRelease = async (timeoutMs = 10_000) => { return true; }; -const main = async () => { - if (!process.env.APPDATA) { - throw new Error("APPDATA is required for the Windows desktop E2E suite."); - } - - await mkdir(artifactsRoot, { recursive: true }); - - const temporaryRoot = await mkdtemp(path.join(tmpdir(), "leafdown-desktop-e2e-")); - const documentPath = path.join(temporaryRoot, "document-lifecycle.md"); - const blocksPath = path.join(temporaryRoot, "block-selection.md"); - const imagesPath = path.join(temporaryRoot, "rendered-images.md"); - const htmlPath = path.join(temporaryRoot, "rendered-html.md"); - const leafImagePath = path.join(temporaryRoot, "leaf.svg"); - const tinyImagePath = path.join(temporaryRoot, "tiny-transparent.svg"); - const folderPath = path.join(temporaryRoot, "folder-context"); +const createWorkerContext = async (workerIndex: number): Promise => { + const label = `worker-${workerIndex + 1}`; + const appIdentifier = `com.azganoth.leafdown.e2e.w${workerIndex + 1}.p${process.pid}`; + const temporaryRoot = await mkdtemp(path.join(tmpdir(), `leafdown-desktop-e2e-${label}-`)); + const fixtureRoot = path.join(temporaryRoot, "fixtures"); + const appDataDirectory = path.join(process.env.APPDATA ?? "", appIdentifier); + const localDataDirectory = path.join(process.env.LOCALAPPDATA ?? "", appIdentifier); + const contextPath = path.join(temporaryRoot, "run-context.json"); + const workerArtifactsRoot = path.join(artifactsRoot, label); + const storeDirectory = path.join(appDataDirectory, "tauri-plugin-zustand"); + const settingsPath = path.join(storeDirectory, "settings.dev.json"); + const documentPath = path.join(fixtureRoot, "document-lifecycle.md"); + const blocksPath = path.join(fixtureRoot, "block-selection.md"); + const imagesPath = path.join(fixtureRoot, "rendered-images.md"); + const htmlPath = path.join(fixtureRoot, "rendered-html.md"); + const leafImagePath = path.join(fixtureRoot, "leaf.svg"); + const tinyImagePath = path.join(fixtureRoot, "tiny-transparent.svg"); + const folderPath = path.join(fixtureRoot, "folder-context"); const initialFolderFileName = "readme.md"; const initialFolderFilePath = path.join(folderPath, initialFolderFileName); const addedFolderFileName = "watcher-added.md"; const addedFolderFilePath = path.join(folderPath, addedFolderFileName); - const missingDocumentPath = path.join(temporaryRoot, "missing-document.md"); + const missingDocumentPath = path.join(fixtureRoot, "missing-document.md"); const savedMarker = "Saved fixture marker."; const context: DesktopE2ERunContext = { + appIdentifier, blocks: { path: blocksPath }, document: { initialMarker: "Initial fixture marker.", @@ -155,9 +143,7 @@ const main = async () => { savedMarkdown: `${savedMarker}\n`, savedMarker, }, - images: { - path: imagesPath, - }, + images: { path: imagesPath }, html: { path: htmlPath }, folder: { addedFileName: addedFolderFileName, @@ -170,46 +156,9 @@ const main = async () => { }, missingDocumentPath, settingsPath, - temporaryRoot, + temporaryRoot: fixtureRoot, }; - await mkdir(folderPath, { recursive: true }); - await copyFile( - path.join(repositoryRoot, "e2e", "desktop", "fixtures", "block-selection.md"), - blocksPath, - ); - await copyFile( - path.join(repositoryRoot, "e2e", "desktop", "fixtures", "rendered-html.md"), - htmlPath, - ); - await copyFile( - path.join(repositoryRoot, "e2e", "desktop", "fixtures", "document-lifecycle.md"), - documentPath, - ); - await copyFile( - path.join(repositoryRoot, "e2e", "desktop", "fixtures", "folder-context", "readme.md"), - initialFolderFilePath, - ); - await copyFile(path.join(repositoryRoot, "corpus", "assets", "leaf.svg"), leafImagePath); - await writeFile( - tinyImagePath, - '', - ); - await writeFile( - imagesPath, - [ - "![Missing SVG](./missing.svg)", - "", - "![Visible SVG](./leaf.svg)", - "", - "[![Linked SVG](./leaf.svg)](https://example.com)", - "", - "![Tiny transparent SVG](./tiny-transparent.svg)", - "", - ].join("\n"), - ); - await writeJson(contextPath, context); - const scenarios: Scenario[] = [ { name: "block-selection", recentFiles: [blocksPath] }, { name: "diagnostics" }, @@ -222,58 +171,296 @@ const main = async () => { { name: "persistence-restart", continues: "persistence-write" }, { name: "window-lifecycle" }, ]; - const selectedScenarios = selectedScenarioNames.map((name) => { - const scenario = scenarios.find((candidate) => candidate.name === name); - if (!scenario) { - throw new Error(`Desktop E2E scenario configuration is missing: ${name}.`); + const worker: WorkerContext = { + appDataDirectory, + artifactsRoot: workerArtifactsRoot, + context, + contextPath, + label, + localDataDirectory, + port: WEBDRIVER_BASE_PORT + workerIndex, + scenarios: new Map(scenarios.map((scenario) => [scenario.name, scenario])), + storeDirectory, + temporaryRoot, + }; + + try { + const staleStateCleanupErrors = await removeDirectories([appDataDirectory, localDataDirectory]); + + if (staleStateCleanupErrors.length > 0) { + throw new AggregateError( + staleStateCleanupErrors, + `Failed to clear stale state for ${label}.`, + ); + } + + await settleTasks(`Failed to create directories for ${label}.`, [ + mkdir(folderPath, { recursive: true }), + mkdir(workerArtifactsRoot, { recursive: true }), + ]); + await settleTasks(`Failed to create fixtures for ${label}.`, [ + copyFile( + path.join(repositoryRoot, "e2e", "desktop", "fixtures", "block-selection.md"), + blocksPath, + ), + copyFile( + path.join(repositoryRoot, "e2e", "desktop", "fixtures", "rendered-html.md"), + htmlPath, + ), + copyFile( + path.join(repositoryRoot, "e2e", "desktop", "fixtures", "document-lifecycle.md"), + documentPath, + ), + copyFile( + path.join(repositoryRoot, "e2e", "desktop", "fixtures", "folder-context", "readme.md"), + initialFolderFilePath, + ), + copyFile(path.join(repositoryRoot, "corpus", "assets", "leaf.svg"), leafImagePath), + writeFile( + tinyImagePath, + '', + ), + writeFile( + imagesPath, + [ + "![Missing SVG](./missing.svg)", + "", + "![Visible SVG](./leaf.svg)", + "", + "[![Linked SVG](./leaf.svg)](https://example.com)", + "", + "![Tiny transparent SVG](./tiny-transparent.svg)", + "", + ].join("\n"), + ), + ]); + await writeJson(contextPath, context); + + return worker; + } catch (error) { + const cleanupErrors = await removeDirectories([ + appDataDirectory, + localDataDirectory, + temporaryRoot, + ]); + + if (cleanupErrors.length > 0) { + throw new AggregateError( + [error, ...cleanupErrors], + `${label} setup failed and its state could not be fully cleaned.`, + { cause: error }, + ); + } + + throw error; + } +}; + +const resetPersistedState = async ( + worker: WorkerContext, + recentFiles: string[] = [], + recentFolders: string[] = [], +) => { + await rm(worker.storeDirectory, { force: true, recursive: true }); + await mkdir(worker.storeDirectory, { recursive: true }); + await writeJson(path.join(worker.storeDirectory, "recent-items.dev.json"), { + recentFiles, + recentFolders, + version: 1, + }); +}; + +const runWdio = (scenario: Scenario, worker: WorkerContext) => + new Promise((resolve, reject) => { + const wdioExecutable = path.join( + repositoryRoot, + "node_modules", + "@wdio", + "cli", + "bin", + "wdio.js", + ); + const child = spawn(process.execPath, [wdioExecutable, "run", "e2e/desktop/wdio.conf.ts"], { + cwd: repositoryRoot, + env: { + ...process.env, + LEAFDOWN_E2E_APP_IDENTIFIER: worker.context.appIdentifier, + LEAFDOWN_E2E_ARTIFACT_RUN: RUN_LABEL, + LEAFDOWN_E2E_CONTEXT_PATH: worker.contextPath, + LEAFDOWN_E2E_SCENARIO: scenario.name, + LEAFDOWN_E2E_SPEC: `e2e/desktop/specs/${scenario.name}.spec.ts`, + LEAFDOWN_E2E_WEBDRIVER_PORT: String(worker.port), + LEAFDOWN_E2E_WORKER: worker.label, + TAURI_WEBDRIVER_PORT: String(worker.port), + }, + stdio: "inherit", + }); + + child.once("error", reject); + child.once("exit", (code, signal) => { + if (code === 0) { + resolve(); + return; + } + + reject( + new Error( + `Desktop E2E scenario ${scenario.name} failed on ${worker.label} (code=${String(code)}, signal=${String(signal)}).`, + ), + ); + }); + }); + +const runScenario = async (scenario: Scenario, worker: WorkerContext) => { + let scenarioError: unknown; + + try { + await runWdio(scenario, worker); + } catch (error) { + scenarioError = error; + } + + const portReleased = await waitForPortRelease(worker.port); + + if (!portReleased) { + const portError = new Error( + `Scenario ${scenario.name} left a listener on port ${worker.port}. Stop it before the next run.`, + ); + + if (scenarioError) { + throw new AggregateError( + [scenarioError, portError], + `${scenario.name} failed and leaked its port.`, + ); } - return scenario; + throw portError; + } + + if (scenarioError) { + throw scenarioError; + } +}; + +const writeFixtureManifest = async ( + worker: WorkerContext, + failedScenario: string, + error: unknown, +) => { + const { context } = worker; + + await writeJson(path.join(worker.artifactsRoot, "fixture-manifest.json"), { + document: await fileEvidence(context.document.path, context.document.savedMarkdown), + error: String(error), + failedAt: new Date().toISOString(), + failedScenario, + folderAddedDocument: await fileEvidence( + context.folder.addedFilePath, + `${context.folder.addedMarker}\n`, + ), + folderInitialDocument: await fileEvidence( + context.folder.initialFilePath, + `${context.folder.initialMarker}\n`, + ), + missingDocument: await fileEvidence(context.missingDocumentPath), + temporaryRoot: context.temporaryRoot, + webdriverPort: worker.port, + worker: worker.label, }); +}; + +const runScenarioGroup = async (scenarioNames: string[], worker: WorkerContext) => { + let activeScenarioName = scenarioNames[0] ?? "unknown"; try { - for (const [index, scenario] of selectedScenarios.entries()) { + if (!(await isPortFree(worker.port))) { + throw new Error(`WebDriver port ${worker.port} for ${worker.label} is already in use.`); + } + + for (const [index, scenarioName] of scenarioNames.entries()) { + activeScenarioName = scenarioName; + const scenario = worker.scenarios.get(scenarioName); + + if (!scenario) { + throw new Error(`Desktop E2E scenario configuration is missing: ${scenarioName}.`); + } + if (scenario.continues) { - if (scenario.continues !== selectedScenarios[index - 1]?.name) { + if (scenario.continues !== scenarioNames[index - 1]) { throw new Error( - `Scenario ${scenario.name} must run directly after ${scenario.continues}.`, + `Scenario ${scenario.name} must run directly after ${scenario.continues} on one worker.`, ); } } else { - await resetPersistedState(scenario.recentFiles, scenario.recentFolders); + await resetPersistedState(worker, scenario.recentFiles, scenario.recentFolders); } - await runWdio(scenario); + await runScenario(scenario, worker); + } + } catch (error) { + await writeFixtureManifest(worker, activeScenarioName, error); + throw error; + } +}; - if (!(await waitForPortRelease())) { - throw new Error( - `Scenario ${scenario.name} left a listener on port ${WEBDRIVER_PORT}. Stop it before the next run.`, - ); - } +const main = async () => { + if (!process.env.APPDATA || !process.env.LOCALAPPDATA) { + throw new Error("APPDATA and LOCALAPPDATA are required for the Windows desktop E2E suite."); + } + + await mkdir(artifactsRoot, { recursive: true }); + + const effectiveWorkerCount = Math.min(selection.workerCount, selection.scenarioGroups.length); + const workers: WorkerContext[] = []; + let runError: unknown; + let runFailed = false; + + try { + for (let index = 0; index < effectiveWorkerCount; index += 1) { + workers.push(await createWorkerContext(index)); + } + + const failures = await runWorkerPool({ + groups: selection.scenarioGroups, + runGroup: (scenarioNames, workerIndex) => + runScenarioGroup(scenarioNames, workers[workerIndex]), + workerCount: effectiveWorkerCount, + }); + + if (failures.length > 0) { + throw new AggregateError( + failures.map(({ error }) => error), + `${failures.length} desktop E2E scenario group${failures.length === 1 ? "" : "s"} failed.`, + ); } } catch (error) { - await writeJson(path.join(artifactsRoot, "fixture-manifest.json"), { - document: await fileEvidence(documentPath, context.document.savedMarkdown), - failedAt: new Date().toISOString(), - folderAddedDocument: await fileEvidence( - addedFolderFilePath, - `${context.folder.addedMarker}\n`, - ), - folderInitialDocument: await fileEvidence( - initialFolderFilePath, - `${context.folder.initialMarker}\n`, - ), - missingDocument: await fileEvidence(missingDocumentPath), + runError = error; + runFailed = true; + } + + const cleanupErrors = await removeDirectories( + workers.flatMap(({ appDataDirectory, localDataDirectory, temporaryRoot }) => [ + appDataDirectory, + localDataDirectory, temporaryRoot, - }); + ]), + ); - throw error; - } finally { - await Promise.all([ - rm(temporaryRoot, { force: true, recursive: true }), - rm(e2eAppDataDirectory, { force: true, recursive: true }), - ]); + if (runFailed && cleanupErrors.length > 0) { + throw new AggregateError( + [runError, ...cleanupErrors], + "The desktop E2E run failed and its state could not be fully cleaned.", + { cause: runError }, + ); + } + + if (runFailed) { + throw runError; + } + + if (cleanupErrors.length > 0) { + throw new AggregateError(cleanupErrors, "The desktop E2E state could not be fully cleaned."); } }; diff --git a/e2e/desktop/specs/diagnostics.spec.ts b/e2e/desktop/specs/diagnostics.spec.ts index 111cb1c5..5cab57d1 100644 --- a/e2e/desktop/specs/diagnostics.spec.ts +++ b/e2e/desktop/specs/diagnostics.spec.ts @@ -1,6 +1,7 @@ import { $, browser, expect } from "@wdio/globals"; import { getDiagnosticsSummary } from "../support/diagnostics.js"; +import { getDesktopE2ERunContext } from "../support/runContext.js"; import { openMenu } from "../support/ui.js"; describe("desktop diagnostics", () => { @@ -17,9 +18,10 @@ describe("desktop diagnostics", () => { await expect(summaryField).toHaveValue(expect.stringContaining("Leafdown diagnostics")); const summary = await getDiagnosticsSummary(); + const { appIdentifier } = await getDesktopE2ERunContext(); const summaryText = await summaryField.getValue(); - expect(summary.appIdentifier).toBe("com.azganoth.leafdown.e2e"); + expect(summary.appIdentifier).toBe(appIdentifier); expect(summary.runId).not.toHaveLength(0); expect(summaryText).toContain(`Identifier: ${summary.appIdentifier}`); expect(summaryText).toContain(`Run: ${summary.runId}`); diff --git a/e2e/desktop/support/artifacts.ts b/e2e/desktop/support/artifacts.ts index 5b9814cc..83c8ac93 100644 --- a/e2e/desktop/support/artifacts.ts +++ b/e2e/desktop/support/artifacts.ts @@ -8,6 +8,11 @@ import { RUN_LABEL } from "./suite.js"; const repositoryRoot = fileURLToPath(new URL("../../..", import.meta.url)); const scenarioLabel = process.env.LEAFDOWN_E2E_SCENARIO; +const workerLabel = process.env.LEAFDOWN_E2E_WORKER; + +if (!scenarioLabel || !workerLabel) { + throw new Error("Desktop E2E artifact capture requires a worker and scenario identity."); +} export const ARTIFACTS_DIR = path.join( repositoryRoot, @@ -15,7 +20,8 @@ export const ARTIFACTS_DIR = path.join( "desktop", "artifacts", RUN_LABEL, - ...(scenarioLabel ? [scenarioLabel] : []), + workerLabel, + scenarioLabel, ); const writeJson = async (fileName: string, value: unknown) => { diff --git a/e2e/desktop/support/runContext.ts b/e2e/desktop/support/runContext.ts index e2bf12a2..fa491efe 100644 --- a/e2e/desktop/support/runContext.ts +++ b/e2e/desktop/support/runContext.ts @@ -1,6 +1,7 @@ import { readFile } from "node:fs/promises"; export interface DesktopE2ERunContext { + appIdentifier: string; blocks: { path: string }; html: { path: string }; document: { diff --git a/e2e/desktop/support/scenarioSelection.test.ts b/e2e/desktop/support/scenarioSelection.test.ts index 62a3920e..93c315c5 100644 --- a/e2e/desktop/support/scenarioSelection.test.ts +++ b/e2e/desktop/support/scenarioSelection.test.ts @@ -1,49 +1,61 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { selectScenarioNames } from "./scenarioSelection.js"; +import { MAX_DESKTOP_E2E_WORKERS, selectDesktopE2ERun } from "./scenarioSelection.js"; void test("runs every scenario when no focused target is requested", () => { - assert.deepEqual(selectScenarioNames([]), [ - "block-selection", - "diagnostics", - "document-lifecycle", - "folder-watcher", - "rendered-images", - "rendered-html", - "missing-document-error", - "persistence-write", - "persistence-restart", - "window-lifecycle", - ]); + assert.deepEqual(selectDesktopE2ERun([]), { + scenarioGroups: [ + ["block-selection"], + ["diagnostics"], + ["document-lifecycle"], + ["folder-watcher"], + ["rendered-images"], + ["rendered-html"], + ["missing-document-error"], + ["persistence-write", "persistence-restart"], + ["window-lifecycle"], + ], + workerCount: 1, + }); }); void test("selects one independent scenario", () => { - assert.deepEqual(selectScenarioNames(["--scenario", "folder-watcher"]), ["folder-watcher"]); + assert.deepEqual(selectDesktopE2ERun(["--scenario", "folder-watcher"]), { + scenarioGroups: [["folder-watcher"]], + workerCount: 1, + }); }); -void test("selects the block-selection scenario", () => { - assert.deepEqual(selectScenarioNames(["--scenario", "block-selection"]), ["block-selection"]); -}); - -void test("selects the persistence write and restart sequence together", () => { - assert.deepEqual(selectScenarioNames(["--scenario", "persistence"]), [ - "persistence-write", - "persistence-restart", - ]); +void test("selects the persistence sequence and a bounded worker count in either order", () => { + for (const args of [ + ["--scenario", "persistence", "--workers", "2"], + ["--workers", "2", "--scenario", "persistence"], + ]) { + assert.deepEqual(selectDesktopE2ERun(args), { + scenarioGroups: [["persistence-write", "persistence-restart"]], + workerCount: 2, + }); + } }); -void test("rejects missing, unknown, and malformed targets with the valid target list", () => { +void test("rejects missing, unknown, duplicate, and malformed options", () => { for (const args of [ ["--scenario"], ["--scenario", "unknown"], ["--scenario=persistence"], ["folder-watcher"], ["--scenario", "folder-watcher", "extra"], + ["--scenario", "folder-watcher", "--scenario", "diagnostics"], + ["--workers", "2", "--workers", "3"], + ["--workers", "0"], + ["--workers", String(MAX_DESKTOP_E2E_WORKERS + 1)], + ["--workers", "1.5"], + ["--workers", "two"], ]) { assert.throws( - () => selectScenarioNames(args), - /Usage: pnpm test:e2e:desktop:run -- --scenario [\s\S]*Valid scenarios:/u, + () => selectDesktopE2ERun(args), + /Usage: pnpm test:e2e:desktop:run -- \[--scenario \] \[--workers <1-4>\][\s\S]*Valid scenarios:/u, ); } }); diff --git a/e2e/desktop/support/scenarioSelection.ts b/e2e/desktop/support/scenarioSelection.ts index c334479b..fb29582b 100644 --- a/e2e/desktop/support/scenarioSelection.ts +++ b/e2e/desktop/support/scenarioSelection.ts @@ -10,27 +10,73 @@ const scenarioTargets = { "window-lifecycle": ["window-lifecycle"], } as const; +export const MAX_DESKTOP_E2E_WORKERS = 4; + const validScenarioTargets = Object.keys(scenarioTargets).join(", "); const selectionError = (reason: string) => new Error( - `${reason}\nUsage: pnpm test:e2e:desktop:run -- --scenario \nValid scenarios: ${validScenarioTargets}`, + `${reason}\nUsage: pnpm test:e2e:desktop:run -- [--scenario ] [--workers <1-${MAX_DESKTOP_E2E_WORKERS}>]\nValid scenarios: ${validScenarioTargets}`, ); -export const selectScenarioNames = (arguments_: readonly string[]) => { - if (arguments_.length === 0) { - return Object.values(scenarioTargets).flat(); - } +export interface DesktopE2ERunSelection { + scenarioGroups: string[][]; + workerCount: number; +} - if (arguments_.length !== 2 || arguments_[0] !== "--scenario" || !arguments_[1]) { - throw selectionError("Expected exactly one --scenario argument."); - } +export const selectDesktopE2ERun = (arguments_: readonly string[]): DesktopE2ERunSelection => { + let scenarioTarget: keyof typeof scenarioTargets | undefined; + let workerCount = 1; + + for (let index = 0; index < arguments_.length; index += 2) { + const option = arguments_[index]; + const value = arguments_[index + 1]; + + if (!value) { + throw selectionError(`Expected a value after ${option ?? "the final option"}.`); + } + + if (option === "--scenario") { + if (scenarioTarget) { + throw selectionError("The --scenario option may only be provided once."); + } + + if (!(value in scenarioTargets)) { + throw selectionError(`Unknown desktop E2E scenario: ${value}.`); + } + + scenarioTarget = value as keyof typeof scenarioTargets; + continue; + } + + if (option === "--workers") { + if (workerCount !== 1 || arguments_.slice(0, index).includes("--workers")) { + throw selectionError("The --workers option may only be provided once."); + } + + const parsedWorkerCount = Number(value); + + if ( + !Number.isInteger(parsedWorkerCount) || + parsedWorkerCount < 1 || + parsedWorkerCount > MAX_DESKTOP_E2E_WORKERS + ) { + throw selectionError( + `Desktop E2E workers must be an integer from 1 to ${MAX_DESKTOP_E2E_WORKERS}.`, + ); + } - const selectedScenarioNames = scenarioTargets[arguments_[1] as keyof typeof scenarioTargets]; + workerCount = parsedWorkerCount; + continue; + } - if (!selectedScenarioNames) { - throw selectionError(`Unknown desktop E2E scenario: ${arguments_[1]}.`); + throw selectionError(`Unknown desktop E2E option: ${option}.`); } - return [...selectedScenarioNames]; + return { + scenarioGroups: scenarioTarget + ? [[...scenarioTargets[scenarioTarget]]] + : Object.values(scenarioTargets).map((scenarioNames) => [...scenarioNames]), + workerCount, + }; }; diff --git a/e2e/desktop/support/suite.ts b/e2e/desktop/support/suite.ts index a9781a79..14da7006 100644 --- a/e2e/desktop/support/suite.ts +++ b/e2e/desktop/support/suite.ts @@ -2,4 +2,19 @@ export const RUN_LABEL = process.env.LEAFDOWN_E2E_ARTIFACT_RUN ?? `${new Date().toISOString().replaceAll(":", "-").replaceAll(".", "-")}-${process.pid}`; -export const WEBDRIVER_PORT = 4445; +export const WEBDRIVER_BASE_PORT = 4445; + +const configuredWebDriverPort = process.env.LEAFDOWN_E2E_WEBDRIVER_PORT; +const parsedWebDriverPort = configuredWebDriverPort + ? Number(configuredWebDriverPort) + : WEBDRIVER_BASE_PORT; + +if ( + !Number.isInteger(parsedWebDriverPort) || + parsedWebDriverPort < 1 || + parsedWebDriverPort > 65_535 +) { + throw new Error("LEAFDOWN_E2E_WEBDRIVER_PORT must be a valid TCP port."); +} + +export const WEBDRIVER_PORT = parsedWebDriverPort; diff --git a/e2e/desktop/support/workerPool.test.ts b/e2e/desktop/support/workerPool.test.ts new file mode 100644 index 00000000..8fc8ada1 --- /dev/null +++ b/e2e/desktop/support/workerPool.test.ts @@ -0,0 +1,69 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { runWorkerPool } from "./workerPool.js"; + +void test("bounds overlapping groups by the requested worker count", async () => { + let activeGroups = 0; + let maximumActiveGroups = 0; + + const failures = await runWorkerPool({ + groups: ["first", "second", "third", "fourth"], + runGroup: async () => { + activeGroups += 1; + maximumActiveGroups = Math.max(maximumActiveGroups, activeGroups); + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + activeGroups -= 1; + }, + workerCount: 2, + }); + + assert.deepEqual(failures, []); + assert.equal(maximumActiveGroups, 2); +}); + +void test("keeps every scenario in a logical group on one worker in order", async () => { + const observed: Array<{ group: string[]; workerIndex: number }> = []; + + await runWorkerPool({ + groups: [["persistence-write", "persistence-restart"], ["diagnostics"]], + runGroup: async (group, workerIndex) => { + observed.push({ group, workerIndex }); + }, + workerCount: 2, + }); + + assert.equal(observed.length, 2); + assert.deepEqual(observed[0], { + group: ["persistence-write", "persistence-restart"], + workerIndex: 0, + }); + assert.deepEqual(observed[1], { group: ["diagnostics"], workerIndex: 1 }); +}); + +void test("lets healthy workers finish queued groups after another worker fails", async () => { + const completed: string[] = []; + + const failures = await runWorkerPool({ + groups: ["fails", "slow", "after-failure"], + runGroup: async (group) => { + if (group === "fails") { + throw new Error("expected failure"); + } + + await new Promise((resolve) => { + setTimeout(resolve, 5); + }); + completed.push(group); + }, + workerCount: 2, + }); + + assert.deepEqual(completed, ["slow", "after-failure"]); + assert.equal(failures.length, 1); + assert.equal(failures[0]?.group, "fails"); + assert.equal(failures[0]?.workerIndex, 0); + assert.match(String(failures[0]?.error), /expected failure/u); +}); diff --git a/e2e/desktop/support/workerPool.ts b/e2e/desktop/support/workerPool.ts new file mode 100644 index 00000000..087fb4d8 --- /dev/null +++ b/e2e/desktop/support/workerPool.ts @@ -0,0 +1,41 @@ +export interface WorkerGroupFailure { + error: unknown; + group: T; + groupIndex: number; + workerIndex: number; +} + +interface RunWorkerPoolOptions { + groups: readonly T[]; + runGroup: (group: T, workerIndex: number) => Promise; + workerCount: number; +} + +export const runWorkerPool = async ({ + groups, + runGroup, + workerCount, +}: RunWorkerPoolOptions) => { + const effectiveWorkerCount = Math.min(workerCount, groups.length); + const failures: WorkerGroupFailure[] = []; + let nextGroupIndex = 0; + + const runWorker = async (workerIndex: number) => { + while (nextGroupIndex < groups.length) { + const groupIndex = nextGroupIndex; + nextGroupIndex += 1; + const group = groups[groupIndex]; + + try { + await runGroup(group, workerIndex); + } catch (error) { + failures.push({ error, group, groupIndex, workerIndex }); + return; + } + } + }; + + await Promise.all(Array.from({ length: effectiveWorkerCount }, (_, index) => runWorker(index))); + + return failures.toSorted((left, right) => left.groupIndex - right.groupIndex); +}; diff --git a/package.json b/package.json index e5b71536..b7e25432 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "build:e2e:desktop": "tauri build --debug --no-bundle --features desktop-e2e --config e2e/desktop/tauri.conf.json -- --target-dir target/desktop-e2e", "test:e2e:desktop": "pnpm build:e2e:desktop && pnpm test:e2e:desktop:run", "test:e2e:desktop:run": "tsx e2e/desktop/run.ts", - "test:e2e:desktop:runner": "tsx --test e2e/desktop/support/scenarioSelection.test.ts", + "test:e2e:desktop:runner": "tsx --test e2e/desktop/support/scenarioSelection.test.ts e2e/desktop/support/workerPool.test.ts", "preview": "vite preview", "tauri": "tauri", "lint": "pnpm lint:frontend && pnpm lint:backend", diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 5e2300d6..a2847de5 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -36,6 +36,19 @@ pub fn run() { .plugin(tauri_plugin_wdio::init()) .plugin(tauri_plugin_wdio_webdriver::init()); + let context = tauri::generate_context!(); + + #[cfg(feature = "desktop-e2e")] + let context = { + let mut context = context; + + if let Ok(identifier) = std::env::var("LEAFDOWN_E2E_APP_IDENTIFIER") { + context.config_mut().identifier = identifier; + } + + context + }; + builder .plugin( tauri_plugin_window_state::Builder::new() @@ -94,7 +107,7 @@ pub fn run() { folder::watch_markdown_folder, folder::unwatch_markdown_folder ]) - .run(tauri::generate_context!()) + .run(context) .unwrap_or_else(|error| { log::error!("error while running tauri application: {error}"); panic!("error while running tauri application: {error}"); From 760629841feddd5c9081607a687e94c1cbc72849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ademir=20Jos=C3=A9=20Ferreira=20J=C3=BAnior?= Date: Tue, 22 Sep 2026 13:55:34 -0300 Subject: [PATCH 2/2] Accept pnpm argument separator in desktop E2E runner --- e2e/desktop/support/scenarioSelection.test.ts | 1 + e2e/desktop/support/scenarioSelection.ts | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/e2e/desktop/support/scenarioSelection.test.ts b/e2e/desktop/support/scenarioSelection.test.ts index 93c315c5..2ed5e767 100644 --- a/e2e/desktop/support/scenarioSelection.test.ts +++ b/e2e/desktop/support/scenarioSelection.test.ts @@ -31,6 +31,7 @@ void test("selects the persistence sequence and a bounded worker count in either for (const args of [ ["--scenario", "persistence", "--workers", "2"], ["--workers", "2", "--scenario", "persistence"], + ["--", "--workers", "2", "--scenario", "persistence"], ]) { assert.deepEqual(selectDesktopE2ERun(args), { scenarioGroups: [["persistence-write", "persistence-restart"]], diff --git a/e2e/desktop/support/scenarioSelection.ts b/e2e/desktop/support/scenarioSelection.ts index fb29582b..6e580ed4 100644 --- a/e2e/desktop/support/scenarioSelection.ts +++ b/e2e/desktop/support/scenarioSelection.ts @@ -25,12 +25,13 @@ export interface DesktopE2ERunSelection { } export const selectDesktopE2ERun = (arguments_: readonly string[]): DesktopE2ERunSelection => { + const options = arguments_[0] === "--" ? arguments_.slice(1) : arguments_; let scenarioTarget: keyof typeof scenarioTargets | undefined; let workerCount = 1; - for (let index = 0; index < arguments_.length; index += 2) { - const option = arguments_[index]; - const value = arguments_[index + 1]; + for (let index = 0; index < options.length; index += 2) { + const option = options[index]; + const value = options[index + 1]; if (!value) { throw selectionError(`Expected a value after ${option ?? "the final option"}.`); @@ -50,7 +51,7 @@ export const selectDesktopE2ERun = (arguments_: readonly string[]): DesktopE2ERu } if (option === "--workers") { - if (workerCount !== 1 || arguments_.slice(0, index).includes("--workers")) { + if (workerCount !== 1 || options.slice(0, index).includes("--workers")) { throw selectionError("The --workers option may only be provided once."); }