diff --git a/docs/operations/scenarios.md b/docs/operations/scenarios.md index c49f23a4b2068..22b61bf25301b 100644 --- a/docs/operations/scenarios.md +++ b/docs/operations/scenarios.md @@ -41,8 +41,10 @@ adds one fixture for what e2e cannot know about T3 Code: - `t3.project(title, files)` creates a git repository and adds it as a project. - `t3.thread({ project, title, messages })` starts a thread and plays one replayed turn per - message before returning, so the thread has real history. `model` picks the recording. -- `t3.open(path)` opens the app, already signed in. + message before returning, so the thread has real history. `model` picks the recording. It + returns the thread, which `t3.open` takes. +- `t3.open(path)` opens the app, already signed in, at a path. `t3.open(thread)` opens it on a + thread. State is created through the app's own CLI, so it is real events and works on a base commit whose schema differs. New threads default to the replay agent's `showcase` recording: type a @@ -61,6 +63,30 @@ could not run. Keep `agent.act` and `agent.assert` out of scenarios: they call a model, and no model is configured. +### Getting to the screen + +Most scenarios that could not run stopped on the way to the screen the bug is on, not on the +bug. Three rules avoid that: + +- **Open by link, not by sidebar.** `await t3.open(thread)` lands on the thread + ([`thread-opens-by-link.e2e.ts`](../../scripts/scenarios/thread-opens-by-link.e2e.ts)), and + `await t3.open("/settings/general")` on a settings page. A sidebar row is a button that covers + its own title, so a tap on `getByText("Earlier work")` is intercepted. When the sidebar is what + the scenario is about, target the row's button: `screen.getByRole("button", "Earlier work")`. +- **Let `t3.open` do the navigating.** It waits until the pairing page has signed in before it + goes anywhere. A second `app.open` straight after the pairing link cancels the sign-in and + shows the pairing page again. +- **Seed before opening.** Create projects and threads first, then call `t3.open`. With no + project the app opens its first-run page instead of the one the scenario expects. + +The app starts with only `PATH`, `HOME` and the temp directory in its environment, whatever the +shell that ran the scenario has set. Name what it needs in `T3_SCENARIO_INHERIT_ENV`: exact +names or prefixes ending in `*`, separated by commas. + +```sh +T3_SCENARIO_INHERIT_ENV='SHELL,MY_FIXTURE_*' vp run scenarios:prove scripts/scenarios/name.e2e.ts --base main +``` + ## Run ```sh @@ -102,12 +128,27 @@ commits and compares the two reports. It prints, and writes to `scripts/.e2e/pro its commit, ready for a pull request description. `--upload-command` names a command that takes the capture paths and prints one URL per line. `--head` defaults to `HEAD`. -It exits 0 only when the scenario fails on base and passes on head, 1 when it ran and did not -prove the fix, and 2 when it could not run. A scenario that passes on base does not reproduce the -bug, and the verdict says so. +It exits 0 only when the scenario fails on base and passes on head, and 1 when it ran and did not +prove the fix. A scenario that passes on base does not reproduce the bug, and the verdict says so. + +Two more codes mean there is no verdict on the behavior, and `couldNotRun` in the proof says +which: + +- **3, `scenario`:** the app ran and the scenario stopped before its assertion, usually on a + control it did not find. Read that side's `message` and `steps`, fix the scenario, and run it + again. +- **2, `app`:** the app did not start or no test ran: a bad revision, a failed install, a port in + use. `scripts/.e2e/app.log` has the app's output. + +Each side's result is also written as it ends, to `scripts/.e2e/prove/base.json` and +`head.json`, so a run that is cut short keeps the side it finished. + +It runs the scenario on one target: iOS when the file imports `t3-mobile.ts`, web otherwise. +`--target web` or `--target ios` overrides that. The scenario and e2e always come from your checkout. Only the app comes from the commit under -test: a temporary worktree beside your checkout, installed with `vp i` and removed afterwards. A +test: a temporary worktree beside your checkout, installed with `vp i` and removed afterwards (one +left by a run that was killed is replaced). A commit that is your clean `HEAD` runs in place. So a new scenario can prove a fix against a base that predates it. diff --git a/scripts/e2e.config.ts b/scripts/e2e.config.ts index 520fc8338da71..989d63326ac87 100644 --- a/scripts/e2e.config.ts +++ b/scripts/e2e.config.ts @@ -31,6 +31,12 @@ const resolved = await Effect.gen(function* () { appDir: path.resolve(scripts, env.T3_SCENARIO_APP_DIR ?? ".."), home, startApp: path.join(scripts, "scenarios/start-app.ts"), + // e2e gives the app only PATH, HOME and the temp directory. A scenario that needs more names + // it here: exact names or prefixes ending in `*`, separated by commas. + inheritEnv: (env.T3_SCENARIO_INHERIT_ENV ?? "") + .split(",") + .map((name) => name.trim()) + .filter((name) => name !== ""), mobile: { host: host || "127.0.0.1", // A mobile target has no `app.url`, so e2e cannot pick these ports. @@ -143,6 +149,7 @@ export default { executable: resolved.node, args: [startApp, "web", T3_HOME, LOOPBACK, "{port}"], cwd: APP_DIR, + inheritEnv: resolved.inheritEnv, startupTimeout: 300_000, log: ".e2e/app.log", }, @@ -170,6 +177,7 @@ export default { MOBILE.host, ], cwd: APP_DIR, + inheritEnv: resolved.inheritEnv, startupTimeout: 300_000, log: ".e2e/mobile-app.log", }, diff --git a/scripts/prove-scenario.test.ts b/scripts/prove-scenario.test.ts index 51fc2f3ece7d0..a22d636b516f0 100644 --- a/scripts/prove-scenario.test.ts +++ b/scripts/prove-scenario.test.ts @@ -91,6 +91,20 @@ describe("prove-scenario", () => { expect(proof.verdict).toMatch(row.verdict); }); + it("tells a scenario that stopped from an app that did not start", () => { + const side = (report: Parameters[2]) => + readSide("aaaaaaaaaaaa", "/out/base/results", report); + const head = readSide("bbbbbbbbbbbb", "/out/head/results", passed); + // A control the scenario did not find: the app ran, so the scenario is what needs fixing. + const stopped = buildProof(side(brokenScenario), head, (path) => path); + expect(stopped.couldNotRun).toBe("scenario"); + expect(stopped.verdict).toMatch(/fix the scenario/); + const noApp = buildProof(side(undefined), head, (path) => path); + expect(noApp.couldNotRun).toBe("app"); + expect(noApp.verdict).not.toMatch(/fix the scenario/); + expect(buildProof(side(failed), head, (path) => path).couldNotRun).toBeNull(); + }); + it("labels captures with their side and commit and keeps the pairing token out", () => { const proof = buildProof( readSide("aaaaaaaaaaaa", "/out/base/results", failed), diff --git a/scripts/prove-scenario.ts b/scripts/prove-scenario.ts index 6b0bb9c1a44f6..68e5433ebef2b 100644 --- a/scripts/prove-scenario.ts +++ b/scripts/prove-scenario.ts @@ -53,6 +53,11 @@ export interface SideResult { readonly status: "passed" | "failed" | "error"; readonly title: string; readonly message?: string; + /** + * For `error`, what stopped: `app` when the app did not start or no test ran, `scenario` when + * the app ran and the scenario stopped before its assertion, such as on a control it did not find. + */ + readonly stopped?: "app" | "scenario"; readonly steps: ReadonlyArray<{ readonly step: string; readonly status: string }>; /** Capture paths under the run's results directory, in the order the scenario took them. */ readonly captures: ReadonlyArray; @@ -75,7 +80,16 @@ export function readSide( executed.length > 1 ? `the scenario file ran ${executed.length} tests; a proof needs exactly one` : "e2e wrote no report: the app did not start, or no test was selected"; - return { sha, status: "error", title: "", message, steps: [], captures: [], videos: [] }; + return { + sha, + status: "error", + stopped: "app", + title: "", + message, + steps: [], + captures: [], + videos: [], + }; } const files = (kind: string) => (attempt.artifacts ?? []).flatMap((artifact) => @@ -84,15 +98,18 @@ export function readSide( : [], ); const error = attempt.error; + // e2e exits 0 for a skipped test too, and a test that did not run proves nothing. + const status = + attempt.status === "passed" + ? "passed" + : error?.code === "ASSERTION_FAILED" + ? "failed" + : "error"; return { sha, - // e2e exits 0 for a skipped test too, and a test that did not run proves nothing. - status: - attempt.status === "passed" - ? "passed" - : error?.code === "ASSERTION_FAILED" - ? "failed" - : "error", + status, + // A test that ended with an error of its own got as far as the app. + ...(status === "error" ? { stopped: error === undefined ? "app" : "scenario" } : {}), title: result.titlePath.join(" > "), ...(error === undefined ? attempt.status === "passed" @@ -123,7 +140,7 @@ export function buildProof(base: SideResult, head: SideResult, link: (path: stri const broken = base.status === "error" ? base : head.status === "error" ? head : undefined; const verdict = broken !== undefined - ? `Not proven: the scenario could not run on ${broken === base ? `base ${baseSha}` : `head ${headSha}`} (${broken.message ?? "unknown error"}).` + ? `Not proven: the scenario could not run on ${broken === base ? `base ${baseSha}` : `head ${headSha}`} (${broken.message ?? "unknown error"}).${broken.stopped === "scenario" ? " The app ran and the scenario stopped before its assertion: fix the scenario and run it again." : ""}` : base.status === "passed" ? head.status === "passed" ? `Not proven: the scenario passes on base ${baseSha} too, so it does not reproduce the bug.` @@ -160,6 +177,8 @@ export function buildProof(base: SideResult, head: SideResult, link: (path: stri scenario: head.title || base.title, failsOnBase: base.status === "failed", passesOnHead: head.status === "passed", + /** Why there is no verdict on the behavior: the app did not run, or the scenario stopped. */ + couldNotRun: broken?.stopped ?? null, baseSha: base.sha, headSha: head.sha, verdict, @@ -192,10 +211,18 @@ const withCheckout = Effect.fn("prove.withCheckout")(function* ( if (clean && (yield* git(["rev-parse", "HEAD"])) === sha) return yield* use(repo); // Beside this checkout, not in the temp directory, so tooling resolves as it does here. const worktree = `${repo}-scenario-${short(sha)}`; + // A run that was killed left its worktree behind, and git refuses to add over it. + const discard = git(["worktree", "remove", "--force", worktree]).pipe( + Effect.ignore, + Effect.andThen(git(["worktree", "prune"])), + Effect.ignore, + ); return yield* Effect.acquireUseRelease( - git(["worktree", "add", "--quiet", "--detach", "--force", worktree, sha]), + discard.pipe( + Effect.andThen(git(["worktree", "add", "--quiet", "--detach", "--force", worktree, sha])), + ), () => run("vp", ["i"], { cwd: worktree, echo: true }).pipe(Effect.andThen(use(worktree))), - () => git(["worktree", "remove", "--force", worktree]).pipe(Effect.ignore), + () => discard, ); }); @@ -204,6 +231,7 @@ const runSide = Effect.fn("prove.runSide")(function* ( side: "base" | "head", rev: string, scenarioPath: string, + target: string, ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -216,7 +244,7 @@ const runSide = Effect.fn("prove.runSide")(function* ( // The scenario and e2e come from this checkout; only the app comes from `appDir`. run( path.join(scripts, "node_modules/.bin/e2e"), - ["run", scenarioPath, "--output", path.relative(scripts, outDir)], + ["run", scenarioPath, "--target", target, "--output", path.relative(scripts, outDir)], { cwd: scripts, env: { ...environment, T3_SCENARIO_APP_DIR: appDir, E2E_TELEMETRY_DISABLED: "1" }, @@ -230,7 +258,22 @@ const runSide = Effect.fn("prove.runSide")(function* ( Effect.flatMap(decodeReport), Effect.orElseSucceed(() => undefined), ); - return readSide(sha, path.join(outDir, "results"), report); + const result = readSide(sha, path.join(outDir, "results"), report); + // Written as each side ends, so a proof whose second side is cut short still has its first. + yield* fs + .writeFileString( + path.join(scripts, ".e2e/prove", `${side}.json`), + `${yield* encodeProof(result)}\n`, + ) + .pipe(Effect.ignore); + return result; +}); + +/** The one target a scenario runs on: an iOS scenario imports `t3-mobile.ts`, a web one does not. */ +const targetOf = Effect.fn("prove.targetOf")(function* (scenarioPath: string) { + const fs = yield* FileSystem.FileSystem; + const source = yield* fs.readFileString(scenarioPath).pipe(Effect.orElseSucceed(() => "")); + return source.includes("t3-mobile") ? "ios" : "web"; }); /** Uploads the captures and maps each to its URL; a failed upload leaves the map empty. */ @@ -262,6 +305,7 @@ const prove = Effect.fn("prove")(function* (input: { readonly base: string; readonly head: string; readonly uploadCommand: Option.Option; + readonly target: Option.Option; }) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -271,8 +315,10 @@ const prove = Effect.fn("prove")(function* (input: { // A proof left by an earlier run must not be read as this one's. yield* fs.remove(proveDir, { recursive: true, force: true }); yield* fs.makeDirectory(proveDir, { recursive: true }); - const base = yield* runSide("base", input.base, scenarioPath); - const head = yield* runSide("head", input.head, scenarioPath); + // Only that target's app starts: a web scenario needs no Metro and no device. + const target = Option.getOrElse(input.target, () => "") || (yield* targetOf(scenarioPath)); + const base = yield* runSide("base", input.base, scenarioPath, target); + const head = yield* runSide("head", input.head, scenarioPath, target); const links = Option.isSome(input.uploadCommand) ? yield* upload(input.uploadCommand.value, [...base.captures, ...head.captures]) : new Map(); @@ -285,8 +331,13 @@ const prove = Effect.fn("prove")(function* (input: { yield* fs.writeFileString(path.join(proveDir, "proof.md"), proof.markdown); yield* fs.writeFileString(path.join(proveDir, "proof.json"), `${json}\n`); yield* Console.log(json); - const couldNotRun = base.status === "error" || head.status === "error"; - return proof.failsOnBase && proof.passesOnHead ? 0 : couldNotRun ? 2 : 1; + return proof.failsOnBase && proof.passesOnHead + ? 0 + : proof.couldNotRun === "scenario" + ? 3 + : proof.couldNotRun === "app" + ? 2 + : 1; }); const command = Command.make( @@ -304,6 +355,12 @@ const command = Command.make( Flag.withDescription("Command that takes capture paths and prints one URL per line."), Flag.optional, ), + target: Flag.String("target").pipe( + Flag.withDescription( + "Target to run on: web or ios. Default: ios when the scenario imports t3-mobile.ts, else web.", + ), + Flag.optional, + ), }, (input) => prove(input).pipe( diff --git a/scripts/scenarios/t3.ts b/scripts/scenarios/t3.ts index 4df481ace8cf2..24bb6e73d38b3 100644 --- a/scripts/scenarios/t3.ts +++ b/scripts/scenarios/t3.ts @@ -2,11 +2,13 @@ // app. `t3-mobile.ts` is the same for the iOS app. import * as NodeServices from "@effect/platform-node/NodeServices"; import { test as base } from "@e2e-dev/web"; +import { expect } from "e2e"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import { FetchHttpClient, HttpClient, HttpClientResponse } from "effect/http"; import { APP_DIR, T3_HOME } from "../e2e.config.ts"; import { run } from "./shell.ts"; @@ -23,12 +25,19 @@ export interface T3State { readonly title: string; readonly messages: ReadonlyArray; readonly model?: string; - }) => Promise; + }) => Promise; +} + +export interface T3Thread { + readonly threadId: string; } export interface T3 extends T3State { - /** Opens the app, signed in, at `path`. */ - readonly open: (path?: string) => Promise; + /** + * Opens the app, signed in, at a path or on a thread `t3.thread` returned. Opening the thread + * by its link is the reliable way to reach it: a sidebar row covers its own title text. + */ + readonly open: (target?: string | T3Thread) => Promise; } /** e2e's fixtures are promise functions; this is the one place an Effect becomes a promise. */ @@ -99,6 +108,7 @@ export function makeState(home: string, env: NodeJS.ProcessEnv = {}) { const started = yield* turn(["start", input.project, first, "--title", input.title, ...replay]); const { threadId } = yield* decodeLaunched(started.stdout); for (const message of rest) yield* turn(["send", threadId, message]); + return { threadId } satisfies T3Thread; }); return { @@ -122,14 +132,31 @@ const readPairingPath = Effect.gen(function* () { return yield* fs.readFileString(path.join(T3_HOME, "pairing-path")); }); +/** The id every thread link starts with, asked of the server the app talks to. */ +const environmentId = (baseUrl: string) => + HttpClient.get(`${baseUrl}/.well-known/t3/environment`).pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap( + HttpClientResponse.schemaBodyJson(Schema.Struct({ environmentId: Schema.String })), + ), + Effect.map((environment) => environment.environmentId), + Effect.provide(FetchHttpClient.layer), + ); + export const test = base.extend<{ t3: T3 }>({ - t3: async ({ app }, use) => { + t3: async ({ app, browser }, use) => { const { state, cleanup } = makeState(T3_HOME); await use({ ...state, - open: async (path = "/") => { + open: async (target = "/") => { await app.open(await runPromise(readPairingPath)); - if (path !== "/") await app.open(path); + // The pairing page signs in and then leaves itself. Going anywhere before it has done + // so cancels the sign-in, and the next page is the pairing page again. + await expect(browser).not.toHaveURL(/\/pair(?:[?#]|$)/, { timeout: 60_000 }); + if (typeof target !== "string") { + const environment = await runPromise(environmentId(app.baseUrl ?? "")); + await app.open(`/${environment}/${target.threadId}`); + } else if (target !== "/") await app.open(target); }, }); await runPromise(cleanup); diff --git a/scripts/scenarios/thread-opens-by-link.e2e.ts b/scripts/scenarios/thread-opens-by-link.e2e.ts new file mode 100644 index 0000000000000..9df71118df880 --- /dev/null +++ b/scripts/scenarios/thread-opens-by-link.e2e.ts @@ -0,0 +1,20 @@ +import { expect } from "e2e"; + +import { test, WEB } from "./t3.ts"; + +test( + "A seeded thread opens by its link and shows its history", + WEB, + async ({ t3, app, screen }) => { + const project = await t3.project("demo"); + const thread = await t3.thread({ + project, + title: "Earlier work", + messages: ["first", "second"], + }); + await t3.open(thread); + + await expect(screen.getByText("second fixture turn complete", { exact: false })).toBeVisible(); + await app.screenshot("thread"); + }, +);