feat(kix): recover jobs from lost and blocked publications

cloudflare/GITHUB.md

@@ -94,7 +94,15 @@ and moves one branch. Before writing anything it checks, in order:
 - The branch is where the job left it: absent before the first publication,
   then the last commit kix published. A `head` that differs gets a reply naming
   the commit to continue from. A branch changed outside the job, or one that
-  earlier work left on the fork, becomes a decision; it is never overwritten.
+  earlier work left on the fork, becomes a decision; publication never
+  overwrites it. On a service fork branch, the decision also offers to start
+  over: at the maintainer's request (`restart: true` in place of an `answer`,
+  after a live authority check), kix replaces the job with a new one at the
+  job's revision, with a new agent session and no recorded pull request, and
+  then force-resets the branch there. Replacing the job first means a
+  concurrent answer or cancel refuses the restart before any commit is
+  discarded. If the reset fails, the new job waits on the same Start over
+  decision. An iterated pull request's head is never reset.
 
 `plan`, `start`, and `bootstrap` jobs write their branch on the service fork
 `GITHUB_FORK_OWNER/<repository>`, whose immediate parent must be the upstream's
@@ -120,7 +128,10 @@ fenced-out attempt's write lands only if it passed the check before the
 reassignment, and the fast-forward check means it can never overwrite newer
 work. Delivery looks for the branch's open pull request before creating one,
 and reconciles a refused creation the same way, so a retry never opens a
-duplicate. A pull request the maintainer closed is not replaced.
+duplicate. Since the branch is verified to be where the job left it, its only
+open pull request is the job's, even one a failed job opened without recording
+it. A pull request opened since the job and then closed by the maintainer is
+not replaced.
 
 Planning pull requests are always drafts, as are those of jobs asked to stop.
 Other jobs set `draft`, which defaults to `true` while decisions or unfinished
@@ -131,8 +142,9 @@ branches. Pull requests use `maintainer_can_modify: false`, because an
 upstream-scoped token cannot grant edits on the fork branch.
 
 A planning pull request's body ends with a decision section the service
-writes from the job's answered `decision` questions, after those of a failed
-job whose pull request a retry continues (`src/jobs/trail.ts`): each round's
+writes from the job's answered `decision` questions, after those of each
+failed job it continues as a retry, whether or not that job opened the pull
+request (`src/jobs/trail.ts`): each round's
 questions, every option with ✅ on each choice, and the
 maintainer's own words. Permission requests and the service's `account` and
 `publication` questions stay out, since they can name runners and accounts.

cloudflare/README.md

@@ -80,7 +80,9 @@ A job is one maintainer-authorized workflow (`plan`, `start`, `iterate`, or
 create, answer, and cancel jobs at `/projects/github/<owner>/<repo>/jobs`, and
 every request rechecks current GitHub authority. `start` takes only quests that
 observations show ready and unclaimed, except that retrying a failed job does
-not count that job's own open pull request as a claim. A job on an issue or pull request
+not count that job's own open pull request as a claim: the one it recorded, or
+else the only open one observed on its branch at a commit it published. A
+retry records the failed job it `continues`. A job on an issue or pull request
 rereads it from GitHub, refusing one that closed or whose head moved since
 observations read it. `iterate` jobs are untrusted and route
 only to Linux runners that declare VM builds. Without a `flake.nix`, the route
@@ -189,6 +191,8 @@ reachable from its list with its earlier answers. A refused action says why
 and links back to its item. Prompts render as escaped text and never
 preselect the recommendation. A `publication` decision lists the refused
 paths or why the branch is blocked, and takes the maintainer's reply as text.
+A service fork branch that changed outside its job also offers Start over,
+which resets the branch to the job's revision and queues a new job there.
 The inbox reads each target's latest job however old it is: every one that
 awaits a decision, and any other whose issue, pull request, or quest is still
 observed, so finished work stays set aside. Retrying a failed planning

cloudflare/dev/github.ts

@@ -12,6 +12,8 @@ export type Pull = Issue & {
 	review: "APPROVED" | "CHANGES_REQUESTED" | "REVIEW_REQUIRED" | null;
 	reviews?: { state: string; author: string }[];
 	draft?: boolean;
+	/** Opened from the service fork, so its branch is not on the upstream. */
+	fork?: boolean;
 	/** A `## Decisions needed` section in it hands a draft back to the maintainer. */
 	body?: string;
 	/** GitHub's mergeStateStatus; only CLEAN merges. Defaults to BLOCKED, or DRAFT for a draft. */
@@ -28,8 +30,17 @@ export type Repo = {
 	pulls: Pull[];
 	/** Quest branches pushed without a pull request, such as `quest/m0/docs`. */
 	claims: string[];
+	/** Branches on the repository's service fork, `<forks>/<name>`, and their commits. */
+	fork: Record<string, string>;
+};
+export type World = {
+	app: { id: number; slug: string };
+	installation: number;
+	/** The service fork namespace, `GITHUB_FORK_OWNER`. */
+	forks: string;
+	users: User[];
+	repos: Repo[];
 };
-export type World = { app: { id: number; slug: string }; installation: number; users: User[]; repos: Repo[] };
 
 const sha1 = (text: string) => createHash("sha1").update(text).digest("hex");
 /** The Git object ID of a file's content, which the Worker verifies. */
@@ -97,7 +108,10 @@ function graphql(repo: Repo, query: string) {
 	if (/(?<!:)pullRequests\(/.test(query)) result.pullRequests = page(repo.pulls.map((pull) => pullNode(repo, pull)));
 	if (query.includes("open:pullRequests(")) result.open = page(open.map((pull) => pullNode(repo, pull)));
 	if (query.includes("refs(")) {
-		const branches = [...repo.claims, ...open.flatMap((p) => (p.branch.startsWith("quest/") ? [p.branch] : []))];
+		const branches = [
+			...repo.claims,
+			...open.flatMap((p) => (p.branch.startsWith("quest/") && !p.fork ? [p.branch] : [])),
+		];
 		result.refs = page(
 			[...new Set(branches.map((b) => b.replace(/^quest\//, "")))].map((name) => ({
 				name,
@@ -180,6 +194,8 @@ export function fakeGitHub(world: World) {
 			return repo ? json(graphql(repo, query)) : json({ errors: [{ message: "Not Found" }] });
 		}
 		const match = path.match(/^\/repos\/([^/]+)\/([^/]+)(\/.*)?$/);
+		const upstream = match?.[1] === world.forks && world.repos.find((r) => r.name === match[2]);
+		if (upstream) return serviceFork(upstream, match![3] ?? "", method, await body());
 		const repo = match && world.repos.find((r) => r.owner === match[1] && r.name === match[2]);
 		if (!repo) return json({ message: "Not Found" }, 404);
 		const rest = match![3] ?? "";
@@ -238,6 +254,52 @@ export function fakeGitHub(world: World) {
 		}
 		return missing();
 	};
+
+	/** The upstream's service fork: its identity and the branch moves that resetting a job's branch makes. */
+	function serviceFork(upstream: Repo, rest: string, method: string, text: string) {
+		const forkId = upstream.id * 10;
+		if (rest === "")
+			return json({
+				id: forkId,
+				name: upstream.name,
+				owner: { login: world.forks },
+				fork: true,
+				parent: { id: upstream.id },
+				private: false,
+				visibility: "public",
+				default_branch: "main",
+				archived: false,
+				disabled: false,
+			});
+		if (rest === "/installation")
+			return json({
+				id: world.installation,
+				app_id: world.app.id,
+				suspended_at: null,
+				permissions: { metadata: "read", contents: "write" },
+			});
+		const ref = rest.match(/^\/git\/refs?\/heads\/(.+)$/)?.[1];
+		const branch = ref && decodeURIComponent(ref);
+		if (branch && method === "GET") {
+			const sha = upstream.fork[branch];
+			return sha ? json({ object: { type: "commit", sha } }) : json({ message: "Not Found" }, 404);
+		}
+		const input = text ? (JSON.parse(text) as { ref?: string; sha: string; force?: boolean }) : null;
+		// Only forced updates are modeled; a fast-forward needs commit history the fake does not keep.
+		if (branch && method === "PATCH" && input?.force) {
+			if (!upstream.fork[branch]) return json({ message: "Reference does not exist" }, 422);
+			upstream.fork[branch] = input.sha;
+			return json({ object: { type: "commit", sha: input.sha } });
+		}
+		if (rest === "/git/refs" && method === "POST" && input?.ref?.startsWith("refs/heads/")) {
+			const name = input.ref.slice("refs/heads/".length);
+			if (upstream.fork[name]) return json({ message: "Reference already exists" }, 422);
+			upstream.fork[name] = input.sha;
+			return json({ object: { type: "commit", sha: input.sha } }, 201);
+		}
+		console.warn(`fake GitHub: no model for ${method} ${world.forks}/${upstream.name}${rest}`);
+		return json({ message: "Not modeled by the fake GitHub" }, 501);
+	}
 }
 
 /** Sign a webhook body the way GitHub does. */

cloudflare/dev/seed.ts

@@ -32,8 +32,10 @@ export function world(): World {
 					"- [Getting started](/quest/m0/docs.md) - a first-run guide",
 					"- [Crash telemetry](/quest/m0/telemetry.md) - learn about crashes",
 					"- [ZIP export](/quest/m0/export.md) - take your data with you",
+					"- [ZIP import](/quest/m0/import.md) - bring your data back",
 				]),
 			},
+			{ path: "quest/m0/import.md", content: quest("[S] Import a project from a ZIP archive", "Restore it.") },
 			{ path: "quest/m0/landing.md", content: quest("[S] Build the landing page", "Explain the app.") },
 			{ path: "quest/m0/sign-in.md", content: quest("[M] Add passkey sign-in", "Sign in without a password.") },
 			{ path: "quest/m0/docs.md", content: quest("[XS] Write the getting-started guide", "A first-run guide.") },
@@ -50,8 +52,10 @@ export function world(): World {
 					"- [Offline mode](/quest/m1/offline.md) - keep working without a network",
 					"- [Themes](/quest/m1/themes.md) - custom colors",
 					"- [Search](/quest/m1/search.md) - find anything",
+					"- [Fonts](/quest/m1/fonts.md) - choose the editor font",
 				]),
 			},
+			{ path: "quest/m1/fonts.md", content: quest("[S] Choose the editor font", "Read comfortably.") },
 			{ path: "quest/m1/search.md", content: quest("[S] Search across projects", "Find anything.") },
 			{
 				path: "quest/m1/offline.md",
@@ -145,7 +149,20 @@ export function world(): World {
 			},
 		],
 		claims: ["quest/m0/docs"],
+		// Changed outside its waiting job, which can start over.
+		fork: { "quest/m1/fonts": "f".repeat(40) },
 	};
+	// kix opened this one for a job that then failed before recording it.
+	rocket.pulls.push({
+		number: 20,
+		title: "Import a project from a ZIP archive",
+		author: "quest-dev[bot]",
+		branch: "quest/m0/import",
+		fork: true,
+		checks: "PENDING",
+		review: null,
+		draft: true,
+	});
 	const archive: Repo = {
 		id: 9102,
 		owner: "example",
@@ -167,10 +184,12 @@ export function world(): World {
 		issues: [{ number: 1, title: "Archive the old site", author: "reporter" }],
 		pulls: [],
 		claims: [],
+		fork: {},
 	};
 	return {
 		app: { id: 123, slug: "quest-dev" },
 		installation: 9201,
+		forks: "quest-dev-forks",
 		users: [{ id: 9001, login: "dev-maintainer", name: "Dev Maintainer", email: "maintainer@example.test" }],
 		repos: [rocket, archive],
 	};
@@ -302,10 +321,12 @@ async function jobs(db: D1Database, user: string, repo: Repo) {
 		revision: string;
 		branch: string;
 		state: "queued" | "running" | "waiting" | "failed";
+		/** The commit the job last published to its branch. */
+		head?: string;
 		runner?: string;
 		account?: string;
 		summary?: string;
-		question?: { kind: "agent" | "account"; prompt: unknown };
+		question?: { kind: "agent" | "account" | "publication"; prompt: unknown };
 	};
 	const list: Job[] = [
 		{
@@ -399,6 +420,39 @@ async function jobs(db: D1Database, user: string, repo: Repo) {
 			account: "claude-max",
 			summary: "just test failed: two sign-in tests time out against the new redirect.",
 		},
+		{
+			id: "seed-start-import",
+			workflow: "start",
+			target: "quest/m0/import.md",
+			revision: commit,
+			branch: "quest/m0/import",
+			state: "failed",
+			// It opened pull request 20 at this commit, then failed before recording it.
+			head: head(20),
+			runner: "workstation",
+			account: "claude-max",
+			summary: "just check failed after the pull request opened: the importer skips empty folders.",
+		},
+		{
+			id: "seed-start-fonts",
+			workflow: "start",
+			target: "quest/m1/fonts.md",
+			revision: commit,
+			branch: "quest/m1/fonts",
+			state: "waiting",
+			head: "e".repeat(40),
+			runner: "workstation",
+			account: "claude-max",
+			summary: "Added the font picker; publishing it found the branch moved.",
+			question: {
+				kind: "publication",
+				prompt: {
+					type: "publication",
+					blocked: "The branch quest/m1/fonts changed outside this job.",
+					restart: true,
+				},
+			},
+		},
 		{
 			id: "seed-plan-settings",
 			workflow: "plan",
@@ -441,7 +495,7 @@ async function jobs(db: D1Database, user: string, repo: Repo) {
 			return [
 				db
 					.prepare(
-						"INSERT INTO jobs(id,repository,workflow,target,revision,branch,actor,trusted,state,attempt,runner,account,delivered,summary,createdAt,updatedAt) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+						"INSERT INTO jobs(id,repository,workflow,target,revision,branch,head,actor,trusted,state,attempt,runner,account,delivered,summary,createdAt,updatedAt) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
 					)
 					.bind(
 						job.id,
@@ -450,6 +504,7 @@ async function jobs(db: D1Database, user: string, repo: Repo) {
 						job.target,
 						job.revision,
 						job.branch,
+						job.head ?? null,
 						user,
 						job.workflow === "iterate" ? 0 : 1,
 						job.state,

cloudflare/dev/serve.ts

@@ -77,7 +77,7 @@ const mf = new Miniflare(
 					GITHUB_APP_ID: String(data.app.id),
 					GITHUB_APP_PRIVATE_KEY: privateKey,
 					GITHUB_WEBHOOK_SECRET: secret,
-					GITHUB_FORK_OWNER: "quest-dev-forks",
+					GITHUB_FORK_OWNER: data.forks,
 					GOOGLE_CLIENT_ID: "",
 					GOOGLE_CLIENT_SECRET: "",
 					// Notifications stay off: a push would leave localhost for a real push service.

cloudflare/migrations/0016_recovery.sql

@@ -0,0 +1,3 @@
+-- The failed job a retry continues. A retry's decision record follows this
+-- chain, so answers survive a failure that opened no pull request.
+ALTER TABLE jobs ADD COLUMN continues TEXT REFERENCES jobs(id) ON DELETE SET NULL;

cloudflare/src/github/service.ts

@@ -82,6 +82,8 @@ export class ProjectChanged extends GitHubError {
 }
 /** A publication the bot cannot make, which needs a maintainer's decision rather than a retry. */
 export class PublicationBlocked extends GitHubError {}
+/** A job's branch is not where the job left it, which starting the job over can reset. */
+export class BranchChanged extends PublicationBlocked {}
 /** A maintainer check that was answered with a denial, rather than one that could not complete. */
 export class NotMaintainer extends GitHubError {}
 
@@ -94,7 +96,7 @@ export type Contribution = {
 	target: string;
 	branch: string;
 	pull: number | null;
-	/** When the job was created, in milliseconds; older pull requests on its branch are not its own. */
+	/** When the job was created, in milliseconds; an older closed pull request on its branch is not its own. */
 	createdAt: number;
 };
 /** The one branch a job may write, on the repository the service resolved for it. */
@@ -106,6 +108,8 @@ export interface Branch {
 	commit(parent: string, changes: Change[], message: string): Promise<string>;
 	/** Move the branch from `from` (null creates it) to `to`; false when it moved first. */
 	move(from: string | null, to: string): Promise<boolean>;
+	/** Point the branch at `to` wherever it is, creating it if it is gone, discarding its commits. */
+	reset(to: string): Promise<void>;
 }
 export type ContributionState = "enabled" | "paused";
 export type Enrollment = GitHubRepository & { state: ContributionState };
@@ -1023,6 +1027,15 @@ export class GitHubService {
 					throw error;
 				}
 			},
+			reset: async (to) => {
+				try {
+					await this.#api(token, `${path}/git/refs/heads/${ref}`, "PATCH", { sha: sha(to), force: true });
+				} catch (error) {
+					// GitHub refuses to update a reference that does not exist.
+					if (!(error instanceof GitHubError) || error.status !== 422) throw error;
+					await this.#api(token, `${path}/git/refs`, "POST", { ref: `refs/heads/${branch}`, sha: sha(to) });
+				}
+			},
 		};
 	}
 
@@ -1045,16 +1058,22 @@ export class GitHubService {
 			const fork = await this.#fork(upstream, upstream.name, {}, async (_token, fork) => fork);
 			const pulls = `${repositoryPath(upstream)}/pulls`;
 			const label = `${fork.owner.login}:${job.branch}`;
-			// An unrecorded earlier delivery may have opened one, which the maintainer may since have closed.
+			// An unrecorded earlier delivery may have opened one. The branch is
+			// where the job left it, so its only open pull request is the job's,
+			// even one a failed job this retry continues opened. Otherwise one
+			// opened since the job is, which the maintainer may since have closed.
 			// GitHub reports whole seconds, so compare against the second the job was created in.
 			const since = Math.floor(job.createdAt / 1000) * 1000;
-			const opened = async () =>
-				(
-					await this.#api<PullHead[]>(
-						token,
-						`${pulls}?state=all&per_page=100&head=${encodeURIComponent(label)}`,
-					)
-				).find((pull) => Date.parse(pull.created_at) >= since) ?? null;
+			const opened = async () => {
+				const found = await this.#api<PullHead[]>(
+					token,
+					`${pulls}?state=all&per_page=100&head=${encodeURIComponent(label)}`,
+				);
+				const open = found.filter((pull) => pull.state === "open");
+				return (
+					(open.length === 1 ? open[0] : found.find((pull) => Date.parse(pull.created_at) >= since)) ?? null
+				);
+			};
 			let pull = job.pull ? await this.#api<PullHead>(token, `${pulls}/${number(job.pull)}`) : await opened();
 			if (pull && (pull.head.repo?.id !== fork.id || pull.head.ref !== job.branch))
 				throw new GitHubError("The job's pull request no longer comes from its branch");