From 59d7ef241ba67bf3025120d80c7d5168230bdd78 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 22:39:18 -0700 Subject: [PATCH 1/3] chore(quest): claim quest/m0/kix/recovery.md Quest-Claim: f5cf5caa948c10bc From cefdbbdab73ad1d9776766716ff6e23704e66231 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 23:31:12 -0700 Subject: [PATCH 2/3] feat(kix): recover jobs from lost and blocked publications A retry records the failed job it continues, and a planning PR's decision record follows that chain, so answers survive a failure that opened no PR. A failed job's PR whose number went unrecorded no longer blocks its retry: the inbox and the start route accept the only open PR observed on the job's branch at a commit it published, and delivery adopts the branch's only open PR. A publication decision on a service fork branch that changed outside its job, or was left by earlier work, offers Start over: kix resets the branch to the job's revision and queues a fresh job there. Co-Authored-By: Claude Opus 5.5 --- cloudflare/GITHUB.md | 17 ++- cloudflare/README.md | 6 +- cloudflare/dev/github.ts | 66 ++++++++++- cloudflare/dev/seed.ts | 57 +++++++++- cloudflare/dev/serve.ts | 2 +- cloudflare/migrations/0015_recovery.sql | 3 + cloudflare/src/github/service.ts | 37 +++++-- cloudflare/src/github/state.ts | 21 ++++ cloudflare/src/inbox/page.tsx | 28 ++++- cloudflare/src/jobs/publish.ts | 42 ++++++- cloudflare/src/jobs/routes.ts | 25 +++-- cloudflare/src/jobs/store.ts | 67 +++++++++--- cloudflare/test/publish.test.ts | 140 +++++++++++++++++++++++- cloudflare/test/seed.test.ts | 78 ++++++++++++- quest/m0/kix/README.md | 1 - quest/m0/kix/recovery.md | 31 ------ 16 files changed, 530 insertions(+), 91 deletions(-) create mode 100644 cloudflare/migrations/0015_recovery.sql delete mode 100644 quest/m0/kix/recovery.md diff --git a/cloudflare/GITHUB.md b/cloudflare/GITHUB.md index 2e7deb4a..9f964dd9 100644 --- a/cloudflare/GITHUB.md +++ b/cloudflare/GITHUB.md @@ -94,7 +94,12 @@ and moves one branch. Before writing anything it checks, in order: - The branch is where the job left it: absent before the first publication, then the last commit kix published. A `head` that differs gets a reply naming the commit to continue from. A branch changed outside the job, or one that - earlier work left on the fork, becomes a decision; it is never overwritten. + earlier work left on the fork, becomes a decision; publication never + overwrites it. On a service fork branch, the decision also offers to start + over: at the maintainer's request (`restart: true` in place of an `answer`, + after a live authority check), kix force-resets the branch to the job's + revision and replaces the job with a new one there, with a new agent session + and no recorded pull request. An iterated pull request's head is never reset. `plan`, `start`, and `bootstrap` jobs write their branch on the service fork `GITHUB_FORK_OWNER/`, whose immediate parent must be the upstream's @@ -120,7 +125,10 @@ fenced-out attempt's write lands only if it passed the check before the reassignment, and the fast-forward check means it can never overwrite newer work. Delivery looks for the branch's open pull request before creating one, and reconciles a refused creation the same way, so a retry never opens a -duplicate. A pull request the maintainer closed is not replaced. +duplicate. Since the branch is verified to be where the job left it, its only +open pull request is the job's, even one a failed job opened without recording +it. A pull request opened since the job and then closed by the maintainer is +not replaced. Planning pull requests are always drafts, as are those of jobs asked to stop. Other jobs set `draft`, which defaults to `true` while decisions or unfinished @@ -131,8 +139,9 @@ branches. Pull requests use `maintainer_can_modify: false`, because an upstream-scoped token cannot grant edits on the fork branch. A planning pull request's body ends with a decision section the service -writes from the job's answered `decision` questions, after those of a failed -job whose pull request a retry continues (`src/jobs/trail.ts`): each round's +writes from the job's answered `decision` questions, after those of each +failed job it continues as a retry, whether or not that job opened the pull +request (`src/jobs/trail.ts`): each round's questions, every option with ✅ on each choice, and the maintainer's own words. Permission requests and the service's `account` and `publication` questions stay out, since they can name runners and accounts. diff --git a/cloudflare/README.md b/cloudflare/README.md index 1ba0b55d..b44582f3 100644 --- a/cloudflare/README.md +++ b/cloudflare/README.md @@ -80,7 +80,9 @@ A job is one maintainer-authorized workflow (`plan`, `start`, `iterate`, or create, answer, and cancel jobs at `/projects/github///jobs`, and every request rechecks current GitHub authority. `start` takes only quests that observations show ready and unclaimed, except that retrying a failed job does -not count that job's own open pull request as a claim. A job on an issue or pull request +not count that job's own open pull request as a claim: the one it recorded, or +else the only open one observed on its branch at a commit it published. A +retry records the failed job it `continues`. A job on an issue or pull request rereads it from GitHub, refusing one that closed or whose head moved since observations read it. `iterate` jobs are untrusted and route only to Linux runners that declare VM builds. Without a `flake.nix`, the route @@ -189,6 +191,8 @@ reachable from its list with its earlier answers. A refused action says why and links back to its item. Prompts render as escaped text and never preselect the recommendation. A `publication` decision lists the refused paths or why the branch is blocked, and takes the maintainer's reply as text. +A service fork branch that changed outside its job also offers Start over, +which resets the branch to the job's revision and queues a new job there. The inbox reads each target's latest job however old it is: every one that awaits a decision, and any other whose issue, pull request, or quest is still observed, so finished work stays set aside. Retrying a failed planning diff --git a/cloudflare/dev/github.ts b/cloudflare/dev/github.ts index de23f825..6a0f8fe8 100644 --- a/cloudflare/dev/github.ts +++ b/cloudflare/dev/github.ts @@ -12,6 +12,8 @@ export type Pull = Issue & { review: "APPROVED" | "CHANGES_REQUESTED" | "REVIEW_REQUIRED" | null; reviews?: { state: string; author: string }[]; draft?: boolean; + /** Opened from the service fork, so its branch is not on the upstream. */ + fork?: boolean; /** GitHub's mergeStateStatus; only CLEAN merges. Defaults to BLOCKED, or DRAFT for a draft. */ merge?: "CLEAN" | "BLOCKED" | "DIRTY" | "UNSTABLE"; }; @@ -26,8 +28,17 @@ export type Repo = { pulls: Pull[]; /** Quest branches pushed without a pull request, such as `quest/m0/docs`. */ claims: string[]; + /** Branches on the repository's service fork, `/`, and their commits. */ + fork: Record; +}; +export type World = { + app: { id: number; slug: string }; + installation: number; + /** The service fork namespace, `GITHUB_FORK_OWNER`. */ + forks: string; + users: User[]; + repos: Repo[]; }; -export type World = { app: { id: number; slug: string }; installation: number; users: User[]; repos: Repo[] }; const sha1 = (text: string) => createHash("sha1").update(text).digest("hex"); /** The Git object ID of a file's content, which the Worker verifies. */ @@ -93,7 +104,10 @@ function graphql(repo: Repo, query: string) { if (/(? pullNode(repo, pull))); if (query.includes("open:pullRequests(")) result.open = page(open.map((pull) => pullNode(repo, pull))); if (query.includes("refs(")) { - const branches = [...repo.claims, ...open.flatMap((p) => (p.branch.startsWith("quest/") ? [p.branch] : []))]; + const branches = [ + ...repo.claims, + ...open.flatMap((p) => (p.branch.startsWith("quest/") && !p.fork ? [p.branch] : [])), + ]; result.refs = page( [...new Set(branches.map((b) => b.replace(/^quest\//, "")))].map((name) => ({ name, @@ -176,6 +190,8 @@ export function fakeGitHub(world: World) { return repo ? json(graphql(repo, query)) : json({ errors: [{ message: "Not Found" }] }); } const match = path.match(/^\/repos\/([^/]+)\/([^/]+)(\/.*)?$/); + const upstream = match?.[1] === world.forks && world.repos.find((r) => r.name === match[2]); + if (upstream) return serviceFork(upstream, match![3] ?? "", method, await body()); const repo = match && world.repos.find((r) => r.owner === match[1] && r.name === match[2]); if (!repo) return json({ message: "Not Found" }, 404); const rest = match![3] ?? ""; @@ -234,6 +250,52 @@ export function fakeGitHub(world: World) { } return missing(); }; + + /** The upstream's service fork: its identity and the branch moves that resetting a job's branch makes. */ + function serviceFork(upstream: Repo, rest: string, method: string, text: string) { + const forkId = upstream.id * 10; + if (rest === "") + return json({ + id: forkId, + name: upstream.name, + owner: { login: world.forks }, + fork: true, + parent: { id: upstream.id }, + private: false, + visibility: "public", + default_branch: "main", + archived: false, + disabled: false, + }); + if (rest === "/installation") + return json({ + id: world.installation, + app_id: world.app.id, + suspended_at: null, + permissions: { metadata: "read", contents: "write" }, + }); + const ref = rest.match(/^\/git\/refs?\/heads\/(.+)$/)?.[1]; + const branch = ref && decodeURIComponent(ref); + if (branch && method === "GET") { + const sha = upstream.fork[branch]; + return sha ? json({ object: { type: "commit", sha } }) : json({ message: "Not Found" }, 404); + } + const input = text ? (JSON.parse(text) as { ref?: string; sha: string; force?: boolean }) : null; + // Only forced updates are modeled; a fast-forward needs commit history the fake does not keep. + if (branch && method === "PATCH" && input?.force) { + if (!upstream.fork[branch]) return json({ message: "Reference does not exist" }, 422); + upstream.fork[branch] = input.sha; + return json({ object: { type: "commit", sha: input.sha } }); + } + if (rest === "/git/refs" && method === "POST" && input?.ref?.startsWith("refs/heads/")) { + const name = input.ref.slice("refs/heads/".length); + if (upstream.fork[name]) return json({ message: "Reference already exists" }, 422); + upstream.fork[name] = input.sha; + return json({ object: { type: "commit", sha: input.sha } }, 201); + } + console.warn(`fake GitHub: no model for ${method} ${world.forks}/${upstream.name}${rest}`); + return json({ message: "Not modeled by the fake GitHub" }, 501); + } } /** Sign a webhook body the way GitHub does. */ diff --git a/cloudflare/dev/seed.ts b/cloudflare/dev/seed.ts index 6de077f8..559586dc 100644 --- a/cloudflare/dev/seed.ts +++ b/cloudflare/dev/seed.ts @@ -32,8 +32,10 @@ export function world(): World { "- [Getting started](/quest/m0/docs.md) - a first-run guide", "- [Crash telemetry](/quest/m0/telemetry.md) - learn about crashes", "- [ZIP export](/quest/m0/export.md) - take your data with you", + "- [ZIP import](/quest/m0/import.md) - bring your data back", ]), }, + { path: "quest/m0/import.md", content: quest("[S] Import a project from a ZIP archive", "Restore it.") }, { path: "quest/m0/landing.md", content: quest("[S] Build the landing page", "Explain the app.") }, { path: "quest/m0/sign-in.md", content: quest("[M] Add passkey sign-in", "Sign in without a password.") }, { path: "quest/m0/docs.md", content: quest("[XS] Write the getting-started guide", "A first-run guide.") }, @@ -111,7 +113,20 @@ export function world(): World { }, ], claims: ["quest/m0/docs"], + // Changed outside its waiting job, which can start over. + fork: { "quest/m1/themes": "f".repeat(40) }, }; + // kix opened this one for a job that then failed before recording it. + rocket.pulls.push({ + number: 17, + title: "Import a project from a ZIP archive", + author: "quest-dev[bot]", + branch: "quest/m0/import", + fork: true, + checks: "PENDING", + review: null, + draft: true, + }); const archive: Repo = { id: 9102, owner: "example", @@ -133,10 +148,12 @@ export function world(): World { issues: [{ number: 1, title: "Archive the old site", author: "reporter" }], pulls: [], claims: [], + fork: {}, }; return { app: { id: 123, slug: "quest-dev" }, installation: 9201, + forks: "quest-dev-forks", users: [{ id: 9001, login: "dev-maintainer", name: "Dev Maintainer", email: "maintainer@example.test" }], repos: [rocket, archive], }; @@ -265,10 +282,12 @@ async function jobs(db: D1Database, user: string, repo: Repo) { revision: string; branch: string; state: "queued" | "running" | "waiting" | "failed"; + /** The commit the job last published to its branch. */ + head?: string; runner?: string; account?: string; summary?: string; - question?: { kind: "agent" | "account"; prompt: unknown }; + question?: { kind: "agent" | "account" | "publication"; prompt: unknown }; }; const list: Job[] = [ { @@ -362,6 +381,39 @@ async function jobs(db: D1Database, user: string, repo: Repo) { account: "claude-max", summary: "just test failed: two sign-in tests time out against the new redirect.", }, + { + id: "seed-start-import", + workflow: "start", + target: "quest/m0/import.md", + revision: commit, + branch: "quest/m0/import", + state: "failed", + // It opened pull request 17 at this commit, then failed before recording it. + head: head(17), + runner: "workstation", + account: "claude-max", + summary: "just check failed after the pull request opened: the importer skips empty folders.", + }, + { + id: "seed-start-themes", + workflow: "start", + target: "quest/m1/themes.md", + revision: commit, + branch: "quest/m1/themes", + state: "waiting", + head: "e".repeat(40), + runner: "workstation", + account: "claude-max", + summary: "Added the theme picker; publishing it found the branch moved.", + question: { + kind: "publication", + prompt: { + type: "publication", + blocked: "The branch quest/m1/themes changed outside this job.", + restart: true, + }, + }, + }, { id: "seed-plan-settings", workflow: "plan", @@ -404,7 +456,7 @@ async function jobs(db: D1Database, user: string, repo: Repo) { return [ db .prepare( - "INSERT INTO jobs(id,repository,workflow,target,revision,branch,actor,trusted,state,attempt,runner,account,delivered,summary,createdAt,updatedAt) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + "INSERT INTO jobs(id,repository,workflow,target,revision,branch,head,actor,trusted,state,attempt,runner,account,delivered,summary,createdAt,updatedAt) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", ) .bind( job.id, @@ -413,6 +465,7 @@ async function jobs(db: D1Database, user: string, repo: Repo) { job.target, job.revision, job.branch, + job.head ?? null, user, job.workflow === "iterate" ? 0 : 1, job.state, diff --git a/cloudflare/dev/serve.ts b/cloudflare/dev/serve.ts index 77f8b3a5..241c895e 100644 --- a/cloudflare/dev/serve.ts +++ b/cloudflare/dev/serve.ts @@ -77,7 +77,7 @@ const mf = new Miniflare( GITHUB_APP_ID: String(data.app.id), GITHUB_APP_PRIVATE_KEY: privateKey, GITHUB_WEBHOOK_SECRET: secret, - GITHUB_FORK_OWNER: "quest-dev-forks", + GITHUB_FORK_OWNER: data.forks, GOOGLE_CLIENT_ID: "", GOOGLE_CLIENT_SECRET: "", // Notifications stay off: a push would leave localhost for a real push service. diff --git a/cloudflare/migrations/0015_recovery.sql b/cloudflare/migrations/0015_recovery.sql new file mode 100644 index 00000000..f30ccaec --- /dev/null +++ b/cloudflare/migrations/0015_recovery.sql @@ -0,0 +1,3 @@ +-- The failed job a retry continues. A retry's decision record follows this +-- chain, so answers survive a failure that opened no pull request. +ALTER TABLE jobs ADD COLUMN continues TEXT REFERENCES jobs(id) ON DELETE SET NULL; diff --git a/cloudflare/src/github/service.ts b/cloudflare/src/github/service.ts index bfca9d67..c7cc5aae 100644 --- a/cloudflare/src/github/service.ts +++ b/cloudflare/src/github/service.ts @@ -82,6 +82,8 @@ export class ProjectChanged extends GitHubError { } /** A publication the bot cannot make, which needs a maintainer's decision rather than a retry. */ export class PublicationBlocked extends GitHubError {} +/** A job's branch is not where the job left it, which starting the job over can reset. */ +export class BranchChanged extends PublicationBlocked {} /** A maintainer check that was answered with a denial, rather than one that could not complete. */ export class NotMaintainer extends GitHubError {} @@ -94,7 +96,7 @@ export type Contribution = { target: string; branch: string; pull: number | null; - /** When the job was created, in milliseconds; older pull requests on its branch are not its own. */ + /** When the job was created, in milliseconds; an older closed pull request on its branch is not its own. */ createdAt: number; }; /** The one branch a job may write, on the repository the service resolved for it. */ @@ -106,6 +108,8 @@ export interface Branch { commit(parent: string, changes: Change[], message: string): Promise; /** Move the branch from `from` (null creates it) to `to`; false when it moved first. */ move(from: string | null, to: string): Promise; + /** Point the branch at `to` wherever it is, creating it if it is gone, discarding its commits. */ + reset(to: string): Promise; } export type ContributionState = "enabled" | "paused"; export type Enrollment = GitHubRepository & { state: ContributionState }; @@ -1023,6 +1027,15 @@ export class GitHubService { throw error; } }, + reset: async (to) => { + try { + await this.#api(token, `${path}/git/refs/heads/${ref}`, "PATCH", { sha: sha(to), force: true }); + } catch (error) { + // GitHub refuses to update a reference that does not exist. + if (!(error instanceof GitHubError) || error.status !== 422) throw error; + await this.#api(token, `${path}/git/refs`, "POST", { ref: `refs/heads/${branch}`, sha: sha(to) }); + } + }, }; } @@ -1045,16 +1058,22 @@ export class GitHubService { const fork = await this.#fork(upstream, upstream.name, {}, async (_token, fork) => fork); const pulls = `${repositoryPath(upstream)}/pulls`; const label = `${fork.owner.login}:${job.branch}`; - // An unrecorded earlier delivery may have opened one, which the maintainer may since have closed. + // An unrecorded earlier delivery may have opened one. The branch is + // where the job left it, so its only open pull request is the job's, + // even one a failed job this retry continues opened. Otherwise one + // opened since the job is, which the maintainer may since have closed. // GitHub reports whole seconds, so compare against the second the job was created in. const since = Math.floor(job.createdAt / 1000) * 1000; - const opened = async () => - ( - await this.#api( - token, - `${pulls}?state=all&per_page=100&head=${encodeURIComponent(label)}`, - ) - ).find((pull) => Date.parse(pull.created_at) >= since) ?? null; + const opened = async () => { + const found = await this.#api( + token, + `${pulls}?state=all&per_page=100&head=${encodeURIComponent(label)}`, + ); + const open = found.filter((pull) => pull.state === "open"); + return ( + (open.length === 1 ? open[0] : found.find((pull) => Date.parse(pull.created_at) >= since)) ?? null + ); + }; let pull = job.pull ? await this.#api(token, `${pulls}/${number(job.pull)}`) : await opened(); if (pull && (pull.head.repo?.id !== fork.id || pull.head.ref !== job.branch)) throw new GitHubError("The job's pull request no longer comes from its branch"); diff --git a/cloudflare/src/github/state.ts b/cloudflare/src/github/state.ts index a0e0e41b..946a200a 100644 --- a/cloudflare/src/github/state.ts +++ b/cloudflare/src/github/state.ts @@ -267,6 +267,27 @@ export function startable(state: ProjectState, quest: string, retry: number | nu ); } +/** + * The open pull request a retry of a failed job continues: the one the job + * recorded, or else the only open pull request observed on the job's branch + * at a commit the job published, which it opened without recording the + * number. Matching the commit, not only the branch name, keeps a + * contributor's pull request from a same-named branch a claim. + */ +export function continued( + state: ProjectState, + job: { branch: string; head: string | null; pending: string | null; pull: number | null }, +) { + if (job.pull !== null || state.open === null) return job.pull; + const open = new Set(state.open); + const published = [job.head, job.pending].filter((commit) => commit !== null); + const pulls = state.activity.filter((item) => { + const pull = item.kind === "pull" && !item.missing && open.has(item.id) ? (item.data as Pull | null) : null; + return pull?.state === "OPEN" && pull.headRefName === job.branch && published.includes(pull.headRefOid); + }); + return pulls.length === 1 ? pulls[0]!.number : null; +} + export async function readState(db: D1Database, repo: GitHubRepository): Promise { const records = await db.batch([ db diff --git a/cloudflare/src/inbox/page.tsx b/cloudflare/src/inbox/page.tsx index ee844133..12f0dde0 100644 --- a/cloudflare/src/inbox/page.tsx +++ b/cloudflare/src/inbox/page.tsx @@ -1,4 +1,4 @@ -import { type Pull, startable } from "../github/state"; +import { type Pull, continued, startable } from "../github/state"; import { type Prompt, parsePrompt } from "../jobs/prompt"; import { type Done, PUBLICATION_REPLY, inboxURL } from "../jobs/routes"; import { Marker, Notifications } from "../layout"; @@ -113,7 +113,8 @@ function History(props: { item: Item }) { if (q.kind === "account") { return
  • Chose an account to continue on.
    • ; } - if (q.kind === "publication") return
  • {typeof q.answer === "string" ? q.answer : ""}
    • ; + if (q.kind === "publication") + return
  • {typeof q.answer === "string" ? q.answer : "Started over from the job's base."}
    • ; const prompt = parsed(q); const replies = (q.answer as { answers?: { choices: string[]; text?: string }[] }).answers ?? []; return ( @@ -184,17 +185,21 @@ function AgentForm(props: { item: Item; prompt: Prompt }) { /** A publication decision's prompt. kix wrote it, but its paths came from the runner. */ function publication(question: QuestionView) { - const prompt = question.prompt as { refused?: unknown; blocked?: unknown } | null; + const prompt = question.prompt as { refused?: unknown; blocked?: unknown; restart?: unknown } | null; return { refused: Array.isArray(prompt?.refused) ? (prompt.refused as { path: string; reason: string }[]) : [], blocked: typeof prompt?.blocked === "string" ? prompt.blocked : null, + restart: prompt?.restart === true, }; } -/** Work kix would not publish as is; the maintainer's reply resumes the agent's session. */ +/** + * Work kix would not publish as is; the maintainer's reply resumes the + * agent's session. A branch that changed outside the job can instead start over. + */ function PublicationForm(props: { item: Item }) { const { item } = props; - const { refused, blocked } = publication(item.question!); + const { refused, blocked, restart } = publication(item.question!); return (
    @@ -216,6 +221,17 @@ function PublicationForm(props: { item: Item }) {