From 00776e8b8185cfdb3d625b6a6d71b08fceff3ee1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 20:07:22 -0700 Subject: [PATCH 1/3] chore: claim project state quest From 2bd21a442467b67f01201839e1dac8fc46c739a5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 20:28:51 -0700 Subject: [PATCH 2/3] feat: observe enrolled GitHub project state and activity --- cloudflare/GITHUB.md | 64 ++++- cloudflare/README.md | 6 +- cloudflare/migrations/0006_project_state.sql | 24 ++ cloudflare/src/board/model.ts | 17 +- cloudflare/src/board/pages.tsx | 23 +- cloudflare/src/board/project.ts | 4 + cloudflare/src/github/service.ts | 135 ++++++++-- cloudflare/src/github/state.ts | 246 +++++++++++++++++ cloudflare/src/github/webhook.ts | 37 +++ cloudflare/src/index.tsx | 67 ++++- cloudflare/src/map.tsx | 2 + cloudflare/src/projects.tsx | 93 +++++++ cloudflare/test/board.test.ts | 2 +- cloudflare/test/github.test.ts | 50 ++++ cloudflare/test/projects.test.ts | 270 ++++++++++++++++++- 15 files changed, 1008 insertions(+), 32 deletions(-) create mode 100644 cloudflare/migrations/0006_project_state.sql create mode 100644 cloudflare/src/github/state.ts diff --git a/cloudflare/GITHUB.md b/cloudflare/GITHUB.md index 56bd651f..cf5eacd9 100644 --- a/cloudflare/GITHUB.md +++ b/cloudflare/GITHUB.md @@ -203,6 +203,66 @@ See the official [GitHub license API](https://docs.github.com/en/rest/licenses/l [GraphQL blobs](https://docs.github.com/en/graphql/reference/git#blob), and [App metadata API](https://docs.github.com/en/rest/apps/apps#get-the-authenticated-app). -The project-state quest owns ongoing snapshots, branches, PRs, and event -reconciliation; this setup preview does not reserve or launch ready quests. +The setup preview does not reserve or launch ready quests. The jobs/funding quests own live eligibility checks and contribution execution. + +## Project state and activity + +An enrolled project's setup page links to `/projects/github///state`. +A maintainer explicitly refreshes observations with its same-origin form. The +service rereads the default branch at an exact commit using the preview's shared +snapshot reader and validates the Quest tree with the pure core. The authenticated +board and nested quest pages use the existing `Project` seam. Source links retain +the exact commit; live project state never replaces GitHub as authority. + +Each refresh reconciles one page of native issues, **all** PRs (including human +contributions and closed/merged PRs), and `quest/` branches. Independent opaque +cursors persist across requests. Continue reconciliation until all connections +finish; candidates remain empty while a scan is incomplete or a webhook has +invalidated it. The next completed scan also detects deleted entities and removed +claims. Refreshing a completed scan starts again from the authoritative entity +connections, so missed webhooks and later changes to old entities are recovered. +This deliberately avoids GitHub's bounded, delayed activity feed; GitHub does +[not automatically redeliver failed webhooks](https://docs.github.com/en/webhooks/using-webhooks/handling-failed-webhook-deliveries). +There is no scheduled background poll in this stage. + +A single GraphQL activity query reads up to 50 issues, 50 PRs and 100 branch +claims, keeping external request count independent of entity/file counts. Bulk +D1 writes use `json_each`, avoiding a query per entity or SQLite's parameter +limit. A revision comparison commits each page and cursor atomically; concurrent +refreshes or an intervening webhook cannot publish over a newer observation. +A webhook restarts an unfinished scan on the next refresh. Installation tokens +remain read-only, repository-scoped, and revoked after use. + +Durable activity items use the GitHub repository ID, entity kind and full database +ID. GitHub's GraphQL `fullDatabaseId` strings and webhook numeric IDs map to the +same stored text key. Duplicate delivery IDs, redelivery with new IDs, later +pushes/comments/reviews, and reconciliation all update the same item. Webhooks +retain identity pointers only, never issue/PR bodies. Authoritative API reads add +escaped titles, state, comment/review counts, current PR head, that head's check +rollup, aggregate review decision, and up to 20 latest reviewer records with an +explicit pagination flag. Raw bodies and individual check logs are not cached. +Unavailable/deleted entities retain their notification identity and are marked +missing rather than disappearing or presenting old state as current. + +A PR associates with a quest only when its head branch exactly matches the quest's +path without `.md` (or an epic's `/README.md`). An unrelated human PR remains in +activity without a guessed association. Open associated PRs and observed quest +branch claims exclude that quest from candidate data; branch claims display as +Claimed on the board. Runner/run associations belong to job coordination. + +Every cached read and refresh proves current maintainer authority, live repository +identity/public visibility, installation access, and enrollment for that +installation. Refresh rechecks those boundaries after provider reads. Paused +projects remain readable but have no candidates. Disconnect cascades cached +snapshots and activity; explicitly enrolling a replacement installation clears +prior observations. No positive authority cache is created. Candidate data is an +observation tied to its commit, **never execution permission**: job coordination +must validate current eligibility, fresh revision, locks, and PRs before a start. +Review/check data is similarly descriptive and cannot approve a merge. + +The `/data` endpoint exposes the same authenticated snapshot/activity with +`Cache-Control: no-store` for future decision and job consumers. The activity +page is a secondary read view; the durable decision inbox and its unread/answer +state belong to the prompts workflow. Neither incoming activity nor refreshing +starts a model, creates a quest/job, spends usage, or writes to GitHub. diff --git a/cloudflare/README.md b/cloudflare/README.md index 55d297d4..be6246e5 100644 --- a/cloudflare/README.md +++ b/cloudflare/README.md @@ -26,8 +26,8 @@ renders the repository's `docs/*.md` into `build/docs.json`, so the Markdown stays the single source and its relative links keep working on GitHub: links to other docs become `/docs/` and other repository paths point at GitHub. It also snapshots this repository's `quest/` tree and finished quests into -`build/board.json`, which is the board's only project until the GitHub project -state integration supplies enrolled repositories. A shallow clone has no +`build/board.json`, which supplies the public board. Enrolled repositories use authenticated +GitHub snapshots through the same board model. A shallow clone has no history, so its board shows no finished work. It also copies `design/theme/` and `src/site.css` into `build/public/`, which the Worker serves as static assets (`/theme/theme.css`, `/theme/logo.svg`, `/theme/og.png`, and `/site.css`). `build/` is ignored by Git. @@ -65,7 +65,7 @@ The passkey relying party is the hostname of `AUTH_URL`, and its exact origin is the only one accepted. Changing the hostname orphans existing passkeys; people enroll new ones after signing in with a provider. -GitHub remains the planned authority for enrolled project state. The Worker +GitHub is authoritative for enrolled project state. The Worker has no Durable Objects; D1 holds accounts, sessions, and GitHub App state. `cloudflare/` is a Bun workspace of the repository root, so the Worker diff --git a/cloudflare/migrations/0006_project_state.sql b/cloudflare/migrations/0006_project_state.sql new file mode 100644 index 00000000..252807f0 --- /dev/null +++ b/cloudflare/migrations/0006_project_state.sql @@ -0,0 +1,24 @@ +-- Cached observations never grant authority. Disconnect removes their retention. +CREATE TABLE github_project_state ( + repository INTEGER PRIMARY KEY REFERENCES github_projects(id) ON DELETE CASCADE, + revision INTEGER NOT NULL DEFAULT 0, + epoch TEXT NOT NULL, + invalidated INTEGER NOT NULL DEFAULT 0 CHECK(invalidated IN (0,1)), + snapshot TEXT, + scan TEXT, + observedAt INTEGER +); +-- One durable notification per provider entity, independent of delivery IDs. +-- Webhooks keep only identity pointers; authenticated reconciliation fills data. +CREATE TABLE github_project_activity ( + repository INTEGER NOT NULL REFERENCES github_projects(id) ON DELETE CASCADE, + kind TEXT NOT NULL CHECK(kind IN ('issue','pull')), + id TEXT NOT NULL, + number INTEGER NOT NULL, + firstSeen INTEGER NOT NULL, + changedAt INTEGER NOT NULL, + data TEXT, + seenIn TEXT, + missing INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(repository,kind,id) +); diff --git a/cloudflare/src/board/model.ts b/cloudflare/src/board/model.ts index 39a5a146..ce85fc64 100644 --- a/cloudflare/src/board/model.ts +++ b/cloudflare/src/board/model.ts @@ -10,7 +10,11 @@ import type { Change, Completed, Project } from "./project"; * A quest's most advanced state, per design/theme.md: Ready to turn in, then * Available or Requires. */ -export type Status = { kind: "turn-in"; change: Change } | { kind: "available" } | { kind: "blocked"; requires: Ref[] }; +export type Status = + | { kind: "turn-in"; change: Change } + | { kind: "available" } + | { kind: "accepted" } + | { kind: "blocked"; requires: Ref[] }; /** A link to another quest, or a blocker's own words when it names none. */ export type Ref = { title: string; href: string | null }; @@ -64,10 +68,14 @@ export function title(text: string): { size: Size | null; name: string } { * A quest's page: its path without `.md`, like its branch name. A README is * its directory's page. */ +export function boardHref(project: Project): string { + return project.href ?? `/repos/${project.name}`; +} + export function questHref(project: Project, path: string): string { - if (path === ROOT) return `/repos/${project.name}`; + if (path === ROOT) return boardHref(project); const page = path.replace(/\/README\.md$/, "").replace(/\.md$/, ""); - return `/repos/${project.name}/${encodePath(page)}`; + return `${boardHref(project)}/${encodePath(page)}`; } /** @@ -124,6 +132,7 @@ export function readBoard(project: Project): Board { const status = (doc: Doc): Status => { const change = changes.get(doc.path); if (change) return { kind: "turn-in", change }; + if (project.claims?.includes(doc.path)) return { kind: "accepted" }; if (open.has(doc.path)) return { kind: "available" }; const requires = blockers(parsed, doc).map((blocker) => blocker.path === null @@ -214,7 +223,7 @@ export function readBoard(project: Project): Board { } const all = milestones.flatMap((milestone) => flatten(milestone.items)); - const counts = { "turn-in": 0, available: 0, blocked: 0 }; + const counts = { "turn-in": 0, available: 0, blocked: 0, accepted: 0 }; for (const q of all) counts[q.status.kind]++; const milestoneOf = (path: string) => milestones.find((c) => c.number !== "" && path.startsWith(`${dir(c.path)}/`)); diff --git a/cloudflare/src/board/pages.tsx b/cloudflare/src/board/pages.tsx index 6120e645..601fde53 100644 --- a/cloudflare/src/board/pages.tsx +++ b/cloudflare/src/board/pages.tsx @@ -5,6 +5,7 @@ import { type Crumb, Difficulty, Marker, Paste, SignIn, Token, type User } from import { render } from "./markdown"; import { type Board, + boardHref, type Milestone, milestoneAnchor, blobHref, @@ -27,6 +28,8 @@ function label(status: Status, limit = Infinity): string { switch (status.kind) { case "turn-in": return "Ready to turn in"; + case "accepted": + return "Claimed"; case "available": return "Available"; case "blocked": { @@ -156,13 +159,14 @@ function FinishedList(props: { finished: Finished[] }) { /** The statuses a visitor can filter the board by, in the order a contributor asks about them. */ export const SHOWN: [Status["kind"], string][] = [ ["available", "Available"], + ["accepted", "Claimed"], ["turn-in", "Ready to turn in"], ["blocked", "Requires others"], ]; // Counts by status, which double as filters and as the legend for the markers. function Tally(props: { board: Board; show: Status["kind"] | null }) { - const base = `/repos/${props.board.project.name}`; + const base = boardHref(props.board.project); return (