From aebca172b5059e88f445f7908b08df03e5d5a152 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 14:04:01 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m0/routing-evidence/online-reviewers.md Quest-Claim: 0e25b102145e6fc1 From 37d401e5980efe6c02894d18431f5b0903face7f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 14:49:56 -0800 Subject: [PATCH 2/3] feat(usage): count the online reviewers on each PR quest usage reads each PR's reviews, review threads, and comments from GitHub's GraphQL API, ten PRs a query, and reports each online reviewer's record: whether it reviewed the final head, and its findings accepted, declined, unclassified, or unanswered. A rollup gives each reviewer's coverage and answers over the merged PRs. Co-Authored-By: Claude Opus 5.5 --- README.md | 27 +- quest/m0/routing-evidence/README.md | 2 +- quest/m0/routing-evidence/online-reviewers.md | 49 ---- src/cli/main.ts | 10 +- src/cli/online-review.ts | 249 ++++++++++++++++++ src/cli/usage.ts | 75 ++++++ test/online-review.test.ts | 154 +++++++++++ test/usage.test.ts | 85 +++++- 8 files changed, 592 insertions(+), 59 deletions(-) delete mode 100644 quest/m0/routing-evidence/online-reviewers.md create mode 100644 src/cli/online-review.ts create mode 100644 test/online-review.test.ts diff --git a/README.md b/README.md index 91c5a2b3..1e54b635 100644 --- a/README.md +++ b/README.md @@ -1501,6 +1501,7 @@ moq-dev/quest, PRs updated and sessions active since 2026-11-06; costs (~$) esti #569 S merged: quest: finish model routing (quest/m0/model-routing/README) implementer: Claude Code, claude-opus-5-5, unknown effort review: Codex, gpt-6.1-sol, medium effort; round 1, score 10/10, 0 open, last round 0 fixed and 0 refuted + online review: review (OpenAI), not at the final head; 0 findings: 0 accepted, 0 declined, 0 unclassified, 0 unanswered land fixes: codex, gpt-6.1-sol, medium effort sessions: implementing claude claude-opus-5-5 low in 50, cache 36.2k written 1.1M read, out 6.1k, ~$0.63 @@ -1521,12 +1522,28 @@ it or Codex marks it a review, implementing when it began before the PR opened, and iterating otherwise. An opencode session, whose log quest does not read, is the reviewing session of the newest PR whose `## Review` record names it. Each PR shows its `## Models` size and implementers; each reviewer's latest `quest review` round's reviewer, round, score, open findings, and the last round's findings fixed or -refuted; the land fixer's fixes; and the later PRs that list it under -`## Caused by`. - -Rollups follow: spend by size, role, harness, model, and effort, and accuracy +refuted; the land fixer's fixes; the later PRs that list it under +`## Caused by`; and each online reviewer's record, read from GitHub. + +An online reviewer is any bot account that posts reviews, or a review whose +body opens with `Automated review by `, as a reviewer posting under a +person's account does. It reviewed the final head when one of its reviews with +a body or a new thread is at the PR's last head. Its findings are the review +threads it opens and the `[P]` (or `P —`) lines in its review bodies, +one finding per title across its reviews. A finding is accepted, declined, or +unclassified by the first word of its first answer: a reply in its thread, or +for a body finding, a PR comment that links its review or opens with an +accepting word (`Fixed`, `Agreed`, `Applied`, `Right`, `Confirmed`, +`Covered`, `Taken`) or a declining one (`Declined`, `Declining`, `Refuted`, +`Not`). One with no answer counts as accepted when the reviewer settled it, by +resolving its own thread or leaving it out of a later review, and as +unanswered otherwise. + +Rollups follow: spend by size, role, harness, model, and effort; accuracy by size and implementer, where an `Implementer:` line's own size, as -`(iterate, XS)`, counts over the PR's. Sessions on no PR here, from any +`(iterate, XS)`, counts over the PR's; and, over the merged PRs, each online +reviewer's coverage, final heads reviewed, findings per PR reviewed, and +answers. Sessions on no PR here, from any repository or agent ccusage reads, total on an `Unattributed` line. `--pr` and `--quest` narrow the report to one PR or one quest's branch, and `--json` prints it as one object. Without `ccusage` on `PATH`, it says how to install it diff --git a/quest/m0/routing-evidence/README.md b/quest/m0/routing-evidence/README.md index 7e397aec..7985caa2 100644 --- a/quest/m0/routing-evidence/README.md +++ b/quest/m0/routing-evidence/README.md @@ -31,7 +31,7 @@ Decided 2026-11-11 (user, /quest-plan, from an audit of PRs merged only; none). - Records? - ✅ [Cumulative review tallies](/quest/m0/routing-evidence/review-tallies.md) - and [online reviewers in usage](/quest/m0/routing-evidence/online-reviewers.md) + and online reviewers in usage - ✅ [Attribution through token-max](/quest/m0/routing-evidence/usage-token-max.md), kept after the maintainer first left it out, because the writer A/B compares cost and `quest usage` crashes on this host diff --git a/quest/m0/routing-evidence/online-reviewers.md b/quest/m0/routing-evidence/online-reviewers.md deleted file mode 100644 index 6e33fb7d..00000000 --- a/quest/m0/routing-evidence/online-reviewers.md +++ /dev/null @@ -1,49 +0,0 @@ -# [M] quest usage counts the online reviewers - -## Goal - -`quest usage` reports, for each online reviewer on each PR: whether it -reviewed the final head, how many findings it raised, and how many were -accepted, declined, or left unanswered. A reviewer is any bot account that -posts reviews, plus any review whose body opens with -`Automated review by `, such as the Dot, ChatGPT's reviewer that -posts as the maintainer. Rollups per reviewer and repository give coverage, -findings per PR, and acceptance. The data comes from GitHub; the PR body -gains nothing, and no reviewer is named in the code. - -## Plan - -Decided 2026-11-11 (user, /quest-plan, -[routing evidence](/quest/m0/routing-evidence/README.md)): measure the -online reviewers in usage instead of recording them in PR bodies. - -What the 2026-11-11 audit saw of today's reviewers, to check the generic -rules against rather than to code in: - -- CodeRabbit (`coderabbitai[bot]`) posts inline threads tagged Major or - Minor. "✅ Addressed in commit(s)" is its own check. "✅ Confirmed as - addressed by @kixelated" only echoes the agent's reply. Its summary ends - in "Review limit reached" when the hourly cap skipped the PR. -- Pullfrog (`pullfrog[bot]`) names its model in each comment's footer - ("Using `...`"), resolves its own threads on re-review, and posts outage - comments, such as a rejected OpenRouter key or exhausted credit, that are - not reviews. -- The Dot posts as the maintainer: a review body that starts with - "Automated review by review (OpenAI)", with findings as `[P1]`-`[P3]` - lines in the body rather than threads. A finding counts as settled when a - later Dot review on the PR no longer lists it. A reply in a thread or - comment counts as an answer. -- Agent replies post as the maintainer too and end in a `(Written by ...)` - line, which tells them apart from the Dot. - -Agent's choices, for the implementer to revisit: - -- Read threads through GraphQL `reviewThreads` in pages, since REST's - hourly quota is shared and an earlier audit exhausted it. -- A finding is a review thread, or a `[P]` line in a review body, the - marker Codex's reviewers use. Comments outside reviews, such as outage - notices, are not findings. -- Classify a reply as accepted or declined from its first word ("Fixed", - "Agreed", "Applied", or "Declined", "Refuted", "Not"). Report a reply - that fits neither as a fourth state, answered but unclassified, so the - rule's misses stay visible. diff --git a/src/cli/main.ts b/src/cli/main.ts index 862c18e2..252b19ee 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -1106,9 +1106,13 @@ export function main(argv: string[], out: Output): number | Promise { "## Review record names it, as reviewing. Tokens and cost come from ccusage (session --json), the " + "cost an estimate at each model's API list price. Each PR also shows its ## Models size and implementers, " + "each reviewer's latest ## Review round, with its round, score, and findings, " + - "and the later PRs that list it under ## Caused by. Rollups follow: spend by size, role, harness, model, and " + - "effort, and accuracy by size and implementer, where an Implementer line's own size, as (iterate, XS), wins " + - "over the PR's. Without ccusage on PATH, it says how to install it and reports the PRs' records alone. " + + "the later PRs that list it under ## Caused by, and each online reviewer's record from GitHub: whether it " + + "reviewed the last head, and its findings accepted, declined, unclassified, or unanswered. An online reviewer " + + "is a bot that posts reviews, or a review whose body opens with Automated review by ; a finding is a " + + "review thread it opens or a [P] line in its review body, and its first answer's first word takes or " + + "declines it. Rollups follow: spend by size, role, harness, model, and " + + "effort, accuracy by size and implementer, where an Implementer line's own size, as (iterate, XS), wins " + + "over the PR's, and each online reviewer's coverage and findings over the merged PRs. Without ccusage on PATH, it says how to install it and reports the PRs' records alone. " + "--json prints the same report as one object.\n\n" + "Uses GH_TOKEN, GITHUB_TOKEN, or gh auth token. Exits 0 when done, 2 for invalid arguments, and 1 for " + "other failures.", diff --git a/src/cli/online-review.ts b/src/cli/online-review.ts new file mode 100644 index 00000000..aede0f56 --- /dev/null +++ b/src/cli/online-review.ts @@ -0,0 +1,249 @@ +// How the online reviewers did on each PR, read from GitHub: any bot account that posts reviews, and any review whose +// body opens with `Automated review by `, as a reviewer posting under a person's account does. A finding is a +// review thread one opens, or a `[P]` line in its review body; whoever answers it says whether it was taken. +import { type GitHub, type ObjectValue, type Repository, array, object } from "./github"; + +/** One online reviewer's record on one PR. */ +export interface OnlineReview { + /** A bot's login, as `name[bot]`, or the name its review bodies give. */ + reviewer: string; + /** It reviewed the PR's last head. */ + finalHead: boolean; + findings: number; + accepted: number; + declined: number; + /** Answered, but the answer opens with neither an accepting nor a declining word. */ + unclassified: number; + unanswered: number; +} + +type Answer = "accepted" | "declined" | "unclassified"; + +const ACCEPTED = new Set(["fixed", "agreed", "applied", "right", "confirmed", "covered", "taken"]); +const DECLINED = new Set(["declined", "declining", "refuted", "not"]); + +/** Whether an answer takes or declines a finding, by its first word. */ +export function classify(body: string): Answer { + const word = /^[^A-Za-z]*([A-Za-z]+)/.exec(body)?.[1]?.toLowerCase() ?? ""; + return ACCEPTED.has(word) ? "accepted" : DECLINED.has(word) ? "declined" : "unclassified"; +} + +/** The finding lines of a review body: `[P1] ...`, or `P1 — ...` with a dash after the bare marker. */ +export function bodyFindings(body: string): string[] { + return [...body.matchAll(/^[ \t]*(?:[-*][ \t]+)?(?:\[P\d\]|P\d[ \t]+[—–-])[ \t:—–-]*(\S.*)$/gm)].map((match) => + // A later round that relists a finding repeats its title, or its location when it leads; the rest may change. + match[1]! + .split(/ [—–] | \(|: |\. /)[0]! + .replaceAll("`", "") + .replace(/\s+/g, " ") + .trim() + .toLowerCase(), + ); +} + +const FIELDS = { + reviews: "id url author { __typename login } body submittedAt commit { oid }", + reviewThreads: + "isResolved resolvedBy { __typename login } " + + "comments(first: 100) { pageInfo { hasNextPage } nodes { author { __typename login } body pullRequestReview { id } } }", + comments: "author { __typename login } body createdAt", +} as const; +type Connection = keyof typeof FIELDS; +const CONNECTIONS = Object.keys(FIELDS) as Connection[]; +const page = (name: Connection, after: string) => + `${name}(first: ${name === "reviewThreads" ? 50 : 100}${after}) { pageInfo { hasNextPage endCursor } nodes { ${FIELDS[name]} } }`; + +/** PRs read per query; each brings at most 100 reviews, 50 threads of 100 comments, and 100 comments. */ +const BATCH = 10; + +/** A PR's last head, and its reviews, review threads, and comments as GitHub's GraphQL API gives them. */ +export type Activity = Record & { head: string }; + +/** Each PR's reviews, review threads, and comments, and its head, read in batches and then in pages. */ +async function read(client: GitHub, repository: Repository, numbers: number[]): Promise> { + const pulls = new Map(); + const more: { number: number; cursors: Partial> }[] = []; + for (let start = 0; start < numbers.length; start += BATCH) { + const batch = numbers.slice(start, start + BATCH); + const fields = `headRefOid ${CONNECTIONS.map((name) => page(name, "")).join(" ")}`; + const data = await client.graphql( + `query($owner: String!, $name: String!) { repository(owner: $owner, name: $name) { ${batch + .map((number) => `p${number}: pullRequest(number: ${number}) { ${fields} }`) + .join(" ")} } }`, + { owner: repository.owner, name: repository.repo }, + ); + const found = object(data.repository); + for (const number of batch) { + const pull = object(found[`p${number}`]); + const nodes = { head: String(pull.headRefOid) } as Activity; + const cursors: Partial> = {}; + for (const name of CONNECTIONS) { + const connection = object(pull[name]); + nodes[name] = array(connection.nodes); + const info = object(connection.pageInfo); + if (info.hasNextPage === true) cursors[name] = String(info.endCursor); + } + pulls.set(number, nodes); + if (Object.keys(cursors).length) more.push({ number, cursors }); + } + } + for (const { number, cursors } of more) { + const nodes = pulls.get(number)!; + for (let left = cursors; Object.keys(left).length;) { + const names = CONNECTIONS.filter((name) => left[name] !== undefined); + const data = await client.graphql( + `query($owner: String!, $name: String!, $number: Int!, ${names.map((name) => `$${name}: String!`).join(", ")}) { ` + + `repository(owner: $owner, name: $name) { pullRequest(number: $number) { ${names + .map((name) => page(name, `, after: $${name}`)) + .join(" ")} } } }`, + { owner: repository.owner, name: repository.repo, number, ...left }, + ); + const pull = object(object(data.repository).pullRequest); + const next: Partial> = {}; + for (const name of names) { + const connection = object(pull[name]); + nodes[name].push(...array(connection.nodes)); + const info = object(connection.pageInfo); + if (info.hasNextPage === true) next[name] = String(info.endCursor); + } + left = next; + } + } + return pulls; +} + +/** An account's login, a bot's as `name[bot]`, which GraphQL gives a bot author without the suffix. */ +function login(actor: unknown): string | null { + if (actor === null || actor === undefined) return null; + const { __typename, login } = object(actor); + if (typeof login !== "string") return null; + return __typename === "Bot" && !login.endsWith("[bot]") ? `${login}[bot]` : login; +} + +/** Whose review this is: the name its body opens with, else its bot author, else nobody's. */ +function reviewerOf(review: ObjectValue): string | null { + const named = /^Automated review by (.+)/.exec(typeof review.body === "string" ? review.body : ""); + if (named) return named[1]!.trim(); + return object(review.author ?? {}).__typename === "Bot" ? login(review.author) : null; +} + +/** The anchor GitHub links a review by. */ +const ANCHOR = /#pullrequestreview-\w+/g; + +/** Each online reviewer's record on one PR, from its reviews, threads, and comments. */ +export function onlineReviews(pull: Activity): OnlineReview[] { + const reviews = pull.reviews.map((review) => ({ + id: String(review.id), + url: String(review.url), + reviewer: reviewerOf(review), + body: typeof review.body === "string" ? review.body : "", + at: Date.parse(String(review.submittedAt)), + commit: review.commit ? String(object(review.commit).oid) : null, + })); + const byId = new Map(reviews.map((review) => [review.id, review])); + const records = new Map(); + const record = (reviewer: string) => { + let found = records.get(reviewer); + if (!found) { + found = { + reviewer, + finalHead: false, + findings: 0, + accepted: 0, + declined: 0, + unclassified: 0, + unanswered: 0, + }; + records.set(reviewer, found); + } + return found; + }; + const tally = (reviewer: string, answer: Answer | "unanswered") => { + const found = record(reviewer); + found.findings++; + found[answer]++; + }; + + // A review that only replies in threads has an empty body; one that opens a thread or has a body is a pass. + const opened = new Set(); + for (const thread of pull.reviewThreads) { + const comments = object(thread.comments); + if (object(comments.pageInfo).hasNextPage === true) + throw new Error("a review thread has over 100 comments, more than quest usage reads"); + const [first, ...replies] = array(comments.nodes); + const opener = first?.pullRequestReview ? byId.get(String(object(first.pullRequestReview).id)) : undefined; + const reviewer = opener?.reviewer; + if (!opener || !reviewer) continue; + opened.add(opener.id); + // A named reviewer posts under a person's account, so only its own reviews tell its replies from an answer. + const own = (comment: ObjectValue) => + opener.body.startsWith("Automated review by") + ? comment.pullRequestReview != null && + byId.get(String(object(comment.pullRequestReview).id))?.reviewer === reviewer + : login(comment.author) === reviewer; + const answer = replies.find((reply) => !own(reply)); + if (answer) tally(reviewer, classify(String(answer.body ?? ""))); + else + tally( + reviewer, + thread.isResolved === true && login(thread.resolvedBy) === reviewer ? "accepted" : "unanswered", + ); + } + + const passes = reviews + .filter((review) => review.reviewer !== null && (review.body.trim() !== "" || opened.has(review.id))) + .sort((a, b) => a.at - b.at); + for (const pass of passes) { + const found = record(pass.reviewer!); + if (pass.commit === pull.head) found.finalHead = true; + } + + // A body finding is one title across a reviewer's passes; a pass after its last listing no longer finds it. + const listed = new Map(); + const lists = passes.map((pass, index) => { + const titles = bodyFindings(pass.body); + for (const title of titles) { + const id = JSON.stringify([pass.reviewer, title]); + listed.set(id, { reviewer: pass.reviewer!, last: index, answer: listed.get(id)?.answer ?? null }); + } + return titles.map((title) => JSON.stringify([pass.reviewer, title])); + }); + // A comment answers the pass it links, else the latest before it with findings, when it links one or its first + // word takes a side; a reviewer bot's own comments, such as its summaries and outage notices, answer nothing. + const reviewers = new Set(passes.map((pass) => pass.reviewer)); + for (const comment of [...pull.comments].sort( + (a, b) => Date.parse(String(a.createdAt)) - Date.parse(String(b.createdAt)), + )) { + const author = login(comment.author); + if (author !== null && reviewers.has(author)) continue; + const body = String(comment.body ?? ""); + const at = Date.parse(String(comment.createdAt)); + const links = new Set([...body.matchAll(ANCHOR)].map((match) => match[0])); + const linked = passes.findIndex((pass) => links.has(pass.url.slice(pass.url.lastIndexOf("#")))); + let target = linked; + if (target < 0) + for (const [index, pass] of passes.entries()) if (pass.at <= at && lists[index]!.length) target = index; + const answer = classify(body); + if (target < 0 || (linked < 0 && answer === "unclassified")) continue; + for (const id of lists[target]!) { + const finding = listed.get(id)!; + finding.answer ??= answer; + } + } + for (const finding of listed.values()) { + const later = passes.some((pass, index) => index > finding.last && pass.reviewer === finding.reviewer); + tally(finding.reviewer, finding.answer ?? (later ? "accepted" : "unanswered")); + } + + return [...records.values()].sort((a, b) => a.reviewer.localeCompare(b.reviewer)); +} + +/** Each PR's online reviews, by number. */ +export async function readOnlineReviews( + client: GitHub, + repository: Repository, + numbers: number[], +): Promise> { + const pulls = await read(client, repository, numbers); + return new Map([...pulls].map(([number, pull]) => [number, onlineReviews(pull)])); +} diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 9a1db4d2..461774ba 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -12,6 +12,7 @@ import { discoveryEnv, git } from "./git"; import { fixes } from "./land-fixer"; import { MERGER } from "./land-watch-fix"; import { type GitHub, type ObjectValue, type Repository, githubRepository, link, requireRemote } from "./github"; +import { type OnlineReview, readOnlineReviews } from "./online-review"; import { type ReviewSummary, parseReview } from "./review-record"; /** A session's tokens, and their cost at the model's API list price, which subscriptions do not bill. */ @@ -72,6 +73,8 @@ export interface PullRow { landFixes: string[]; /** Later PRs that list it under `## Caused by`. */ fixedBy: number[]; + /** Each online reviewer's passes and findings, as GitHub shows them. */ + online: OnlineReview[]; sessions: SessionRow[]; } @@ -109,6 +112,21 @@ export interface AccuracyRow { fixes: number; } +/** One online reviewer over the merged PRs reported. */ +export interface OnlineRow { + reviewer: string; + /** Merged PRs reported, whether or not it reviewed them. */ + merged: number; + /** Those it reviewed, and those whose last head it reviewed. */ + reviewed: number; + finalHead: number; + findings: number; + accepted: number; + declined: number; + unclassified: number; + unanswered: number; +} + export interface Usage { repository: string; since: string; @@ -117,6 +135,7 @@ export interface Usage { pulls: PullRow[]; spend: SpendRow[]; accuracy: AccuracyRow[]; + online: OnlineRow[]; /** The sessions on no PR reported here, from every repository and agent ccusage reads; null when narrowed. */ unattributed: ({ sessions: number } & Tokens) | null; } @@ -412,6 +431,7 @@ function pullRows(pulls: Pull[], repository: Repository): PullRow[] { review: parseReview(each.body), landFixes: fixes(each.body), fixedBy: (fixedBy.get(each.number) ?? []).sort((a, b) => a - b), + online: [], sessions: [], }; }); @@ -624,6 +644,37 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { ); } +function onlineRows(pulls: PullRow[]): OnlineRow[] { + const merged = pulls.filter((each) => each.state === "merged"); + const rows = new Map(); + for (const each of merged) + for (const review of each.online) { + let row = rows.get(review.reviewer); + if (!row) { + row = { + reviewer: review.reviewer, + merged: merged.length, + reviewed: 0, + finalHead: 0, + findings: 0, + accepted: 0, + declined: 0, + unclassified: 0, + unanswered: 0, + }; + rows.set(review.reviewer, row); + } + row.reviewed++; + if (review.finalHead) row.finalHead++; + row.findings += review.findings; + row.accepted += review.accepted; + row.declined += review.declined; + row.unclassified += review.unclassified; + row.unanswered += review.unanswered; + } + return [...rows.values()].sort((a, b) => a.reviewer.localeCompare(b.reviewer)); +} + /** Rollup rows by size, smallest first and unsized last, then by each further field. */ function order( [size, ...a]: [Size | null, ...(string | number | null)[]], @@ -672,6 +723,12 @@ export async function usage(root: string, options: UsageOptions, deps: UsageDepe for (const session of left) for (const spent of session.models) add(unattributed, spent); } } + const online = await readOnlineReviews( + client, + repository, + rows.map((row) => row.number), + ); + for (const row of rows) row.online = online.get(row.number) ?? []; return { repository: key(repository), since: options.since, @@ -679,6 +736,7 @@ export async function usage(root: string, options: UsageOptions, deps: UsageDepe pulls: rows, spend: spendRows(rows), accuracy: accuracyRows(rows), + online: onlineRows(rows), unattributed, }; } @@ -694,6 +752,9 @@ const compact = (n: number) => const tokens = (t: Tokens) => `in ${compact(t.input)}, cache ${compact(t.cacheWrite)} written ${compact(t.cacheRead)} read, out ${compact(t.output)}, ~$${t.cost.toFixed(2)}`; const mean = (sum: number, count: number) => (count ? (sum / count).toFixed(1) : "-"); +const percent = (part: number, whole: number) => (whole ? `${Math.round((100 * part) / whole)}%` : "-"); +const answers = (r: Omit) => + `${r.findings} findings: ${r.accepted} accepted, ${r.declined} declined, ${r.unclassified} unclassified, ${r.unanswered} unanswered`; /** Rows of cells, each column padded to its widest cell. */ function table(rows: string[][]): string[] { @@ -720,6 +781,8 @@ export function usageLines(report: Usage): string[] { lines.push( ` review: ${r.harness}, ${r.model}, ${r.effort} effort; round ${r.round}, score ${r.score ?? "-"}/10, ${r.findings} open, last round ${r.fixed} fixed and ${r.refuted} refuted`, ); + for (const o of row.online) + lines.push(` online review: ${o.reviewer}, ${o.finalHead ? "" : "not "}at the final head; ${answers(o)}`); if (row.landFixes.length) lines.push(` land fixes: ${row.landFixes.join("; ")}`); if (row.fixedBy.length) lines.push(` fixed later by: ${row.fixedBy.map((n) => `#${n}`).join(", ")}`); const sessions = row.sessions.flatMap((s) => @@ -759,6 +822,18 @@ export function usageLines(report: Usage): string[] { ]), ), ); + if (report.online.length) + lines.push( + "", + "Online reviewers over merged PRs:", + ...table( + report.online.map((o) => [ + " ", + o.reviewer, + `reviewed ${o.reviewed}/${o.merged} (${percent(o.reviewed, o.merged)}), final head ${o.finalHead} (${percent(o.finalHead, o.merged)}), ${mean(o.findings, o.reviewed)} findings per PR reviewed, ${percent(o.accepted, o.findings)} accepted; ${answers(o)}`, + ]), + ), + ); const u = report.unattributed; if (u) lines.push("", `Unattributed: ${u.sessions} sessions on no PR here, any repository or agent: ${tokens(u)}`); return lines; diff --git a/test/online-review.test.ts b/test/online-review.test.ts new file mode 100644 index 00000000..e2447ff2 --- /dev/null +++ b/test/online-review.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, test } from "vitest"; + +import { type Activity, bodyFindings, classify, onlineReviews } from "../src/cli/online-review"; + +const HEAD = "h".repeat(40); +const bot = (login: string) => ({ __typename: "Bot", login }); +const person = { __typename: "User", login: "maintainer" }; +const review = (id: string, author: object, body: string, at: string, commit = HEAD) => ({ + id, + url: `https://github.com/acme/proj/pull/1#pullrequestreview-${id}`, + author, + body, + submittedAt: at, + commit: { oid: commit }, +}); +const comment = (author: object, body: string, review: string) => ({ author, body, pullRequestReview: { id: review } }); +const thread = (comments: object[], resolvedBy: object | null = null) => ({ + isResolved: resolvedBy !== null, + resolvedBy, + comments: { pageInfo: { hasNextPage: false }, nodes: comments }, +}); +const activity = (pull: Partial): Activity => ({ + head: HEAD, + reviews: [], + reviewThreads: [], + comments: [], + ...pull, +}); + +describe("classify", () => { + test("reads an answer's first word", () => { + expect(classify("Fixed in abc123: done.")).toBe("accepted"); + expect(classify("**Agreed**, applied.")).toBe("accepted"); + expect(classify("Declining: the caller already checks.")).toBe("declined"); + expect(classify("Not quite: the window opens later.")).toBe("declined"); + expect(classify("Moot after merging main.")).toBe("unclassified"); + expect(classify("")).toBe("unclassified"); + }); +}); + +describe("bodyFindings", () => { + test("reads bracketed and dashed markers, keyed by title", () => { + const body = [ + "Automated review by review (OpenAI)", + "", + "[P2] Keep `sampling` when unavailable — rs/a.rs:138. Long text.", + "P3 — src/b.ts:10–12: preserve output. More.", + "- [P1] Restore the hierarchy (quest/x.md:4)", + "The previous P2 is fixed.", + "P2 is a prose word here.", + ].join("\n"); + expect(bodyFindings(body)).toEqual([ + "keep sampling when unavailable", + "src/b.ts:10–12", + "restore the hierarchy", + ]); + }); +}); + +describe("onlineReviews", () => { + test("a bot's threads are taken, declined, unclassified, or unanswered by their first answer", () => { + const pull = activity({ + reviews: [ + review("r1", bot("rabbit"), "Actionable comments posted: 5", "2026-11-06T10:00:00Z", "a".repeat(40)), + // A reply-only review at the head is no pass over it. + review("r2", bot("rabbit"), "", "2026-11-06T12:00:00Z"), + review("r3", person, "", "2026-11-06T11:00:00Z"), + // A person's own review opens no online reviewer's thread. + review("r4", person, "Looks off.", "2026-11-06T11:00:00Z"), + ], + reviewThreads: [ + thread([ + comment(bot("rabbit"), "Bug.", "r1"), + comment(bot("rabbit"), "Also.", "r2"), + comment(person, "Fixed in abc.", "r3"), + ]), + thread([comment(bot("rabbit"), "Bug.", "r1"), comment(person, "Declining: holds.", "r3")]), + thread([comment(bot("rabbit"), "Bug.", "r1"), comment(person, "Moot now.", "r3")]), + thread([comment(bot("rabbit"), "Bug.", "r1")]), + // Resolved by the reviewer itself, which reports a bot's login with its suffix there. + thread([comment(bot("rabbit"), "Bug.", "r1")], { __typename: "User", login: "rabbit[bot]" }), + thread([comment(person, "Nit.", "r4")]), + ], + }); + expect(onlineReviews(pull)).toEqual([ + { + reviewer: "rabbit[bot]", + finalHead: false, + findings: 5, + accepted: 2, + declined: 1, + unclassified: 1, + unanswered: 1, + }, + ]); + }); + + test("a named reviewer's body findings are settled by a later pass or answered in a comment", () => { + const named = (id: string, body: string, at: string, commit?: string) => + review(id, person, `Automated review by dot (X)\n\n${body}`, at, commit); + const pull = activity({ + reviews: [ + named("d1", "[P2] Keep it — a.rs:1.\n[P3] Rename it — b.rs:2.", "2026-11-06T10:00:00Z", "a".repeat(40)), + named( + "d2", + "[P2] Keep it — a.rs:3, still.\n[P1] Guard it — c.rs:4.", + "2026-11-06T12:00:00Z", + "b".repeat(40), + ), + named("d3", "[P1] Guard it — c.rs:4.\n[P3] Docs — d.md:1.", "2026-11-06T14:00:00Z"), + review("p1", bot("frog"), "No issues.", "2026-11-06T09:00:00Z", "a".repeat(40)), + // The agent's reply in the Dot's thread posts under the same account, in a review of its own. + review("x1", person, "", "2026-11-06T15:00:00Z"), + ], + reviewThreads: [ + thread([comment(person, "Bug.", "d3"), comment(person, "Agreed and fixed.", "x1")]), + thread([comment(person, "Bug.", "d3"), comment(person, "Still.", "d3")]), + ], + comments: [ + // The bot's own notice answers nothing, nor does a comment that neither links a pass nor takes a side. + { author: bot("frog"), body: "Fixed: credit restored.", createdAt: "2026-11-06T12:30:00Z" }, + { author: person, body: "Design change: narrowed.", createdAt: "2026-11-06T12:40:00Z" }, + // Answers d2's findings, which d2 links, though d3 came between. + { + author: person, + body: "Re [review](https://github.com/acme/proj/pull/1#pullrequestreview-d2): declining both.", + createdAt: "2026-11-06T14:30:00Z", + }, + ], + }); + expect(onlineReviews(pull)).toEqual([ + { + reviewer: "dot (X)", + finalHead: true, + // Keep it (relisted, answered unclassified), Rename it (settled), Guard it (answered unclassified), + // Docs (open in the last pass), and the two threads. + findings: 6, + accepted: 2, + declined: 0, + unclassified: 2, + unanswered: 2, + }, + { + reviewer: "frog[bot]", + finalHead: false, + findings: 0, + accepted: 0, + declined: 0, + unclassified: 0, + unanswered: 0, + }, + ]); + }); +}); diff --git a/test/usage.test.ts b/test/usage.test.ts index b632fd4e..00dc32e5 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -406,9 +406,61 @@ describe("usage", () => { pr(8, "old", "", "2026-10-01T00:00:00Z", { updated_at: "2026-10-01T00:00:00Z" }), ]; const requests: string[] = []; - const fetch = async (input: string | URL | Request) => { + const connection = (nodes: object[]) => ({ pageInfo: { hasNextPage: false, endCursor: null }, nodes }); + // #10's final head got a bot's review with one thread, which the agent fixed. + const activity = (number: number) => ({ + headRefOid: "b".repeat(40), + reviews: connection( + number === 10 + ? [ + { + id: "r1", + url: "https://github.com/acme/proj/pull/10#pullrequestreview-1", + author: { __typename: "Bot", login: "rabbit" }, + body: "One finding.", + submittedAt: "2026-11-06T15:00:00Z", + commit: { oid: "b".repeat(40) }, + }, + ] + : [], + ), + reviewThreads: connection( + number === 10 + ? [ + { + isResolved: false, + resolvedBy: null, + comments: { + pageInfo: { hasNextPage: false }, + nodes: [ + { + author: { __typename: "Bot", login: "rabbit" }, + body: "Bug.", + pullRequestReview: { id: "r1" }, + }, + { + author: { __typename: "User", login: "acme" }, + body: "Fixed.", + pullRequestReview: { id: "r2" }, + }, + ], + }, + }, + ] + : [], + ), + comments: connection([]), + }); + const fetch = async (input: string | URL | Request, init?: RequestInit) => { const url = new URL(String(input)); requests.push(url.pathname); + if (url.pathname === "/graphql") { + const { query } = JSON.parse(String(init?.body)) as { query: string }; + const numbers = [...query.matchAll(/p(\d+): pullRequest/g)].map((match) => Number(match[1])); + return json({ + data: { repository: Object.fromEntries(numbers.map((number) => [`p${number}`, activity(number)])) }, + }); + } if (url.pathname === "/repos/acme/proj") return json({ owner: { login: "acme" }, @@ -496,8 +548,39 @@ describe("usage", () => { ["S", "gpt-6.1-sol", 1, 1, 17, 2, 1], ["L", "claude-opus-5-5", 1, 1, 17, 2, 1], ]); + expect(ten.online).toEqual([ + { + reviewer: "rabbit[bot]", + finalHead: true, + findings: 1, + accepted: 1, + declined: 0, + unclassified: 0, + unanswered: 0, + }, + ]); + // #11 and #10 merged; #9 closed and #7 is open, so neither counts. + expect(report.online).toEqual([ + { + reviewer: "rabbit[bot]", + merged: 2, + reviewed: 1, + finalHead: 1, + findings: 1, + accepted: 1, + declined: 0, + unclassified: 0, + unanswered: 0, + }, + ]); const text = usageLines(report).join("\n"); expect(text).toContain("#10 L merged: PR 10 (quest/m0/a)"); + expect(text).toContain( + "online review: rabbit[bot], at the final head; 1 findings: 1 accepted, 0 declined, 0 unclassified, 0 unanswered", + ); + expect(text).toContain( + "rabbit[bot] reviewed 1/2 (50%), final head 1 (50%), 1.0 findings per PR reviewed, 100% accepted;", + ); expect(text).toContain("fixed later by: #11"); expect(text).not.toContain("SECRET"); expect(JSON.stringify(report)).not.toContain("SECRET"); From 141fae1a3ceee6683582dada662bea346c3876da Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 11 Nov 2026 15:21:54 -0800 Subject: [PATCH 3/3] fix(usage): answer every linked review, page long threads, keep repeated titles apart Co-Authored-By: Claude Opus 5.5 --- README.md | 12 ++++--- src/cli/main.ts | 2 +- src/cli/online-review.ts | 72 ++++++++++++++++++++++++++------------ src/cli/usage.ts | 2 +- test/online-review.test.ts | 18 ++++++---- test/usage.test.ts | 30 +++++++++++----- 6 files changed, 92 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 1e54b635..42210c86 100644 --- a/README.md +++ b/README.md @@ -1525,18 +1525,20 @@ reviewer, round, score, open findings, and the last round's findings fixed or refuted; the land fixer's fixes; the later PRs that list it under `## Caused by`; and each online reviewer's record, read from GitHub. -An online reviewer is any bot account that posts reviews, or a review whose -body opens with `Automated review by `, as a reviewer posting under a -person's account does. It reviewed the final head when one of its reviews with +An online reviewer is any bot account whose review has a body or opens a +thread, as a reply-only review does not, or a review whose body opens with +`Automated review by `, as a reviewer posting under a person's account +does. It reviewed the final head when one of its reviews with a body or a new thread is at the PR's last head. Its findings are the review threads it opens and the `[P]` (or `P —`) lines in its review bodies, -one finding per title across its reviews. A finding is accepted, declined, or +one finding per title across its reviews (a repeated title in one review is +another finding). A finding is accepted, declined, or unclassified by the first word of its first answer: a reply in its thread, or for a body finding, a PR comment that links its review or opens with an accepting word (`Fixed`, `Agreed`, `Applied`, `Right`, `Confirmed`, `Covered`, `Taken`) or a declining one (`Declined`, `Declining`, `Refuted`, `Not`). One with no answer counts as accepted when the reviewer settled it, by -resolving its own thread or leaving it out of a later review, and as +a bot resolving its own thread or a later review leaving it out, and as unanswered otherwise. Rollups follow: spend by size, role, harness, model, and effort; accuracy diff --git a/src/cli/main.ts b/src/cli/main.ts index 252b19ee..59a89c83 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -1108,7 +1108,7 @@ export function main(argv: string[], out: Output): number | Promise { "each reviewer's latest ## Review round, with its round, score, and findings, " + "the later PRs that list it under ## Caused by, and each online reviewer's record from GitHub: whether it " + "reviewed the last head, and its findings accepted, declined, unclassified, or unanswered. An online reviewer " + - "is a bot that posts reviews, or a review whose body opens with Automated review by ; a finding is a " + + "is a bot whose review has a body or opens a thread, or a review whose body opens with Automated review by ; a finding is a " + "review thread it opens or a [P] line in its review body, and its first answer's first word takes or " + "declines it. Rollups follow: spend by size, role, harness, model, and " + "effort, accuracy by size and implementer, where an Implementer line's own size, as (iterate, XS), wins " + diff --git a/src/cli/online-review.ts b/src/cli/online-review.ts index aede0f56..3f64eb12 100644 --- a/src/cli/online-review.ts +++ b/src/cli/online-review.ts @@ -1,4 +1,5 @@ -// How the online reviewers did on each PR, read from GitHub: any bot account that posts reviews, and any review whose +// How the online reviewers did on each PR, read from GitHub: any bot account whose review has a body or opens a +// thread, and any review whose // body opens with `Automated review by `, as a reviewer posting under a person's account does. A finding is a // review thread one opens, or a `[P]` line in its review body; whoever answers it says whether it was taken. import { type GitHub, type ObjectValue, type Repository, array, object } from "./github"; @@ -28,24 +29,32 @@ export function classify(body: string): Answer { return ACCEPTED.has(word) ? "accepted" : DECLINED.has(word) ? "declined" : "unclassified"; } -/** The finding lines of a review body: `[P1] ...`, or `P1 — ...` with a dash after the bare marker. */ +/** + * The finding lines of a review body, `[P1] ...` or `P1 — ...` with a dash after the bare marker, each keyed by its + * title and how many earlier lines of the body share that title. + */ export function bodyFindings(body: string): string[] { - return [...body.matchAll(/^[ \t]*(?:[-*][ \t]+)?(?:\[P\d\]|P\d[ \t]+[—–-])[ \t:—–-]*(\S.*)$/gm)].map((match) => + const seen = new Map(); + return [...body.matchAll(/^[ \t]*(?:[-*][ \t]+)?(?:\[P\d\]|P\d[ \t]+[—–-])[ \t:—–-]*(\S.*)$/gm)].map((match) => { // A later round that relists a finding repeats its title, or its location when it leads; the rest may change. - match[1]! + const title = match[1]! .split(/ [—–] | \(|: |\. /)[0]! .replaceAll("`", "") .replace(/\s+/g, " ") .trim() - .toLowerCase(), - ); + .toLowerCase(); + const count = seen.get(title) ?? 0; + seen.set(title, count + 1); + return count ? `${title} #${count + 1}` : title; + }); } +const THREAD_COMMENT = "author { __typename login } body pullRequestReview { id }"; const FIELDS = { reviews: "id url author { __typename login } body submittedAt commit { oid }", reviewThreads: - "isResolved resolvedBy { __typename login } " + - "comments(first: 100) { pageInfo { hasNextPage } nodes { author { __typename login } body pullRequestReview { id } } }", + "id isResolved resolvedBy { __typename login } " + + `comments(first: 100) { pageInfo { hasNextPage endCursor } nodes { ${THREAD_COMMENT} } }`, comments: "author { __typename login } body createdAt", } as const; type Connection = keyof typeof FIELDS; @@ -74,7 +83,10 @@ async function read(client: GitHub, repository: Repository, numbers: number[]): ); const found = object(data.repository); for (const number of batch) { - const pull = object(found[`p${number}`]); + const value = found[`p${number}`]; + if (value === null || value === undefined) + throw new Error(`GitHub has no PR #${number} in ${repository.owner}/${repository.repo} for this token`); + const pull = object(value); const nodes = { head: String(pull.headRefOid) } as Activity; const cursors: Partial> = {}; for (const name of CONNECTIONS) { @@ -109,6 +121,23 @@ async function read(client: GitHub, repository: Repository, numbers: number[]): left = next; } } + // A long thread's later comments matter when none of its first hundred answers it. + for (const nodes of pulls.values()) + for (const thread of nodes.reviewThreads) { + const comments = object(thread.comments); + const all = array(comments.nodes); + for (let info = object(comments.pageInfo); info.hasNextPage === true;) { + const data = await client.graphql( + "query($id: ID!, $after: String!) { node(id: $id) { ... on PullRequestReviewThread { " + + `comments(first: 100, after: $after) { pageInfo { hasNextPage endCursor } nodes { ${THREAD_COMMENT} } } } } }`, + { id: String(thread.id), after: String(info.endCursor) }, + ); + const page = object(object(data.node).comments); + all.push(...array(page.nodes)); + info = object(page.pageInfo); + } + thread.comments = { nodes: all }; + } return pulls; } @@ -167,10 +196,7 @@ export function onlineReviews(pull: Activity): OnlineReview[] { // A review that only replies in threads has an empty body; one that opens a thread or has a body is a pass. const opened = new Set(); for (const thread of pull.reviewThreads) { - const comments = object(thread.comments); - if (object(comments.pageInfo).hasNextPage === true) - throw new Error("a review thread has over 100 comments, more than quest usage reads"); - const [first, ...replies] = array(comments.nodes); + const [first, ...replies] = array(object(thread.comments).nodes); const opener = first?.pullRequestReview ? byId.get(String(object(first.pullRequestReview).id)) : undefined; const reviewer = opener?.reviewer; if (!opener || !reviewer) continue; @@ -208,7 +234,7 @@ export function onlineReviews(pull: Activity): OnlineReview[] { } return titles.map((title) => JSON.stringify([pass.reviewer, title])); }); - // A comment answers the pass it links, else the latest before it with findings, when it links one or its first + // A comment answers each pass it links, else the latest before it with findings, when it links one or its first // word takes a side; a reviewer bot's own comments, such as its summaries and outage notices, answer nothing. const reviewers = new Set(passes.map((pass) => pass.reviewer)); for (const comment of [...pull.comments].sort( @@ -219,16 +245,18 @@ export function onlineReviews(pull: Activity): OnlineReview[] { const body = String(comment.body ?? ""); const at = Date.parse(String(comment.createdAt)); const links = new Set([...body.matchAll(ANCHOR)].map((match) => match[0])); - const linked = passes.findIndex((pass) => links.has(pass.url.slice(pass.url.lastIndexOf("#")))); - let target = linked; - if (target < 0) - for (const [index, pass] of passes.entries()) if (pass.at <= at && lists[index]!.length) target = index; const answer = classify(body); - if (target < 0 || (linked < 0 && answer === "unclassified")) continue; - for (const id of lists[target]!) { - const finding = listed.get(id)!; - finding.answer ??= answer; + let targets = passes.flatMap((pass, index) => + links.has(pass.url.slice(pass.url.lastIndexOf("#"))) ? [index] : [], + ); + if (!targets.length && answer !== "unclassified") { + for (const [index, pass] of passes.entries()) if (pass.at <= at && lists[index]!.length) targets = [index]; } + for (const target of targets) + for (const id of lists[target]!) { + const finding = listed.get(id)!; + finding.answer ??= answer; + } } for (const finding of listed.values()) { const later = passes.some((pass, index) => index > finding.last && pass.reviewer === finding.reviewer); diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 461774ba..8e25ccdf 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -754,7 +754,7 @@ const tokens = (t: Tokens) => const mean = (sum: number, count: number) => (count ? (sum / count).toFixed(1) : "-"); const percent = (part: number, whole: number) => (whole ? `${Math.round((100 * part) / whole)}%` : "-"); const answers = (r: Omit) => - `${r.findings} findings: ${r.accepted} accepted, ${r.declined} declined, ${r.unclassified} unclassified, ${r.unanswered} unanswered`; + `${r.findings} finding${r.findings === 1 ? "" : "s"}: ${r.accepted} accepted, ${r.declined} declined, ${r.unclassified} unclassified, ${r.unanswered} unanswered`; /** Rows of cells, each column padded to its widest cell. */ function table(rows: string[][]): string[] { diff --git a/test/online-review.test.ts b/test/online-review.test.ts index e2447ff2..25faf4e7 100644 --- a/test/online-review.test.ts +++ b/test/online-review.test.ts @@ -39,6 +39,10 @@ describe("classify", () => { }); describe("bodyFindings", () => { + test("two findings that share a title in one body stay two", () => { + expect(bodyFindings("[P1] Guard null (GET)\n[P2] Guard null (POST)")).toEqual(["guard null", "guard null #2"]); + }); + test("reads bracketed and dashed markers, keyed by title", () => { const body = [ "Automated review by review (OpenAI)", @@ -120,10 +124,10 @@ describe("onlineReviews", () => { // The bot's own notice answers nothing, nor does a comment that neither links a pass nor takes a side. { author: bot("frog"), body: "Fixed: credit restored.", createdAt: "2026-11-06T12:30:00Z" }, { author: person, body: "Design change: narrowed.", createdAt: "2026-11-06T12:40:00Z" }, - // Answers d2's findings, which d2 links, though d3 came between. + // Answers the findings of both reviews it links, though d3 came between. { author: person, - body: "Re [review](https://github.com/acme/proj/pull/1#pullrequestreview-d2): declining both.", + body: "Declining: [one](https://github.com/acme/proj/pull/1#pullrequestreview-d1), [two](https://github.com/acme/proj/pull/1#pullrequestreview-d2).", createdAt: "2026-11-06T14:30:00Z", }, ], @@ -132,12 +136,12 @@ describe("onlineReviews", () => { { reviewer: "dot (X)", finalHead: true, - // Keep it (relisted, answered unclassified), Rename it (settled), Guard it (answered unclassified), - // Docs (open in the last pass), and the two threads. + // Keep it (relisted), Rename it, and Guard it, all declined though Rename it was settled; Docs (open in + // the last pass); and the two threads. findings: 6, - accepted: 2, - declined: 0, - unclassified: 2, + accepted: 1, + declined: 3, + unclassified: 0, unanswered: 2, }, { diff --git a/test/usage.test.ts b/test/usage.test.ts index 00dc32e5..17908863 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -430,19 +430,16 @@ describe("usage", () => { { isResolved: false, resolvedBy: null, + id: "t1", + // The answer is on the thread's second page of comments. comments: { - pageInfo: { hasNextPage: false }, + pageInfo: { hasNextPage: true, endCursor: "c1" }, nodes: [ { author: { __typename: "Bot", login: "rabbit" }, body: "Bug.", pullRequestReview: { id: "r1" }, }, - { - author: { __typename: "User", login: "acme" }, - body: "Fixed.", - pullRequestReview: { id: "r2" }, - }, ], }, }, @@ -455,7 +452,24 @@ describe("usage", () => { const url = new URL(String(input)); requests.push(url.pathname); if (url.pathname === "/graphql") { - const { query } = JSON.parse(String(init?.body)) as { query: string }; + const { query, variables } = JSON.parse(String(init?.body)) as { + query: string; + variables: Record; + }; + if (variables.id === "t1" && variables.after === "c1") + return json({ + data: { + node: { + comments: connection([ + { + author: { __typename: "User", login: "acme" }, + body: "Fixed.", + pullRequestReview: { id: "r2" }, + }, + ]), + }, + }, + }); const numbers = [...query.matchAll(/p(\d+): pullRequest/g)].map((match) => Number(match[1])); return json({ data: { repository: Object.fromEntries(numbers.map((number) => [`p${number}`, activity(number)])) }, @@ -576,7 +590,7 @@ describe("usage", () => { const text = usageLines(report).join("\n"); expect(text).toContain("#10 L merged: PR 10 (quest/m0/a)"); expect(text).toContain( - "online review: rabbit[bot], at the final head; 1 findings: 1 accepted, 0 declined, 0 unclassified, 0 unanswered", + "online review: rabbit[bot], at the final head; 1 finding: 1 accepted, 0 declined, 0 unclassified, 0 unanswered", ); expect(text).toContain( "rabbit[bot] reviewed 1/2 (50%), final head 1 (50%), 1.0 findings per PR reviewed, 100% accepted;",