diff --git a/changelog.d/794.changed.md b/changelog.d/794.changed.md new file mode 100644 index 00000000..ed853d16 --- /dev/null +++ b/changelog.d/794.changed.md @@ -0,0 +1 @@ +The job page names the revision an adversarial review covered, beside the packet hash, and says so when an older record carries no revision instead of printing a hash that belongs to none. The repository review trend names its first-decision latency (one sample per work item, at its first review) instead of counting every review, and adds an override rate with its own denominator: merges that went past a recorded changes-requested decision, each naming the review job, the model and the tip it overturned. A review callback's reviewer words now reach the follow-up goal and the job page, bounded to one line; a review with no body produces exactly the goal it produced before. diff --git a/docs/architecture.md b/docs/architecture.md index dce90ce1..1bead8f1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -785,7 +785,11 @@ recorded". `scripts/restore-drill.mjs --status-file` is the writer. the linked work item is `changes_requested` or the newest succeeded review job for that PR has a score below `AUTO_MERGE_MIN_REVIEW_SCORE` (default 80). A manual maintainer merge may override the review-score gate, and the override - is audited as `job.merge_review_override`. The agent prompt always forbids + is audited as `job.merge_review_override`. The row names the decision it + overturned — the review job, its score, the model that answered and the tip + that review covered — so a reader can tell which review a merge went past; a + decision that recorded no revision leaves that field unknown rather than + borrowing the merge tip. The agent prompt always forbids merging. The switch is stored on the repository and included in the job payload for reporting. A stale or downgraded role blocks the merge. A merge also needs the newest `iterate` job for that pull request to have @@ -1107,6 +1111,20 @@ fewer than `REVIEW_TREND_SMALL_SAMPLE` (5) reviews are shown with a `Small sample n=` badge rather than smoothed into a trend. What each cell means is fixed in [ui.md](ui.md). +The identity of a decision travels with it. The `work_item.review_outcome` row +names the review job, the score, the model that answered and the tip that review +covered (`formatReviewOutcomeDetail`/`parseReviewOutcomeDetail`, in `src/db.ts` +beside the trend that reads them). A row written before those fields existed +still parses, with the newer fields unknown, and the page says `revision not +recorded` rather than printing a hash or an id that belongs to nothing. The +PR→first-decision median takes one sample per work item, at its first successful +review, so a PR reviewed four times is one first decision and not four; the +override rate is `overrides / recorded changes-requested decisions` in the same +window, its numerator being merges that named such a decision and went ahead. +A threshold-only block is derived at merge time and has no decision row: its +override is audited on the job page but has no denominator here, and the card +says so rather than folding it into a rate it cannot explain. + `GET /api/read/metrics` exposes the same read model as JSON for an external control room: a normal user gets `personal`, a super admin gets the global sections. The route is read-only and session-scoped. @@ -1338,8 +1356,11 @@ named fields only — base sha, tip sha, the pull-request patches, the verifier result at the tip and the repository policy — so the author's summary and its claimed test output are not in the request at all; a caller that still holds them cannot pass them through. Each automatic merge attempt records one -`job.adversarial` event with the model, the prompt hash and the evidence the -reviewer saw. +`job.adversarial` event with the model, the prompt hash, the base and tip sha and +the evidence the reviewer saw. The job page reads the tip back beside the hash, +so a verdict cannot be read against the wrong revision; a record written before +the reader carried the tip says `revision not recorded` instead of printing a +hash with no revision. The answer must be one of two shapes: `Concrete risk: :` with a failure mode and a repro, or `No concrete risk` with the checks it ran. A bare diff --git a/docs/ui.md b/docs/ui.md index 33482699..7e92d125 100644 --- a/docs/ui.md +++ b/docs/ui.md @@ -466,9 +466,17 @@ moves on with the next fix and is not a review outcome. | Reviews | completed review jobs with a score, terminated inside the window | no such review | | Average score | mean of those scores, one decimal | no scored review | | Changes requested | `changes_requested outcomes / outcomes recorded`, `x% (a/b)` | no outcome recorded in the window | -| PR→review | median `review finished_at − work item created_at` for reviews linked to a PR | no linked PR review | +| PR→first decision | median `first review finished_at − work item created_at`, one sample per work item | no linked PR review | | PR→merge | median `merged_at − created_at` for work items merged in the window | no merge | | Model | the model each review ran, `label ×n` | no recorded model | +| Overrides | `overrides / recorded changes-requested decisions`, `x% (a/b)`, plus each override's review job, model and tip | no recorded blocking decision in the window | + +An override is a merge that named a recorded `changes_requested` decision and +went ahead; the denominator is those decisions in the same window, so the number +cannot be read as a share of all reviews. A threshold-only block has no decision +row and is not in the rate. Each override names ` · · `, +and a field a row does not carry prints `not recorded` rather than an empty +placeholder. Rules the card follows, because a reader cannot tell a confident zero from an unknown one: diff --git a/mock/console.html b/mock/console.html index c0b1f2ba..d9a793d8 100644 --- a/mock/console.html +++ b/mock/console.html @@ -71,7 +71,7 @@

@HugoXK

需要你处理
-

FlyBot 在等你回答 0

+

工作区在等你回答 0

回答会直接发送给正在运行的团队,不需要进入仓库工作台。

查看仓库 diff --git a/mock/dashboard.html b/mock/dashboard.html index 0884aad5..4e3f38db 100644 --- a/mock/dashboard.html +++ b/mock/dashboard.html @@ -90,7 +90,7 @@

第一个任务已经开出 PR

需要你处理
-

FlyBot 在等你回答 0

+

工作区在等你回答 0

回答会直接发送给正在运行的团队,不需要进入仓库工作台。

diff --git a/mock/job.html b/mock/job.html index 0f1e50cd..2e28fff1 100644 --- a/mock/job.html +++ b/mock/job.html @@ -149,6 +149,7 @@

结果

+

—

diff --git a/mock/repository-demo.html b/mock/repository-demo.html index 4f4e3d97..09071e4d 100644 --- a/mock/repository-demo.html +++ b/mock/repository-demo.html @@ -47,7 +47,7 @@
- +
@@ -59,7 +59,7 @@

HugoXK/demo-app

发现间隔每小时 下次约自动工作已关闭 排期控制台 - FlyBot未启用 + 工作区未启用 GitHub 凭据正常 空闲
@@ -93,7 +93,7 @@

分诊这个仓库

需要你处理
-

FlyBot 在等你回答 0

+

工作区在等你回答 0

回答会直接发送给正在运行的团队,不需要进入仓库工作台。

查看仓库 @@ -109,7 +109,7 @@

和 Chief 沟通

还没有对话。给 Chief 发第一句。

-

FlyBot 实例就绪后可以和 Chief 沟通。

+

工作区就绪后可以和 Chief 沟通。

项目记忆

这个仓库的版本化记忆:决策、约定和踩过的坑,按角色可见,每次读到的是确定的版本而不是摘要。

@@ -150,8 +150,8 @@

给这个仓库打版本标签

故障复盘

未关闭的复盘

每个失败、停滞或取消的任务有一条复盘;同一复发键只保留一个计数,并在第二次出现时升级一次。缺少证据时标记为未核实,不猜原因。

0 个未关闭

没有未关闭的复盘。

-
审核质量

审核趋势

只统计这个仓库已完成的审核。空窗口显示 —;少于 5 个审核标注为小样本;要求修改率的分母是记录了结果的审核,不是全部审核。

-
窗口审核数平均分要求修改率PR→审核PR→合并模型
7 天——————
30 天——————
+
审核质量

审核趋势

只统计这个仓库已完成的审核。空窗口显示 —;少于 5 个审核标注为小样本;要求修改率的分母是记录了结果的审核,不是全部审核。PR→首次决定每个工作项只取第一次审核;改判率的分母是窗口内记录了「要求修改」的决定,分子是这些决定之后仍然合并的次数。

+
窗口审核数平均分要求修改率PR→首次决定PR→合并模型改判率
7 天———————
30 天———————

最近 7 天没有审核,所以显示 —,不是 0。 最近 30 天没有审核,所以显示 —,不是 0。

待处理 PR

合并前会重新检查 PR 状态、checks、commit status、reviews 和 branch protection。每个 PR 的合并在它自己的任务页『审查 → 合并』卡片里,只有那一个合并控件。

@@ -215,9 +215,9 @@

工作项

-
每仓库 FlyBot
-

FlyBot 状态

-

未启用 · 这个部署没有启用每仓库 FlyBot 实例。

+
每仓库工作区
+

工作区状态

+

未启用 · 这个部署没有启用每仓库工作区。

最近检查 未知

@@ -227,7 +227,7 @@

FlyBot 状态

迭代目标

-

已用仓库 2/10,FlyBot 实例 0/3

+

已用仓库 2/10,工作区 0/3

@@ -334,7 +334,7 @@

连接

危险操作

-

这里的操作只影响 Fledge 连接,不会删除 GitHub 仓库。停止调度会保留实例和历史;停止 FlyBot 会停止容器;删除会移除连接、实例、任务和 token。

+

这里的操作只影响 Fledge 连接,不会删除 GitHub 仓库。停止调度会保留实例和历史;停止工作区会停止容器;删除会移除连接、实例、任务和 token。

diff --git a/mock/repository.html b/mock/repository.html index 567d8a35..78f45997 100644 --- a/mock/repository.html +++ b/mock/repository.html @@ -47,7 +47,7 @@
- +
@@ -59,7 +59,7 @@

HugoXK/fledge

发现间隔每天 下次约2026-11-16 01:00:00 UTC 排期控制台 - FlyBot未启用 + 工作区未启用 GitHub 凭据正常 空闲
@@ -93,7 +93,7 @@

分诊这个仓库

需要你处理
-

FlyBot 在等你回答 0

+

工作区在等你回答 0

回答会直接发送给正在运行的团队,不需要进入仓库工作台。

查看仓库 @@ -109,7 +109,7 @@

和 Chief 沟通

还没有对话。给 Chief 发第一句。

-

FlyBot 实例就绪后可以和 Chief 沟通。

+

工作区就绪后可以和 Chief 沟通。

项目记忆

这个仓库的版本化记忆:决策、约定和踩过的坑,按角色可见,每次读到的是确定的版本而不是摘要。

@@ -154,8 +154,8 @@

给这个仓库打版本标签

故障复盘

未关闭的复盘

每个失败、停滞或取消的任务有一条复盘;同一复发键只保留一个计数,并在第二次出现时升级一次。缺少证据时标记为未核实,不猜原因。

0 个未关闭

没有未关闭的复盘。

-
审核质量

审核趋势

只统计这个仓库已完成的审核。空窗口显示 —;少于 5 个审核标注为小样本;要求修改率的分母是记录了结果的审核,不是全部审核。

-
窗口审核数平均分要求修改率PR→审核PR→合并模型
7 天——————
30 天——————
+
审核质量

审核趋势

只统计这个仓库已完成的审核。空窗口显示 —;少于 5 个审核标注为小样本;要求修改率的分母是记录了结果的审核,不是全部审核。PR→首次决定每个工作项只取第一次审核;改判率的分母是窗口内记录了「要求修改」的决定,分子是这些决定之后仍然合并的次数。

+
窗口审核数平均分要求修改率PR→首次决定PR→合并模型改判率
7 天———————
30 天———————

最近 7 天没有审核,所以显示 —,不是 0。 最近 30 天没有审核,所以显示 —,不是 0。

待处理 PR

合并前会重新检查 PR 状态、checks、commit status、reviews 和 branch protection。每个 PR 的合并在它自己的任务页『审查 → 合并』卡片里,只有那一个合并控件。

@@ -247,9 +247,9 @@

工作项

-
每仓库 FlyBot
-

FlyBot 状态

-

未启用 · 这个部署没有启用每仓库 FlyBot 实例。

+
每仓库工作区
+

工作区状态

+

未启用 · 这个部署没有启用每仓库工作区。

最近检查 未知

@@ -259,7 +259,7 @@

FlyBot 状态

迭代目标

-

已用仓库 2/10,FlyBot 实例 0/3

+

已用仓库 2/10,工作区 0/3

@@ -366,7 +366,7 @@

连接

危险操作

-

这里的操作只影响 Fledge 连接,不会删除 GitHub 仓库。停止调度会保留实例和历史;停止 FlyBot 会停止容器;删除会移除连接、实例、任务和 token。

+

这里的操作只影响 Fledge 连接,不会删除 GitHub 仓库。停止调度会保留实例和历史;停止工作区会停止容器;删除会移除连接、实例、任务和 token。

diff --git a/mock/schedule-console.html b/mock/schedule-console.html index 827a2018..38c8ee8e 100644 --- a/mock/schedule-console.html +++ b/mock/schedule-console.html @@ -58,7 +58,7 @@
上次成功
2026-11-14 01:00:00 UTC
最后决定
skipped · active-job
租约
未被持有(尝试 1 次)
-
投递结果
unknown(这一片还没有投递箱)
+
投递结果
未排队(这次运行没有外部通知)
有效期
不限 → 不限
时序版本
1 · skip
宽限
不限(默认)
diff --git a/mock/settings.html b/mock/settings.html index eb28eb37..144a735a 100644 --- a/mock/settings.html +++ b/mock/settings.html @@ -148,7 +148,7 @@

邮件通知

API token

-

给脚本或 FlyBot 调用的网关 token(flg_…)。它不是你的 GitHub token,也不是模型 key。

+

给脚本或工作区调用的网关 token(flg_…)。它不是你的 GitHub token,也不是模型 key。

前缀名称范围有效期最近使用
flg_mock…laptop-cligateway2026-12-15—
diff --git a/src/db.ts b/src/db.ts index ca062db8..468a83b9 100644 --- a/src/db.ts +++ b/src/db.ts @@ -83,9 +83,128 @@ export interface ReviewTrendWindow { medianPrOpenToReviewMs: number | null medianPrOpenToMergeMs: number | null models: Array<{ label: string; reviews: number }> + // #794: the decision, and the merge that went past it. A blocking decision is + // a recorded `changes_requested` outcome; an override is a merge that named + // that decision and went ahead. Both are audit rows, so the rate and its + // details come from one place and cannot be read two ways. + blockingDecisions: number + overrides: number + overrideRate: number | null + overrideDetails: ReviewOverrideRecord[] smallSample: boolean } +// #794: the reviewer packet identity is the decision's own record. The audit +// detail is the one row both the trend and a merge override read, so the format +// and its reader live together; a second copy in the service would drift and the +// override would start naming a different decision from the trend. +export interface ReviewDecisionRecord { + workItemId: string | null + outcome: 'approved' | 'changes_requested' + score: number | null + reviewJobId: string | null + model: string | null + tip: string | null +} + +export interface ReviewOverrideRecord { + reviewJobId: string | null + score: number | null + model: string | null + tip: string | null + reason: string +} + +function detailValue(value: string | number | null | undefined): string { + if (value === null || value === undefined) return '-' + const text = String(value).trim() + return text === '' ? '-' : text +} + +function detailField(text: string, key: string, next: string | null): string | null { + const start = text.indexOf(key) + if (start < 0) return null + const from = start + key.length + const end = next === null ? text.length : text.indexOf(next, from) + if (end < 0) return null + const value = text.slice(from, end).trim() + return value === '' || value === '-' ? null : value +} + +function detailNumber(value: string | null): number | null { + if (value === null) return null + const parsed = Number(value) + return Number.isFinite(parsed) ? parsed : null +} + +/** The `work_item.review_outcome` detail both the trend and an override read. */ +export function formatReviewOutcomeDetail(input: { + workItemId: string + outcome: 'approved' | 'changes_requested' + score: number | null + reviewJobId: string | null + model: string | null + tip: string | null +}): string { + return `${input.workItemId}: ${input.outcome} score=${detailValue(input.score)} review=${detailValue(input.reviewJobId)} tip=${detailValue(input.tip)} model=${detailValue(input.model)}` +} + +/** + * Reads a decision row. A row written before the identity was recorded has no + * `review=`/`tip=`/`model=` field; it still parses, and the caller shows + * "not recorded" rather than inventing a revision. + */ +export function parseReviewOutcomeDetail(detail: string): ReviewDecisionRecord | undefined { + const separator = detail.indexOf(': ') + if (separator < 0) return undefined + const body = detail.slice(separator + 2) + const outcome = body.startsWith('changes_requested ') ? 'changes_requested' : body.startsWith('approved ') ? 'approved' : undefined + if (outcome === undefined) return undefined + const scoreAt = body.indexOf(' score=') + if (scoreAt < 0) return undefined + const tail = body.slice(scoreAt + 1) + return { + workItemId: detail.slice(0, separator), + outcome, + // The score is the one field every row has carried since #361, so it is read + // from the prefix rather than between two markers a newer row introduced. + score: detailNumber(/^score=(\S+)/.exec(tail)?.[1] ?? null), + reviewJobId: detailField(tail, 'review=', ' tip='), + tip: detailField(tail, 'tip=', ' model='), + model: detailField(tail, 'model=', null), + } +} + +/** The `job.merge_review_override` detail, built beside its reader. */ +export function formatReviewOverrideDetail(input: { + jobId: string + reviewJobId: string | null + score: number | null + model: string | null + tip: string | null + reason: string +}): string { + return `${input.jobId}: review ${detailValue(input.reviewJobId)} score=${detailValue(input.score)} tip=${detailValue(input.tip)} model=${detailValue(input.model)} reason=${input.reason}` +} + +/** A row in the shape this build never wrote returns undefined, not a guess. */ +export function parseReviewOverrideDetail(detail: string): ReviewOverrideRecord | undefined { + const separator = detail.indexOf(': ') + if (separator < 0) return undefined + const body = detail.slice(separator + 2) + if (!body.startsWith('review ')) return undefined + const scoreAt = body.indexOf(' score=') + if (scoreAt < 0) return undefined + const tail = body.slice(scoreAt + 1) + return { + reviewJobId: detailField(body, 'review ', ' score='), + score: detailNumber(/^score=(\S+)/.exec(tail)?.[1] ?? null), + tip: detailField(tail, 'tip=', ' model='), + model: detailField(tail, 'model=', ' reason='), + reason: detailField(body, 'reason=', null) ?? '', + } +} + // Below this many completed reviews the page says "small sample" instead of // letting one score read as a trend. export const REVIEW_TREND_SMALL_SAMPLE = 5 @@ -4734,6 +4853,31 @@ export class Store { } } + /** + * #794: the last decision a review recorded for one work item, in the shape + * `parseReviewOutcomeDetail` reads. A merge override names the same decision, + * so it reads it back instead of re-deriving a review id and a tip from + * whatever job happens to be newest at merge time. + */ + latestReviewOutcome(itemId: string, outcome?: 'approved' | 'changes_requested'): ReviewDecisionRecord | undefined { + // #980: compare the item id literally instead of interpolating it into a + // LIKE pattern. `_` is a single-character wildcard there, so `item_a` + // matched a newer `itemXa` row and the override named another item's + // decision. The prefix before the first `:` is the id the writer stored, + // which is what the trend reader beside this one already compares. + const row = outcome === undefined + ? this.get<{ detail: string }>( + "SELECT detail FROM audit_log WHERE action = 'work_item.review_outcome' AND substr(detail, 1, instr(detail, ':') - 1) = ? ORDER BY id DESC LIMIT 1", + itemId, + ) + : this.get<{ detail: string }>( + "SELECT detail FROM audit_log WHERE action = 'work_item.review_outcome' AND substr(detail, 1, instr(detail, ':') - 1) = ? AND substr(detail, instr(detail, ':') + 2) LIKE ? ORDER BY id DESC LIMIT 1", + itemId, + `${outcome} score=%`, + ) + return row === undefined ? undefined : parseReviewOutcomeDetail(row.detail) + } + reviewTrend(repoId: string, sinceIso: string): ReviewTrendWindow { const reviewJobs = this.all( `SELECT * FROM jobs @@ -4744,8 +4888,9 @@ export class Store { ) const scores = reviewJobs.flatMap((job) => (job.review_score === null ? [] : [job.review_score])) const averageScore = scores.length === 0 ? null : Math.round((scores.reduce((sum, score) => sum + score, 0) / scores.length) * 10) / 10 - const outcomes = this.all<{ outcome: string }>( - `SELECT CASE WHEN a.detail LIKE '%: changes_requested score=%' THEN 'changes_requested' ELSE 'approved' END AS outcome + const outcomes = this.all<{ item: string; outcome: string }>( + `SELECT substr(a.detail, 1, instr(a.detail, ':') - 1) AS item, + CASE WHEN a.detail LIKE '%: changes_requested score=%' THEN 'changes_requested' ELSE 'approved' END AS outcome FROM audit_log a JOIN work_items w ON w.id = substr(a.detail, 1, instr(a.detail, ':') - 1) WHERE a.action = 'work_item.review_outcome' AND a.at >= ? AND w.repo_id = ? @@ -4754,18 +4899,43 @@ export class Store { repoId, ) const changesRequested = outcomes.filter((row) => row.outcome === 'changes_requested').length + // #794: one sample per work item, taken at its first successful review. The + // old join pushed one sample per review job, so a PR reviewed four times + // contributed four samples and the median was of every decision, not the + // first. A first decision is the first review the work item finished with. const reviewDelays = this.all<{ opened: string; reviewed: string }>( - `SELECT w.created_at AS opened, j.finished_at AS reviewed + `SELECT w.created_at AS opened, MIN(j.finished_at) AS reviewed FROM jobs j JOIN work_items w ON w.id = j.work_item_id WHERE j.repo_id = ? AND j.kind = 'review' AND j.status = 'succeeded' - AND j.finished_at IS NOT NULL AND j.finished_at >= ? AND w.pr_url IS NOT NULL`, + AND j.finished_at IS NOT NULL AND j.finished_at >= ? AND w.pr_url IS NOT NULL + GROUP BY w.id`, repoId, sinceIso, ).flatMap((row) => { const value = durationMs(row.opened, row.reviewed) return value === null ? [] : [value] }) + // #794: an override counts only when the decision it overturned is a + // recorded `changes_requested` outcome in the same window and the item it + // belongs to merged. A threshold-only block is derived at merge time and has + // no decision row of its own: it is audited on the job page, but it has no + // denominator here, and inventing one would make the rate read two ways. + const blockingItems = new Set(outcomes.filter((row) => row.outcome === 'changes_requested').map((row) => row.item)) + const overrideDetails = this.all<{ detail: string; item: string }>( + `SELECT a.detail AS detail, w.id AS item + FROM audit_log a + JOIN jobs j ON j.id = substr(a.detail, 1, instr(a.detail, ':') - 1) + JOIN work_items w ON w.id = j.work_item_id + WHERE a.action = 'job.merge_review_override' AND a.at >= ? AND j.repo_id = ? AND w.pr_state = 'merged'`, + sinceIso, + repoId, + ).flatMap((row) => { + if (!blockingItems.has(row.item)) return [] + const parsed = parseReviewOverrideDetail(row.detail) + return parsed === undefined ? [] : [parsed] + }) + const overrides = overrideDetails.length const mergeDelays = this.all('SELECT * FROM work_items WHERE repo_id = ? AND merged_at IS NOT NULL AND merged_at >= ?', repoId, sinceIso).flatMap((item) => { const value = durationMs(item.created_at, item.merged_at) return value === null ? [] : [value] @@ -4784,6 +4954,10 @@ export class Store { changesRequestedRate: outcomes.length === 0 ? null : Math.round((changesRequested / outcomes.length) * 1000) / 10, medianPrOpenToReviewMs: median(reviewDelays), medianPrOpenToMergeMs: median(mergeDelays), + blockingDecisions: changesRequested, + overrides, + overrideRate: changesRequested === 0 ? null : Math.round((overrides / changesRequested) * 1000) / 10, + overrideDetails: overrideDetails.slice(0, 5), models: [...byModel.entries()] .map(([label, reviews]) => ({ label, reviews })) .sort((a, b) => b.reviews - a.reviews || a.label.localeCompare(b.label)), diff --git a/src/jobs/service.ts b/src/jobs/service.ts index 2d636b43..db1ed06d 100644 --- a/src/jobs/service.ts +++ b/src/jobs/service.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto' import type { Config } from '../config.js' import type { JobRow, RepoRow, ScheduleOccurrenceRow, SecurityFindingWrite, Store, UserRow } from '../db.js' +import { formatReviewOutcomeDetail, formatReviewOverrideDetail } from '../db.js' import type { Mailer, MailMessage } from '../email/mailer.js' import { QuotaNotifier } from '../gateway/notifier.js' import { quotaView } from '../gateway/quota.js' @@ -44,7 +45,7 @@ import { } from '../lib/adversarialReview.js' import { evaluateTestEvidence, type VerificationResult } from '../lib/verification.js' import { observedAttribution } from '../lib/teamStats.js' -import { reviewInstructionLine } from '../lib/followUpInstruction.js' +import { normalizeReviewInstruction, reviewInstructionLine } from '../lib/followUpInstruction.js' import { decideMention, mentionIdentity, mentionJobKind, type MentionTarget } from '../lib/mentionGovernance.js' import { DEFAULT_MAX_REVISIONS, decideRevision, maxRevisionsSettingKey, parseMaxRevisions, revisionCounterKey, type ReviewAuthor } from '../lib/revisionBudget.js' import { sweepExpiredTrials, type TrialSweepResult } from '../users/trial.js' @@ -2469,7 +2470,23 @@ export class Jobs { : `最新审核分 ${reviewBlock.score} 低于 ${reviewBlock.minScore},自动合并已阻止。`, } } - this.store.audit(job.user_id, 'job.merge_review_override', `${job.id}: ${detail}`) + // #794: the override names the decision it overturned — the review job, + // its score, the model that answered and the tip that review covered — + // instead of a bare sentence. The decision row is the one source; a + // threshold-only block has no such row and the record says so. + const decision = linkedItem === undefined + ? undefined + : this.store.latestReviewOutcome(linkedItem.id, reviewBlock.kind === 'changes_requested' ? 'changes_requested' : undefined) + const overrideModel = decision?.model ?? (review === undefined ? null : parseModelRun(review.model_json)?.model?.label ?? null) + const overrideScore = decision?.score ?? (reviewBlock.kind === 'score' ? reviewBlock.score : review?.review_score ?? null) + this.store.audit(job.user_id, 'job.merge_review_override', formatReviewOverrideDetail({ + jobId: job.id, + reviewJobId: decision?.reviewJobId ?? review?.id ?? null, + score: overrideScore, + model: overrideModel, + tip: decision?.tip ?? null, + reason: detail, + })) } if (!manual) { // Defence in depth for the automatic path: the job must be the newest @@ -3691,6 +3708,13 @@ export class Jobs { ...(typeof value.baseBranch === 'string' ? { baseBranch: value.baseBranch.slice(0, 300) } : {}), ...(typeof value.headSha === 'string' ? { headSha: value.headSha.slice(0, 100) } : {}), ...(typeof value.reason === 'string' ? { reason: value.reason.slice(0, 300) } : {}), + // #794: the reviewer's own words are parsed, not dropped. The carrier + // (#251) bounds the line at REVIEW_INSTRUCTION_MAX and collapses it to + // one line, so the callback accepts the same normalised form the + // webhook path writes; a field the parser omits is a silent loss for + // every follow-up parked behind a review job. Redaction runs before + // the slice, like every other callback string. + ...(typeof value.instruction === 'string' ? { instruction: normalizeReviewInstruction(redact(value.instruction)) } : {}), }] }).slice(0, 10) : [] @@ -3857,7 +3881,23 @@ export class Jobs { if (item !== undefined) { const outcome = followUps.length > 0 || (reviewScore !== undefined && reviewScore < 80) ? 'changes_requested' : 'approved' this.store.updateWorkItem(item.id, { state: outcome }) - this.store.audit(job.user_id, 'work_item.review_outcome', `${item.id}: ${outcome} score=${reviewScore ?? '-'}`) + // #794: the recorded decision carries its own identity — the review job, + // the model that answered, and the tip that review covered. The review's + // own follow-up is the only place the tip is reported for this PR; a + // review with no body has none, and then the row says `-` instead of + // borrowing the merge tip. + const reviewModel = parseModelRun(job.model_json)?.model?.label ?? null + const reviewedTip = item.pr_url === null + ? null + : followUps.find((follow) => follow.prUrl === item.pr_url)?.headSha ?? null + this.store.audit(job.user_id, 'work_item.review_outcome', formatReviewOutcomeDetail({ + workItemId: item.id, + outcome, + score: reviewScore ?? null, + reviewJobId: job.id, + model: reviewModel, + tip: reviewedTip, + })) } } if (status === 'succeeded') await this.verifyAndMaybeMerge(jobId) diff --git a/src/lib/adversarialReview.ts b/src/lib/adversarialReview.ts index 6a65b89e..c99f5496 100644 --- a/src/lib/adversarialReview.ts +++ b/src/lib/adversarialReview.ts @@ -275,6 +275,12 @@ export interface AdversarialRecord { reason: string | null model: string | null promptHash: string | null + // #794: the packet identity is the revision it was computed for. The pass has + // recorded both shas all along; dropping them here let the page print a hash + // that names no revision at all. A record written before this reader existed + // returns null for both rather than borrowing a revision from somewhere else. + baseSha: string | null + tipSha: string | null checks: string[] } @@ -306,6 +312,8 @@ export function parseAdversarialRecord(meta: string | null | undefined): Adversa reason: text(verdict.reason) ?? text(value.reason), model: text(value.model), promptHash: text(value.promptHash), + baseSha: text(value.baseSha), + tipSha: text(value.tipSha), checks: Array.isArray(verdict.checks) ? verdict.checks.filter((entry): entry is string => typeof entry === 'string').slice(0, 20) : [], } } diff --git a/src/views/pages.ts b/src/views/pages.ts index 6986d0bf..038c2f05 100644 --- a/src/views/pages.ts +++ b/src/views/pages.ts @@ -1365,8 +1365,20 @@ export function repoPage(input: { config: Config; locale: Locale; viewer: Viewer window.models.length === 0 ? '—' : window.models.map((model) => `${model.label} ×${model.reviews}`).join(', ') const trendSample = (window: ReviewTrendWindow): string => window.smallSample ? ` ${escapeHtml(t(`小样本 n=${window.reviews}`, `Small sample n=${window.reviews}`))}` : '' + // #794: an override is not a bare count. The rate names its own denominator + // and the row names the decision it went past — review job, model and tip — + // or says that a row predates the identity rather than printing an empty one. + const trendOverrides = (window: ReviewTrendWindow): string => { + if (window.overrideRate === null) return '—' + const rate = `${window.overrideRate}% (${window.overrides}/${window.blockingDecisions})` + if (window.overrideDetails.length === 0) return rate + const details = window.overrideDetails.map((entry) => `${entry.reviewJobId === null + ? escapeHtml(t('审核未记录', 'review not recorded')) + : `${escapeHtml(entry.reviewJobId)}`} · ${entry.model === null ? escapeHtml(t('模型未记录', 'model not recorded')) : escapeHtml(entry.model)} · ${entry.tip === null ? escapeHtml(t('修订未记录', 'revision not recorded')) : `${escapeHtml(entry.tip.slice(0, 12))}`}`).join(' ') + return `${rate}
${details}
` + } const reviewTrendRows = trendWindows - .map(({ key, zh, en, window }) => `${escapeHtml(t(zh, en))}${window.reviews === 0 ? '—' : `${window.reviews}${trendSample(window)}`}${window.averageScore === null ? '—' : window.averageScore}${escapeHtml(trendRate(window))}${escapeHtml(durationText(window.medianPrOpenToReviewMs))}${escapeHtml(durationText(window.medianPrOpenToMergeMs))}${escapeHtml(trendModels(window))}`) + .map(({ key, zh, en, window }) => `${escapeHtml(t(zh, en))}${window.reviews === 0 ? '—' : `${window.reviews}${trendSample(window)}`}${window.averageScore === null ? '—' : window.averageScore}${escapeHtml(trendRate(window))}${escapeHtml(durationText(window.medianPrOpenToReviewMs))}${escapeHtml(durationText(window.medianPrOpenToMergeMs))}${escapeHtml(trendModels(window))}${window.overrideRate === null ? '—' : trendOverrides(window)}`) .join('') const reviewTrendEmpty = trendWindows .filter(({ window }) => window.reviews === 0) @@ -1426,8 +1438,8 @@ export function repoPage(input: { config: Config; locale: Locale; viewer: Viewer : `
${postmortemRows}
${escapeHtml(t('任务', 'Job'))}${escapeHtml(t('分类', 'Category'))}${escapeHtml(t('疑似原因', 'Suspected cause'))}${escapeHtml(t('复发', 'Recurrence'))}${escapeHtml(t('首次 / 最近', 'First / last'))}${escapeHtml(t('预防措施', 'Preventive action'))}
`}
` const reviewTrendCard = reviewTrend === undefined ? '' : `
-
${escapeHtml(t('审核质量', 'Review quality'))}

${escapeHtml(t('审核趋势', 'Review trend'))}

${escapeHtml(t('只统计这个仓库已完成的审核。空窗口显示 —;少于 5 个审核标注为小样本;要求修改率的分母是记录了结果的审核,不是全部审核。', 'Completed reviews for this repository only. An empty window shows —; fewer than 5 reviews is labelled a small sample; the changes-requested rate is over reviews that recorded an outcome, not all reviews.'))}

-
${reviewTrendRows}
${escapeHtml(t('窗口', 'Window'))}${escapeHtml(t('审核数', 'Reviews'))}${escapeHtml(t('平均分', 'Average score'))}${escapeHtml(t('要求修改率', 'Changes requested'))}${escapeHtml(t('PR→审核', 'PR→review'))}${escapeHtml(t('PR→合并', 'PR→merge'))}${escapeHtml(t('模型', 'Model'))}
+
${escapeHtml(t('审核质量', 'Review quality'))}

${escapeHtml(t('审核趋势', 'Review trend'))}

${escapeHtml(t('只统计这个仓库已完成的审核。空窗口显示 —;少于 5 个审核标注为小样本;要求修改率的分母是记录了结果的审核,不是全部审核。PR→首次决定每个工作项只取第一次审核;改判率的分母是窗口内记录了「要求修改」的决定,分子是这些决定之后仍然合并的次数。', 'Completed reviews for this repository only. An empty window shows —; fewer than 5 reviews is labelled a small sample; the changes-requested rate is over reviews that recorded an outcome, not all reviews. PR→first decision takes one sample per work item at its first review; the override rate is over recorded changes-requested decisions in the window, and its numerator is merges that went past one.'))}

+
${reviewTrendRows}
${escapeHtml(t('窗口', 'Window'))}${escapeHtml(t('审核数', 'Reviews'))}${escapeHtml(t('平均分', 'Average score'))}${escapeHtml(t('要求修改率', 'Changes requested'))}${escapeHtml(t('PR→首次决定', 'PR→first decision'))}${escapeHtml(t('PR→合并', 'PR→merge'))}${escapeHtml(t('模型', 'Model'))}${escapeHtml(t('改判率', 'Overrides'))}
${reviewTrendEmpty === '' ? '' : `

${reviewTrendEmpty}

`}
` const advanced = advancedViewer(viewer) @@ -3742,7 +3754,9 @@ export function jobPage(input: { : escapeHtml(t(`${pathGuard.violations.length} 个文件违反路径约束。`, `${pathGuard.violations.length} file(s) violate the path constraints.`))}${pathGuard.allow.length === 0 && pathGuard.deny.length === 0 ? ` · ${escapeHtml(t('未设置约束', 'no constraints set'))}` : ''}

${pathGuard.violations.length === 0 ? '' : `
${pathGuard.violations.map((violation) => ``).join('')}
${escapeHtml(t('文件', 'File'))}${escapeHtml(t('原因', 'Reason'))}
${escapeHtml(violation.file)}${escapeHtml(violation.reason === 'denied' ? t('命中禁止路径', 'Denied path') : t('不在允许路径', 'Not allowed'))}
`}`} ${adversarial === null ? '' : `

${escapeHtml(t('独立对抗式审核', 'Independent adversarial review'))}

-

${escapeHtml(adversarialText(adversarial))}${adversarial.model === null ? '' : ` · ${escapeHtml(t('模型', 'Model'))} ${escapeHtml(adversarial.model)}`}${adversarial.promptHash === null ? '' : ` · ${escapeHtml(t('提示哈希', 'Prompt hash'))} ${escapeHtml(adversarial.promptHash.slice(0, 12))}`}

`} +

${escapeHtml(adversarialText(adversarial))}${adversarial.model === null ? '' : ` · ${escapeHtml(t('模型', 'Model'))} ${escapeHtml(adversarial.model)}`}${adversarial.promptHash === null ? '' : ` · ${escapeHtml(t('提示哈希', 'Prompt hash'))} ${escapeHtml(adversarial.promptHash.slice(0, 12))}`}${adversarial.tipSha === null + ? (adversarial.promptHash === null ? '' : ` · ${escapeHtml(t('修订未记录', 'revision not recorded'))}`) + : ` · ${escapeHtml(t('修订', 'Revision'))} ${escapeHtml(adversarial.tipSha.slice(0, 12))}`}

`}
`}

${escapeHtml(t('时间线', 'Timeline'))}

diff --git a/test/jobs.test.ts b/test/jobs.test.ts index dc970169..86596a76 100644 --- a/test/jobs.test.ts +++ b/test/jobs.test.ts @@ -1,6 +1,6 @@ import test from 'node:test' import assert from 'node:assert/strict' -import { Store } from '../src/db.js' +import { Store, formatReviewOutcomeDetail, parseReviewOverrideDetail } from '../src/db.js' import type { JobRow, RepoRow, WorkItemRow } from '../src/db.js' import { InstanceManager } from '../src/flybot/instances.js' import { PLAYBOOK_VERSION, playbookFingerprint } from '../src/flybot/playbook.js' @@ -3803,3 +3803,79 @@ test('a retried job records attempt N+1 and links it to the attempt it continues store.close() } }) + +test('an override records the decision it overturned — job, review, score, model and tip', async () => { + // #980: the override write was only asserted to exist. A detail that lost the + // decision identity (job, model, tip) kept the suite green, so this reads the + // structured record the page and the trend both consume. + const merges: string[] = [] + const github = await startFakeServer((request, response) => { + const method = request.method ?? 'GET' + const url = request.url ?? '' + if (method === 'GET' && url === '/repos/o/r/pulls/1') { + json(response, 200, { number: 1, html_url: 'https://github.com/o/r/pull/1', title: 'fix', state: 'open', head: { ref: 'fledge/fix', sha: 'abc' }, base: { ref: 'main' }, mergeable: true, mergeable_state: 'clean', draft: false, merged: false }) + return + } + if (method === 'GET' && url === '/repos/o/r/commits/abc/check-runs?per_page=100') { + json(response, 200, { check_runs: [{ name: 'ci', status: 'completed', conclusion: 'success' }] }) + return + } + if (method === 'GET' && url === '/repos/o/r/commits/abc/status') { + json(response, 200, { state: 'success', statuses: [] }) + return + } + if (method === 'GET' && url === '/repos/o/r/pulls/1/reviews?per_page=100') { + json(response, 200, []) + return + } + if (method === 'GET' && url === '/repos/o/r/branches/main/protection') { + json(response, 200, { required_status_checks: { contexts: ['ci'] }, required_pull_request_reviews: { required_approving_review_count: 0 } }) + return + } + if (method === 'PUT' && url === '/repos/o/r/pulls/1/merge') { + merges.push('merge') + json(response, 200, { merged: true, sha: 'def' }) + return + } + json(response, 404, { message: 'Not Found' }) + }) + const base = testConfig() + const config = testConfig({ github: { ...base.github, apiBase: github.url } }) + const store = new Store(':memory:') + const jobs = new Jobs(store, config, { name: 'noop', async run() { return { status: 'running' } } }, new MemoryMailer()) + const tip = 'c'.repeat(40) + try { + const user = makeUser(store, { login: 'override-record-user', status: 'approved' }) + const repo = store.createRepo({ id: 'repo_override_record', userId: user.id, owner: 'o', name: 'r', defaultBranch: 'main', tokenEnc: encryptSecret('gh-token', config.tokenEncryptionKey), permission: 'admin', repoRole: 'maintainer', autoMerge: true, upstreamHtmlUrl: 'https://github.com/o/r', forkHtmlUrl: 'https://github.com/o/r', goal: '', autoIterate: false, intervalMinutes: 1440 }) + const item = store.createWorkItem({ id: 'work_override_record', repoId: repo.id, userId: user.id, title: 'fix', prUrl: 'https://github.com/o/r/pull/1', state: 'changes_requested' }) + const review = store.createJob({ id: 'job_override_review', repoId: repo.id, userId: user.id, trigger: 'manual', goal: 'review', callbackHash: 'review-hash', platformTokenHash: '', kind: 'review', workItemId: item.id }) + store.updateJob(review.id, { status: 'succeeded', review_score: 60, finished_at: new Date().toISOString() }) + // The decision row is the one source the override names, so seed it with the + // identity the record has to carry through. + store.audit(user.id, 'work_item.review_outcome', formatReviewOutcomeDetail({ + workItemId: item.id, outcome: 'changes_requested', score: 60, reviewJobId: review.id, model: 'model-980', tip, + })) + const secret = 'override-record-secret' + store.createJob({ id: 'job_override_iterate', repoId: repo.id, userId: user.id, trigger: 'manual', goal: 'g', callbackHash: sha256Hex(secret), platformTokenHash: '', kind: 'iterate', baseBranch: 'main', workItemId: item.id }) + store.updateJob('job_override_iterate', { status: 'running', branch: 'fledge/fix', pr_url: 'https://github.com/o/r/pull/1', started_at: new Date().toISOString() }) + + const manual = await jobs.manualMerge(repo, store.getJob('job_override_iterate')!) + assert.equal(manual.ok, true) + assert.equal(merges.length, 1) + + const row = store.listAudit().find((entry) => entry.action === 'job.merge_review_override') + assert.ok(row, 'the override is audited') + const detail = row.detail ?? '' + assert.ok(detail.startsWith('job_override_iterate: review '), 'the detail names the iterate job that merged') + const parsed = parseReviewOverrideDetail(detail) + assert.ok(parsed, 'the detail parses as the structured record, not a bare sentence') + assert.equal(parsed.reviewJobId, 'job_override_review', 'the review job that made the decision') + assert.equal(parsed.score, 60, 'the score that decision recorded') + assert.equal(parsed.model, 'model-980', 'the model that answered the review') + assert.equal(parsed.tip, tip, 'the tip that review covered') + assert.match(parsed.reason, /changes_requested|review gate/i, 'the reason still names the block') + } finally { + store.close() + await github.close() + } +}) diff --git a/test/reviewDecisionTrend.test.ts b/test/reviewDecisionTrend.test.ts new file mode 100644 index 00000000..c6061a59 --- /dev/null +++ b/test/reviewDecisionTrend.test.ts @@ -0,0 +1,311 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { Store, formatReviewOutcomeDetail, parseReviewOutcomeDetail } from '../src/db.js' +import type { RepoRow, UserRow } from '../src/db.js' +import { repoPage } from '../src/views/pages.js' +import { makeUser, testConfig } from './helpers.js' + +// #794: two numbers on the review trend were not the numbers they claimed. +// `medianPrOpenToReviewMs` joined every succeeded review job to its work item, +// so a PR reviewed three times contributed three samples and the "first +// decision" latency was really the median of every decision. And an override was +// audited as a bare sentence with no way to name the decision it overturned. +// These cases read the same stored rows the page reads: `jobs`, `work_items` and +// `audit_log`. + +const DAY = 86_400_000 +const now = Date.now() + +function ago(days: number): string { + return new Date(now - days * DAY).toISOString() +} + +const TIP_A = 'a'.repeat(40) +const TIP_B = 'b'.repeat(40) + +interface Seeded { + store: Store + user: UserRow + repo: RepoRow +} + +function seed(prefix: string): Seeded { + const store = new Store(':memory:') + const user = makeUser(store, { login: `${prefix}-owner`, status: 'approved' }) + const repo = store.createRepo({ + id: `repo_${prefix}`, userId: user.id, owner: 'acme', name: prefix, defaultBranch: 'main', tokenEnc: 'v1:test', + goal: 'keep the build green', autoIterate: false, intervalMinutes: 1440, + }) + return { store, user, repo } +} + +function seedWorkItem( + store: Store, + user: UserRow, + repo: RepoRow, + input: { id: string; prNumber: number; openedDaysAgo: number; mergedDaysAgo?: number; state: string }, +): void { + store.createWorkItem({ + id: input.id, + repoId: repo.id, + userId: user.id, + title: `pull request ${input.prNumber}`, + prUrl: `https://github.com/acme/${repo.name}/pull/${input.prNumber}`, + state: input.state, + prState: input.mergedDaysAgo === undefined ? 'open' : 'merged', + ...(input.mergedDaysAgo === undefined ? {} : { mergedAt: ago(input.mergedDaysAgo) }), + }) + store.run('UPDATE work_items SET created_at = ? WHERE id = ?', ago(input.openedDaysAgo), input.id) +} + +function seedReview( + store: Store, + user: UserRow, + repo: RepoRow, + input: { id: string; finishedDaysAgo: number; score: number; model: string; workItemId?: string }, +): void { + store.createJob({ + id: input.id, + repoId: repo.id, + userId: user.id, + trigger: 'review', + goal: 'review the open pull requests', + callbackHash: `hash_${input.id}`, + platformTokenHash: `token_${input.id}`, + kind: 'review', + workItemId: input.workItemId ?? null, + createdAt: ago(input.finishedDaysAgo + 1), + }) + store.updateJob(input.id, { + status: 'succeeded', + finished_at: ago(input.finishedDaysAgo), + review_score: input.score, + model_json: JSON.stringify({ model: { id: `provider:${input.model}`, label: input.model }, attempts: [] }), + }) +} + +function page(seeded: Seeded): string { + const week = seeded.store.reviewTrend(seeded.repo.id, ago(7)) + const month = seeded.store.reviewTrend(seeded.repo.id, ago(30)) + return repoPage({ + config: testConfig(), + locale: 'en', + viewer: { user: seeded.user, csrfToken: 'csrf' }, + repo: seeded.repo, + jobs: [], + reviewTrend: { week, month }, + }) +} + +test('the trend takes one first-decision sample per work item, and says so', () => { + const seeded = seed('first-decision') + const { store, user, repo } = seeded + try { + // item_a: opened ten days ago, reviewed once one day ago -> 9 days. + // item_b: opened eight days ago, reviewed three times (7, 6, 5 days ago); + // the first decision is the 1-day one, not 2 and 3 as well. + // item_c: opened six days ago, reviewed once one day ago -> 5 days. + seedWorkItem(store, user, repo, { id: 'item_a', prNumber: 1, openedDaysAgo: 10, state: 'changes_requested' }) + seedWorkItem(store, user, repo, { id: 'item_b', prNumber: 2, openedDaysAgo: 8, mergedDaysAgo: 4, state: 'merged' }) + seedWorkItem(store, user, repo, { id: 'item_c', prNumber: 3, openedDaysAgo: 6, state: 'approved' }) + seedReview(store, user, repo, { id: 'job_ra', finishedDaysAgo: 1, score: 50, model: 'model-a', workItemId: 'item_a' }) + seedReview(store, user, repo, { id: 'job_rb1', finishedDaysAgo: 7, score: 40, model: 'model-b', workItemId: 'item_b' }) + seedReview(store, user, repo, { id: 'job_rb2', finishedDaysAgo: 6, score: 55, model: 'model-b', workItemId: 'item_b' }) + seedReview(store, user, repo, { id: 'job_rb3', finishedDaysAgo: 5, score: 60, model: 'model-b', workItemId: 'item_b' }) + seedReview(store, user, repo, { id: 'job_rc', finishedDaysAgo: 1, score: 91, model: 'model-a', workItemId: 'item_c' }) + + const week = store.reviewTrend(repo.id, ago(7)) + assert.equal(week.reviews, 5, 'five reviews are still five reviews') + assert.equal(week.medianPrOpenToReviewMs, 5 * DAY, 'samples are 9, 1 (item_b first) and 5 days; the median is 5') + assert.notEqual(week.medianPrOpenToReviewMs, 3 * DAY, 'counting every review of item_b would move the median to 3 days') + + const html = page(seeded) + assert.match(html, /PR→first decision/, 'the column says which latency it is') + } finally { + seeded.store.close() + } +}) + +test('a decision row round-trips, and a row written before the identity still reads', () => { + // The formatter and its reader live together so the trend and an override + // cannot read one audit row two ways. These cases pin both shapes the store + // holds: the one this build writes, and the one it wrote before #794. + const detail = formatReviewOutcomeDetail({ + workItemId: 'item_x', outcome: 'changes_requested', score: 61, + reviewJobId: 'job_decision', tip: TIP_A, model: 'model-a', + }) + assert.equal(detail, `item_x: changes_requested score=61 review=job_decision tip=${TIP_A} model=model-a`) + assert.deepEqual(parseReviewOutcomeDetail(detail), { + workItemId: 'item_x', outcome: 'changes_requested', score: 61, + reviewJobId: 'job_decision', tip: TIP_A, model: 'model-a', + }) + + // A row written before the identity existed carries the score and nothing + // else: it parses with the missing fields null instead of a borrowed revision. + assert.deepEqual(parseReviewOutcomeDetail('item_old: changes_requested score=55'), { + workItemId: 'item_old', outcome: 'changes_requested', score: 55, + reviewJobId: null, tip: null, model: null, + }) + + // Anything that is not that shape is unknown, not guessed. + assert.equal(parseReviewOutcomeDetail('no separator here'), undefined, 'no work item id, no row') + assert.equal(parseReviewOutcomeDetail('item_x: something score=5'), undefined, 'an outcome this build never wrote is not a decision') + assert.equal(parseReviewOutcomeDetail('item_x: approved'), undefined, 'a row with no score is not a decision') +}) + +test('the override rate counts a merge past a recorded blocking decision', () => { + const seeded = seed('override-rate') + const { store, user, repo } = seeded + try { + seedWorkItem(store, user, repo, { id: 'item_a', prNumber: 1, openedDaysAgo: 10, state: 'changes_requested' }) + seedWorkItem(store, user, repo, { id: 'item_b', prNumber: 2, openedDaysAgo: 8, mergedDaysAgo: 4, state: 'merged' }) + seedWorkItem(store, user, repo, { id: 'item_c', prNumber: 3, openedDaysAgo: 6, state: 'approved' }) + seedReview(store, user, repo, { id: 'job_ra', finishedDaysAgo: 2, score: 50, model: 'model-a', workItemId: 'item_a' }) + seedReview(store, user, repo, { id: 'job_rb', finishedDaysAgo: 5, score: 60, model: 'model-b', workItemId: 'item_b' }) + seedReview(store, user, repo, { id: 'job_rc', finishedDaysAgo: 1, score: 91, model: 'model-a', workItemId: 'item_c' }) + // The decision row is the record both the trend and an override read. It + // names the review that made the decision, its score, its model and the tip + // that review covered. + store.audit(user.id, 'work_item.review_outcome', `item_a: changes_requested score=50 review=job_ra tip=${TIP_A} model=model-a`) + store.audit(user.id, 'work_item.review_outcome', `item_b: changes_requested score=60 review=job_rb tip=${TIP_B} model=model-b`) + store.audit(user.id, 'work_item.review_outcome', 'item_c: approved score=91 review=job_rc tip=- model=model-a') + // The merge that went past item_b's blocking decision, audited against the + // iterate job it merged. + store.createJob({ + id: 'job_fix_b', repoId: repo.id, userId: user.id, trigger: 'manual', goal: 'fix it', callbackHash: 'hash_fix_b', + platformTokenHash: '', kind: 'iterate', workItemId: 'item_b', createdAt: ago(4), + }) + store.updateJob('job_fix_b', { + status: 'succeeded', finished_at: ago(4), pr_url: 'https://github.com/acme/override-rate/pull/2', branch: 'fledge/fix-b', + }) + store.audit(user.id, 'job.merge_review_override', `job_fix_b: review job_rb score=60 tip=${TIP_B} model=model-b reason=review 60 < 80`) + + const week = store.reviewTrend(repo.id, ago(7)) + assert.equal(week.blockingDecisions, 2, 'two recorded changes_requested decisions in the window') + assert.equal(week.overrides, 1, 'one merge went past a recorded blocking decision') + assert.equal(week.overrideRate, 50, 'one override over two blocking decisions is 50%, not 100% and not 0') + assert.equal(week.overrideDetails.length, 1) + assert.equal(week.overrideDetails[0]?.reviewJobId, 'job_rb') + assert.equal(week.overrideDetails[0]?.score, 60) + assert.equal(week.overrideDetails[0]?.model, 'model-b') + assert.equal(week.overrideDetails[0]?.tip, TIP_B) + + const html = page(seeded) + assert.match(html, /data-trend-override/) + assert.match(html, /job_rb/, 'the trend row names the review job the override overturned') + assert.match(html, /model-b/, 'and the model that made the decision') + assert.match(html, new RegExp(TIP_B.slice(0, 12)), 'and the tip it covered') + assert.match(html, /50% \(1\/2\)/, 'the override rate names its own denominator') + } finally { + seeded.store.close() + } +}) + +test('an override with no recorded blocking decision stays out of the rate', () => { + const seeded = seed('override-nothing') + const { store, user, repo } = seeded + try { + seedWorkItem(store, user, repo, { id: 'item_a', prNumber: 1, openedDaysAgo: 10, state: 'changes_requested' }) + seedWorkItem(store, user, repo, { id: 'item_b', prNumber: 2, openedDaysAgo: 8, mergedDaysAgo: 4, state: 'merged' }) + seedReview(store, user, repo, { id: 'job_ra', finishedDaysAgo: 2, score: 50, model: 'model-a', workItemId: 'item_a' }) + seedReview(store, user, repo, { id: 'job_rb', finishedDaysAgo: 5, score: 72, model: 'model-b', workItemId: 'item_b' }) + store.audit(user.id, 'work_item.review_outcome', `item_a: changes_requested score=50 review=job_ra tip=${TIP_A} model=model-a`) + // item_b's recorded outcome is `approved`: a threshold-only block is derived + // at merge time and has no decision row, so it is audited on the job page but + // is not part of the trend's denominator. + store.audit(user.id, 'work_item.review_outcome', `item_b: approved score=72 review=job_rb tip=${TIP_B} model=model-b`) + store.createJob({ + id: 'job_fix_b', repoId: repo.id, userId: user.id, trigger: 'manual', goal: 'fix it', callbackHash: 'hash_fix_b', + platformTokenHash: '', kind: 'iterate', workItemId: 'item_b', createdAt: ago(4), + }) + store.updateJob('job_fix_b', { + status: 'succeeded', finished_at: ago(4), pr_url: 'https://github.com/acme/override-nothing/pull/2', branch: 'fledge/fix-b', + }) + store.audit(user.id, 'job.merge_review_override', `job_fix_b: review job_rb score=72 tip=${TIP_B} model=model-b reason=review 72 < 80`) + + const week = store.reviewTrend(repo.id, ago(7)) + assert.equal(week.blockingDecisions, 1) + assert.equal(week.overrides, 0, 'no recorded blocking decision was overturned') + assert.equal(week.overrideRate, 0, 'the denominator is the recorded decision, not the derived block') + } finally { + seeded.store.close() + } +}) + +test('an empty window shows — for the override rate, never 0', () => { + const seeded = seed('override-empty') + try { + const week = seeded.store.reviewTrend(seeded.repo.id, ago(7)) + assert.equal(week.blockingDecisions, 0) + assert.equal(week.overrides, 0) + assert.equal(week.overrideRate, null, 'no blocking decision means the rate is unknown, not zero') + assert.deepEqual(week.overrideDetails, []) + + const html = page(seeded) + assert.match(html, /data-trend-empty="week"/) + assert.doesNotMatch(html, /data-trend-override-rate="week"[^<]*>0%/) + } finally { + seeded.store.close() + } +}) + +test('an override whose decision predates the identity says so on the page', () => { + const seeded = seed('override-unrecorded') + const { store, user, repo } = seeded + try { + // The decision row was written before #794, so it names no review job; the + // override row is from the same build and carries `-` in every identity + // field. The page must label the gap, not print three empty slots. + seedWorkItem(store, user, repo, { id: 'item_a', prNumber: 1, openedDaysAgo: 10, state: 'changes_requested' }) + seedWorkItem(store, user, repo, { id: 'item_b', prNumber: 2, openedDaysAgo: 8, mergedDaysAgo: 4, state: 'merged' }) + seedReview(store, user, repo, { id: 'job_ra', finishedDaysAgo: 2, score: 50, model: 'model-a', workItemId: 'item_a' }) + seedReview(store, user, repo, { id: 'job_rb', finishedDaysAgo: 5, score: 60, model: 'model-b', workItemId: 'item_b' }) + store.audit(user.id, 'work_item.review_outcome', 'item_a: changes_requested score=50') + store.audit(user.id, 'work_item.review_outcome', 'item_b: changes_requested score=60') + store.createJob({ + id: 'job_fix_b', repoId: repo.id, userId: user.id, trigger: 'manual', goal: 'fix it', callbackHash: 'hash_fix_b', + platformTokenHash: '', kind: 'iterate', workItemId: 'item_b', createdAt: ago(4), + }) + store.updateJob('job_fix_b', { + status: 'succeeded', finished_at: ago(4), pr_url: 'https://github.com/acme/override-unrecorded/pull/2', branch: 'fledge/fix-b', + }) + store.audit(user.id, 'job.merge_review_override', 'job_fix_b: review - score=60 tip=- model=- reason=review 60 < 80') + + const week = store.reviewTrend(repo.id, ago(7)) + assert.equal(week.overrides, 1, 'the merge still counts: the decision itself is recorded') + assert.equal(week.overrideDetails[0]?.reviewJobId, null, 'the row names no review job') + assert.equal(week.overrideDetails[0]?.tip, null) + assert.equal(week.overrideDetails[0]?.model, null) + + const html = page(seeded) + assert.match(html, /review not recorded/i, 'the review identity is labelled missing') + assert.match(html, /model not recorded/i) + assert.match(html, /revision not recorded/i) + } finally { + seeded.store.close() + } +}) + +test('a look-alike item id does not lend its newer decision to the item an override names', () => { + // #980: the lookup interpolated the item id into a LIKE pattern, so `_` acted + // as a single-character wildcard. `item_a` matched `itemXa`, and because the + // query took the newest row, the override named the look-alike's decision. + const seeded = seed('look-alike') + const { store, user } = seeded + try { + store.audit(user.id, 'work_item.review_outcome', `item_a: changes_requested score=41 review=job_real tip=${TIP_A} model=model-real`) + store.audit(user.id, 'work_item.review_outcome', `itemXa: changes_requested score=99 review=job_other tip=${TIP_B} model=model-other`) + + const anyOutcome = store.latestReviewOutcome('item_a') + assert.equal(anyOutcome?.workItemId, 'item_a', 'the newest decision for item_a must belong to item_a') + assert.equal(anyOutcome?.score, 41, 'and carry item_a\'s score, not the look-alike\'s') + assert.equal(anyOutcome?.reviewJobId, 'job_real', 'and the review job that decided item_a') + + const blocking = store.latestReviewOutcome('item_a', 'changes_requested') + assert.equal(blocking?.workItemId, 'item_a', 'the outcome filter must not widen the item id either') + assert.equal(blocking?.score, 41) + assert.equal(blocking?.model, 'model-real', 'the override would have named the look-alike\'s model') + } finally { + seeded.store.close() + } +}) diff --git a/test/reviewInstructionCallback.test.ts b/test/reviewInstructionCallback.test.ts new file mode 100644 index 00000000..fc648a9c --- /dev/null +++ b/test/reviewInstructionCallback.test.ts @@ -0,0 +1,193 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { Store } from '../src/db.js' +import { Jobs } from '../src/jobs/service.js' +import { jobPage } from '../src/views/pages.js' +import { sha256Hex } from '../src/lib/secrets.js' +import { REVIEW_INSTRUCTION_MAX } from '../src/lib/followUpInstruction.js' +import { MemoryMailer, makeUser, testConfig } from './helpers.js' + +// #794: the review callback parses its `followUps` field by field, so the +// reviewer's `instruction` was accepted by nobody and dropped before the goal +// was built — the carrier (#251) and the page read-back already existed, and the +// line was empty for every review-driven follow-up. These cases pin the missing +// half: a callback's instruction reaches the goal, the page reads it back, and a +// review without one produces exactly the goal it produced before. + +const RUNNER = { name: 'capture', async run() { return { status: 'running' as const } } } + +function repoFor(store: Store, id: string): ReturnType { + const user = makeUser(store, { login: `packet-${id}`, status: 'approved' }) + return store.createRepo({ + id, userId: user.id, owner: 'o', name: id, defaultBranch: 'main', tokenEnc: 'x', + permission: 'admin', repoRole: 'maintainer', goal: '', autoIterate: false, intervalMinutes: 1440, + }) +} + +function reviewJob(store: Store, repo: ReturnType, id: string, secret: string): void { + store.createJob({ + id, repoId: repo.id, userId: repo.user_id, trigger: 'schedule', goal: 'review the pull request', + callbackHash: sha256Hex(secret), platformTokenHash: '', kind: 'review', baseBranch: 'main', + }) + store.updateJob(id, { status: 'running', branch: 'fledge/callback', started_at: new Date().toISOString() }) +} + +// `enqueue` appends a carry-over block to every goal; it is not the instruction's +// business, so the comparisons below read the directives the follow-up wrote. +function directives(goal: string): string { + return goal.split('\n\n[Carry-over')[0] ?? goal +} + +function goalWithoutInstruction(prUrl: string, branch: string): string { + return [ + `Follow-up on the existing pull request ${prUrl} (branch ${branch}).`, + 'Reason: review requested changes.', + 'Continue on the same branch and update the existing PR. Do not open a second PR or create a new issue.', + 'Make the smallest correct fix, add or adjust tests, run the relevant checks, and callback with the branch and PR URL.', + ].join('\n') +} + +test('a review callback carries the reviewer’s words into the follow-up goal and the page', async () => { + const store = new Store(':memory:') + const jobs = new Jobs(store, testConfig(), RUNNER, new MemoryMailer()) + const secret = 'packet-callback-secret' + const prUrl = 'https://github.com/o/packet-callback/pull/7' + const words = 'the retry loop swallows the error; surface it and add a test for the second attempt' + try { + const repo = repoFor(store, 'packet-callback') + reviewJob(store, repo, 'job_packet_callback', secret) + const result = await jobs.handleCallback('job_packet_callback', secret, { + status: 'succeeded', + reviewScore: 55, + followUps: [{ prUrl, branch: 'fledge/packet-callback', headSha: 'ccc', reason: 'review requested changes', instruction: words }], + }) + assert.equal(result.ok, true, result.error ?? 'no error') + + const fix = store.listJobsPageForUser(repo.user_id, 50, 0).find((job) => job.trigger === 'followup') + assert.ok(fix !== undefined, 'the review follow-up starts its fix') + assert.ok(fix.goal.includes(`Reviewer asked: ${words}`), `the goal carries the reviewer’s words verbatim; got:\n${fix.goal}`) + assert.equal( + directives(fix.goal), + [ + `Follow-up on the existing pull request ${prUrl} (branch fledge/packet-callback).`, + 'Reason: review requested changes.', + `Reviewer asked: ${words}`, + 'Continue on the same branch and update the existing PR. Do not open a second PR or create a new issue.', + 'Make the smallest correct fix, add or adjust tests, run the relevant checks, and callback with the branch and PR URL.', + ].join('\n'), + 'the carried line sits between the reason and the standing directives, not after them', + ) + + const html = jobPage({ + config: testConfig(), + locale: 'en', + viewer: { user: store.userById(repo.user_id)!, csrfToken: 'csrf' }, + repo, + job: fix, + }) + assert.match(html, /data-reviewer-instruction/) + assert.match(html, new RegExp(words.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').slice(0, 40))) + } finally { + store.close() + } +}) + +test('a review with no body produces exactly the goal it produced before', async () => { + const store = new Store(':memory:') + const jobs = new Jobs(store, testConfig(), RUNNER, new MemoryMailer()) + const secret = 'packet-callback-empty' + const prUrl = 'https://github.com/o/packet-empty/pull/8' + try { + const repo = repoFor(store, 'packet-empty') + reviewJob(store, repo, 'job_packet_empty', secret) + const result = await jobs.handleCallback('job_packet_empty', secret, { + status: 'succeeded', + reviewScore: 62, + followUps: [{ prUrl, branch: 'fledge/packet-empty', headSha: 'ddd', reason: 'review requested changes' }], + }) + assert.equal(result.ok, true, result.error ?? 'no error') + const fix = store.listJobsPageForUser(repo.user_id, 50, 0).find((job) => job.trigger === 'followup') + assert.ok(fix !== undefined) + assert.equal(directives(fix.goal), goalWithoutInstruction(prUrl, 'fledge/packet-empty'), 'no instruction, no extra line') + const html = jobPage({ + config: testConfig(), + locale: 'en', + viewer: { user: store.userById(repo.user_id)!, csrfToken: 'csrf' }, + repo, + job: fix, + }) + assert.doesNotMatch(html, /data-reviewer-instruction/) + assert.doesNotMatch(html, /Reviewer asked:/) + } finally { + store.close() + } +}) + +test('a callback instruction is bounded by the carrier’s own limit', async () => { + const store = new Store(':memory:') + const jobs = new Jobs(store, testConfig(), RUNNER, new MemoryMailer()) + const secret = 'packet-callback-long' + const prUrl = 'https://github.com/o/packet-long/pull/9' + try { + const repo = repoFor(store, 'packet-long') + reviewJob(store, repo, 'job_packet_long', secret) + const result = await jobs.handleCallback('job_packet_long', secret, { + status: 'succeeded', + reviewScore: 40, + followUps: [{ prUrl, branch: 'fledge/packet-long', headSha: 'eee', instruction: 'x'.repeat(REVIEW_INSTRUCTION_MAX + 500) }], + }) + assert.equal(result.ok, true, result.error ?? 'no error') + const fix = store.listJobsPageForUser(repo.user_id, 50, 0).find((job) => job.trigger === 'followup') + assert.ok(fix !== undefined) + const line = directives(fix.goal).split('\n').find((entry) => entry.startsWith('Reviewer asked: ')) + assert.ok(line !== undefined, 'the bounded instruction still reaches the goal') + assert.ok(line.length <= 'Reviewer asked: '.length + REVIEW_INSTRUCTION_MAX, 'the line stops at the carrier’s bound') + assert.ok(line.endsWith('…'), 'a clipped instruction ends in an ellipsis, not a silent cut') + } finally { + store.close() + } +}) + +test('a review that decides a work item records the job, the model and the tip it covered', async () => { + // #794: the decision row is the record an override reads back. Before this + // test the row was only ever written by hand in a fixture, so the callback + // path — which is the only real writer — was unexecuted: it could name the + // wrong job, drop the model, or borrow the merge tip and nothing failed. + const store = new Store(':memory:') + const jobs = new Jobs(store, testConfig(), RUNNER, new MemoryMailer()) + const secret = 'packet-callback-decision' + const prUrl = 'https://github.com/o/packet-decision/pull/11' + const tip = 'f'.repeat(40) + try { + const repo = repoFor(store, 'packet-decision') + store.createWorkItem({ + id: 'item_decision', repoId: repo.id, userId: repo.user_id, title: 'pull request 11', + prUrl, state: 'in_review', + }) + store.createJob({ + id: 'job_decision', repoId: repo.id, userId: repo.user_id, trigger: 'schedule', goal: 'review the pull request', + callbackHash: sha256Hex(secret), platformTokenHash: '', kind: 'review', baseBranch: 'main', workItemId: 'item_decision', + }) + store.updateJob('job_decision', { + status: 'running', branch: 'fledge/decision', started_at: new Date().toISOString(), + model_json: JSON.stringify({ model: { id: 'provider:model-a', label: 'model-a' }, attempts: [] }), + }) + const result = await jobs.handleCallback('job_decision', secret, { + status: 'succeeded', + reviewScore: 55, + followUps: [{ prUrl, branch: 'fledge/decision', headSha: tip, reason: 'review requested changes' }], + }) + assert.equal(result.ok, true, result.error ?? 'no error') + + const decision = store.latestReviewOutcome('item_decision') + assert.ok(decision !== undefined, 'the callback wrote a decision row this build can read') + assert.equal(decision.outcome, 'changes_requested') + assert.equal(decision.score, 55) + assert.equal(decision.reviewJobId, 'job_decision', 'the row names the review job that decided') + assert.equal(decision.model, 'model-a', 'and the model that answered') + assert.equal(decision.tip, tip, 'and the tip that review covered, taken from its own follow-up') + assert.equal(store.latestReviewOutcome('item_decision', 'approved'), undefined, 'a decision does not answer to the outcome it is not') + } finally { + store.close() + } +}) diff --git a/test/reviewPacketRecord.test.ts b/test/reviewPacketRecord.test.ts new file mode 100644 index 00000000..c504541b --- /dev/null +++ b/test/reviewPacketRecord.test.ts @@ -0,0 +1,101 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { Store } from '../src/db.js' +import { parseAdversarialRecord } from '../src/lib/adversarialReview.js' +import { jobPage } from '../src/views/pages.js' +import { makeUser, testConfig } from './helpers.js' + +// #794: a packet hash with no revision is a verdict a maintainer cannot place. +// The adversarial pass already records `tipSha` and `baseSha` in the +// `job.adversarial` event; the reader dropped both, so the page could print a +// hash that belongs to no revision. These cases pin the reader and the page +// against the record the pass actually writes. + +const TIP = '40eb9d7bac304cb95b6249eddb2d67f90277f208' +const BASE = 'c3d16bbc51bc04e0fc405f1f4e6c0e7bc1868439' + +test('a recorded packet identity exposes the revision it covers', () => { + const record = parseAdversarialRecord(JSON.stringify({ + status: 'risk', + tipSha: TIP, + baseSha: BASE, + model: 'fake/model', + promptHash: 'abcdef0123456789', + target: 'src/x.ts:9', + verdict: { kind: 'risk', file: 'src/x.ts', line: 9, failureMode: 'stale read', repro: 'open twice', checks: ['read the diff'] }, + })) + assert.equal(record?.tipSha, TIP, 'the tip the verdict covers is read back, not dropped') + assert.equal(record?.baseSha, BASE, 'the base it was read against is read back') + + // A record written before the fields were read back is not an error; it says + // "unknown" rather than borrowing a revision from somewhere else. + const older = parseAdversarialRecord(JSON.stringify({ + status: 'none', + model: 'fake/model', + promptHash: 'abcdef0123456789', + verdict: { kind: 'none', checks: [] }, + })) + assert.equal(older?.tipSha, null) + assert.equal(older?.baseSha, null) +}) + +function pageFor(events: Array<{ meta: string }>): string { + const store = new Store(':memory:') + try { + const user = makeUser(store, { login: 'packet-record-page', status: 'approved' }) + const repo = store.createRepo({ + id: 'repo_packet_record', userId: user.id, owner: 'o', name: 'pr', defaultBranch: 'main', tokenEnc: 'x', + permission: 'admin', repoRole: 'maintainer', goal: '', autoIterate: false, intervalMinutes: 1440, + }) + const job = store.createJob({ id: 'job_packet_record', repoId: repo.id, userId: user.id, trigger: 'manual', goal: 'g', callbackHash: 'cb', platformTokenHash: '' }) + return jobPage({ + config: testConfig(), + locale: 'en', + viewer: { user: store.userById(user.id)!, csrfToken: 'csrf' }, + repo, + job, + events: events.map((event, index) => ({ + id: index + 1, + at: new Date().toISOString(), + job_id: job.id, + repo_id: repo.id, + user_id: user.id, + kind: 'job.adversarial', + role: null, + step: 'risk', + meta: event.meta, + })), + }) + } finally { + store.close() + } +} + +test('the job page names the tip beside the packet hash', () => { + const html = pageFor([{ + meta: JSON.stringify({ + status: 'risk', + tipSha: TIP, + baseSha: BASE, + model: 'fake/model', + promptHash: 'abcdef0123456789', + target: 'src/x.ts:9', + verdict: { kind: 'risk', file: 'src/x.ts', line: 9, failureMode: 'stale read', repro: 'open twice', checks: [] }, + }), + }]) + assert.match(html, new RegExp(TIP), 'the revision the hash belongs to is on the page') + assert.match(html, /data-adversarial-tip/) +}) + +test('a record without a revision says so instead of printing a bare hash', () => { + const html = pageFor([{ + meta: JSON.stringify({ + status: 'none', + model: 'fake/model', + promptHash: 'abcdef0123456789', + verdict: { kind: 'none', checks: [] }, + }), + }]) + assert.match(html, /abcdef012345/, 'the hash it did record is still shown') + assert.match(html, /revision not recorded/i, 'a hash with no revision is labelled, not left to imply one') +})