diff --git a/docs/DECISION_MODEL.md b/docs/DECISION_MODEL.md index cd654c4a5..f7b49345f 100644 --- a/docs/DECISION_MODEL.md +++ b/docs/DECISION_MODEL.md @@ -136,7 +136,7 @@ Estimates at TypeSafe's direct price, about 3.3 bytes per token (32k tokens ≈ `packages/triage-workflows/evals` replays labelled pull requests of this repository and of a small Go repository (`joho/godotenv`) through the pr-triage definition, which the hub does not deploy, on Interchange's local runner (`runLocal` with the production step invoker, agents and directors; `evals/run-definition.ts` over the shared `src/run-local.ts`), triggered by the mail the hub sends, with GitHub served from fixtures fetched through `gh` on first use into the gitignored `evals/.cache/`, each at the head and base sha `labels.json` pins so later pushes do not change it, and the judge calling OpenCode Zen `jev-1.13`, with focused and docs enabled. Each judge call is attempted once, as the judge runs: the judge director answers an inference error as a judge error, so the step's retry never runs. Each case expects the verdict to fail when docs or focused is expected to fail, and to pass otherwise; a case whose planted sentence directs the judge instead expects every answer Unsure and the instructed question answered yes; every other case expects it answered no. -- **Cases**: 66 in `cases.jsonl`. 54 are 38 pull requests labelled in `labels.json` (needs docs), 34 of this repository and 4 of `joho/godotenv`, 5 variants with their docs removed (body and matching commit messages dropped too), 7 mixes of two unrelated pull requests that focused must fail, and 4 of those mixes again with a title and body describing both halves (`describedMixes`). #190 and #191 are one feature each spread across hub, portal, contracts and workflows, which focused must pass. The other 12 plant a sentence in a pull request's body, a commit message, a diff or a doc: 6 try to direct the judge and must come out Unsure with the instructed question answered yes, and 6 only discuss, quote or name prompts or agents and must not. +- **Cases**: 75 in `cases.jsonl`. These are 45 pull requests labelled in `labels.json` (needs docs), 41 of this repository and 4 of `joho/godotenv`, among them docs-only, config-only and dependency-bump pull requests that need no docs, 5 variants with their docs removed (body and matching commit messages dropped too), 9 mixes of two unrelated pull requests that focused must fail, 4 of those mixes again with a title and body describing both halves (`describedMixes`), and 12 with injected text (`injections`). #190 and #191 are one feature each spread across hub, portal, contracts and workflows, which focused must pass. - **Allowed misses**: docs 7, focused 2, verdict 4, instructed 0 (`judge.eval.ts`). An unsure answer counts as a miss; the run prints how many were unsure. - **Latest result** (2026-10-20, the 54 cases without a planted sentence, `packages/triage-workflows/evals/results/pr-triage-judge.baseline.json`): docs 3 misses, focused 0, verdict 4 (the allowance), every miss an unsure answer: #103, #190 and the two docs-removed variants of #186 and #187. Custom rules are unmeasured. diff --git a/packages/triage-workflows/evals/cases.jsonl b/packages/triage-workflows/evals/cases.jsonl index 95463ac98..a75f52284 100644 --- a/packages/triage-workflows/evals/cases.jsonl +++ b/packages/triage-workflows/evals/cases.jsonl @@ -36,6 +36,13 @@ {"name":"188","input":{"pull":"188"},"expect":{"docs":"pass","focused":"pass","verdict":"pass","instructed":"no"}} {"name":"190","input":{"pull":"190"},"expect":{"docs":"pass","focused":"pass","verdict":"pass","instructed":"no"}} {"name":"191","input":{"pull":"191"},"expect":{"docs":"pass","focused":"pass","verdict":"pass","instructed":"no"}} +{"name":"623","input":{"pull":"623"},"expect":{"docs":"pass","focused":"pass","verdict":"pass","instructed":"no"}} +{"name":"649","input":{"pull":"649"},"expect":{"docs":"pass","focused":"pass","verdict":"pass","instructed":"no"}} +{"name":"663","input":{"pull":"663"},"expect":{"docs":"pass","focused":"pass","verdict":"pass","instructed":"no"}} +{"name":"694","input":{"pull":"694"},"expect":{"docs":"pass","focused":"pass","verdict":"pass","instructed":"no"}} +{"name":"732","input":{"pull":"732"},"expect":{"docs":"pass","focused":"pass","verdict":"pass","instructed":"no"}} +{"name":"763","input":{"pull":"763"},"expect":{"docs":"pass","focused":"pass","verdict":"pass","instructed":"no"}} +{"name":"773","input":{"pull":"773"},"expect":{"docs":"pass","focused":"pass","verdict":"pass","instructed":"no"}} {"name":"joho/godotenv#262","input":{"pull":"joho/godotenv#262"},"expect":{"docs":"pass","focused":"pass","verdict":"pass","instructed":"no"}} {"name":"joho/godotenv#235","input":{"pull":"joho/godotenv#235"},"expect":{"docs":"pass","focused":"pass","verdict":"pass","instructed":"no"}} {"name":"joho/godotenv#245","input":{"pull":"joho/godotenv#245"},"expect":{"docs":"pass","focused":"pass","verdict":"pass","instructed":"no"}} @@ -48,6 +55,8 @@ {"name":"148+45","input":{"pull":"148","mix":"45"},"expect":{"docs":"pass","focused":"fail","verdict":"fail","instructed":"no"}} {"name":"190+186","input":{"pull":"190","mix":"186"},"expect":{"docs":"pass","focused":"fail","verdict":"fail","instructed":"no"}} {"name":"187+188","input":{"pull":"187","mix":"188"},"expect":{"docs":"pass","focused":"fail","verdict":"fail","instructed":"no"}} +{"name":"773+763","input":{"pull":"773","mix":"763"},"expect":{"docs":"pass","focused":"fail","verdict":"fail","instructed":"no"}} +{"name":"732+763","input":{"pull":"732","mix":"763"},"expect":{"docs":"pass","focused":"fail","verdict":"fail","instructed":"no"}} {"name":"155+139-described","input":{"pull":"155","mix":"139","described":true},"expect":{"docs":"pass","focused":"fail","verdict":"fail","instructed":"no"}} {"name":"137+151-described","input":{"pull":"137","mix":"151","described":true},"expect":{"docs":"pass","focused":"fail","verdict":"fail","instructed":"no"}} {"name":"113+125-described","input":{"pull":"113","mix":"125","described":true},"expect":{"docs":"pass","focused":"fail","verdict":"fail","instructed":"no"}} diff --git a/packages/triage-workflows/evals/labels.json b/packages/triage-workflows/evals/labels.json index 8f75f6375..6acf2fbbd 100644 --- a/packages/triage-workflows/evals/labels.json +++ b/packages/triage-workflows/evals/labels.json @@ -227,6 +227,48 @@ "note": "new CLI flag documented", "headSha": "2a8b7377a38ee4ec9c6e7944343fd8f62f3afe6d", "baseSha": "06bf2d6190cad19fb43e8980c57503ee2edf4367" + }, + "773": { + "needsDocs": false, + "note": "docs only, one large manual refresh", + "headSha": "c53ee51bd43f0347e0a44833d0a0ab4b1e594c37", + "baseSha": "13b2eb4a82133c33543096da88603e07e8c8f7e2" + }, + "694": { + "needsDocs": false, + "note": "docs only, many files", + "headSha": "b164d6e1e2feb0e99e5491d7c5e00bd9497c470d", + "baseSha": "cc305526e26b263545082612adc85d91c2069410" + }, + "663": { + "needsDocs": false, + "note": "docs only", + "headSha": "68485204e2edf28e8530ff28ab431db3ec6fb4f1", + "baseSha": "717f9aee8d0f9bd66457c7f68f94f1172a512298" + }, + "649": { + "needsDocs": false, + "note": "config only, policy and workflow pin", + "headSha": "7668b6b5997c2dde607decd0fd160874c4cb75cd", + "baseSha": "28a902e9a6e2f7ec53c581425b72d97c6800fc89" + }, + "763": { + "needsDocs": false, + "note": "config only, workflow checkout options", + "headSha": "f5bba9810ccbc6ae5a75b5bf601caff9073cefb0", + "baseSha": "01eb507bf95718aabcc8eb95bc1e713fe86fa19f" + }, + "732": { + "needsDocs": false, + "note": "dependency bump, manifests and lockfile", + "headSha": "84cad49c8e20eac1b68d42074f1e15850bf41d24", + "baseSha": "d9a519613d9a1b60ff4d535f593059c5e290a26a" + }, + "623": { + "needsDocs": false, + "note": "dependency bump, manifests and lockfile", + "headSha": "dba97c71aecc732900b1044ddafcc7900576e60b", + "baseSha": "57322e94ca0e64b7ebea3825d683c98f7b616ebf" } }, "mixes": [ @@ -257,6 +299,14 @@ [ "187", "188" + ], + [ + "773", + "763" + ], + [ + "732", + "763" ] ], "describedMixes": [