From 9477471ff7febd7d2bd00c5cd64ae412998f164a Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sun, 11 Oct 2026 22:41:59 -0700 Subject: [PATCH 1/6] feat(evals): wire @corbits/evals and add dispatch-routing eval --- .gitignore | 1 + bun.lock | 5 + evals/prompts/dispatch-routing/README.md | 75 +++++++ evals/prompts/dispatch-routing/cases.jsonl | 9 + package.json | 2 + scripts/eval-prompts-dispatch.ts | 215 +++++++++++++++++++++ 6 files changed, 307 insertions(+) create mode 100644 evals/prompts/dispatch-routing/README.md create mode 100644 evals/prompts/dispatch-routing/cases.jsonl create mode 100644 scripts/eval-prompts-dispatch.ts diff --git a/.gitignore b/.gitignore index 046e458b7..9d0bbc004 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ worktrees evals/capability/results/* !evals/capability/results/baseline-0286.json evals/public/results/* +evals/prompts/dispatch-routing/last.json local-harness-bench/ subagents/ diff --git a/bun.lock b/bun.lock index 2ffe3abfc..fa5ed5229 100644 --- a/bun.lock +++ b/bun.lock @@ -35,6 +35,7 @@ "isomorphic-git": "catalog:", }, "devDependencies": { + "@corbits/evals": "^0.1.1", "@intx/inference-testing": "0.3.0", "@types/bun": "1.4.3", "knip": "^6.41.0", @@ -302,6 +303,8 @@ "@corbits/codex-provider": ["@corbits/codex-provider@0.1.2", "", { "dependencies": { "arktype": "^2.2.3" }, "peerDependencies": { "@corbits/oauth-core": "^0.2.0", "@corbits/openai-responses": "^0.2.0", "@intx/inference": "^0.4.0", "@intx/types": "^0.4.0" } }, "sha512-jIKDuydaAOF1xLC6iZgtSEAIcuaXtGGxsvFUm4ZBH30wDDVZDGIAokPHtROZGlbImQ9ElaoTShDfHSY2iP5kdQ=="], + "@corbits/evals": ["@corbits/evals@0.1.1", "", { "dependencies": { "@corbits/system-one": "^0.3.0" }, "peerDependencies": { "@intx/workflow": "^0.4.0" }, "optionalPeers": ["@intx/workflow"] }, "sha512-JjBbLVZn/xtrdQiOr/yglsBfiaa8fAksCaZ+kiu+bRK0MQAhUjUKRmp1G06JouHagAUsAlqhcbagI91gsoEASQ=="], + "@corbits/first-class-providers": ["@corbits/first-class-providers@workspace:packages/first-class-providers"], "@corbits/oauth-core": ["@corbits/oauth-core@0.3.0", "", { "dependencies": { "@intx/db": "^0.4.0", "@intx/hub-api": "^0.4.0", "@intx/hub-common": "^0.4.0", "@intx/types": "^0.4.0", "arktype": "^2.2.3", "drizzle-orm": "^0.45.2", "hono": "^4.11.9" } }, "sha512-Al6l7FXCSDVboMCujr8piMmudQ6W+7wVRZKsZZ8Elpy5BbW4+gRGjrXSdplGILiEl/YedoWi66hGF/DpLUjVoA=="], @@ -314,6 +317,8 @@ "@corbits/provider-zen": ["@corbits/provider-zen@workspace:packages/zen"], + "@corbits/system-one": ["@corbits/system-one@0.3.1", "", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": "^0.4.0", "@intx/types": "^0.4.0" } }, "sha512-YvYaP6CY5VGM7MPJW6JPU4d5uPESMCX1xspddch0NGCBNB4Nz4yAR1JOFz6gYyVy0JyYHqtNYArXwFB5kUetug=="], + "@corbits/xai-provider": ["@corbits/xai-provider@0.1.2", "", { "dependencies": { "arktype": "^2.2.3" }, "peerDependencies": { "@corbits/oauth-core": "^0.2.0", "@corbits/openai-responses": "^0.2.0", "@intx/inference": "^0.4.0", "@intx/types": "^0.4.0" } }, "sha512-4gCSLehuE922UG5DeAQLMA3iaq5MzSEgKNuq2Lb4aJqEYsQ85t8rTmu0DXs+T6mHNq2cbCI9eCF4aXuyv420iA=="], "@emnapi/core": ["@emnapi/core@1.11.2", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" } }, "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA=="], diff --git a/evals/prompts/dispatch-routing/README.md b/evals/prompts/dispatch-routing/README.md new file mode 100644 index 000000000..300315bfe --- /dev/null +++ b/evals/prompts/dispatch-routing/README.md @@ -0,0 +1,75 @@ +# Dispatch routing eval (first `@corbits/evals` eval) + +First eval wired through the published [`@corbits/evals`](https://github.com/corbitsdev/corbits-evals#readme) +package (CL-10452). It exercises the **real dispatch director system prompt** +(`agents/dispatch/src/prompt.ts`, exported by `@corbits/code-agent-dispatch`) +against **brand-new routing requests**: each case is a user request that the +dispatch director must route to one specialist. + +## Run + +```sh +bun scripts/eval-prompts-dispatch.ts +# or +bun run eval:prompts-dispatch +``` + +The run calls a model through local Ollama by default (matching the package's +own examples). Override with env: + +```sh +EVAL_MODEL=tev1:0.8b bun scripts/eval-prompts-dispatch.ts +EVAL_OLLAMA_URL=http://localhost:11434 EVAL_MODEL=qwen3:0.6b bun scripts/eval-prompts-dispatch.ts +``` + +## What it measures + +The run composes the dispatch card **verbatim** + the case request + a fixed +reply-format stanza (harness glue so the routing decision is measurable), then +asks the model which specialist it would spawn. Checks: + +| Check | Helper | Passes when | +| -------- | ---------- | -------------------------------------------------- | +| `routes` | `mentions` | the reply names the expected specialist | +| `noDiy` | `avoids` | the reply does not volunteer to do the work itself | +| `fast` | `under` | the run finishes within 120s | + +## Cases + +`cases.jsonl` — 9 brand-new routing prompts. Each case: + +```json +{ + "name": "map-retry-backoff", + "input": "Where does the repo handle retry backoff? Find every call site and sketch the flow between them.", + "expect": { "route": "explorer", "decoys": ["coder", "planner"] } +} +``` + +`expect.route` is the correct specialist; `expect.decoys` lists the likely +wrong picks for humans scanning the report. + +## Gates + +Per check, `allow` in `scripts/eval-prompts-dispatch.ts`: + +- `routes: { miss: 6, drop: 0 }` — the gate trips only when fewer than 3 of 9 + cases route correctly. Calibrated to the local smoke model (`qwen3:0.6b`, + observed 3/9); **tighten when running a capable model** (e.g. `miss: 1`). +- `noDiy: { miss: 2 }` +- `fast: 0` + +`compare` + `save` point at `last.json`: each run compares drops against the +previous stored run, then replaces it. Until a capable-model baseline is +stored, drops are not gated. + +## Integration notes + +- `@corbits/evals@0.1.1` is a devDependency (LGPL-2.1-only — see the PR body). +- The package's `evaluate()` is the whole harness: run, cases, checks, and + per-check miss/drop allowances in one call. It rejects with `EvalFailure` + when a check breaks its allowance; `.report.markdown()` prints the summary. +- Jev (LLM-as-judge string/`ask` checks) needs the `@corbits/system-one` + gateway; this eval uses code checks only so it runs offline. +- Trajectory scoring (`ranSteps` / `@corbits/evals/workflow`) is the + CL-10452 follow-up: this first eval uses a plain function run. diff --git a/evals/prompts/dispatch-routing/cases.jsonl b/evals/prompts/dispatch-routing/cases.jsonl new file mode 100644 index 000000000..496b56f99 --- /dev/null +++ b/evals/prompts/dispatch-routing/cases.jsonl @@ -0,0 +1,9 @@ +{"name":"map-retry-backoff","input":"Where does the repo handle retry backoff? Find every call site and sketch the flow between them.","expect":{"route":"explorer","decoys":["coder","planner"]}} +{"name":"plan-offline-mode","input":"We need offline mode. Write a build plan with phases, risks, and acceptance criteria.","expect":{"route":"planner","decoys":["coder","explorer"]}} +{"name":"fix-token-refresh","input":"There's a bug in the login flow: the token refresh silently fails. Fix it and add a regression test.","expect":{"route":"coder","decoys":["explorer","reviewer"]}} +{"name":"review-ship-diff","input":"Review the attached diff for correctness and edge cases before we ship it.","expect":{"route":"reviewer","decoys":["coder"]}} +{"name":"design-settings-cramped","input":"The settings screen looks cramped. Update the design tokens and spacing rules.","expect":{"route":"designer","decoys":["coder","explorer"]}} +{"name":"doc-storage-layer","input":"Write the architecture doc for the new storage layer, covering the write path and failure modes.","expect":{"route":"shakespeare","decoys":["explorer","planner"]}} +{"name":"audit-permissions-boundary","input":"Audit the new permissions boundary for trust issues and secret exposure.","expect":{"route":"warden","decoys":["reviewer"]}} +{"name":"probe-latency-percentiles","input":"Measure latency percentiles for the new model endpoint across 200 requests and report the distribution.","expect":{"route":"prober","decoys":["qa-lead","reviewer"]}} +{"name":"qa-onboarding-run","input":"Run the CLI through the onboarding flow end to end and show it works with a pass/fail record.","expect":{"route":"qa-lead","decoys":["prober","coder"]}} \ No newline at end of file diff --git a/package.json b/package.json index f6e8313ee..b9f1e5980 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "check": "bun run lint && bun run typecheck && bun run check:dead-exports && bun run build && bun run check:projects-dir-guard", "start": "bun run build && bun ./dist/index.js", "eval:capability": "bun scripts/eval-capability.ts", + "eval:prompts-dispatch": "bun scripts/eval-prompts-dispatch.ts", "eval:public-swe-one": "bun scripts/eval-public-swe-one.ts", "tui:smoke": "bun ./src/tui/smoke.ts" }, @@ -116,6 +117,7 @@ "@corbits/code-agent-explorer": "workspace:*" }, "devDependencies": { + "@corbits/evals": "^0.1.1", "@intx/inference-testing": "0.3.0", "@types/bun": "1.4.3", "knip": "^6.41.0", diff --git a/scripts/eval-prompts-dispatch.ts b/scripts/eval-prompts-dispatch.ts new file mode 100644 index 000000000..b51807425 --- /dev/null +++ b/scripts/eval-prompts-dispatch.ts @@ -0,0 +1,215 @@ +#!/usr/bin/env bun +/** + * FIRST @corbits/evals eval: dispatch director routing (CL-10452). + * + * Exercises the real product dispatch director system prompt + * (`agents/dispatch/src/prompt.ts`, exported by `@corbits/code-agent-dispatch`) + * against brand-new routing requests. Each case is a user request that must be + * routed to one specialist; the checks and the per-check miss allowances gate + * the run through `evaluate()` from `@corbits/evals`. + * + * The run composes the dispatch card (unchanged) + the case request + a fixed + * reply-format stanza (harness glue so the routing decision is measurable), + * and calls a model. Defaults to local Ollama, matching the package's own + * examples; override with EVAL_OLLAMA_URL / EVAL_MODEL. + * + * Run: + * bun scripts/eval-prompts-dispatch.ts + * EVAL_MODEL=tev1:0.8b bun scripts/eval-prompts-dispatch.ts + * + * See evals/prompts/dispatch-routing/README.md for gates and calibration. + */ +import { readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { systemPrompt } from "@corbits/code-agent-dispatch"; +import { EvalFailure, evaluate, avoids, mentions, under } from "@corbits/evals"; +import type { Case, Run } from "@corbits/evals"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = join(HERE, ".."); +const CASES_PATH = join( + REPO_ROOT, + "evals/prompts/dispatch-routing/cases.jsonl", +); +const LAST_PATH = join(REPO_ROOT, "evals/prompts/dispatch-routing/last.json"); + +const OLLAMA_URL = process.env.EVAL_OLLAMA_URL ?? "http://localhost:11434"; +const MODEL = process.env.EVAL_MODEL ?? "qwen3:0.6b"; + +/** Harness glue appended after the case request; the dispatch card is verbatim. */ +const REPLY_FORMAT = `\n\n# Reply format +Reply with exactly one line: the specialist you would spawn, one of: explorer, planner, coder, reviewer, designer, artist, warden, shakespeare, prober, qa-lead. Then a second line with a one-sentence brief.`; + +/** Route one request: dispatch card + request (+ reply format) through the model. */ +async function dispatchRoute( + input: string, + signal: AbortSignal, +): Promise { + const prompt = `${systemPrompt.build()}\n\n# Request\n${input}${REPLY_FORMAT}`; + const res = await fetch(`${OLLAMA_URL}/api/chat`, { + method: "POST", + headers: { "content-type": "application/json" }, + signal, + body: JSON.stringify({ + model: MODEL, + messages: [{ role: "system", content: prompt }], + stream: false, + options: { temperature: 0 }, + }), + }); + if (!res.ok) { + throw new Error(`model call failed (${res.status}): ${await res.text()}`); + } + const data = (await res.json()) as { + message?: { content?: string }; + error?: string; + }; + if (data.error !== undefined) { + throw new Error(`model call failed: ${data.error}`); + } + const reply = data.message?.content?.trim() ?? ""; + if (reply === "") { + throw new Error("model returned an empty reply"); + } + return reply; +} + +/** Per-case expected specialist and likely-wrong decoys (for humans; route gates). */ +interface RoutingExpect { + route: string; + decoys: readonly string[]; +} + +function loadCases(path: string): Promise[]> { + return readFile(path, "utf8").then((text) => { + const cases: Case[] = []; + const lines = text.split(/\r?\n/); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]?.trim(); + if (line === undefined || line === "") { + continue; + } + let raw: unknown; + try { + raw = JSON.parse(line); + } catch (cause) { + throw new Error( + `${path}:${i + 1}: invalid JSON (${(cause as Error).message})`, + ); + } + if (typeof raw !== "object" || raw === null || Array.isArray(raw)) { + throw new Error(`${path}:${i + 1}: expected an object`); + } + const rec = raw as Record; + const input = rec["input"]; + if (typeof input !== "string") { + throw new Error(`${path}:${i + 1}: missing string "input"`); + } + const expect = rec["expect"]; + if ( + typeof expect !== "object" || + expect === null || + Array.isArray(expect) + ) { + throw new Error(`${path}:${i + 1}: missing object "expect"`); + } + const exp = expect as Record; + const route = exp["route"]; + const decoys = exp["decoys"]; + if ( + typeof route !== "string" || + !Array.isArray(decoys) || + !decoys.every((d) => typeof d === "string") + ) { + throw new Error( + `${path}:${i + 1}: expect needs string "route" and string[] "decoys"`, + ); + } + const evalCase: Case = { + input, + expect: { route, decoys: decoys as string[] }, + }; + const name = rec["name"]; + if (typeof name === "string" && name !== "") { + evalCase.name = name; + } + cases.push(evalCase); + } + if (cases.length === 0) { + throw new Error(`${path}: no cases`); + } + return cases; + }); +} + +/** Fail-closed word list: never matches real output, so a missing expect fails. */ +const routeWords = ( + run: Run, +): readonly string[] => + run.expect === undefined ? ["__missing_expect__"] : [run.expect.route]; + +/** The dispatch card says delegate by default and never DIY-map explorer work. */ +const DIY_PHRASES = [ + "I will fix", + "I'll fix", + "I will implement", + "I'll implement", + "I can fix", + "I can implement", + "I'll write", + "I will write", + "I'll do it", + "I can do it", + "I'll handle", + "I will handle", + "I'm going to implement", +] as const; + +async function main(): Promise { + const cases = await loadCases(CASES_PATH); + const report = await evaluate( + "dispatch-routing", + { + run: dispatchRoute, + cases, + checks: { + routes: mentions(routeWords), + noDiy: avoids(DIY_PHRASES), + fast: under({ ms: 120_000 }), + }, + // Gates: calibrated to the local smoke model (qwen3:0.6b) so the harness + // trips only on a routing collapse; tighten when running a capable model. + // `last.json` (compare + save) is the regression/drop gate once a strong + // model baseline is stored. + allow: { + routes: { miss: 6, drop: 0 }, + noDiy: { miss: 2, drop: 0 }, + fast: 0, + }, + compare: LAST_PATH, + save: LAST_PATH, + about: { + prompt: "agents/dispatch/src/prompt.ts (dispatch director)", + model: MODEL, + ollamaUrl: OLLAMA_URL, + }, + }, + ); + console.log(report.markdown()); + return 0; +} + +main() + .then((code) => { + process.exitCode = code; + }) + .catch((err: unknown) => { + if (err instanceof EvalFailure) { + console.log(err.report.markdown()); + process.exitCode = 1; + return; + } + console.error(err); + process.exitCode = 1; + }); From 9e31b296e41bcf1137521aa2d31a6e6095975d4f Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 12 Oct 2026 16:30:37 -0700 Subject: [PATCH 2/6] refactor(evals): drop dispatch-routing narration --- evals/prompts/dispatch-routing/README.md | 65 +----------------------- scripts/eval-prompts-dispatch.ts | 29 ----------- 2 files changed, 2 insertions(+), 92 deletions(-) diff --git a/evals/prompts/dispatch-routing/README.md b/evals/prompts/dispatch-routing/README.md index 300315bfe..d2b684df0 100644 --- a/evals/prompts/dispatch-routing/README.md +++ b/evals/prompts/dispatch-routing/README.md @@ -1,12 +1,4 @@ -# Dispatch routing eval (first `@corbits/evals` eval) - -First eval wired through the published [`@corbits/evals`](https://github.com/corbitsdev/corbits-evals#readme) -package (CL-10452). It exercises the **real dispatch director system prompt** -(`agents/dispatch/src/prompt.ts`, exported by `@corbits/code-agent-dispatch`) -against **brand-new routing requests**: each case is a user request that the -dispatch director must route to one specialist. - -## Run +# Dispatch routing eval ```sh bun scripts/eval-prompts-dispatch.ts @@ -14,62 +6,9 @@ bun scripts/eval-prompts-dispatch.ts bun run eval:prompts-dispatch ``` -The run calls a model through local Ollama by default (matching the package's -own examples). Override with env: +Local Ollama by default. Override with `EVAL_OLLAMA_URL` and `EVAL_MODEL`: ```sh EVAL_MODEL=tev1:0.8b bun scripts/eval-prompts-dispatch.ts EVAL_OLLAMA_URL=http://localhost:11434 EVAL_MODEL=qwen3:0.6b bun scripts/eval-prompts-dispatch.ts ``` - -## What it measures - -The run composes the dispatch card **verbatim** + the case request + a fixed -reply-format stanza (harness glue so the routing decision is measurable), then -asks the model which specialist it would spawn. Checks: - -| Check | Helper | Passes when | -| -------- | ---------- | -------------------------------------------------- | -| `routes` | `mentions` | the reply names the expected specialist | -| `noDiy` | `avoids` | the reply does not volunteer to do the work itself | -| `fast` | `under` | the run finishes within 120s | - -## Cases - -`cases.jsonl` — 9 brand-new routing prompts. Each case: - -```json -{ - "name": "map-retry-backoff", - "input": "Where does the repo handle retry backoff? Find every call site and sketch the flow between them.", - "expect": { "route": "explorer", "decoys": ["coder", "planner"] } -} -``` - -`expect.route` is the correct specialist; `expect.decoys` lists the likely -wrong picks for humans scanning the report. - -## Gates - -Per check, `allow` in `scripts/eval-prompts-dispatch.ts`: - -- `routes: { miss: 6, drop: 0 }` — the gate trips only when fewer than 3 of 9 - cases route correctly. Calibrated to the local smoke model (`qwen3:0.6b`, - observed 3/9); **tighten when running a capable model** (e.g. `miss: 1`). -- `noDiy: { miss: 2 }` -- `fast: 0` - -`compare` + `save` point at `last.json`: each run compares drops against the -previous stored run, then replaces it. Until a capable-model baseline is -stored, drops are not gated. - -## Integration notes - -- `@corbits/evals@0.1.1` is a devDependency (LGPL-2.1-only — see the PR body). -- The package's `evaluate()` is the whole harness: run, cases, checks, and - per-check miss/drop allowances in one call. It rejects with `EvalFailure` - when a check breaks its allowance; `.report.markdown()` prints the summary. -- Jev (LLM-as-judge string/`ask` checks) needs the `@corbits/system-one` - gateway; this eval uses code checks only so it runs offline. -- Trajectory scoring (`ranSteps` / `@corbits/evals/workflow`) is the - CL-10452 follow-up: this first eval uses a plain function run. diff --git a/scripts/eval-prompts-dispatch.ts b/scripts/eval-prompts-dispatch.ts index b51807425..fea7b00d6 100644 --- a/scripts/eval-prompts-dispatch.ts +++ b/scripts/eval-prompts-dispatch.ts @@ -1,24 +1,4 @@ #!/usr/bin/env bun -/** - * FIRST @corbits/evals eval: dispatch director routing (CL-10452). - * - * Exercises the real product dispatch director system prompt - * (`agents/dispatch/src/prompt.ts`, exported by `@corbits/code-agent-dispatch`) - * against brand-new routing requests. Each case is a user request that must be - * routed to one specialist; the checks and the per-check miss allowances gate - * the run through `evaluate()` from `@corbits/evals`. - * - * The run composes the dispatch card (unchanged) + the case request + a fixed - * reply-format stanza (harness glue so the routing decision is measurable), - * and calls a model. Defaults to local Ollama, matching the package's own - * examples; override with EVAL_OLLAMA_URL / EVAL_MODEL. - * - * Run: - * bun scripts/eval-prompts-dispatch.ts - * EVAL_MODEL=tev1:0.8b bun scripts/eval-prompts-dispatch.ts - * - * See evals/prompts/dispatch-routing/README.md for gates and calibration. - */ import { readFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -37,11 +17,9 @@ const LAST_PATH = join(REPO_ROOT, "evals/prompts/dispatch-routing/last.json"); const OLLAMA_URL = process.env.EVAL_OLLAMA_URL ?? "http://localhost:11434"; const MODEL = process.env.EVAL_MODEL ?? "qwen3:0.6b"; -/** Harness glue appended after the case request; the dispatch card is verbatim. */ const REPLY_FORMAT = `\n\n# Reply format Reply with exactly one line: the specialist you would spawn, one of: explorer, planner, coder, reviewer, designer, artist, warden, shakespeare, prober, qa-lead. Then a second line with a one-sentence brief.`; -/** Route one request: dispatch card + request (+ reply format) through the model. */ async function dispatchRoute( input: string, signal: AbortSignal, @@ -75,7 +53,6 @@ async function dispatchRoute( return reply; } -/** Per-case expected specialist and likely-wrong decoys (for humans; route gates). */ interface RoutingExpect { route: string; decoys: readonly string[]; @@ -143,13 +120,11 @@ function loadCases(path: string): Promise[]> { }); } -/** Fail-closed word list: never matches real output, so a missing expect fails. */ const routeWords = ( run: Run, ): readonly string[] => run.expect === undefined ? ["__missing_expect__"] : [run.expect.route]; -/** The dispatch card says delegate by default and never DIY-map explorer work. */ const DIY_PHRASES = [ "I will fix", "I'll fix", @@ -178,10 +153,6 @@ async function main(): Promise { noDiy: avoids(DIY_PHRASES), fast: under({ ms: 120_000 }), }, - // Gates: calibrated to the local smoke model (qwen3:0.6b) so the harness - // trips only on a routing collapse; tighten when running a capable model. - // `last.json` (compare + save) is the regression/drop gate once a strong - // model baseline is stored. allow: { routes: { miss: 6, drop: 0 }, noDiy: { miss: 2, drop: 0 }, From 973466f2b930d4adbe3c091885711243fabd17e3 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 12 Oct 2026 16:58:21 -0700 Subject: [PATCH 3/6] fix(evals): score dispatch route on the first line --- scripts/eval-prompts-dispatch.test.ts | 221 ++++++++++++++++++++++++++ scripts/eval-prompts-dispatch.ts | 141 ++++++++++++---- 2 files changed, 329 insertions(+), 33 deletions(-) create mode 100644 scripts/eval-prompts-dispatch.test.ts diff --git a/scripts/eval-prompts-dispatch.test.ts b/scripts/eval-prompts-dispatch.test.ts new file mode 100644 index 000000000..0720de81d --- /dev/null +++ b/scripts/eval-prompts-dispatch.test.ts @@ -0,0 +1,221 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "bun:test"; +import { evaluate, type CheckValue, type Run } from "@corbits/evals"; +import { + ROUTE_BUDGET_MS, + evaluateRouting, + noDiyCheck, + routeLine, + routesCheck, +} from "./eval-prompts-dispatch.js"; + +const ROSTER = + "explorer, planner, coder, reviewer, designer, artist, warden, shakespeare, prober, qa-lead"; + +function passed(value: CheckValue | Promise): boolean { + if (typeof value === "object" && value !== null && "then" in value) { + throw new Error("expected a sync check"); + } + if (typeof value === "boolean") return value; + if (typeof value === "number") return value === 1; + return value.pass; +} + +function routingRun( + output: string, + expect: { route: string; decoys: readonly string[] }, +): Run { + return { + input: "task", + output, + expect, + steps: [], + ms: 1, + }; +} + +describe("routes scores the first line", () => { + const coder = { route: "coder", decoys: ["explorer", "reviewer"] }; + + test("a roster-only reply does not pass", () => { + expect(passed(routesCheck(routingRun(ROSTER, coder)))).toBe(false); + expect( + passed(routesCheck(routingRun(`${ROSTER}\nI would spawn coder.`, coder))), + ).toBe(false); + }); + + test("a decoy on the first line does not pass", () => { + expect( + passed( + routesCheck(routingRun("reviewer\nI will not change code.", coder)), + ), + ).toBe(false); + expect( + passed( + routesCheck( + routingRun("I would not spawn explorer; spawn coder.", { + route: "explorer", + decoys: ["coder", "planner"], + }), + ), + ), + ).toBe(false); + }); + + test("the specialist alone on the first line passes", () => { + expect( + passed( + routesCheck( + routingRun( + "coder\nI will just patch the login flow and add a test.", + coder, + ), + ), + ), + ).toBe(true); + expect(passed(routesCheck(routingRun("Coder\nbrief", coder)))).toBe(true); + }); +}); + +describe("noDiy ignores the brief line", () => { + test("DIY verbs in the brief do not fail", () => { + expect( + passed( + noDiyCheck( + routingRun( + "coder\nI will just patch the login flow and add a test.", + { + route: "coder", + decoys: ["explorer"], + }, + ), + ), + ), + ).toBe(true); + expect( + passed( + noDiyCheck( + routingRun("shakespeare\nI'll write the architecture doc", { + route: "shakespeare", + decoys: ["explorer", "planner"], + }), + ), + ), + ).toBe(true); + }); + + test("DIY on the route line still fails", () => { + expect( + passed( + noDiyCheck( + routingRun("I'll write the architecture doc\nshakespeare", { + route: "shakespeare", + decoys: ["explorer"], + }), + ), + ), + ).toBe(false); + expect( + passed( + noDiyCheck( + routingRun("I'll fix the login bug", { + route: "coder", + decoys: ["reviewer"], + }), + ), + ), + ).toBe(false); + expect(routeLine("coder\nI'll fix it")).toBe("coder"); + }); +}); + +describe("drop baseline", () => { + const coderCase = { + name: "fix-token-refresh", + input: "Fix the login flow.", + expect: { route: "coder", decoys: ["explorer", "reviewer"] }, + }; + + function stored(routesPass: boolean): string { + return `${JSON.stringify( + { + format: 1, + eval: "dispatch-routing", + cases: { + "fix-token-refresh": { + routes: { pass: routesPass, value: routesPass ? 1 : 0 }, + noDiy: { pass: true, value: 1 }, + fast: { pass: true, value: 1 }, + }, + }, + }, + null, + 2, + )}\n`; + } + + test("a failing run does not replace the baseline and the next run still drops", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + const before = stored(true); + await writeFile(baselinePath, before); + const timeouts: (number | string | undefined)[] = []; + const run = async () => "reviewer\nI refuse."; + try { + const first = await evaluateRouting({ + cases: [coderCase], + run, + lastPath, + baselinePath, + evaluateFn: (name, options) => { + timeouts.push(options.timeout); + return evaluate(name, options); + }, + }); + expect(first).toBe(1); + expect(await readFile(baselinePath, "utf8")).toBe(before); + expect(await readFile(lastPath, "utf8")).not.toBe(before); + + const second = await evaluateRouting({ + cases: [coderCase], + run, + lastPath, + baselinePath, + }); + expect(second).toBe(1); + expect(await readFile(baselinePath, "utf8")).toBe(before); + expect(timeouts).toEqual([ROUTE_BUDGET_MS]); + expect(ROUTE_BUDGET_MS).toBe(120_000); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("a passing run updates the baseline", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + await writeFile(baselinePath, stored(false)); + try { + const code = await evaluateRouting({ + cases: [coderCase], + run: async () => "coder\nAdd a regression test for token refresh.", + lastPath, + baselinePath, + }); + expect(code).toBe(0); + const after = JSON.parse(await readFile(baselinePath, "utf8")) as { + cases: { "fix-token-refresh": { routes: { pass: boolean } } }; + }; + expect(after.cases["fix-token-refresh"].routes.pass).toBe(true); + expect(await readFile(lastPath, "utf8")).toBe( + await readFile(baselinePath, "utf8"), + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/scripts/eval-prompts-dispatch.ts b/scripts/eval-prompts-dispatch.ts index fea7b00d6..731432164 100644 --- a/scripts/eval-prompts-dispatch.ts +++ b/scripts/eval-prompts-dispatch.ts @@ -1,10 +1,16 @@ #!/usr/bin/env bun -import { readFile } from "node:fs/promises"; +import { copyFile, readFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { systemPrompt } from "@corbits/code-agent-dispatch"; -import { EvalFailure, evaluate, avoids, mentions, under } from "@corbits/evals"; -import type { Case, Run } from "@corbits/evals"; +import { EvalFailure, evaluate, avoids, under } from "@corbits/evals"; +import type { + Case, + CheckValue, + EvaluateOptions, + Report, + Run, +} from "@corbits/evals"; const HERE = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = join(HERE, ".."); @@ -13,10 +19,16 @@ const CASES_PATH = join( "evals/prompts/dispatch-routing/cases.jsonl", ); const LAST_PATH = join(REPO_ROOT, "evals/prompts/dispatch-routing/last.json"); +const BASELINE_PATH = join( + REPO_ROOT, + "evals/prompts/dispatch-routing/baseline.json", +); const OLLAMA_URL = process.env.EVAL_OLLAMA_URL ?? "http://localhost:11434"; const MODEL = process.env.EVAL_MODEL ?? "qwen3:0.6b"; +export const ROUTE_BUDGET_MS = 120_000; + const REPLY_FORMAT = `\n\n# Reply format Reply with exactly one line: the specialist you would spawn, one of: explorer, planner, coder, reviewer, designer, artist, warden, shakespeare, prober, qa-lead. Then a second line with a one-sentence brief.`; @@ -120,10 +132,37 @@ function loadCases(path: string): Promise[]> { }); } -const routeWords = ( +export function routeLine(output: string): string { + const line = output.split(/\r?\n/, 1)[0] ?? ""; + return line.trim(); +} + +function chosenRoute(output: string): string { + return routeLine(output).toLowerCase(); +} + +export function routesCheck( run: Run, -): readonly string[] => - run.expect === undefined ? ["__missing_expect__"] : [run.expect.route]; +): CheckValue { + const expected = run.expect; + if (expected === undefined) { + return { pass: false, value: 0, explanation: "case has no expect" }; + } + const chosen = chosenRoute(run.output); + const route = expected.route.trim().toLowerCase(); + const decoys = expected.decoys.map((decoy) => decoy.trim().toLowerCase()); + if (decoys.includes(chosen)) { + return { pass: false, value: 0, explanation: `decoy route: ${chosen}` }; + } + if (chosen !== route) { + return { + pass: false, + value: 0, + explanation: `expected ${route}, got ${chosen === "" ? "(empty)" : chosen}`, + }; + } + return true; +} const DIY_PHRASES = [ "I will fix", @@ -141,46 +180,82 @@ const DIY_PHRASES = [ "I'm going to implement", ] as const; -async function main(): Promise { - const cases = await loadCases(CASES_PATH); - const report = await evaluate( - "dispatch-routing", - { - run: dispatchRoute, - cases, +const diyOnText = avoids(DIY_PHRASES); + +export function noDiyCheck( + run: Run, +): CheckValue | Promise { + return diyOnText({ ...run, output: routeLine(run.output) }); +} + +const DISPATCH_EVAL = "dispatch-routing"; + +type DispatchEvaluate = ( + name: string, + options: EvaluateOptions, +) => Promise>; + +export async function evaluateRouting(args: { + cases: readonly Case[]; + run: (input: string, signal: AbortSignal) => Promise; + lastPath: string; + baselinePath: string; + evaluateFn?: DispatchEvaluate; +}): Promise { + const evaluateFn: DispatchEvaluate = + args.evaluateFn ?? ((name, options) => evaluate(name, options)); + try { + const report = await evaluateFn(DISPATCH_EVAL, { + run: args.run, + cases: args.cases, checks: { - routes: mentions(routeWords), - noDiy: avoids(DIY_PHRASES), - fast: under({ ms: 120_000 }), + routes: routesCheck, + noDiy: noDiyCheck, + fast: under({ ms: ROUTE_BUDGET_MS }), }, allow: { routes: { miss: 6, drop: 0 }, noDiy: { miss: 2, drop: 0 }, fast: 0, }, - compare: LAST_PATH, - save: LAST_PATH, + compare: args.baselinePath, + save: args.lastPath, + timeout: ROUTE_BUDGET_MS, about: { prompt: "agents/dispatch/src/prompt.ts (dispatch director)", model: MODEL, ollamaUrl: OLLAMA_URL, }, - }, - ); - console.log(report.markdown()); - return 0; -} - -main() - .then((code) => { - process.exitCode = code; - }) - .catch((err: unknown) => { + }); + await copyFile(args.lastPath, args.baselinePath); + console.log(report.markdown()); + return 0; + } catch (err: unknown) { if (err instanceof EvalFailure) { console.log(err.report.markdown()); - process.exitCode = 1; - return; + return 1; } - console.error(err); - process.exitCode = 1; + throw err; + } +} + +async function main(): Promise { + const cases = await loadCases(CASES_PATH); + return evaluateRouting({ + cases, + run: dispatchRoute, + lastPath: LAST_PATH, + baselinePath: BASELINE_PATH, }); +} + +if (import.meta.main) { + main() + .then((code) => { + process.exitCode = code; + }) + .catch((err: unknown) => { + console.error(err); + process.exitCode = 1; + }); +} From f8e0c45f45306bbdc125131e1855afcccf2c85e5 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 12 Oct 2026 17:06:15 -0700 Subject: [PATCH 4/6] fix(evals): fail closed when the route baseline is missing --- scripts/eval-prompts-dispatch.test.ts | 30 ++++++++++++++++++++++++++- scripts/eval-prompts-dispatch.ts | 24 +++++++++++++++++++-- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/scripts/eval-prompts-dispatch.test.ts b/scripts/eval-prompts-dispatch.test.ts index 0720de81d..06a59d4b7 100644 --- a/scripts/eval-prompts-dispatch.test.ts +++ b/scripts/eval-prompts-dispatch.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { access, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, test } from "bun:test"; @@ -218,4 +218,32 @@ describe("drop baseline", () => { await rm(dir, { recursive: true, force: true }); } }); + + test("a missing baseline fails closed and is not created", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + const run = async () => "reviewer\nI refuse."; + try { + const first = await evaluateRouting({ + cases: [coderCase], + run, + lastPath, + baselinePath, + }); + expect(first).not.toBe(0); + await expect(access(baselinePath)).rejects.toThrow(); + + const second = await evaluateRouting({ + cases: [coderCase], + run, + lastPath, + baselinePath, + }); + expect(second).not.toBe(0); + await expect(access(baselinePath)).rejects.toThrow(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); }); diff --git a/scripts/eval-prompts-dispatch.ts b/scripts/eval-prompts-dispatch.ts index 731432164..bf58e2457 100644 --- a/scripts/eval-prompts-dispatch.ts +++ b/scripts/eval-prompts-dispatch.ts @@ -1,5 +1,5 @@ #!/usr/bin/env bun -import { copyFile, readFile } from "node:fs/promises"; +import { access, copyFile, readFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { systemPrompt } from "@corbits/code-agent-dispatch"; @@ -195,6 +195,15 @@ type DispatchEvaluate = ( options: EvaluateOptions, ) => Promise>; +async function baselinePresent(path: string): Promise { + try { + await access(path); + return true; + } catch { + return false; + } +} + export async function evaluateRouting(args: { cases: readonly Case[]; run: (input: string, signal: AbortSignal) => Promise; @@ -202,6 +211,7 @@ export async function evaluateRouting(args: { baselinePath: string; evaluateFn?: DispatchEvaluate; }): Promise { + const hadBaseline = await baselinePresent(args.baselinePath); const evaluateFn: DispatchEvaluate = args.evaluateFn ?? ((name, options) => evaluate(name, options)); try { @@ -227,8 +237,18 @@ export async function evaluateRouting(args: { ollamaUrl: OLLAMA_URL, }, }); - await copyFile(args.lastPath, args.baselinePath); console.log(report.markdown()); + const routeMisses = report.checks["routes"]?.missed; + if (!hadBaseline) { + console.error( + `eval "${DISPATCH_EVAL}" failed: no baseline at ${args.baselinePath}`, + ); + return 1; + } + if (routeMisses !== 0) { + return 0; + } + await copyFile(args.lastPath, args.baselinePath); return 0; } catch (err: unknown) { if (err instanceof EvalFailure) { From af2a4251d48753fcd124a8d1826121a492157046 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 12 Oct 2026 17:09:00 -0700 Subject: [PATCH 5/6] fix(evals): seed the route baseline only on a clean pass --- scripts/eval-prompts-dispatch.test.ts | 51 +++++++++++++++++++++++++++ scripts/eval-prompts-dispatch.ts | 15 ++++---- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/scripts/eval-prompts-dispatch.test.ts b/scripts/eval-prompts-dispatch.test.ts index 06a59d4b7..a24001957 100644 --- a/scripts/eval-prompts-dispatch.test.ts +++ b/scripts/eval-prompts-dispatch.test.ts @@ -246,4 +246,55 @@ describe("drop baseline", () => { await rm(dir, { recursive: true, force: true }); } }); + + test("a clean pass seeds a missing baseline and the next run compares", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + try { + const seeded = await evaluateRouting({ + cases: [coderCase], + run: async () => "coder\nAdd a regression test for token refresh.", + lastPath, + baselinePath, + }); + expect(seeded).toBe(0); + const seededText = await readFile(baselinePath, "utf8"); + const parsed = JSON.parse(seededText) as { + cases: { "fix-token-refresh": { routes: { pass: boolean } } }; + }; + expect(parsed.cases["fix-token-refresh"].routes.pass).toBe(true); + + const dropped = await evaluateRouting({ + cases: [coderCase], + run: async () => "reviewer\nI refuse.", + lastPath, + baselinePath, + }); + expect(dropped).not.toBe(0); + expect(await readFile(baselinePath, "utf8")).toBe(seededText); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("a within-budget miss leaves an existing baseline unchanged", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + const before = stored(false); + await writeFile(baselinePath, before); + try { + const code = await evaluateRouting({ + cases: [coderCase], + run: async () => "reviewer\nI refuse.", + lastPath, + baselinePath, + }); + expect(code).toBe(0); + expect(await readFile(baselinePath, "utf8")).toBe(before); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); }); diff --git a/scripts/eval-prompts-dispatch.ts b/scripts/eval-prompts-dispatch.ts index bf58e2457..323a761a0 100644 --- a/scripts/eval-prompts-dispatch.ts +++ b/scripts/eval-prompts-dispatch.ts @@ -238,14 +238,13 @@ export async function evaluateRouting(args: { }, }); console.log(report.markdown()); - const routeMisses = report.checks["routes"]?.missed; - if (!hadBaseline) { - console.error( - `eval "${DISPATCH_EVAL}" failed: no baseline at ${args.baselinePath}`, - ); - return 1; - } - if (routeMisses !== 0) { + if (report.checks["routes"]?.missed !== 0) { + if (!hadBaseline) { + console.error( + `eval "${DISPATCH_EVAL}" failed: no baseline at ${args.baselinePath}`, + ); + return 1; + } return 0; } await copyFile(args.lastPath, args.baselinePath); From 6ae164bd853c357bd092e52c740c184c73629648 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 16 Oct 2026 17:54:29 -0700 Subject: [PATCH 6/6] fix(evals): fail unbaselined route misses and drop noDiy --- scripts/eval-prompts-dispatch.test.ts | 134 ++++++++++++--------- scripts/eval-prompts-dispatch.ts | 166 +++++++++++++++++++------- 2 files changed, 205 insertions(+), 95 deletions(-) diff --git a/scripts/eval-prompts-dispatch.test.ts b/scripts/eval-prompts-dispatch.test.ts index a24001957..10537a78c 100644 --- a/scripts/eval-prompts-dispatch.test.ts +++ b/scripts/eval-prompts-dispatch.test.ts @@ -6,8 +6,6 @@ import { evaluate, type CheckValue, type Run } from "@corbits/evals"; import { ROUTE_BUDGET_MS, evaluateRouting, - noDiyCheck, - routeLine, routesCheck, } from "./eval-prompts-dispatch.js"; @@ -79,58 +77,6 @@ describe("routes scores the first line", () => { }); }); -describe("noDiy ignores the brief line", () => { - test("DIY verbs in the brief do not fail", () => { - expect( - passed( - noDiyCheck( - routingRun( - "coder\nI will just patch the login flow and add a test.", - { - route: "coder", - decoys: ["explorer"], - }, - ), - ), - ), - ).toBe(true); - expect( - passed( - noDiyCheck( - routingRun("shakespeare\nI'll write the architecture doc", { - route: "shakespeare", - decoys: ["explorer", "planner"], - }), - ), - ), - ).toBe(true); - }); - - test("DIY on the route line still fails", () => { - expect( - passed( - noDiyCheck( - routingRun("I'll write the architecture doc\nshakespeare", { - route: "shakespeare", - decoys: ["explorer"], - }), - ), - ), - ).toBe(false); - expect( - passed( - noDiyCheck( - routingRun("I'll fix the login bug", { - route: "coder", - decoys: ["reviewer"], - }), - ), - ), - ).toBe(false); - expect(routeLine("coder\nI'll fix it")).toBe("coder"); - }); -}); - describe("drop baseline", () => { const coderCase = { name: "fix-token-refresh", @@ -146,7 +92,6 @@ describe("drop baseline", () => { cases: { "fix-token-refresh": { routes: { pass: routesPass, value: routesPass ? 1 : 0 }, - noDiy: { pass: true, value: 1 }, fast: { pass: true, value: 1 }, }, }, @@ -297,4 +242,83 @@ describe("drop baseline", () => { await rm(dir, { recursive: true, force: true }); } }); + + test("an unbaselined routes miss fails and is not stored as a pass", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + const before = stored(true); + await writeFile(baselinePath, before); + const unknown = { + name: "map-the-repo", + input: "Where is the login handler?", + expect: { route: "explorer", decoys: ["coder", "reviewer"] }, + }; + try { + const code = await evaluateRouting({ + cases: [coderCase, unknown], + run: async (input) => + input === unknown.input + ? "coder\nwrong specialist" + : "coder\nAdd a regression test for token refresh.", + lastPath, + baselinePath, + }); + expect(code).not.toBe(0); + const after = JSON.parse(await readFile(baselinePath, "utf8")) as { + cases: Record; + }; + expect(after.cases["map-the-repo"]?.routes?.pass).not.toBe(true); + expect(after.cases["fix-token-refresh"]?.routes?.pass).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("a new pass beside a baselined miss is stored and a later flip drops", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + await writeFile(baselinePath, stored(false)); + const added = { + name: "map-the-repo", + input: "Where is the login handler?", + expect: { route: "explorer", decoys: ["coder", "reviewer"] }, + }; + try { + const storedPass = await evaluateRouting({ + cases: [coderCase, added], + run: async (input) => + input === added.input + ? "explorer\nLook up the handler." + : "reviewer\nI refuse.", + lastPath, + baselinePath, + }); + expect(storedPass).toBe(0); + const after = JSON.parse(await readFile(baselinePath, "utf8")) as { + cases: { + "fix-token-refresh": { routes: { pass: boolean } }; + "map-the-repo": { routes: { pass: boolean } }; + }; + }; + expect(after.cases["fix-token-refresh"].routes.pass).toBe(false); + expect(after.cases["map-the-repo"].routes.pass).toBe(true); + const passedText = await readFile(baselinePath, "utf8"); + + const flipped = await evaluateRouting({ + cases: [coderCase, added], + run: async (input) => + input === added.input + ? "coder\nwrong specialist" + : "reviewer\nI refuse.", + lastPath, + baselinePath, + }); + expect(flipped).not.toBe(0); + expect(await readFile(baselinePath, "utf8")).toBe(passedText); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); }); diff --git a/scripts/eval-prompts-dispatch.ts b/scripts/eval-prompts-dispatch.ts index 323a761a0..3b01b5d16 100644 --- a/scripts/eval-prompts-dispatch.ts +++ b/scripts/eval-prompts-dispatch.ts @@ -1,9 +1,9 @@ #!/usr/bin/env bun -import { access, copyFile, readFile } from "node:fs/promises"; +import { access, copyFile, readFile, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { systemPrompt } from "@corbits/code-agent-dispatch"; -import { EvalFailure, evaluate, avoids, under } from "@corbits/evals"; +import { EvalFailure, evaluate, under } from "@corbits/evals"; import type { Case, CheckValue, @@ -132,7 +132,7 @@ function loadCases(path: string): Promise[]> { }); } -export function routeLine(output: string): string { +function routeLine(output: string): string { const line = output.split(/\r?\n/, 1)[0] ?? ""; return line.trim(); } @@ -164,30 +164,6 @@ export function routesCheck( return true; } -const DIY_PHRASES = [ - "I will fix", - "I'll fix", - "I will implement", - "I'll implement", - "I can fix", - "I can implement", - "I'll write", - "I will write", - "I'll do it", - "I can do it", - "I'll handle", - "I will handle", - "I'm going to implement", -] as const; - -const diyOnText = avoids(DIY_PHRASES); - -export function noDiyCheck( - run: Run, -): CheckValue | Promise { - return diyOnText({ ...run, output: routeLine(run.output) }); -} - const DISPATCH_EVAL = "dispatch-routing"; type DispatchEvaluate = ( @@ -195,6 +171,71 @@ type DispatchEvaluate = ( options: EvaluateOptions, ) => Promise>; +interface StoredCheck { + pass: boolean; + value: number; +} + +interface StoredRun { + format: 1; + eval: string; + about?: Record; + cases: Record>; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function parseStoredRun(text: string, source: string): StoredRun { + let raw: unknown; + try { + raw = JSON.parse(text); + } catch (cause) { + throw new Error(`${source}: invalid JSON (${(cause as Error).message})`); + } + if (!isRecord(raw) || !isRecord(raw["cases"])) { + throw new Error(`${source}: expected a stored run`); + } + const format = raw["format"]; + const evalName = raw["eval"]; + if (format !== 1 || typeof evalName !== "string" || evalName === "") { + throw new Error(`${source}: expected a format 1 stored run`); + } + const cases: Record> = {}; + for (const [name, checks] of Object.entries(raw["cases"])) { + if (!isRecord(checks)) { + throw new Error(`${source}: case "${name}" must be an object`); + } + const parsed: Record = {}; + for (const [check, outcome] of Object.entries(checks)) { + if ( + !isRecord(outcome) || + typeof outcome["pass"] !== "boolean" || + typeof outcome["value"] !== "number" || + !Number.isFinite(outcome["value"]) + ) { + throw new Error( + `${source}: case "${name}" check "${check}" is invalid`, + ); + } + parsed[check] = { pass: outcome["pass"], value: outcome["value"] }; + } + cases[name] = parsed; + } + const stored: StoredRun = { format: 1, eval: evalName, cases }; + if (isRecord(raw["about"])) { + stored.about = raw["about"]; + } + return stored; +} + +function routesStoredPass( + checks: Record | undefined, +): boolean { + return checks?.["routes"]?.pass === true; +} + async function baselinePresent(path: string): Promise { try { await access(path); @@ -204,6 +245,59 @@ async function baselinePresent(path: string): Promise { } } +// A miss is a drop only when the stored routes pass is true, so the miss +// budget would keep a case that has never been baselined green. Passes still +// merge: a miss is never written as a pass, and a stored pass is never +// replaced by a miss, so a later flip of a case that passed beside a sibling +// miss is a drop. +async function commitRoutingBaseline(args: { + hadBaseline: boolean; + lastPath: string; + baselinePath: string; +}): Promise { + const last = parseStoredRun( + await readFile(args.lastPath, "utf8"), + args.lastPath, + ); + const missed = Object.entries(last.cases).filter( + ([, checks]) => !routesStoredPass(checks), + ); + if (missed.length === 0) { + await copyFile(args.lastPath, args.baselinePath); + return 0; + } + if (!args.hadBaseline) { + console.error( + `eval "${DISPATCH_EVAL}" failed: no baseline at ${args.baselinePath}`, + ); + return 1; + } + const beforeText = await readFile(args.baselinePath, "utf8"); + const baseline = parseStoredRun(beforeText, args.baselinePath); + const unbaselined = missed + .filter(([name]) => baseline.cases[name] === undefined) + .map(([name]) => name); + const cases = { ...baseline.cases }; + let changed = false; + for (const [name, checks] of Object.entries(last.cases)) { + if (!routesStoredPass(checks)) continue; + if (routesStoredPass(cases[name])) continue; + cases[name] = checks; + changed = true; + } + if (changed) { + const next: StoredRun = { ...baseline, cases }; + await writeFile(args.baselinePath, `${JSON.stringify(next, null, 2)}\n`); + } + if (unbaselined.length !== 0) { + console.error( + `eval "${DISPATCH_EVAL}" failed: unbaselined routes miss (${unbaselined.join(", ")})`, + ); + return 1; + } + return 0; +} + export async function evaluateRouting(args: { cases: readonly Case[]; run: (input: string, signal: AbortSignal) => Promise; @@ -220,12 +314,10 @@ export async function evaluateRouting(args: { cases: args.cases, checks: { routes: routesCheck, - noDiy: noDiyCheck, fast: under({ ms: ROUTE_BUDGET_MS }), }, allow: { routes: { miss: 6, drop: 0 }, - noDiy: { miss: 2, drop: 0 }, fast: 0, }, compare: args.baselinePath, @@ -238,17 +330,11 @@ export async function evaluateRouting(args: { }, }); console.log(report.markdown()); - if (report.checks["routes"]?.missed !== 0) { - if (!hadBaseline) { - console.error( - `eval "${DISPATCH_EVAL}" failed: no baseline at ${args.baselinePath}`, - ); - return 1; - } - return 0; - } - await copyFile(args.lastPath, args.baselinePath); - return 0; + return await commitRoutingBaseline({ + hadBaseline, + lastPath: args.lastPath, + baselinePath: args.baselinePath, + }); } catch (err: unknown) { if (err instanceof EvalFailure) { console.log(err.report.markdown());