diff --git a/.gitignore b/.gitignore index 046e458b7..9d0bbc004 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ worktrees evals/capability/results/* !evals/capability/results/baseline-0286.json evals/public/results/* +evals/prompts/dispatch-routing/last.json local-harness-bench/ subagents/ diff --git a/bun.lock b/bun.lock index 2ffe3abfc..fa5ed5229 100644 --- a/bun.lock +++ b/bun.lock @@ -35,6 +35,7 @@ "isomorphic-git": "catalog:", }, "devDependencies": { + "@corbits/evals": "^0.1.1", "@intx/inference-testing": "0.3.0", "@types/bun": "1.4.3", "knip": "^6.41.0", @@ -302,6 +303,8 @@ "@corbits/codex-provider": ["@corbits/codex-provider@0.1.2", "", { "dependencies": { "arktype": "^2.2.3" }, "peerDependencies": { "@corbits/oauth-core": "^0.2.0", "@corbits/openai-responses": "^0.2.0", "@intx/inference": "^0.4.0", "@intx/types": "^0.4.0" } }, "sha512-jIKDuydaAOF1xLC6iZgtSEAIcuaXtGGxsvFUm4ZBH30wDDVZDGIAokPHtROZGlbImQ9ElaoTShDfHSY2iP5kdQ=="], + "@corbits/evals": ["@corbits/evals@0.1.1", "", { "dependencies": { "@corbits/system-one": "^0.3.0" }, "peerDependencies": { "@intx/workflow": "^0.4.0" }, "optionalPeers": ["@intx/workflow"] }, "sha512-JjBbLVZn/xtrdQiOr/yglsBfiaa8fAksCaZ+kiu+bRK0MQAhUjUKRmp1G06JouHagAUsAlqhcbagI91gsoEASQ=="], + "@corbits/first-class-providers": ["@corbits/first-class-providers@workspace:packages/first-class-providers"], "@corbits/oauth-core": ["@corbits/oauth-core@0.3.0", "", { "dependencies": { "@intx/db": "^0.4.0", "@intx/hub-api": "^0.4.0", "@intx/hub-common": "^0.4.0", "@intx/types": "^0.4.0", "arktype": "^2.2.3", "drizzle-orm": "^0.45.2", "hono": "^4.11.9" } }, "sha512-Al6l7FXCSDVboMCujr8piMmudQ6W+7wVRZKsZZ8Elpy5BbW4+gRGjrXSdplGILiEl/YedoWi66hGF/DpLUjVoA=="], @@ -314,6 +317,8 @@ "@corbits/provider-zen": ["@corbits/provider-zen@workspace:packages/zen"], + "@corbits/system-one": ["@corbits/system-one@0.3.1", "", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": "^0.4.0", "@intx/types": "^0.4.0" } }, "sha512-YvYaP6CY5VGM7MPJW6JPU4d5uPESMCX1xspddch0NGCBNB4Nz4yAR1JOFz6gYyVy0JyYHqtNYArXwFB5kUetug=="], + "@corbits/xai-provider": ["@corbits/xai-provider@0.1.2", "", { "dependencies": { "arktype": "^2.2.3" }, "peerDependencies": { "@corbits/oauth-core": "^0.2.0", "@corbits/openai-responses": "^0.2.0", "@intx/inference": "^0.4.0", "@intx/types": "^0.4.0" } }, "sha512-4gCSLehuE922UG5DeAQLMA3iaq5MzSEgKNuq2Lb4aJqEYsQ85t8rTmu0DXs+T6mHNq2cbCI9eCF4aXuyv420iA=="], "@emnapi/core": ["@emnapi/core@1.11.2", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" } }, "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA=="], diff --git a/evals/prompts/dispatch-routing/README.md b/evals/prompts/dispatch-routing/README.md new file mode 100644 index 000000000..d2b684df0 --- /dev/null +++ b/evals/prompts/dispatch-routing/README.md @@ -0,0 +1,14 @@ +# Dispatch routing eval + +```sh +bun scripts/eval-prompts-dispatch.ts +# or +bun run eval:prompts-dispatch +``` + +Local Ollama by default. Override with `EVAL_OLLAMA_URL` and `EVAL_MODEL`: + +```sh +EVAL_MODEL=tev1:0.8b bun scripts/eval-prompts-dispatch.ts +EVAL_OLLAMA_URL=http://localhost:11434 EVAL_MODEL=qwen3:0.6b bun scripts/eval-prompts-dispatch.ts +``` diff --git a/evals/prompts/dispatch-routing/cases.jsonl b/evals/prompts/dispatch-routing/cases.jsonl new file mode 100644 index 000000000..496b56f99 --- /dev/null +++ b/evals/prompts/dispatch-routing/cases.jsonl @@ -0,0 +1,9 @@ +{"name":"map-retry-backoff","input":"Where does the repo handle retry backoff? Find every call site and sketch the flow between them.","expect":{"route":"explorer","decoys":["coder","planner"]}} +{"name":"plan-offline-mode","input":"We need offline mode. Write a build plan with phases, risks, and acceptance criteria.","expect":{"route":"planner","decoys":["coder","explorer"]}} +{"name":"fix-token-refresh","input":"There's a bug in the login flow: the token refresh silently fails. Fix it and add a regression test.","expect":{"route":"coder","decoys":["explorer","reviewer"]}} +{"name":"review-ship-diff","input":"Review the attached diff for correctness and edge cases before we ship it.","expect":{"route":"reviewer","decoys":["coder"]}} +{"name":"design-settings-cramped","input":"The settings screen looks cramped. Update the design tokens and spacing rules.","expect":{"route":"designer","decoys":["coder","explorer"]}} +{"name":"doc-storage-layer","input":"Write the architecture doc for the new storage layer, covering the write path and failure modes.","expect":{"route":"shakespeare","decoys":["explorer","planner"]}} +{"name":"audit-permissions-boundary","input":"Audit the new permissions boundary for trust issues and secret exposure.","expect":{"route":"warden","decoys":["reviewer"]}} +{"name":"probe-latency-percentiles","input":"Measure latency percentiles for the new model endpoint across 200 requests and report the distribution.","expect":{"route":"prober","decoys":["qa-lead","reviewer"]}} +{"name":"qa-onboarding-run","input":"Run the CLI through the onboarding flow end to end and show it works with a pass/fail record.","expect":{"route":"qa-lead","decoys":["prober","coder"]}} \ No newline at end of file diff --git a/package.json b/package.json index f6e8313ee..b9f1e5980 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "check": "bun run lint && bun run typecheck && bun run check:dead-exports && bun run build && bun run check:projects-dir-guard", "start": "bun run build && bun ./dist/index.js", "eval:capability": "bun scripts/eval-capability.ts", + "eval:prompts-dispatch": "bun scripts/eval-prompts-dispatch.ts", "eval:public-swe-one": "bun scripts/eval-public-swe-one.ts", "tui:smoke": "bun ./src/tui/smoke.ts" }, @@ -116,6 +117,7 @@ "@corbits/code-agent-explorer": "workspace:*" }, "devDependencies": { + "@corbits/evals": "^0.1.1", "@intx/inference-testing": "0.3.0", "@types/bun": "1.4.3", "knip": "^6.41.0", diff --git a/scripts/eval-prompts-dispatch.test.ts b/scripts/eval-prompts-dispatch.test.ts new file mode 100644 index 000000000..10537a78c --- /dev/null +++ b/scripts/eval-prompts-dispatch.test.ts @@ -0,0 +1,324 @@ +import { access, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "bun:test"; +import { evaluate, type CheckValue, type Run } from "@corbits/evals"; +import { + ROUTE_BUDGET_MS, + evaluateRouting, + routesCheck, +} from "./eval-prompts-dispatch.js"; + +const ROSTER = + "explorer, planner, coder, reviewer, designer, artist, warden, shakespeare, prober, qa-lead"; + +function passed(value: CheckValue | Promise): boolean { + if (typeof value === "object" && value !== null && "then" in value) { + throw new Error("expected a sync check"); + } + if (typeof value === "boolean") return value; + if (typeof value === "number") return value === 1; + return value.pass; +} + +function routingRun( + output: string, + expect: { route: string; decoys: readonly string[] }, +): Run { + return { + input: "task", + output, + expect, + steps: [], + ms: 1, + }; +} + +describe("routes scores the first line", () => { + const coder = { route: "coder", decoys: ["explorer", "reviewer"] }; + + test("a roster-only reply does not pass", () => { + expect(passed(routesCheck(routingRun(ROSTER, coder)))).toBe(false); + expect( + passed(routesCheck(routingRun(`${ROSTER}\nI would spawn coder.`, coder))), + ).toBe(false); + }); + + test("a decoy on the first line does not pass", () => { + expect( + passed( + routesCheck(routingRun("reviewer\nI will not change code.", coder)), + ), + ).toBe(false); + expect( + passed( + routesCheck( + routingRun("I would not spawn explorer; spawn coder.", { + route: "explorer", + decoys: ["coder", "planner"], + }), + ), + ), + ).toBe(false); + }); + + test("the specialist alone on the first line passes", () => { + expect( + passed( + routesCheck( + routingRun( + "coder\nI will just patch the login flow and add a test.", + coder, + ), + ), + ), + ).toBe(true); + expect(passed(routesCheck(routingRun("Coder\nbrief", coder)))).toBe(true); + }); +}); + +describe("drop baseline", () => { + const coderCase = { + name: "fix-token-refresh", + input: "Fix the login flow.", + expect: { route: "coder", decoys: ["explorer", "reviewer"] }, + }; + + function stored(routesPass: boolean): string { + return `${JSON.stringify( + { + format: 1, + eval: "dispatch-routing", + cases: { + "fix-token-refresh": { + routes: { pass: routesPass, value: routesPass ? 1 : 0 }, + fast: { pass: true, value: 1 }, + }, + }, + }, + null, + 2, + )}\n`; + } + + test("a failing run does not replace the baseline and the next run still drops", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + const before = stored(true); + await writeFile(baselinePath, before); + const timeouts: (number | string | undefined)[] = []; + const run = async () => "reviewer\nI refuse."; + try { + const first = await evaluateRouting({ + cases: [coderCase], + run, + lastPath, + baselinePath, + evaluateFn: (name, options) => { + timeouts.push(options.timeout); + return evaluate(name, options); + }, + }); + expect(first).toBe(1); + expect(await readFile(baselinePath, "utf8")).toBe(before); + expect(await readFile(lastPath, "utf8")).not.toBe(before); + + const second = await evaluateRouting({ + cases: [coderCase], + run, + lastPath, + baselinePath, + }); + expect(second).toBe(1); + expect(await readFile(baselinePath, "utf8")).toBe(before); + expect(timeouts).toEqual([ROUTE_BUDGET_MS]); + expect(ROUTE_BUDGET_MS).toBe(120_000); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("a passing run updates the baseline", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + await writeFile(baselinePath, stored(false)); + try { + const code = await evaluateRouting({ + cases: [coderCase], + run: async () => "coder\nAdd a regression test for token refresh.", + lastPath, + baselinePath, + }); + expect(code).toBe(0); + const after = JSON.parse(await readFile(baselinePath, "utf8")) as { + cases: { "fix-token-refresh": { routes: { pass: boolean } } }; + }; + expect(after.cases["fix-token-refresh"].routes.pass).toBe(true); + expect(await readFile(lastPath, "utf8")).toBe( + await readFile(baselinePath, "utf8"), + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("a missing baseline fails closed and is not created", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + const run = async () => "reviewer\nI refuse."; + try { + const first = await evaluateRouting({ + cases: [coderCase], + run, + lastPath, + baselinePath, + }); + expect(first).not.toBe(0); + await expect(access(baselinePath)).rejects.toThrow(); + + const second = await evaluateRouting({ + cases: [coderCase], + run, + lastPath, + baselinePath, + }); + expect(second).not.toBe(0); + await expect(access(baselinePath)).rejects.toThrow(); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("a clean pass seeds a missing baseline and the next run compares", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + try { + const seeded = await evaluateRouting({ + cases: [coderCase], + run: async () => "coder\nAdd a regression test for token refresh.", + lastPath, + baselinePath, + }); + expect(seeded).toBe(0); + const seededText = await readFile(baselinePath, "utf8"); + const parsed = JSON.parse(seededText) as { + cases: { "fix-token-refresh": { routes: { pass: boolean } } }; + }; + expect(parsed.cases["fix-token-refresh"].routes.pass).toBe(true); + + const dropped = await evaluateRouting({ + cases: [coderCase], + run: async () => "reviewer\nI refuse.", + lastPath, + baselinePath, + }); + expect(dropped).not.toBe(0); + expect(await readFile(baselinePath, "utf8")).toBe(seededText); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("a within-budget miss leaves an existing baseline unchanged", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + const before = stored(false); + await writeFile(baselinePath, before); + try { + const code = await evaluateRouting({ + cases: [coderCase], + run: async () => "reviewer\nI refuse.", + lastPath, + baselinePath, + }); + expect(code).toBe(0); + expect(await readFile(baselinePath, "utf8")).toBe(before); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("an unbaselined routes miss fails and is not stored as a pass", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + const before = stored(true); + await writeFile(baselinePath, before); + const unknown = { + name: "map-the-repo", + input: "Where is the login handler?", + expect: { route: "explorer", decoys: ["coder", "reviewer"] }, + }; + try { + const code = await evaluateRouting({ + cases: [coderCase, unknown], + run: async (input) => + input === unknown.input + ? "coder\nwrong specialist" + : "coder\nAdd a regression test for token refresh.", + lastPath, + baselinePath, + }); + expect(code).not.toBe(0); + const after = JSON.parse(await readFile(baselinePath, "utf8")) as { + cases: Record; + }; + expect(after.cases["map-the-repo"]?.routes?.pass).not.toBe(true); + expect(after.cases["fix-token-refresh"]?.routes?.pass).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("a new pass beside a baselined miss is stored and a later flip drops", async () => { + const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-")); + const baselinePath = join(dir, "baseline.json"); + const lastPath = join(dir, "last.json"); + await writeFile(baselinePath, stored(false)); + const added = { + name: "map-the-repo", + input: "Where is the login handler?", + expect: { route: "explorer", decoys: ["coder", "reviewer"] }, + }; + try { + const storedPass = await evaluateRouting({ + cases: [coderCase, added], + run: async (input) => + input === added.input + ? "explorer\nLook up the handler." + : "reviewer\nI refuse.", + lastPath, + baselinePath, + }); + expect(storedPass).toBe(0); + const after = JSON.parse(await readFile(baselinePath, "utf8")) as { + cases: { + "fix-token-refresh": { routes: { pass: boolean } }; + "map-the-repo": { routes: { pass: boolean } }; + }; + }; + expect(after.cases["fix-token-refresh"].routes.pass).toBe(false); + expect(after.cases["map-the-repo"].routes.pass).toBe(true); + const passedText = await readFile(baselinePath, "utf8"); + + const flipped = await evaluateRouting({ + cases: [coderCase, added], + run: async (input) => + input === added.input + ? "coder\nwrong specialist" + : "reviewer\nI refuse.", + lastPath, + baselinePath, + }); + expect(flipped).not.toBe(0); + expect(await readFile(baselinePath, "utf8")).toBe(passedText); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/scripts/eval-prompts-dispatch.ts b/scripts/eval-prompts-dispatch.ts new file mode 100644 index 000000000..3b01b5d16 --- /dev/null +++ b/scripts/eval-prompts-dispatch.ts @@ -0,0 +1,366 @@ +#!/usr/bin/env bun +import { access, copyFile, readFile, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { systemPrompt } from "@corbits/code-agent-dispatch"; +import { EvalFailure, evaluate, under } from "@corbits/evals"; +import type { + Case, + CheckValue, + EvaluateOptions, + Report, + Run, +} from "@corbits/evals"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = join(HERE, ".."); +const CASES_PATH = join( + REPO_ROOT, + "evals/prompts/dispatch-routing/cases.jsonl", +); +const LAST_PATH = join(REPO_ROOT, "evals/prompts/dispatch-routing/last.json"); +const BASELINE_PATH = join( + REPO_ROOT, + "evals/prompts/dispatch-routing/baseline.json", +); + +const OLLAMA_URL = process.env.EVAL_OLLAMA_URL ?? "http://localhost:11434"; +const MODEL = process.env.EVAL_MODEL ?? "qwen3:0.6b"; + +export const ROUTE_BUDGET_MS = 120_000; + +const REPLY_FORMAT = `\n\n# Reply format +Reply with exactly one line: the specialist you would spawn, one of: explorer, planner, coder, reviewer, designer, artist, warden, shakespeare, prober, qa-lead. Then a second line with a one-sentence brief.`; + +async function dispatchRoute( + input: string, + signal: AbortSignal, +): Promise { + const prompt = `${systemPrompt.build()}\n\n# Request\n${input}${REPLY_FORMAT}`; + const res = await fetch(`${OLLAMA_URL}/api/chat`, { + method: "POST", + headers: { "content-type": "application/json" }, + signal, + body: JSON.stringify({ + model: MODEL, + messages: [{ role: "system", content: prompt }], + stream: false, + options: { temperature: 0 }, + }), + }); + if (!res.ok) { + throw new Error(`model call failed (${res.status}): ${await res.text()}`); + } + const data = (await res.json()) as { + message?: { content?: string }; + error?: string; + }; + if (data.error !== undefined) { + throw new Error(`model call failed: ${data.error}`); + } + const reply = data.message?.content?.trim() ?? ""; + if (reply === "") { + throw new Error("model returned an empty reply"); + } + return reply; +} + +interface RoutingExpect { + route: string; + decoys: readonly string[]; +} + +function loadCases(path: string): Promise[]> { + return readFile(path, "utf8").then((text) => { + const cases: Case[] = []; + const lines = text.split(/\r?\n/); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]?.trim(); + if (line === undefined || line === "") { + continue; + } + let raw: unknown; + try { + raw = JSON.parse(line); + } catch (cause) { + throw new Error( + `${path}:${i + 1}: invalid JSON (${(cause as Error).message})`, + ); + } + if (typeof raw !== "object" || raw === null || Array.isArray(raw)) { + throw new Error(`${path}:${i + 1}: expected an object`); + } + const rec = raw as Record; + const input = rec["input"]; + if (typeof input !== "string") { + throw new Error(`${path}:${i + 1}: missing string "input"`); + } + const expect = rec["expect"]; + if ( + typeof expect !== "object" || + expect === null || + Array.isArray(expect) + ) { + throw new Error(`${path}:${i + 1}: missing object "expect"`); + } + const exp = expect as Record; + const route = exp["route"]; + const decoys = exp["decoys"]; + if ( + typeof route !== "string" || + !Array.isArray(decoys) || + !decoys.every((d) => typeof d === "string") + ) { + throw new Error( + `${path}:${i + 1}: expect needs string "route" and string[] "decoys"`, + ); + } + const evalCase: Case = { + input, + expect: { route, decoys: decoys as string[] }, + }; + const name = rec["name"]; + if (typeof name === "string" && name !== "") { + evalCase.name = name; + } + cases.push(evalCase); + } + if (cases.length === 0) { + throw new Error(`${path}: no cases`); + } + return cases; + }); +} + +function routeLine(output: string): string { + const line = output.split(/\r?\n/, 1)[0] ?? ""; + return line.trim(); +} + +function chosenRoute(output: string): string { + return routeLine(output).toLowerCase(); +} + +export function routesCheck( + run: Run, +): CheckValue { + const expected = run.expect; + if (expected === undefined) { + return { pass: false, value: 0, explanation: "case has no expect" }; + } + const chosen = chosenRoute(run.output); + const route = expected.route.trim().toLowerCase(); + const decoys = expected.decoys.map((decoy) => decoy.trim().toLowerCase()); + if (decoys.includes(chosen)) { + return { pass: false, value: 0, explanation: `decoy route: ${chosen}` }; + } + if (chosen !== route) { + return { + pass: false, + value: 0, + explanation: `expected ${route}, got ${chosen === "" ? "(empty)" : chosen}`, + }; + } + return true; +} + +const DISPATCH_EVAL = "dispatch-routing"; + +type DispatchEvaluate = ( + name: string, + options: EvaluateOptions, +) => Promise>; + +interface StoredCheck { + pass: boolean; + value: number; +} + +interface StoredRun { + format: 1; + eval: string; + about?: Record; + cases: Record>; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function parseStoredRun(text: string, source: string): StoredRun { + let raw: unknown; + try { + raw = JSON.parse(text); + } catch (cause) { + throw new Error(`${source}: invalid JSON (${(cause as Error).message})`); + } + if (!isRecord(raw) || !isRecord(raw["cases"])) { + throw new Error(`${source}: expected a stored run`); + } + const format = raw["format"]; + const evalName = raw["eval"]; + if (format !== 1 || typeof evalName !== "string" || evalName === "") { + throw new Error(`${source}: expected a format 1 stored run`); + } + const cases: Record> = {}; + for (const [name, checks] of Object.entries(raw["cases"])) { + if (!isRecord(checks)) { + throw new Error(`${source}: case "${name}" must be an object`); + } + const parsed: Record = {}; + for (const [check, outcome] of Object.entries(checks)) { + if ( + !isRecord(outcome) || + typeof outcome["pass"] !== "boolean" || + typeof outcome["value"] !== "number" || + !Number.isFinite(outcome["value"]) + ) { + throw new Error( + `${source}: case "${name}" check "${check}" is invalid`, + ); + } + parsed[check] = { pass: outcome["pass"], value: outcome["value"] }; + } + cases[name] = parsed; + } + const stored: StoredRun = { format: 1, eval: evalName, cases }; + if (isRecord(raw["about"])) { + stored.about = raw["about"]; + } + return stored; +} + +function routesStoredPass( + checks: Record | undefined, +): boolean { + return checks?.["routes"]?.pass === true; +} + +async function baselinePresent(path: string): Promise { + try { + await access(path); + return true; + } catch { + return false; + } +} + +// A miss is a drop only when the stored routes pass is true, so the miss +// budget would keep a case that has never been baselined green. Passes still +// merge: a miss is never written as a pass, and a stored pass is never +// replaced by a miss, so a later flip of a case that passed beside a sibling +// miss is a drop. +async function commitRoutingBaseline(args: { + hadBaseline: boolean; + lastPath: string; + baselinePath: string; +}): Promise { + const last = parseStoredRun( + await readFile(args.lastPath, "utf8"), + args.lastPath, + ); + const missed = Object.entries(last.cases).filter( + ([, checks]) => !routesStoredPass(checks), + ); + if (missed.length === 0) { + await copyFile(args.lastPath, args.baselinePath); + return 0; + } + if (!args.hadBaseline) { + console.error( + `eval "${DISPATCH_EVAL}" failed: no baseline at ${args.baselinePath}`, + ); + return 1; + } + const beforeText = await readFile(args.baselinePath, "utf8"); + const baseline = parseStoredRun(beforeText, args.baselinePath); + const unbaselined = missed + .filter(([name]) => baseline.cases[name] === undefined) + .map(([name]) => name); + const cases = { ...baseline.cases }; + let changed = false; + for (const [name, checks] of Object.entries(last.cases)) { + if (!routesStoredPass(checks)) continue; + if (routesStoredPass(cases[name])) continue; + cases[name] = checks; + changed = true; + } + if (changed) { + const next: StoredRun = { ...baseline, cases }; + await writeFile(args.baselinePath, `${JSON.stringify(next, null, 2)}\n`); + } + if (unbaselined.length !== 0) { + console.error( + `eval "${DISPATCH_EVAL}" failed: unbaselined routes miss (${unbaselined.join(", ")})`, + ); + return 1; + } + return 0; +} + +export async function evaluateRouting(args: { + cases: readonly Case[]; + run: (input: string, signal: AbortSignal) => Promise; + lastPath: string; + baselinePath: string; + evaluateFn?: DispatchEvaluate; +}): Promise { + const hadBaseline = await baselinePresent(args.baselinePath); + const evaluateFn: DispatchEvaluate = + args.evaluateFn ?? ((name, options) => evaluate(name, options)); + try { + const report = await evaluateFn(DISPATCH_EVAL, { + run: args.run, + cases: args.cases, + checks: { + routes: routesCheck, + fast: under({ ms: ROUTE_BUDGET_MS }), + }, + allow: { + routes: { miss: 6, drop: 0 }, + fast: 0, + }, + compare: args.baselinePath, + save: args.lastPath, + timeout: ROUTE_BUDGET_MS, + about: { + prompt: "agents/dispatch/src/prompt.ts (dispatch director)", + model: MODEL, + ollamaUrl: OLLAMA_URL, + }, + }); + console.log(report.markdown()); + return await commitRoutingBaseline({ + hadBaseline, + lastPath: args.lastPath, + baselinePath: args.baselinePath, + }); + } catch (err: unknown) { + if (err instanceof EvalFailure) { + console.log(err.report.markdown()); + return 1; + } + throw err; + } +} + +async function main(): Promise { + const cases = await loadCases(CASES_PATH); + return evaluateRouting({ + cases, + run: dispatchRoute, + lastPath: LAST_PATH, + baselinePath: BASELINE_PATH, + }); +} + +if (import.meta.main) { + main() + .then((code) => { + process.exitCode = code; + }) + .catch((err: unknown) => { + console.error(err); + process.exitCode = 1; + }); +}