diff --git a/README.md b/README.md index 2396211..392c03d 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,13 @@ Bun only (>=1.2.0). ## Example ```ts -import { evaluate, mentions, under } from "@corbits/evals"; +import { mentions, runEvaluation, under } from "@corbits/evals"; -const report = await evaluate("refund-agent", { - run: refundAgent, // an @intx/agent Agent, workflow(), or (input) => output - cases: [{ name: "duplicate charge", input: "I was charged twice" }], - checks: { +const report = await runEvaluation({ + name: "refund-agent", + task: refundAgent, // an @intx/agent Agent, workflow(), or (input) => output + data: [{ name: "duplicate charge", input: "I was charged twice" }], + scorers: { refunds: mentions(["refund"]), readyToSend: "Is this reply ready to send to the customer?", fast: under({ ms: 5_000 }), @@ -118,12 +119,13 @@ compared runs and defaults to the input. ## Passing ```ts -await evaluate("judge", { - run, - cases, // a case can say { must: ["tests"], skip: ["docs"] } - checks, +await runEvaluation({ + name: "judge", + task: run, + data, // a case can say { must: ["tests"], skip: ["docs"] } + scorers, compare: "evals/judge.last.json", // read: measure drops against it - save: "evals/judge.last.json", // write: store this run, pass or fail + save: "evals/judge.last.json", // write: store this experiment, pass or fail allow: { tests: { miss: 1, drop: 0 }, docs: { miss: 5, drop: 2 }, @@ -141,16 +143,16 @@ Per check, `allow` sets (default 0): A case's `must` checks must pass whatever `allow` says; its `skip` checks are not run or counted. -`evaluate` prints each check's passes, misses and drops, and rejects with -`EvalFailure` naming the limit each failing check broke and its failed +`runEvaluation` prints each scorer's passes, misses and drops, and rejects with +`EvalFailure` naming the limit each failing scorer broke and its failed cases; `.report` holds every result. -A stored run is each case × check pass and value — plus a Jev check's +An experiment is each case × scorer pass and value — plus a Jev scorer's probability distribution and confidence — and `about`, no times. `compare` only reads; a missing file skips drops. `save` only writes, on -every run. One path for both compares with the last run, then replaces -it. `compare` also takes a report from an earlier `evaluate` of the same -eval, with no file: `evaluate(name, { ...options, compare: before })`. +every run. One path for both compares with the last experiment, then replaces +it. `compare` also takes an experiment from an earlier `runEvaluation` of the same +evaluation, with no file: `runEvaluation({ ...spec, compare: before.experiment })`. Flips print as `fixed` / `broke`, new and gone cases as `added` / `removed`. `quiet: true` skips the summary. diff --git a/e2e/evaluate.e2e.test.ts b/e2e/evaluate.e2e.test.ts index e8e8ae1..3a58acb 100644 --- a/e2e/evaluate.e2e.test.ts +++ b/e2e/evaluate.e2e.test.ts @@ -7,7 +7,7 @@ import { type } from "arktype"; import { avoids, EvalFailure, - evaluate, + runEvaluation, fitsSchema, matches, mentions, @@ -30,14 +30,15 @@ async function failure(pending: Promise): Promise { if (cause instanceof EvalFailure) return cause; throw cause; } - throw new Error("expected evaluate to fail"); + throw new Error("expected runEvaluation to fail"); } describe("runs", () => { test("a function run with the default check", async () => { - const report = await evaluate("upper", { - run: upper, - cases: [ + const report = await runEvaluation({ + name: "upper", + task: upper, + data: [ { input: "hi", expect: "HI" }, { name: "yo", input: "yo", expect: "YO" }, ], @@ -56,15 +57,20 @@ describe("runs", () => { return { type: "reply" as const, reply: `re: ${content}` }; }, }; - const report = await evaluate("agent", { - run: agent, - cases: [{ input: "refund", expect: "re: refund" }], + const report = await runEvaluation({ + name: "agent", + task: agent, + data: [{ input: "refund", expect: "re: refund" }], }); expect(report.ok).toBe(true); expect(seen[0]).toBeInstanceOf(AbortSignal); const suspended = { send: async () => ({ type: "suspended" as const }) }; const err = await failure( - evaluate("agent-suspended", { run: suspended, cases: ["x"].map((input) => ({ input })) }), + runEvaluation({ + name: "agent-suspended", + task: suspended, + data: ["x"].map((input) => ({ input })), + }), ); expect(err.report.cases[0]?.error).toContain('did not reply (got "suspended")'); }); @@ -72,15 +78,16 @@ describe("runs", () => { test("a raw workflow definition is refused with the fix", async () => { const definition = { id: "w", triggers: [], steps: {}, stepOrder: [] }; await expect( - evaluate("raw", { run: definition as never, cases: [{ input: "x" }] }), + runEvaluation({ name: "raw", task: definition as never, data: [{ input: "x" }] }), ).rejects.toThrow("@corbits/evals/workflow"); }); test("objects stay raw and match deeply", async () => { - const report = await evaluate("objects", { - run: (input: { amount: number }) => ({ decision: "refund", amount: input.amount }), - cases: [{ input: { amount: 5 }, expect: { decision: "refund", amount: 5 } }], - checks: { + const report = await runEvaluation({ + name: "objects", + task: (input: { amount: number }) => ({ decision: "refund", amount: input.amount }), + data: [{ input: { amount: 5 }, expect: { decision: "refund", amount: 5 } }], + scorers: { whole: matches(), decision: matches("decision"), amount: matches((v: { amount: number }) => v.amount), @@ -97,11 +104,11 @@ describe("runs", () => { path, '{"name":"a","input":"hi","expect":"HI"}\n\n{"input":"yo","expect":"YO"}\n', ); - const report = await evaluate("jsonl", { run: upper, cases: path }); + const report = await runEvaluation({ name: "jsonl", task: upper, data: path }); expect(report.cases.map((c) => c.name)).toEqual(["a", "yo"]); expect(report.passed).toBe(2); await Bun.write(path, '{"input":"x"}\n{"name":"b"}\n'); - await expect(evaluate("jsonl-bad", { run: upper, cases: path })).rejects.toThrow( + await expect(runEvaluation({ name: "jsonl-bad", task: upper, data: path })).rejects.toThrow( `${path}:2: missing "input"`, ); }); @@ -109,10 +116,11 @@ describe("runs", () => { describe("checks", () => { test("code checks return booleans or 0..1 numbers", async () => { - const report = await evaluate("code", { - run: upper, - cases: [{ input: "hi" }], - checks: { short: (r) => r.output.length < 5, half: () => 0.5, slow: async () => true }, + const report = await runEvaluation({ + name: "code", + task: upper, + data: [{ input: "hi" }], + scorers: { short: (r) => r.output.length < 5, half: () => 0.5, slow: async () => true }, allow: { half: 1 }, }); const checks = report.cases[0]?.checks; @@ -131,10 +139,11 @@ describe("checks", () => { ], }), }; - const report = await evaluate("helpers", { - run: steps, - cases: [{ input: "order 42" }], - checks: { + const report = await runEvaluation({ + name: "helpers", + task: steps, + data: [{ input: "order 42" }], + scorers: { mentionsAll: mentions(["refund", "ORDER 42"]), mentionsSome: mentions(["refund", "credit"]), mentionsHalf: mentions(["refund", "credit"], { atLeast: 0.5 }), @@ -178,16 +187,19 @@ describe("checks", () => { }); test("matches fails a case without expect", async () => { - const err = await failure(evaluate("no-expect", { run: upper, cases: [{ input: "hi" }] })); + const err = await failure( + runEvaluation({ name: "no-expect", task: upper, data: [{ input: "hi" }] }), + ); expect(err.report.cases[0]?.checks["matches"]?.explanation).toBe("case has no expect"); }); test("a throwing, hanging or malformed check fails itself closed, not its siblings", async () => { const err = await failure( - evaluate("bad-checks", { - run: upper, - cases: [{ input: "hi", expect: "HI" }], - checks: { + runEvaluation({ + name: "bad-checks", + task: upper, + data: [{ input: "hi", expect: "HI" }], + scorers: { exact: matches(), boom: () => { throw new Error("kaput"); @@ -209,8 +221,9 @@ describe("checks", () => { test("a run error or timeout fails every check and aborts the run", async () => { let aborted = false; const err = await failure( - evaluate("bad-runs", { - run: (input: string, signal) => { + runEvaluation({ + name: "bad-runs", + task: (input: string, signal) => { if (input === "boom") throw new Error("kaput"); return new Promise((_, reject) => { signal.addEventListener("abort", () => { @@ -219,8 +232,8 @@ describe("checks", () => { }); }); }, - cases: [{ input: "boom" }, { input: "hang" }], - checks: { fine: () => true }, + data: [{ input: "boom" }, { input: "hang" }], + scorers: { fine: () => true }, timeout: 30, }), ); @@ -236,10 +249,11 @@ describe("gates", () => { test("repeats fail a check when any attempt fails and report the deciding one", async () => { let calls = 0; const err = await failure( - evaluate("flaky", { - run: upper, - cases: [{ input: "hi" }], - checks: { + runEvaluation({ + name: "flaky", + task: upper, + data: [{ input: "hi" }], + scorers: { flaky: () => { calls += 1; return calls === 1 @@ -259,25 +273,31 @@ describe("gates", () => { }); test("bad options are rejected up front", async () => { - const base = { run: upper, cases: [{ input: "a", expect: "A" }] }; - await expect(evaluate("o", { ...base, concurrency: Number.NaN })).rejects.toThrow( + const base = { task: upper, data: [{ input: "a", expect: "A" }] }; + await expect(runEvaluation({ name: "o", ...base, concurrency: Number.NaN })).rejects.toThrow( "concurrency", ); - await expect(evaluate("o", { ...base, repeats: 0 })).rejects.toThrow("repeats"); - await expect(evaluate("o", { ...base, timeout: "soon" as never })).rejects.toThrow("timeout"); - await expect(evaluate("o", { ...base, checks: {} })).rejects.toThrow("no checks"); - await expect(evaluate("o", { ...base, cases: [] })).rejects.toThrow("no cases"); + await expect(runEvaluation({ name: "o", ...base, repeats: 0 })).rejects.toThrow("repeats"); + await expect(runEvaluation({ name: "o", ...base, timeout: "soon" as never })).rejects.toThrow( + "timeout", + ); + await expect(runEvaluation({ name: "o", ...base, scorers: {} })).rejects.toThrow("no scorers"); + await expect(runEvaluation({ name: "o", ...base, data: [] })).rejects.toThrow("no data"); await expect( - evaluate("o", { ...base, cases: [...base.cases, { input: "a" }] }), + runEvaluation({ name: "o", ...base, data: [...base.data, { input: "a" }] }), ).rejects.toThrow('duplicate case "a"'); }); test("markdown escapes cells", async () => { const err = await failure( - evaluate("md", { run: upper, cases: [{ name: "a|b\nc", input: "", expect: "no" }] }), + runEvaluation({ + name: "md", + task: upper, + data: [{ name: "a|b\nc", input: "", expect: "no" }], + }), ); const md = err.report.markdown(); - expect(md).toContain("# eval: md"); + expect(md).toContain("# evaluation: md"); expect(md).toContain("| matches | 0/1 | 1 (0 allowed) | FAIL |"); expect(md).toContain("| a\\|b c | FAIL"); expect(md).not.toContain(""); @@ -293,19 +313,33 @@ describe("passing", () => { const checks = { exact: matches(), short: (r: { output: string }) => r.output.length === 1 }; test("each check may miss as many cases as allowed", async () => { - const strict = await failure(evaluate("strict", { run: upper, cases, checks })); + const strict = await failure( + runEvaluation({ name: "strict", task: upper, data: cases, scorers: checks }), + ); expect(strict.message).toContain("exact: 1 missed, 0 allowed"); expect(strict.message).toContain('c / exact: expected "nope", got "C"'); expect(strict.report.checks["exact"]).toMatchObject({ passed: 2, counted: 3, missed: 1 }); expect(strict.report.checks["short"]?.ok).toBe(true); - const report = await evaluate("lenient", { run: upper, cases, checks, allow: { exact: 1 } }); + const report = await runEvaluation({ + name: "lenient", + task: upper, + data: cases, + scorers: checks, + allow: { exact: 1 }, + }); expect(report.ok).toBe(true); expect(report.checks["exact"]?.allow).toEqual({ miss: 1, drop: 0 }); const long = (input: string) => input.toUpperCase().repeat(2); const over = await failure( - evaluate("over", { run: long, cases, checks, allow: { exact: { miss: 3 }, short: 2 } }), + runEvaluation({ + name: "over", + task: long, + data: cases, + scorers: checks, + allow: { exact: { miss: 3 }, short: 2 }, + }), ); expect(over.message).toContain("short: 3 missed, 2 allowed"); expect(over.message).not.toContain("exact:"); @@ -315,10 +349,11 @@ describe("passing", () => { test("must fails a check despite its allowance; skip leaves a case out", async () => { const must = await failure( - evaluate("must", { - run: upper, - cases: cases.map((c) => (c.name === "c" ? { ...c, must: ["exact"] } : c)), - checks, + runEvaluation({ + name: "must", + task: upper, + data: cases.map((c) => (c.name === "c" ? { ...c, must: ["exact"] } : c)), + scorers: checks, allow: { exact: 1 }, }), ); @@ -326,10 +361,11 @@ describe("passing", () => { expect(must.report.checks["exact"]).toMatchObject({ missed: 1, mustFailed: ["c"], ok: false }); const seen: string[] = []; - const report = await evaluate("skip", { - run: upper, - cases: cases.map((c) => (c.name === "c" ? { ...c, skip: ["exact"] } : c)), - checks: { + const report = await runEvaluation({ + name: "skip", + task: upper, + data: cases.map((c) => (c.name === "c" ? { ...c, skip: ["exact"] } : c)), + scorers: { exact: (r) => { seen.push(r.input); return r.output === r.expect; @@ -348,38 +384,44 @@ describe("passing", () => { path, '{"name":"a","input":"a","expect":"A","must":["exact"]}\n{"name":"c","input":"c","expect":"nope","skip":["exact"]}\n', ); - const report = await evaluate("jsonl-must", { run: upper, cases: path, checks }); + const report = await runEvaluation({ + name: "jsonl-must", + task: upper, + data: path, + scorers: checks, + }); expect(report.checks["exact"]).toMatchObject({ passed: 1, counted: 1 }); await Bun.write(path, '{"input":"a","skip":"exact"}\n'); - await expect(evaluate("jsonl-must", { run: upper, cases: path, checks })).rejects.toThrow( - `${path}:1: "skip" must be an array of check names`, - ); + await expect( + runEvaluation({ name: "jsonl-must", task: upper, data: path, scorers: checks }), + ).rejects.toThrow(`${path}:1: "skip" must be an array of check names`); }); test("bad allow, must and skip names are rejected", async () => { - const base = { run: upper, cases, checks }; - await expect(evaluate("o", { ...base, allow: { exakt: 1 } })).rejects.toThrow( - 'allow names unknown check "exakt"', + const base = { task: upper, data: cases, scorers: checks }; + await expect(runEvaluation({ name: "o", ...base, allow: { exakt: 1 } })).rejects.toThrow( + 'allow names unknown scorer "exakt"', ); - await expect(evaluate("o", { ...base, allow: { exact: -1 } })).rejects.toThrow( + await expect(runEvaluation({ name: "o", ...base, allow: { exact: -1 } })).rejects.toThrow( "allow.exact must be a whole number >= 0", ); - await expect(evaluate("o", { ...base, allow: { exact: { drop: 0.5 } } })).rejects.toThrow( - "allow.exact.drop must be a whole number >= 0", - ); await expect( - evaluate("o", { ...base, allow: { exact: { misses: 1 } as never } }), + runEvaluation({ name: "o", ...base, allow: { exact: { drop: 0.5 } } }), + ).rejects.toThrow("allow.exact.drop must be a whole number >= 0"); + await expect( + runEvaluation({ name: "o", ...base, allow: { exact: { misses: 1 } as never } }), ).rejects.toThrow('allow.exact has unknown "misses"'); await expect( - evaluate("o", { ...base, cases: [{ name: "a", input: "a", must: ["nope"] }] }), - ).rejects.toThrow('case "a" must names unknown check "nope"'); + runEvaluation({ name: "o", ...base, data: [{ name: "a", input: "a", must: ["nope"] }] }), + ).rejects.toThrow('case "a" must names unknown scorer "nope"'); await expect( - evaluate("o", { ...base, cases: [{ name: "a", input: "a", skip: ["nope"] }] }), - ).rejects.toThrow('case "a" skip names unknown check "nope"'); + runEvaluation({ name: "o", ...base, data: [{ name: "a", input: "a", skip: ["nope"] }] }), + ).rejects.toThrow('case "a" skip names unknown scorer "nope"'); await expect( - evaluate("o", { + runEvaluation({ + name: "o", ...base, - cases: [{ name: "a", input: "a", must: ["exact"], skip: ["exact"] }], + data: [{ name: "a", input: "a", must: ["exact"], skip: ["exact"] }], }), ).rejects.toThrow('case "a" both must and skip "exact"'); }); @@ -408,15 +450,22 @@ describe("compare and save", () => { } test("without either nothing is read or written", async () => { - await touchesNoFile(() => evaluate("upper", { run: upper, cases, allow })); + await touchesNoFile(() => runEvaluation({ name: "upper", task: upper, data: cases, allow })); }); test("save writes the run, passing or failing", async () => { const path = join(await tempDir(), "upper.json"); - await evaluate("upper", { run: upper, cases, allow, save: path, about: { model: "m-1" } }); + await runEvaluation({ + name: "upper", + task: upper, + data: cases, + allow, + save: path, + about: { model: "m-1" }, + }); expect(JSON.parse(await Bun.file(path).text())).toEqual({ - format: 1, - eval: "upper", + format: 2, + name: "upper", about: { model: "m-1" }, cases: { a: { matches: { pass: true, value: 1 } }, @@ -425,7 +474,7 @@ describe("compare and save", () => { }, }); - await failure(evaluate("upper", { run: upper, cases, save: path })); + await failure(runEvaluation({ name: "upper", task: upper, data: cases, save: path })); expect(JSON.parse(await Bun.file(path).text()).about).toBeUndefined(); }); @@ -433,7 +482,13 @@ describe("compare and save", () => { const path = join(await tempDir(), "missing.json"); const log = spyOn(console, "log").mockImplementation(() => {}); try { - const report = await evaluate("upper", { run: swap, cases, allow, compare: path }); + const report = await runEvaluation({ + name: "upper", + task: swap, + data: cases, + allow, + compare: path, + }); expect(report.checks["matches"]?.drop).toBeUndefined(); expect(log).toHaveBeenCalledWith(` no compare run at ${path}`); } finally { @@ -441,19 +496,26 @@ describe("compare and save", () => { } expect(await Bun.file(path).exists()).toBe(false); - await evaluate("upper", { run: upper, cases, allow, save: path }); + await runEvaluation({ name: "upper", task: upper, data: cases, allow, save: path }); const stored = await Bun.file(path).text(); - await evaluate("upper", { run: upper, cases, allow, compare: path }); - await failure(evaluate("upper", { run: swap, cases, allow, compare: path })); + await runEvaluation({ name: "upper", task: upper, data: cases, allow, compare: path }); + await failure(runEvaluation({ name: "upper", task: swap, data: cases, allow, compare: path })); expect(await Bun.file(path).text()).toBe(stored); }); test("one path for both compares, then overwrites", async () => { const path = join(await tempDir(), "upper.last.json"); - await evaluate("upper", { run: upper, cases, allow, compare: path, save: path }); + await runEvaluation({ + name: "upper", + task: upper, + data: cases, + allow, + compare: path, + save: path, + }); const err = await failure( - evaluate("upper", { run: swap, cases, allow, compare: path, save: path }), + runEvaluation({ name: "upper", task: swap, data: cases, allow, compare: path, save: path }), ); expect(err.report.changes).toEqual([ { case: "b", check: "matches", change: "broke" }, @@ -464,32 +526,54 @@ describe("compare and save", () => { expect(err.message).toContain("b / matches"); expect(JSON.parse(await Bun.file(path).text()).cases.b.matches.pass).toBe(false); - const again = await evaluate("upper", { run: swap, cases, allow, compare: path, save: path }); + const again = await runEvaluation({ + name: "upper", + task: swap, + data: cases, + allow, + compare: path, + save: path, + }); expect(again.checks["matches"]?.drop).toBe(0); }); test("an earlier report gates drops with no file", async () => { - const before = await evaluate("upper", { run: upper, cases, allow }); + const before = await runEvaluation({ name: "upper", task: upper, data: cases, allow }); await touchesNoFile(async () => { - const err = await failure(evaluate("upper", { run: swap, cases, allow, compare: before })); + const err = await failure( + runEvaluation({ + name: "upper", + task: swap, + data: cases, + allow, + compare: before.experiment, + }), + ); expect(err.report.checks["matches"]).toMatchObject({ drop: 1, ok: false }); - const within = await evaluate("upper", { - run: swap, - cases, + const within = await runEvaluation({ + name: "upper", + task: swap, + data: cases, allow: { matches: { miss: 1, drop: 1 } }, - compare: before, + compare: before.experiment, }); expect(within.ok).toBe(true); }); - await expect(evaluate("other", { run: upper, cases, allow, compare: before })).rejects.toThrow( - 'compare is a report of eval "upper"', - ); + await expect( + runEvaluation({ name: "other", task: upper, data: cases, allow, compare: before.experiment }), + ).rejects.toThrow('compare is an experiment of "upper"'); }); test("added and removed cases are reported, not counted as drops", async () => { - const before = await evaluate("upper", { run: upper, cases, allow }); + const before = await runEvaluation({ name: "upper", task: upper, data: cases, allow }); const changed = [...cases.slice(1), { name: "d", input: "d", expect: "nope" }]; - const report = await evaluate("upper", { run: upper, cases: changed, allow, compare: before }); + const report = await runEvaluation({ + name: "upper", + task: upper, + data: changed, + allow, + compare: before.experiment, + }); expect(report.added).toEqual(["d"]); expect(report.removed).toEqual(["a"]); expect(report.checks["matches"]?.drop).toBe(0); @@ -497,16 +581,16 @@ describe("compare and save", () => { test("refuses another eval's run and malformed files; reads 0.1.0 files", async () => { const path = join(await tempDir(), "b.json"); - await evaluate("one", { run: upper, cases, allow, save: path }); - await expect(evaluate("two", { run: upper, cases, allow, compare: path })).rejects.toThrow( - 'stored run of eval "one", not "two"', - ); + await runEvaluation({ name: "one", task: upper, data: cases, allow, save: path }); + await expect( + runEvaluation({ name: "two", task: upper, data: cases, allow, compare: path }), + ).rejects.toThrow('stored experiment "one", not "two"'); const old = { ...JSON.parse(await Bun.file(path).text()), passRate: 0.667 }; await Bun.write(path, JSON.stringify(old)); - await evaluate("one", { run: upper, cases, allow, compare: path }); - await Bun.write(path, JSON.stringify({ format: 2 })); - await expect(evaluate("one", { run: upper, cases, allow, compare: path })).rejects.toThrow( - "unknown format 2", - ); + await runEvaluation({ name: "one", task: upper, data: cases, allow, compare: path }); + await Bun.write(path, JSON.stringify({ format: 1, name: "one", cases: {} })); + await expect( + runEvaluation({ name: "one", task: upper, data: cases, allow, compare: path }), + ).rejects.toThrow("expected 2"); }); }); diff --git a/e2e/jev.e2e.test.ts b/e2e/jev.e2e.test.ts index 29eb7c0..4926f72 100644 --- a/e2e/jev.e2e.test.ts +++ b/e2e/jev.e2e.test.ts @@ -7,9 +7,9 @@ import type { EvaluateDeps } from "@corbits/system-one"; import { ask, EvalFailure, - evaluate, + runEvaluation, jev, - type Check, + type Scorer, type Escalation, type Judge, type Outcome, @@ -44,13 +44,14 @@ function stubJev(answer: unknown) { return { bodies, judge: jev({ apiKey: "test-key", deps: stubDeps(fetch) }) }; } -async function judged(check: Check, judge: Judge) { +async function judged(check: Scorer, judge: Judge) { let report: Report; try { - report = await evaluate("jev", { - run: (input: string) => `reply to ${input}`, - cases: [{ input: "refund?", expect: "yes" }], - checks: { judged: check }, + report = await runEvaluation({ + name: "jev", + task: (input: string) => `reply to ${input}`, + data: [{ input: "refund?", expect: "yes" }], + scorers: { judged: check }, judge, }); } catch (cause) { @@ -60,7 +61,7 @@ async function judged(check: Check, judge: Judge) { return { outcome: report.cases[0]?.checks["judged"] }; } -async function answered(answer: unknown, check: Check) { +async function answered(answer: unknown, check: Scorer) { const { bodies, judge } = stubJev(answer); return { ...(await judged(check, judge)), bodies }; } @@ -116,10 +117,11 @@ describe("sentence checks", () => { describe("batching", () => { test("a case's Jev checks answer in one call and map back by name", async () => { const { bodies, judge } = stubJev({ type: "noul", noul: 0.9 }); - const report = await evaluate("jev", { - run: (input: string) => `reply to ${input}`, - cases: [{ input: "refund?" }], - checks: { ready: "Ready to send?", polite: "Is the tone polite?" }, + const report = await runEvaluation({ + name: "jev", + task: (input: string) => `reply to ${input}`, + data: [{ input: "refund?" }], + scorers: { ready: "Ready to send?", polite: "Is the tone polite?" }, judge, }); expect(bodies).toHaveLength(1); @@ -142,10 +144,11 @@ describe("batching", () => { }) as unknown as EvaluateDeps["fetch"]; let report: Report; try { - await evaluate("jev", { - run: (input: string) => input, - cases: [{ input: "a" }], - checks: { ready: "Ready to send?", polite: "Is the tone polite?" }, + await runEvaluation({ + name: "jev", + task: (input: string) => input, + data: [{ input: "a" }], + scorers: { ready: "Ready to send?", polite: "Is the tone polite?" }, judge: jev({ apiKey: "test-key", deps: stubDeps(failing) }), }); throw new Error("expected failure"); @@ -162,10 +165,11 @@ describe("batching", () => { test("a case that skips one Jev check keeps it out of the call", async () => { const { bodies, judge } = stubJev({ type: "noul", noul: 0.9 }); - await evaluate("jev", { - run: (input: string) => input, - cases: [{ input: "a", skip: ["polite"] }], - checks: { ready: "Ready to send?", polite: "Is the tone polite?" }, + await runEvaluation({ + name: "jev", + task: (input: string) => input, + data: [{ input: "a", skip: ["polite"] }], + scorers: { ready: "Ready to send?", polite: "Is the tone polite?" }, judge, }); expect(Object.keys(bodies[0]?.["questions"] as object)).toEqual(["ready"]); @@ -240,7 +244,7 @@ describe("thresholds", () => { await expect(judged({ question, yes: 0.2, no: 0.5 }, judge)).rejects.toThrow( "no must not exceed yes", ); - // Even a hand-built ask (bypassing ask()) is validated at evaluate time. + // Even a hand-built ask (bypassing ask()) is validated at runEvaluation time. await expect( judged({ question: "Ready?", options: { a: "a" }, pass: ["a"], sure: 0 }, judge), ).rejects.toThrow("sure must be in (0, 1]"); @@ -514,10 +518,11 @@ describe("stored reports", () => { probabilities: { ready: 0.8, revise: 0.1, escalate: 0.1 }, confidence: 0.8, }; - await evaluate("jev", { - run: (input: string) => `reply to ${input}`, - cases: [{ input: "refund?" }], - checks: { + await runEvaluation({ + name: "jev", + task: (input: string) => `reply to ${input}`, + data: [{ input: "refund?" }], + scorers: { judged: ask("Ready to send?", { options: { ready: "send", revise: "work", escalate: "human" }, pass: "ready", diff --git a/e2e/workflow.e2e.test.ts b/e2e/workflow.e2e.test.ts index b5782a6..00724b0 100644 --- a/e2e/workflow.e2e.test.ts +++ b/e2e/workflow.e2e.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from "bun:test"; import { defineAgent } from "@intx/agent"; import { defineWorkflow, step, type StepInvoker } from "@intx/workflow"; -import { EvalFailure, evaluate, matches, ranSteps, under } from "../src/index.js"; +import { EvalFailure, runEvaluation, matches, ranSteps, under } from "../src/index.js"; import { workflow } from "../src/workflow.js"; function agent(id: string) { @@ -36,10 +36,11 @@ const invokeStep: StepInvoker = async ({ agent: a, input }) => { describe("workflow runs", () => { test("output defaults to the last step and every step is recorded", async () => { - const report = await evaluate("workflow", { - run: workflow(pipeline, { invokeStep }), - cases: [{ input: "refund me", expect: "DRAFT: REFUND ME" }], - checks: { + const report = await runEvaluation({ + name: "workflow", + task: workflow(pipeline, { invokeStep }), + data: [{ input: "refund me", expect: "DRAFT: REFUND ME" }], + scorers: { exact: matches(), order: (r) => r.steps.map((s) => s.name).join(">") === "drafter>reviewer", steps: ranSteps(["drafter", "reviewer"]), @@ -59,13 +60,14 @@ describe("workflow runs", () => { }); test("output and trigger pick object values", async () => { - const report = await evaluate("workflow-object", { - run: workflow(pipeline, { + const report = await runEvaluation({ + name: "workflow-object", + task: workflow(pipeline, { invokeStep, trigger: (input: { order: number }) => `order ${input.order}`, output: (run) => ({ draft: run.outputs["draft"], review: run.outputs["review"] }), }), - cases: [ + data: [ { input: { order: 42 }, expect: { draft: "draft: order 42", review: "DRAFT: ORDER 42" } }, ], }); @@ -74,14 +76,15 @@ describe("workflow runs", () => { test("a failing step fails the case closed", async () => { try { - await evaluate("workflow-fail", { - run: workflow(pipeline, { + await runEvaluation({ + name: "workflow-fail", + task: workflow(pipeline, { invokeStep: async ({ agent: a }) => { if (a.id === "reviewer") throw new Error("reviewer down"); return { output: "draft" }; }, }), - cases: [{ input: "x", expect: "X" }], + data: [{ input: "x", expect: "X" }], }); throw new Error("expected a failure"); } catch (cause) { diff --git a/examples/local-jev.ts b/examples/local-jev.ts index ffe8775..d9949b2 100644 --- a/examples/local-jev.ts +++ b/examples/local-jev.ts @@ -1,6 +1,6 @@ // Local Jev judge over canned support drafts (good, invented-compensation, // refusal). Run: bun examples/local-jev.ts -import { ask, evaluate } from "../src/index.js"; +import { ask, runEvaluation } from "../src/index.js"; import { localJev } from "./ollama.js"; const DRAFTS: Record = { @@ -12,10 +12,11 @@ const DRAFTS: Record = { escalate: "I can't help with that. Goodbye.", }; -const report = await evaluate("local-jev-support", { - run: (id: string) => DRAFTS[id] ?? "", - cases: Object.keys(DRAFTS).map((id) => ({ input: id })), - checks: { +const report = await runEvaluation({ + name: "local-jev-support", + task: (id: string) => DRAFTS[id] ?? "", + data: Object.keys(DRAFTS).map((id) => ({ input: id })), + scorers: { ready: ask( "Classify the draft reply. Ready means accurate, answers the request, invents nothing. Unsafe means it invents refunds, credits, or policies.", { diff --git a/examples/local-loop.ts b/examples/local-loop.ts index c5e5be0..ed9a04b 100644 --- a/examples/local-loop.ts +++ b/examples/local-loop.ts @@ -1,19 +1,20 @@ // Full local loop: qwen2.5:7b writes replies, clef-flash:9b grades them. // Run: bun examples/local-loop.ts -import { ask, evaluate } from "../src/index.js"; +import { ask, runEvaluation } from "../src/index.js"; import { localJev, ollamaGenerate } from "./ollama.js"; const SYSTEM = `You are a support agent. Rules: never promise refunds, credits, or compensation beyond what the context states. Verified context is ground truth. Reply in 2 sentences max.`; type Ticket = { message: string; verified: string }; -const report = await evaluate("local-loop-support", { - run: (ticket: Ticket) => +const report = await runEvaluation({ + name: "local-loop-support", + task: (ticket: Ticket) => ollamaGenerate( "qwen2.5:7b", `${SYSTEM}\n\nVerified context: ${ticket.verified}\nCustomer: ${ticket.message}`, ), - cases: [ + data: [ { name: "dup-charge", input: { @@ -36,7 +37,7 @@ const report = await evaluate("local-loop-support", { }, }, ], - checks: { + scorers: { ready: ask( "Classify the reply against the verified context. Ready = accurate, answers the request, invents nothing beyond context. Revise otherwise.", { options: { ready: "sendable", revise: "inaccurate or invents" }, pass: "ready" }, diff --git a/examples/ports/battle.ts b/examples/ports/battle.ts index fc8b52f..3557175 100644 --- a/examples/ports/battle.ts +++ b/examples/ports/battle.ts @@ -1,6 +1,6 @@ // Pairwise beats absolute 1-10 for small judges. // Run: bun examples/ports/battle.ts -import { ask, evaluate } from "../../src/index.js"; +import { ask, runEvaluation } from "../../src/index.js"; import { localJev } from "../ollama.js"; const QUESTION = "What is the capital of France?"; @@ -21,13 +21,14 @@ const PAIRS: [string, string][] = [ ]; // Jev sees the case input, so each pair goes in as { question, A, B }. -const report = await evaluate("port-battle", { - run: (pair: { question: string; A: string; B: string }) => pair, - cases: PAIRS.map(([a, b]) => ({ +const report = await runEvaluation({ + name: "port-battle", + task: (pair: { question: string; A: string; B: string }) => pair, + data: PAIRS.map(([a, b]) => ({ name: `${a}-vs-${b}`, input: { question: QUESTION, A: ANSWERS[a] ?? "", B: ANSWERS[b] ?? "" }, })), - checks: { + scorers: { battle: ask("Which answer better serves the question? Correctness first, then completeness.", { options: { A: "answer A is better", diff --git a/examples/ports/factuality.ts b/examples/ports/factuality.ts index 1c14f0d..c448467 100644 --- a/examples/ports/factuality.ts +++ b/examples/ports/factuality.ts @@ -1,7 +1,7 @@ // ABCDE factuality rubric: A, B, C and E count as factual, D does not. // Agreement vs ground truth is the metric; exact match is the baseline. // Run: bun examples/ports/factuality.ts -import { ask, evaluate, matches } from "../../src/index.js"; +import { ask, runEvaluation, matches } from "../../src/index.js"; import { localJev } from "../ollama.js"; const CRITERIA = { @@ -74,10 +74,11 @@ const CASES: { id: string; question: string; expert: string; submission: string; }, ]; -const report = await evaluate("port-factuality", { - run: (question: string) => CASES.find((c) => c.question === question)?.submission ?? "", - cases: CASES.map((c) => ({ name: c.id, input: c.question, expect: c.expert })), - checks: { +const report = await runEvaluation({ + name: "port-factuality", + task: (question: string) => CASES.find((c) => c.question === question)?.submission ?? "", + data: CASES.map((c) => ({ name: c.id, input: c.question, expect: c.expert })), + scorers: { baseline_exact: matches(), jev_factuality: ask( "Compare the factual content of the output with the expert answer in expect. Ignore style, grammar, punctuation. Pick one option.", diff --git a/examples/ports/faithfulness.ts b/examples/ports/faithfulness.ts index 75c8487..1c379a6 100644 --- a/examples/ports/faithfulness.ts +++ b/examples/ports/faithfulness.ts @@ -1,7 +1,7 @@ // Claim-checking (extract claims, verify each against context) as one // Jev choice. Keyword coverage passes the vocab-reuse trap; the judge // must catch it. Run: bun examples/ports/faithfulness.ts -import { ask, evaluate } from "../../src/index.js"; +import { ask, runEvaluation } from "../../src/index.js"; import { localJev } from "../ollama.js"; import { coverage } from "../words.js"; @@ -53,10 +53,11 @@ const CASES: { id: string; context: string; answer: string; truth: Truth }[] = [ ]; // The context goes in as the input, so Jev sees it next to the answer. -const report = await evaluate("port-faithfulness", { - run: (context: string) => CASES.find((c) => c.context === context)?.answer ?? "", - cases: CASES.map((c) => ({ name: c.id, input: c.context })), - checks: { +const report = await runEvaluation({ + name: "port-faithfulness", + task: (context: string) => CASES.find((c) => c.context === context)?.answer ?? "", + data: CASES.map((c) => ({ name: c.id, input: c.context })), + scorers: { baseline_coverage: (r) => coverage(r.input, r.output) >= 0.5, jev_faithfulness: ask( "Is every claim in the output grounded in the context given as input? Pick one option.", diff --git a/examples/ports/keyword-coverage.ts b/examples/ports/keyword-coverage.ts index 2c016db..f87f75a 100644 --- a/examples/ports/keyword-coverage.ts +++ b/examples/ports/keyword-coverage.ts @@ -1,6 +1,6 @@ // Keyword overlap: exact match fails every paraphrase; coverage passes // them. Run: bun examples/ports/keyword-coverage.ts -import { evaluate, matches } from "../../src/index.js"; +import { runEvaluation, matches } from "../../src/index.js"; import { coverage } from "../words.js"; const CASES: { id: string; reference: string; output: string; truth: boolean }[] = [ @@ -36,10 +36,11 @@ const CASES: { id: string; reference: string; output: string; truth: boolean }[] }, ]; -const report = await evaluate("port-keyword-coverage", { - run: (id: string) => CASES.find((c) => c.id === id)?.output ?? "", - cases: CASES.map((c) => ({ input: c.id, expect: c.reference })), - checks: { +const report = await runEvaluation({ + name: "port-keyword-coverage", + task: (id: string) => CASES.find((c) => c.id === id)?.output ?? "", + data: CASES.map((c) => ({ input: c.id, expect: c.reference })), + scorers: { baseline_exact: matches(), port_coverage: (r) => coverage(r.expect ?? "", r.output) >= 0.5, }, diff --git a/examples/ports/mtbench-score.ts b/examples/ports/mtbench-score.ts index f3bfd17..fece94e 100644 --- a/examples/ports/mtbench-score.ts +++ b/examples/ports/mtbench-score.ts @@ -1,6 +1,6 @@ // 1-10 helpfulness as Jev levels. Rank order of four answers is the // ground truth. Run: bun examples/ports/mtbench-score.ts -import { ask, evaluate } from "../../src/index.js"; +import { ask, runEvaluation } from "../../src/index.js"; import { localJev } from "../ollama.js"; const LEVELS = [ @@ -25,11 +25,12 @@ const CASES: { id: string; rank: number; answer: string }[] = [ { id: "wrong", rank: 4, answer: "The capital of France is Lyon, a lovely city." }, ]; -const report = await evaluate("port-mtbench-score", { - run: (input: { question: string; id: string }) => +const report = await runEvaluation({ + name: "port-mtbench-score", + task: (input: { question: string; id: string }) => CASES.find((c) => c.id === input.id)?.answer ?? "", - cases: CASES.map((c) => ({ name: c.id, input: { question: QUESTION, id: c.id } })), - checks: { + data: CASES.map((c) => ({ name: c.id, input: { question: QUESTION, id: c.id } })), + scorers: { jev_helpfulness: ask( "Rate the output's helpfulness and correctness for the question. A bare fragment with no explanation caps at 5. A complete correct answer with supporting detail scores 8+. A wrong answer scores 1-2.", { levels: LEVELS, pass: "6: good" }, diff --git a/package.json b/package.json index 33ccb92..4cc752a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@corbits/evals", "version": "0.1.2", - "description": "Evals for agents and workflows in one evaluate call: code, Jev and helper checks over JSONL or inline cases, gated per check on allowed misses and drops.", + "description": "Evals for agents and workflows in one runEvaluation call: code, Jev and helper scorers over JSONL or inline data, gated per scorer on allowed misses and drops.", "keywords": [ "agents", "corbits", diff --git a/src/calibrate.ts b/src/calibrate.ts index 34872ad..3afed93 100644 --- a/src/calibrate.ts +++ b/src/calibrate.ts @@ -1,6 +1,6 @@ import type { Decision } from "@corbits/system-one"; -import { caseName } from "./evaluate.js"; +import { caseName } from "./evaluation.js"; import { DEFAULTS, decideBatch, toOutcome, type Ask, type Judge, type Sentence } from "./jev.js"; import { mapPool, runOnce, type RunSpec } from "./runner.js"; import { toTraced, type Runnable } from "./runs.js"; diff --git a/src/evaluate.ts b/src/evaluation.ts similarity index 68% rename from src/evaluate.ts rename to src/evaluation.ts index ec6b7b0..e88d040 100644 --- a/src/evaluate.ts +++ b/src/evaluation.ts @@ -8,7 +8,7 @@ import { toMarkdown, toStored, toSummary, - type Stored, + type Experiment, } from "./reports.js"; import { runCases, type NamedCase } from "./runner.js"; import { toTraced, type AgentLike, type Runnable } from "./runs.js"; @@ -16,7 +16,7 @@ import { toMs, type Duration } from "./timeout.js"; import type { Case, CaseResult, CheckFn, CheckResult, Report, Run } from "./types.js"; /** Code, a yes/no question for Jev, or `ask(...)`. */ -export type Check = +export type Scorer = | CheckFn | string | Ask @@ -25,34 +25,35 @@ export type Check = /** Cases a check may miss, or `{ miss, drop }`; both default to 0. */ export type Allow = number | { miss?: number; drop?: number }; -export type EvaluateOptions = { - run: Runnable; +export type Evaluation = { + name: string; + task: Runnable; /** Case objects, or the path of a JSONL file of them. */ - cases: readonly Case[] | string; + data: readonly Case[] | string; /** Defaults to `{ matches: matches() }`. */ - checks?: Record>; - /** Per check name. Unlisted checks may miss no case and drop nothing. */ + scorers?: Record>; + /** Per scorer name. Unlisted scorers may miss no case and drop nothing. */ allow?: Record; /** - * A stored run's path or a previous report to measure drops against. + * An experiment file's path, or an experiment, to measure drops against. * Never written; a missing file means drops are not gated. */ - compare?: string | Report; - /** Path to store this run at, pass or fail. */ + compare?: string | Experiment; + /** Path to store this experiment at, pass or fail. */ save?: string; repeats?: number; concurrency?: number; - /** Bounds each run and each check. A run that times out is aborted via its signal. */ + /** Bounds each task and each scorer. A task that times out is aborted via its signal. */ timeout?: Duration; - /** Stamped into the stored run, e.g. `{ model }`. */ + /** Stamped into the experiment, e.g. `{ model }`. */ about?: Record; - /** Jev for string and `ask` checks. Defaults to `jev()`. */ + /** Jev for string and `ask` scorers. Defaults to `jev()`. */ judge?: Judge; /** Skip the console summary. */ quiet?: boolean; }; -/** Thrown by `evaluate` when the gate fails; `report` holds every result. */ +/** Thrown by `runEvaluation` when the gate fails; `report` holds every result. */ export class EvalFailure extends Error { readonly report: Report; constructor(message: string, report: Report) { @@ -65,7 +66,7 @@ export class EvalFailure extends Error { function positiveInt(name: string, key: string, raw: number | undefined, fallback: number): number { if (raw === undefined) return fallback; if (!Number.isInteger(raw) || raw < 1) { - throw new Error(`eval "${name}": ${key} must be a whole number >= 1`); + throw new Error(`evaluation "${name}": ${key} must be a whole number >= 1`); } return raw; } @@ -73,7 +74,7 @@ function positiveInt(name: string, key: string, raw: number | undefined, fallbac function count(name: string, key: string, raw: unknown): number { if (raw === undefined) return 0; if (typeof raw !== "number" || !Number.isInteger(raw) || raw < 0) { - throw new Error(`eval "${name}": ${key} must be a whole number >= 0`); + throw new Error(`evaluation "${name}": ${key} must be a whole number >= 0`); } return raw; } @@ -87,7 +88,7 @@ function toAllowances( ): Record { const unknown = Object.keys(allow).find((key) => !checks.includes(key)); if (unknown !== undefined) { - throw new Error(`eval "${name}": allow names unknown check "${unknown}"`); + throw new Error(`evaluation "${name}": allow names unknown scorer "${unknown}"`); } return Object.fromEntries( checks.map((check) => { @@ -97,7 +98,7 @@ function toAllowances( } const extra = Object.keys(raw).find((key) => key !== "miss" && key !== "drop"); if (extra !== undefined) { - throw new Error(`eval "${name}": allow.${check} has unknown "${extra}"`); + throw new Error(`evaluation "${name}": allow.${check} has unknown "${extra}"`); } const { miss, drop } = raw as { miss?: unknown; drop?: unknown }; return [ @@ -125,21 +126,23 @@ function named( cases: readonly Case[], checks: readonly string[], ): Named[] { - if (cases.length === 0) throw new Error(`eval "${name}" has no cases`); + if (cases.length === 0) throw new Error(`evaluation "${name}" has no data`); const seen = new Set(); return cases.map((evalCase) => { const key = caseName(evalCase); - if (seen.has(key)) throw new Error(`eval "${name}" has duplicate case "${key}"`); + if (seen.has(key)) throw new Error(`evaluation "${name}" has duplicate case "${key}"`); seen.add(key); for (const field of ["must", "skip"] as const) { const unknown = evalCase[field]?.find((check) => !checks.includes(check)); if (unknown !== undefined) { - throw new Error(`eval "${name}": case "${key}" ${field} names unknown check "${unknown}"`); + throw new Error( + `evaluation "${name}": case "${key}" ${field} names unknown scorer "${unknown}"`, + ); } } const both = evalCase.must?.find((check) => evalCase.skip?.includes(check)); if (both !== undefined) { - throw new Error(`eval "${name}": case "${key}" both must and skip "${both}"`); + throw new Error(`evaluation "${name}": case "${key}" both must and skip "${both}"`); } return { ...evalCase, name: key }; }); @@ -147,13 +150,13 @@ function named( function toCheckParts( name: string, - checks: Record>, + checks: Record>, ): { code: Record) => unknown>; jev: Record; } { const entries = Object.entries(checks); - if (entries.length === 0) throw new Error(`eval "${name}" has no checks`); + if (entries.length === 0) throw new Error(`evaluation "${name}" has no scorers`); const code: Record) => unknown> = {}; const jev: Record = {}; for (const [key, check] of entries) { @@ -163,7 +166,7 @@ function toCheckParts( } const valid = typeof check === "string" ? check.trim() !== "" : typeof check?.question === "string"; - if (!valid) throw new Error(`eval "${name}": check "${key}" is not a check`); + if (!valid) throw new Error(`evaluation "${name}": check "${key}" is not a scorer`); checkThresholds(check as Ask | Sentence); jev[key] = check as Ask | Sentence | string; } @@ -174,7 +177,7 @@ function score( results: readonly CaseResult[], cases: readonly Named[], allowances: Record, - stored: Stored | undefined, + stored: Experiment | undefined, ): { checks: Record; reasons: string[] } { const must = new Map(cases.map((c) => [c.name, c.must ?? []])); const checks: Record = {}; @@ -217,20 +220,20 @@ function score( async function toCompared( name: string, - compare: string | Report | undefined, -): Promise { + compare: string | Experiment | undefined, +): Promise { if (compare === undefined) return undefined; if (typeof compare !== "string") { if (compare.name !== name) { - throw new Error(`eval "${name}": compare is a report of eval "${compare.name}"`); + throw new Error(`evaluation "${name}": compare is an experiment of "${compare.name}"`); } - return fromReport(compare); + return compare; } const file = Bun.file(compare); if (!(await file.exists())) return undefined; const stored = parseStored(await file.text(), compare); - if (stored.eval !== name) { - throw new Error(`${compare}: stored run of eval "${stored.eval}", not "${name}"`); + if (stored.name !== name) { + throw new Error(`${compare}: stored experiment "${stored.name}", not "${name}"`); } return stored; } @@ -253,41 +256,35 @@ function failures(cases: readonly CaseResult[], checks: Record( - name: string, - options: EvaluateOptions & { run: AgentLike }, +export async function runEvaluation( + spec: Evaluation & { task: AgentLike }, ): Promise>; -export async function evaluate( - name: string, - options: EvaluateOptions, +export async function runEvaluation( + spec: Evaluation, ): Promise>; -export async function evaluate( - name: string, - options: EvaluateOptions, -): Promise> { - if (name.trim() === "") throw new Error("evaluate needs a name"); - const repeats = positiveInt(name, "repeats", options.repeats, 1); - const concurrency = positiveInt(name, "concurrency", options.concurrency, 4); - const timeoutMs = options.timeout === undefined ? undefined : toMs(options.timeout, "timeout"); - let judge: Judge | undefined = options.judge; - const defined = options.checks ?? { matches: matches() as Check }; +export async function runEvaluation(spec: Evaluation): Promise> { + const name = spec.name; + if (typeof name !== "string" || name.trim() === "") throw new Error("runEvaluation needs a name"); + const repeats = positiveInt(name, "repeats", spec.repeats, 1); + const concurrency = positiveInt(name, "concurrency", spec.concurrency, 4); + const timeoutMs = spec.timeout === undefined ? undefined : toMs(spec.timeout, "timeout"); + let judge: Judge | undefined = spec.judge; + const defined = spec.scorers ?? { matches: matches() as Scorer }; const { code, jev: jevChecks } = toCheckParts(name, defined); const checkNames = Object.keys(defined); const jevGroup = Object.keys(jevChecks).length === 0 ? undefined : { judge: (judge ??= jev()), checks: jevChecks }; - const allowances = toAllowances(name, options.allow ?? {}, checkNames); + const allowances = toAllowances(name, spec.allow ?? {}, checkNames); const cases = named( name, - typeof options.cases === "string" - ? ((await fromJsonl(options.cases)) as Case[]) - : options.cases, + typeof spec.data === "string" ? ((await fromJsonl(spec.data)) as Case[]) : spec.data, checkNames, ); - const run = toTraced(options.run); + const run = toTraced(spec.task, "task"); - const stored = await toCompared(name, options.compare); + const stored = await toCompared(name, spec.compare); const results = await runCases({ cases, @@ -299,7 +296,7 @@ export async function evaluate( ...(timeoutMs === undefined ? {} : { timeoutMs }), }); const scored = score(results, cases, allowances, stored); - const data: Omit, "markdown"> = { + const data: Omit, "markdown" | "experiment"> = { name, ok: scored.reasons.length === 0, total: results.length, @@ -307,7 +304,7 @@ export async function evaluate( checks: scored.checks, cases: results, }; - if (options.about !== undefined) data.about = { ...options.about }; + if (spec.about !== undefined) data.about = { ...spec.about }; if (stored !== undefined) { const before = stored.cases; const now = new Set(results.map((c) => c.name)); @@ -315,20 +312,24 @@ export async function evaluate( data.added = results.filter((c) => before[c.name] === undefined).map((c) => c.name); data.removed = Object.keys(before).filter((c) => !now.has(c)); } - const report: Report = { ...data, markdown: () => toMarkdown(data) }; - const log = options.quiet === true ? () => {} : console.log; + const report: Report = { + ...data, + experiment: fromReport(data), + markdown: () => toMarkdown(data), + }; + const log = spec.quiet === true ? () => {} : console.log; log(toSummary(data)); - if (typeof options.compare === "string" && stored === undefined) { - log(` no compare run at ${options.compare}`); + if (typeof spec.compare === "string" && stored === undefined) { + log(` no compare run at ${spec.compare}`); } - if (options.save !== undefined) { - await Bun.write(options.save, toStored(data)); - log(` wrote ${options.save}`); + if (spec.save !== undefined) { + await Bun.write(spec.save, toStored(data)); + log(` wrote ${spec.save}`); } if (!report.ok) { throw new EvalFailure( [ - `eval "${name}" failed: ${scored.reasons.join("; ")}`, + `evaluation "${name}" failed: ${scored.reasons.join("; ")}`, ...failures(results, scored.checks), ].join("\n"), report, diff --git a/src/index.ts b/src/index.ts index dd86892..efe1428 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,7 +31,14 @@ export { type CalibrationRow, type Labeled, } from "./calibrate.js"; -export { EvalFailure, evaluate, type Allow, type Check, type EvaluateOptions } from "./evaluate.js"; +export { + EvalFailure, + runEvaluation, + type Allow, + type Evaluation, + type Scorer, +} from "./evaluation.js"; +export type { Experiment, ExperimentRow } from "./reports.js"; export { ask, jev, diff --git a/src/reports.ts b/src/reports.ts index ad05f51..f4c47ec 100644 --- a/src/reports.ts +++ b/src/reports.ts @@ -2,32 +2,32 @@ import type { CaseResult, Change, CheckResult, Outcome, Report } from "./types.j type Row = Record; -/** A report without its methods. */ -export type ReportData = Omit; +/** A report without its methods or its experiment. */ +export type ReportData = Omit; -/** Per case × check pass and value, without times, so it diffs cleanly in git. */ -export type Stored = { - eval: string; +/** Per case × scorer pass and value, without times, so it diffs cleanly in git. */ +export type Experiment = { + name: string; about?: Record; - cases: Record>; + cases: Record>; }; -/** One check's stored answer: the outcome plus a Jev distribution when present. */ -export type StoredRow = { +/** One scorer's stored answer: the outcome plus a Jev distribution when present. */ +export type ExperimentRow = { pass: boolean; value: number; probabilities?: Record; confidence?: number; }; -const STORED_FORMAT = 1; +const STORED_FORMAT = 2; function round(n: number): number { return Math.round(n * 1000) / 1000; } -export function fromReport(report: ReportData): Stored { - const cases: Stored["cases"] = {}; +export function fromReport(report: ReportData): Experiment { + const cases: Experiment["cases"] = {}; for (const c of report.cases) { cases[c.name] = Object.fromEntries( Object.entries(c.checks).map(([name, o]) => [ @@ -48,7 +48,7 @@ export function fromReport(report: ReportData): Stored { ); } return { - eval: report.name, + name: report.name, ...(report.about === undefined ? {} : { about: report.about }), cases, }; @@ -62,7 +62,7 @@ function isObject(value: unknown): value is Row { return typeof value === "object" && value !== null && !Array.isArray(value); } -function stored(raw: unknown, where: string): StoredRow { +function stored(raw: unknown, where: string): ExperimentRow { if (!isObject(raw) || typeof raw["pass"] !== "boolean") { throw new Error(`${where}: "pass" must be a boolean`); } @@ -70,7 +70,7 @@ function stored(raw: unknown, where: string): StoredRow { if (typeof value !== "number" || !Number.isFinite(value)) { throw new Error(`${where}: "value" must be a finite number`); } - const row: StoredRow = { pass: raw["pass"], value }; + const row: ExperimentRow = { pass: raw["pass"], value }; const probabilities = raw["probabilities"]; if (probabilities !== undefined) { if (!isObject(probabilities)) { @@ -93,7 +93,7 @@ function stored(raw: unknown, where: string): StoredRow { return row; } -export function parseStored(text: string, source: string): Stored { +export function parseStored(text: string, source: string): Experiment { let raw: unknown; try { raw = JSON.parse(text); @@ -106,12 +106,12 @@ export function parseStored(text: string, source: string): Stored { `${source}: unknown format ${JSON.stringify(raw["format"])}, expected ${STORED_FORMAT}`, ); } - const name = raw["eval"]; + const name = raw["name"]; if (typeof name !== "string" || name === "") { - throw new Error(`${source}: "eval" must be a non-empty string`); + throw new Error(`${source}: "name" must be a non-empty string`); } if (!isObject(raw["cases"])) throw new Error(`${source}: "cases" must be an object`); - const cases: Stored["cases"] = {}; + const cases: Experiment["cases"] = {}; for (const [caseName, checks] of Object.entries(raw["cases"])) { if (!isObject(checks)) throw new Error(`${source}: case "${caseName}" must be an object`); cases[caseName] = Object.fromEntries( @@ -121,7 +121,7 @@ export function parseStored(text: string, source: string): Stored { ]), ); } - const run: Stored = { eval: name, cases }; + const run: Experiment = { name, cases }; const about = raw["about"]; if (about !== undefined) { if (!isObject(about)) throw new Error(`${source}: "about" must be an object`); @@ -131,7 +131,7 @@ export function parseStored(text: string, source: string): Stored { } /** Case × check pass flips. Cases or checks on one side only are not changes. */ -export function flips(stored: Stored, cases: readonly CaseResult[]): Change[] { +export function flips(stored: Experiment, cases: readonly CaseResult[]): Change[] { const changes: Change[] = []; for (const c of cases) { const before = stored.cases[c.name]; @@ -166,7 +166,7 @@ export function toMarkdown(report: ReportData): string { const names = Object.keys(report.checks); const comparing = report.changes !== undefined; const lines = [ - `# eval: ${cell(report.name)}`, + `# evaluation: ${cell(report.name)}`, ``, `- cases: ${report.total}, passed: ${report.passed}`, `- result: ${report.ok ? "ok" : "FAILED"}`, diff --git a/src/runs.ts b/src/runs.ts index 08c5f42..f446d98 100644 --- a/src/runs.ts +++ b/src/runs.ts @@ -22,7 +22,7 @@ export type Traced = { export type Runnable = RunFn | Traced | AgentLike; -const NOT_RUNNABLE = "run must be a function, an @intx/agent Agent, or workflow()"; +const NOT_RUNNABLE = "must be a function, an @intx/agent Agent, or workflow()"; function isWorkflowDefinition(value: object): boolean { return "stepOrder" in value && "steps" in value && "triggers" in value; @@ -34,12 +34,13 @@ function isWorkflowDefinition(value: object): boolean { */ export function toTraced( run: Runnable, + noun = "run", ): (input: I, signal: AbortSignal) => Promise> { if (typeof run === "function") { return async (input, signal) => ({ output: await run(input, signal), steps: [] }); } if (typeof run !== "object" || run === null) { - throw new Error(NOT_RUNNABLE); + throw new Error(`${noun} ${NOT_RUNNABLE}`); } if (TRACE in run) return run[TRACE]; if ("send" in run && typeof run.send === "function") { @@ -52,7 +53,7 @@ export function toTraced( }; } if (isWorkflowDefinition(run)) { - throw new Error('run: wrap a workflow with workflow() from "@corbits/evals/workflow"'); + throw new Error(`${noun}: wrap a workflow with workflow() from "@corbits/evals/workflow"`); } - throw new Error(NOT_RUNNABLE); + throw new Error(`${noun} ${NOT_RUNNABLE}`); } diff --git a/src/types.ts b/src/types.ts index 632ec8d..756c72b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,3 +1,5 @@ +import type { Experiment } from "./reports.js"; + export type Step = { name: string; kind: "llm" | "tool" | "step"; @@ -90,7 +92,9 @@ export type Report = { changes?: Change[]; /** Cases only in this run. Present when comparing. */ added?: string[]; - /** Cases only in the compared run. Present when comparing. */ + /** Cases only in the compared experiment. Present when comparing. */ removed?: string[]; + /** The persisted experiment for this run. */ + experiment: Experiment; markdown(): string; };