diff --git a/README.md b/README.md index 95eb742..1e88556 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,35 @@ evaluation, with no file: `runEvaluation({ ...spec, compare: before.experiment } Flips print as `fixed` / `broke`, new and gone cases as `added` / `removed`. `quiet: true` skips the summary. +## CLI + +The package bin is `corbits-eval` (`./dist/cli/index.js`). + +```sh +corbits-eval run ./evals/refund.ts +corbits-eval run ./evals/refund.ts --format json +``` + +The file is an ES module path resolved from the current working directory. +`corbits-eval` loads the `evaluation` export when it is a spec object, otherwise +the default export when that is a spec object, otherwise a zero-argument +function on either of those exports. The function must return a spec. No other +exports are read. + +A spec is the object `runEvaluation` accepts (`name`, `data`, `task`, +`scorers`, and the rest). Case rows use `expectedOutput`. The CLI calls +`runEvaluation`; it does not score on its own. + +Exit codes: + +| code | when | +| ---- | ---------------------------------------------------------------------------------------------- | +| 0 | the run passed its gate | +| 1 | the evaluation failed its gate | +| 2 | the file is missing, the import failed, there is no spec export, or the evaluation did not run | + +`--format json` (and `--json`) come from incur and print only that result as JSON on stdout, with no human summary. Exit 1 is only a failed gate (`EVAL_FAILED`). Config that stops the run before it starts, such as empty `data` or a string or `ask` scorer with no `judge.invoke`, exits 2. + ## Calibrating Thresholds are per-workload starting points, not constants. `calibrate` diff --git a/bun.lock b/bun.lock index 78def42..9de4ace 100644 --- a/bun.lock +++ b/bun.lock @@ -6,6 +6,7 @@ "name": "@corbits/evals", "dependencies": { "@corbits/system-one": "^0.3.0", + "incur": "^0.7.1", }, "devDependencies": { "@intx/agent": "^0.5.0", @@ -30,6 +31,8 @@ "@ark/util": ["@ark/util@0.56.2", "", {}, "sha512-9kU2sUE38FZEGG7l3hamYMBieLYEJh2L1mrYD2eXpT+78EnQSV1bhjxJhnxGBMSTbtwpBSDNSK+K60WvaI/DTQ=="], + "@cfworker/json-schema": ["@cfworker/json-schema@4.1.1", "", {}, "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og=="], + "@corbits/system-one": ["@corbits/system-one@0.3.0", "", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": "^0.4.0", "@intx/types": "^0.4.0" } }, "sha512-rH1MSTyTph1O3wRjVuh2gbunfPLXK0HFDjOym9CkXtcDpRa6J+sWiDsnHdIla3KnVxPWQ2dTbYYck6KlTghtIw=="], "@intx/agent": ["@intx/agent@0.5.0", "", { "dependencies": { "@intx/inference": "0.5.0", "@intx/log": "0.5.0", "@intx/mime": "0.5.0", "@intx/types": "0.5.0", "arktype": "^2.1.29" } }, "sha512-EWkRfduYJii67goq0cdO6qgRtV0aLe2jGk2aZJhurOivluy3tx0SlhiO2YaQmA4du+qBNzlMV9MMfKkmQ/zs+Q=="], @@ -50,6 +53,8 @@ "@logtape/logtape": ["@logtape/logtape@2.3.11", "", {}, "sha512-T7T+IShQBxofHCs7J1kVMZPLLHYAXAOqPEc/xwzd6/2q8va/D/t/HP7uCeJzTQOywpXF1tbyz15O4VHT8NF2hw=="], + "@modelcontextprotocol/server": ["@modelcontextprotocol/server@2.0.0-alpha.4", "", { "dependencies": { "zod": "^4.2.0" } }, "sha512-/KEo3ZJ50HlagHp0lz2vPgfBZFtXHu6zTBXT9XqPc+4O9i4+IbBVWKq/a9yAfv/ifp0u3+mRo8SUk5kMKzhN8A=="], + "@oxfmt/binding-android-arm-eabi": ["@oxfmt/binding-android-arm-eabi@0.70.0", "", { "os": "android", "cpu": "arm" }, "sha512-Xd7YO4/T2axEj6FTLcj4Why3mTBqFMg+x24xtorT4Lb2+1g82090GH0a/4U1m0pABGYiix2bq1pqkYrmV3f0Sw=="], "@oxfmt/binding-android-arm64": ["@oxfmt/binding-android-arm64@0.70.0", "", { "os": "android", "cpu": "arm64" }, "sha512-x9rlMYyKXdgKdYyUJzGsK1ZV8P4di/J32ipzcS6Jet6p9r9UAh28neXIMtdlSaJJycdi61Z4YkcLKLpk8ueFjg=="], @@ -126,6 +131,10 @@ "@oxlint/binding-win32-x64-msvc": ["@oxlint/binding-win32-x64-msvc@1.85.0", "", { "os": "win32", "cpu": "x64" }, "sha512-pBebIPUpKKhWrhSMWhy8TdAZBewiXnfxmaAGxhzxM1068GagqFaTwgKlU6e+UyJ2sPR+VoHouhXuGJkQjsrDvA=="], + "@scalar/openapi-types": ["@scalar/openapi-types@0.8.0", "", {}, "sha512-WmaxVSfvY5K/TwcG2B2TU1WOe1As1uc2s7myswtP6dBlcjU3hM08SApxv/jmyGaCE8t4gO5BBhmHY4pDUfmr2g=="], + + "@toon-format/toon": ["@toon-format/toon@2.3.1", "", {}, "sha512-sWqEMeAJGMda9qRrfPKrX3C4c33CO9SJuvJzzrcBEn5sbDB+k6pSLkDsfN0rVnLQwR97MV3sDgVbcxQNsq8xVw=="], + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], "@types/node": ["@types/node@26.6.4", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg=="], @@ -138,6 +147,8 @@ "hono": ["hono@4.13.13", "", {}, "sha512-CQ46U0ZkAGmbT/4UxdzzGJpacP2IeKgY4a5/tOI9AABbpOMfK739wfDXmv1usCk+3RkKj1hQy4/fjhiwa2xlrA=="], + "incur": ["incur@0.7.1", "", { "dependencies": { "@cfworker/json-schema": "^4.1.1", "@modelcontextprotocol/server": "2.0.0-alpha.4", "@scalar/openapi-types": "^0.8.0", "@toon-format/toon": "^2.3.1", "tokenx": "^1.3.0", "yaml": "^2.8.2", "zod": "^4.3.6" }, "bin": { "incur": "dist/cli/index.js", "incur.src": "src/cli/index.ts" } }, "sha512-tXEStxASXtVWOjY7wNBBMy03+3HPpiLPkrEaqNb5iYerRNRWqTYqwdAEgLfU6Ga2OUrz9LpRuMWFvdj5kIaUlg=="], + "oxfmt": ["oxfmt@0.70.0", "", { "dependencies": { "tinypool": "2.1.2" }, "optionalDependencies": { "@oxfmt/binding-android-arm-eabi": "0.70.0", "@oxfmt/binding-android-arm64": "0.70.0", "@oxfmt/binding-darwin-arm64": "0.70.0", "@oxfmt/binding-darwin-x64": "0.70.0", "@oxfmt/binding-freebsd-x64": "0.70.0", "@oxfmt/binding-linux-arm-gnueabihf": "0.70.0", "@oxfmt/binding-linux-arm-musleabihf": "0.70.0", "@oxfmt/binding-linux-arm64-gnu": "0.70.0", "@oxfmt/binding-linux-arm64-musl": "0.70.0", "@oxfmt/binding-linux-ppc64-gnu": "0.70.0", "@oxfmt/binding-linux-riscv64-gnu": "0.70.0", "@oxfmt/binding-linux-riscv64-musl": "0.70.0", "@oxfmt/binding-linux-s390x-gnu": "0.70.0", "@oxfmt/binding-linux-x64-gnu": "0.70.0", "@oxfmt/binding-linux-x64-musl": "0.70.0", "@oxfmt/binding-openharmony-arm64": "0.70.0", "@oxfmt/binding-win32-arm64-msvc": "0.70.0", "@oxfmt/binding-win32-ia32-msvc": "0.70.0", "@oxfmt/binding-win32-x64-msvc": "0.70.0" }, "peerDependencies": { "svelte": "^5.0.0", "vite-plus": "*" }, "optionalPeers": ["svelte", "vite-plus"], "bin": { "oxfmt": "bin/oxfmt" } }, "sha512-IsHxZ4y0wQLLMhnrJblBJgZsLDzfULrJnAw5j/QqsTlMa/m3AqsbToi+W71uhBGaqlqq/PbbjvHc09TJwdv3Tw=="], "oxlint": ["oxlint@1.85.0", "", { "optionalDependencies": { "@oxlint/binding-android-arm-eabi": "1.85.0", "@oxlint/binding-android-arm64": "1.85.0", "@oxlint/binding-darwin-arm64": "1.85.0", "@oxlint/binding-darwin-x64": "1.85.0", "@oxlint/binding-freebsd-x64": "1.85.0", "@oxlint/binding-linux-arm-gnueabihf": "1.85.0", "@oxlint/binding-linux-arm-musleabihf": "1.85.0", "@oxlint/binding-linux-arm64-gnu": "1.85.0", "@oxlint/binding-linux-arm64-musl": "1.85.0", "@oxlint/binding-linux-ppc64-gnu": "1.85.0", "@oxlint/binding-linux-riscv64-gnu": "1.85.0", "@oxlint/binding-linux-riscv64-musl": "1.85.0", "@oxlint/binding-linux-s390x-gnu": "1.85.0", "@oxlint/binding-linux-x64-gnu": "1.85.0", "@oxlint/binding-linux-x64-musl": "1.85.0", "@oxlint/binding-openharmony-arm64": "1.85.0", "@oxlint/binding-win32-arm64-msvc": "1.85.0", "@oxlint/binding-win32-ia32-msvc": "1.85.0", "@oxlint/binding-win32-x64-msvc": "1.85.0" }, "peerDependencies": { "oxlint-tsgolint": ">=7.0.2001", "vite-plus": "*" }, "optionalPeers": ["oxlint-tsgolint", "vite-plus"], "bin": { "oxlint": "bin/oxlint" } }, "sha512-bc26s97nuvPj1ViyPsqmKecVkUWFMEdtayO8MaQ6oiLfs1pj94cQlZZhrh4BPNlr9HQosjhIlwgZKsfcwmcNgg=="], @@ -146,10 +157,16 @@ "tinypool": ["tinypool@2.1.2", "", {}, "sha512-9YodfrxS9g9IbFr/KOjE5bAeJ0p61n3bW6mqvy0jtoeKd1kTW1Cxm0oulm6KX2lyM9Gl6WIe8nEbY7LWv5ZJww=="], + "tokenx": ["tokenx@1.6.0", "", {}, "sha512-CKTjk345ajvBAUp5xUI9a5KKN0zU0lBueVHQbCskH1Hp6WkUKsPW2qGCYNs0pxNyfzxfo+IIjdt2W4sMbw/qBw=="], + "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], "undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="], + "yaml": ["yaml@2.9.1", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw=="], + + "zod": ["zod@4.6.7", "", {}, "sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg=="], + "@corbits/system-one/@intx/inference": ["@intx/inference@0.4.0", "", { "dependencies": { "@intx/log": "0.4.0", "@intx/types": "0.4.0", "arktype": "^2.1.29" } }, "sha512-cuaqCqotmjg0OzGUm8QmFJr1xX5wPVS0LeuhrxHsj0X3ImoCzVB4e/ttTc+DaP3K1tTMQ/8Q8HHCAPX36mOecQ=="], "@corbits/system-one/@intx/inference/@intx/log": ["@intx/log@0.4.0", "", { "dependencies": { "@logtape/hono": "^2.0.2", "@logtape/logtape": "^2.0.2" }, "peerDependencies": { "hono": "^4.0.0" }, "optionalPeers": ["hono"] }, "sha512-f1IPZ/9YpEqlA0I2UZPOlRJOTDoOFEZRwZoUVuw8DpgR1WatOMm4hA0E8o6Mtz+pA0ux7gVdls4y/JZcgypsfQ=="], diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts new file mode 100644 index 0000000..e49c7bd --- /dev/null +++ b/e2e/cli.e2e.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; + +const root = join(import.meta.dir, ".."); +const cli = join(root, "src/cli/index.ts"); +const fixtures = join(import.meta.dir, "fixtures"); + +async function run( + file: string, + args: readonly string[] = [], +): Promise<{ code: number; stdout: string; stderr: string }> { + const proc = Bun.spawn(["bun", cli, "run", file, ...args], { + cwd: root, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + await proc.exited; + return { code: proc.exitCode ?? 1, stdout, stderr }; +} + +function jsonBody(stdout: string): unknown { + return JSON.parse(stdout); +} + +describe("corbits-eval run", () => { + test("a passing evaluation exits 0", async () => { + expect((await run(join(fixtures, "pass.ts"))).code).toBe(0); + }); + + test("a failed gate exits 1", async () => { + expect((await run(join(fixtures, "fail.ts"))).code).toBe(1); + }); + + test("a module with no spec exits 2", async () => { + expect((await run(join(fixtures, "missing-export.ts"))).code).toBe(2); + }); + + test("a missing file exits 2", async () => { + expect((await run(join(fixtures, "does-not-exist.ts"))).code).toBe(2); + }); + + test("--format json stdout is only JSON for a pass and a gate failure", async () => { + const passed = await run(join(fixtures, "pass.ts"), ["--format", "json"]); + expect(passed.code).toBe(0); + expect(passed.stdout).not.toContain("cases passed"); + expect(jsonBody(passed.stdout)).toMatchObject({ name: "pass", ok: true }); + + const failed = await run(join(fixtures, "fail.ts"), ["--format", "json"]); + expect(failed.code).toBe(1); + expect(failed.stdout).not.toContain("cases passed"); + expect(jsonBody(failed.stdout)).toMatchObject({ code: "EVAL_FAILED" }); + }); + + test("--json stdout is only JSON for a pass and a gate failure", async () => { + const passed = await run(join(fixtures, "pass.ts"), ["--json"]); + expect(passed.code).toBe(0); + expect(passed.stdout).not.toContain("cases passed"); + expect(jsonBody(passed.stdout)).toMatchObject({ name: "pass", ok: true }); + + const failed = await run(join(fixtures, "fail.ts"), ["--json"]); + expect(failed.code).toBe(1); + expect(failed.stdout).not.toContain("cases passed"); + expect(jsonBody(failed.stdout)).toMatchObject({ code: "EVAL_FAILED" }); + }); + + test("a non-spec export exits 2", async () => { + const result = await run(join(fixtures, "not-a-spec.ts")); + expect(result.code).toBe(2); + expect(result.stdout).toContain("no evaluation spec"); + expect(result.stdout).not.toContain("cases.map is not a function"); + }); + + test("empty data exits 2", async () => { + const result = await run(join(fixtures, "empty.ts")); + expect(result.code).toBe(2); + expect(result.stdout).toContain("EVAL_CONFIG"); + expect(result.stdout).toContain("has no data"); + }); + + test("a string scorer with no judge exits 2", async () => { + const result = await run(join(fixtures, "needs-judge.ts")); + expect(result.code).toBe(2); + expect(result.stdout).toContain("EVAL_CONFIG"); + expect(result.stdout).toContain("string and ask checks need judge.invoke"); + }); +}); diff --git a/e2e/fixtures/empty.ts b/e2e/fixtures/empty.ts new file mode 100644 index 0000000..7002e16 --- /dev/null +++ b/e2e/fixtures/empty.ts @@ -0,0 +1,5 @@ +export const evaluation = { + name: "empty", + task: (input: string) => input, + data: [], +}; diff --git a/e2e/fixtures/fail.ts b/e2e/fixtures/fail.ts new file mode 100644 index 0000000..aee0080 --- /dev/null +++ b/e2e/fixtures/fail.ts @@ -0,0 +1,5 @@ +export const evaluation = { + name: "fail", + task: (input: string) => input.toUpperCase(), + data: [{ input: "hi", expectedOutput: "nope" }], +}; diff --git a/e2e/fixtures/missing-export.ts b/e2e/fixtures/missing-export.ts new file mode 100644 index 0000000..6437c87 --- /dev/null +++ b/e2e/fixtures/missing-export.ts @@ -0,0 +1 @@ +export const note = "not a spec"; diff --git a/e2e/fixtures/needs-judge.ts b/e2e/fixtures/needs-judge.ts new file mode 100644 index 0000000..3ed3db4 --- /dev/null +++ b/e2e/fixtures/needs-judge.ts @@ -0,0 +1,6 @@ +export const evaluation = { + name: "needs-judge", + task: (input: string) => input, + data: [{ input: "hi" }], + scorers: { fine: "is it fine?" }, +}; diff --git a/e2e/fixtures/not-a-spec.ts b/e2e/fixtures/not-a-spec.ts new file mode 100644 index 0000000..a68f193 --- /dev/null +++ b/e2e/fixtures/not-a-spec.ts @@ -0,0 +1 @@ +export const evaluation = { name: "bad", data: 1, task: "nope" }; diff --git a/e2e/fixtures/pass.ts b/e2e/fixtures/pass.ts new file mode 100644 index 0000000..a0732e6 --- /dev/null +++ b/e2e/fixtures/pass.ts @@ -0,0 +1,5 @@ +export const evaluation = { + name: "pass", + task: (input: string) => input.toUpperCase(), + data: [{ input: "hi", expectedOutput: "HI" }], +}; diff --git a/package.json b/package.json index 4cc752a..6fc7452 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,9 @@ "type": "git", "url": "git+https://github.com/corbitsdev/corbits-evals.git" }, + "bin": { + "corbits-eval": "./dist/cli/index.js" + }, "files": [ "dist", "README.md", @@ -51,7 +54,8 @@ "check": "bun run typecheck && bun run lint && bun run format:check && bun run test && bun run test:e2e" }, "dependencies": { - "@corbits/system-one": "^0.3.0" + "@corbits/system-one": "^0.3.0", + "incur": "^0.7.1" }, "devDependencies": { "@intx/agent": "^0.5.0", @@ -72,6 +76,7 @@ } }, "engines": { - "bun": ">=1.2.0" + "bun": ">=1.2.0", + "node": ">=22" } } diff --git a/src/cli/index.ts b/src/cli/index.ts new file mode 100644 index 0000000..573f412 --- /dev/null +++ b/src/cli/index.ts @@ -0,0 +1,12 @@ +#!/usr/bin/env bun +import { Cli } from "incur"; + +const cli = Cli.create("corbits-eval", { + description: "Run a corbits evaluation file", +}).fs(new URL("./", import.meta.url)); + +if (import.meta.main) { + await cli.serve(); +} + +export { cli }; diff --git a/src/cli/run.ts b/src/cli/run.ts new file mode 100644 index 0000000..241e551 --- /dev/null +++ b/src/cli/run.ts @@ -0,0 +1,31 @@ +import { Cli, z } from "incur"; + +import { loadEvalFile } from "../evalFile.js"; +import { EvalFailure, runEvaluation } from "../evaluation.js"; + +export default Cli.command({ + description: "Run an evaluation module", + args: z.object({ + file: z.string().describe("Evaluation module path, resolved from the current directory"), + }), + async run(c) { + let spec; + try { + spec = await loadEvalFile(c.args.file); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + return c.error({ code: "EVAL_FILE", message, exitCode: 2 }); + } + + try { + const report = await runEvaluation(c.format === "json" ? { ...spec, quiet: true } : spec); + return { name: report.name, ok: report.ok, passed: report.passed, total: report.total }; + } catch (cause) { + if (cause instanceof EvalFailure) { + return c.error({ code: "EVAL_FAILED", message: cause.message, exitCode: 1 }); + } + const message = cause instanceof Error ? cause.message : String(cause); + return c.error({ code: "EVAL_CONFIG", message, exitCode: 2 }); + } + }, +}); diff --git a/src/evalFile.ts b/src/evalFile.ts new file mode 100644 index 0000000..430c0b9 --- /dev/null +++ b/src/evalFile.ts @@ -0,0 +1,75 @@ +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +import type { Evaluation } from "./evaluation.js"; +import { TRACE } from "./runs.js"; + +/** A missing file, a failed import, or a module with no evaluation spec. */ +export class EvalFileError extends Error { + constructor(message: string) { + super(message); + this.name = "EvalFileError"; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +/** A task `toTraced` can run: a function, `workflow()`, or an agent with `send`. */ +function isRunnable(value: unknown): boolean { + if (typeof value === "function") return true; + if (!isRecord(value)) return false; + if (TRACE in value) return true; + return typeof value["send"] === "function"; +} + +function isData(value: unknown): boolean { + return Array.isArray(value) || typeof value === "string"; +} + +function isSpec(value: unknown): value is Evaluation { + return ( + isRecord(value) && + typeof value["name"] === "string" && + isData(value["data"]) && + isRunnable(value["task"]) + ); +} + +function isZeroArg(value: unknown): value is () => unknown { + return typeof value === "function" && value.length === 0; +} + +async function fromFactory( + value: unknown, +): Promise | undefined> { + if (!isZeroArg(value)) return undefined; + const produced: unknown = await value(); + if (isSpec(produced)) return produced; + throw new EvalFileError("evaluation function did not return a spec"); +} + +/** + * Load an evaluation module. Uses `evaluation` when it is a spec, else `default` + * when it is a spec, else a zero-arg function on either export. + */ +export async function loadEvalFile(file: string): Promise> { + const href = pathToFileURL(resolve(file)).href; + let mod: Record; + try { + mod = (await import(href)) as Record; + } catch (cause) { + const detail = cause instanceof Error ? cause.message : String(cause); + throw new EvalFileError(`could not import ${file}: ${detail}`); + } + + const evaluation = mod["evaluation"]; + const fallback = mod["default"]; + if (isSpec(evaluation)) return evaluation; + if (isSpec(fallback)) return fallback; + + const produced = (await fromFactory(evaluation)) ?? (await fromFactory(fallback)); + if (produced !== undefined) return produced; + throw new EvalFileError(`${file} has no evaluation spec export`); +}