feat: add the corbits-eval CLI

README.md

@@ -163,6 +163,35 @@ evaluation, with no file: `runEvaluation({ ...spec, compare: before.experiment }
 Flips print as `fixed` / `broke`, new and gone cases as `added` /
 `removed`. `quiet: true` skips the summary.
 
+## CLI
+
+The package bin is `corbits-eval` (`./dist/cli/index.js`).
+
+```sh
+corbits-eval run ./evals/refund.ts
+corbits-eval run ./evals/refund.ts --format json
+```
+
+The file is an ES module path resolved from the current working directory.
+`corbits-eval` loads the `evaluation` export when it is a spec object, otherwise
+the default export when that is a spec object, otherwise a zero-argument
+function on either of those exports. The function must return a spec. No other
+exports are read.
+
+A spec is the object `runEvaluation` accepts (`name`, `data`, `task`,
+`scorers`, and the rest). Case rows use `expectedOutput`. The CLI calls
+`runEvaluation`; it does not score on its own.
+
+Exit codes:
+
+| code | when                                                                                           |
+| ---- | ---------------------------------------------------------------------------------------------- |
+| 0    | the run passed its gate                                                                        |
+| 1    | the evaluation failed its gate                                                                 |
+| 2    | the file is missing, the import failed, there is no spec export, or the evaluation did not run |
+
+`--format json` (and `--json`) come from incur and print only that result as JSON on stdout, with no human summary. Exit 1 is only a failed gate (`EVAL_FAILED`). Config that stops the run before it starts, such as empty `data` or a string or `ask` scorer with no `judge.invoke`, exits 2.
+
 ## Calibrating
 
 Thresholds are per-workload starting points, not constants. `calibrate`

e2e/cli.e2e.test.ts

@@ -0,0 +1,90 @@
+import { describe, expect, test } from "bun:test";
+import { join } from "node:path";
+
+const root = join(import.meta.dir, "..");
+const cli = join(root, "src/cli/index.ts");
+const fixtures = join(import.meta.dir, "fixtures");
+
+async function run(
+  file: string,
+  args: readonly string[] = [],
+): Promise<{ code: number; stdout: string; stderr: string }> {
+  const proc = Bun.spawn(["bun", cli, "run", file, ...args], {
+    cwd: root,
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const [stdout, stderr] = await Promise.all([
+    new Response(proc.stdout).text(),
+    new Response(proc.stderr).text(),
+  ]);
+  await proc.exited;
+  return { code: proc.exitCode ?? 1, stdout, stderr };
+}
+
+function jsonBody(stdout: string): unknown {
+  return JSON.parse(stdout);
+}
+
+describe("corbits-eval run", () => {
+  test("a passing evaluation exits 0", async () => {
+    expect((await run(join(fixtures, "pass.ts"))).code).toBe(0);
+  });
+
+  test("a failed gate exits 1", async () => {
+    expect((await run(join(fixtures, "fail.ts"))).code).toBe(1);
+  });
+
+  test("a module with no spec exits 2", async () => {
+    expect((await run(join(fixtures, "missing-export.ts"))).code).toBe(2);
+  });
+
+  test("a missing file exits 2", async () => {
+    expect((await run(join(fixtures, "does-not-exist.ts"))).code).toBe(2);
+  });
+
+  test("--format json stdout is only JSON for a pass and a gate failure", async () => {
+    const passed = await run(join(fixtures, "pass.ts"), ["--format", "json"]);
+    expect(passed.code).toBe(0);
+    expect(passed.stdout).not.toContain("cases passed");
+    expect(jsonBody(passed.stdout)).toMatchObject({ name: "pass", ok: true });
+
+    const failed = await run(join(fixtures, "fail.ts"), ["--format", "json"]);
+    expect(failed.code).toBe(1);
+    expect(failed.stdout).not.toContain("cases passed");
+    expect(jsonBody(failed.stdout)).toMatchObject({ code: "EVAL_FAILED" });
+  });
+
+  test("--json stdout is only JSON for a pass and a gate failure", async () => {
+    const passed = await run(join(fixtures, "pass.ts"), ["--json"]);
+    expect(passed.code).toBe(0);
+    expect(passed.stdout).not.toContain("cases passed");
+    expect(jsonBody(passed.stdout)).toMatchObject({ name: "pass", ok: true });
+
+    const failed = await run(join(fixtures, "fail.ts"), ["--json"]);
+    expect(failed.code).toBe(1);
+    expect(failed.stdout).not.toContain("cases passed");
+    expect(jsonBody(failed.stdout)).toMatchObject({ code: "EVAL_FAILED" });
+  });
+
+  test("a non-spec export exits 2", async () => {
+    const result = await run(join(fixtures, "not-a-spec.ts"));
+    expect(result.code).toBe(2);
+    expect(result.stdout).toContain("no evaluation spec");
+    expect(result.stdout).not.toContain("cases.map is not a function");
+  });
+
+  test("empty data exits 2", async () => {
+    const result = await run(join(fixtures, "empty.ts"));
+    expect(result.code).toBe(2);
+    expect(result.stdout).toContain("EVAL_CONFIG");
+    expect(result.stdout).toContain("has no data");
+  });
+
+  test("a string scorer with no judge exits 2", async () => {
+    const result = await run(join(fixtures, "needs-judge.ts"));
+    expect(result.code).toBe(2);
+    expect(result.stdout).toContain("EVAL_CONFIG");
+    expect(result.stdout).toContain("string and ask checks need judge.invoke");
+  });
+});

e2e/fixtures/empty.ts

@@ -0,0 +1,5 @@
+export const evaluation = {
+  name: "empty",
+  task: (input: string) => input,
+  data: [],
+};

e2e/fixtures/fail.ts

@@ -0,0 +1,5 @@
+export const evaluation = {
+  name: "fail",
+  task: (input: string) => input.toUpperCase(),
+  data: [{ input: "hi", expectedOutput: "nope" }],
+};

e2e/fixtures/missing-export.ts

@@ -0,0 +1 @@
+export const note = "not a spec";

e2e/fixtures/needs-judge.ts

@@ -0,0 +1,6 @@
+export const evaluation = {
+  name: "needs-judge",
+  task: (input: string) => input,
+  data: [{ input: "hi" }],
+  scorers: { fine: "is it fine?" },
+};

e2e/fixtures/not-a-spec.ts

@@ -0,0 +1 @@
+export const evaluation = { name: "bad", data: 1, task: "nope" };

e2e/fixtures/pass.ts

@@ -0,0 +1,5 @@
+export const evaluation = {
+  name: "pass",
+  task: (input: string) => input.toUpperCase(),
+  data: [{ input: "hi", expectedOutput: "HI" }],
+};

package.json

@@ -19,6 +19,9 @@
     "type": "git",
     "url": "git+https://github.com/corbitsdev/corbits-evals.git"
   },
+  "bin": {
+    "corbits-eval": "./dist/cli/index.js"
+  },
   "files": [
     "dist",
     "README.md",
@@ -51,7 +54,8 @@
     "check": "bun run typecheck && bun run lint && bun run format:check && bun run test && bun run test:e2e"
   },
   "dependencies": {
-    "@corbits/system-one": "^0.3.0"
+    "@corbits/system-one": "^0.3.0",
+    "incur": "^0.7.1"
   },
   "devDependencies": {
     "@intx/agent": "^0.5.0",
@@ -72,6 +76,7 @@
     }
   },
   "engines": {
-    "bun": ">=1.2.0"
+    "bun": ">=1.2.0",
+    "node": ">=22"
   }
 }

src/cli/index.ts

@@ -0,0 +1,12 @@
+#!/usr/bin/env bun
+import { Cli } from "incur";
+
+const cli = Cli.create("corbits-eval", {
+  description: "Run a corbits evaluation file",
+}).fs(new URL("./", import.meta.url));
+
+if (import.meta.main) {
+  await cli.serve();
+}
+
+export { cli };

src/cli/run.ts

@@ -0,0 +1,31 @@
+import { Cli, z } from "incur";
+
+import { loadEvalFile } from "../evalFile.js";
+import { EvalFailure, runEvaluation } from "../evaluation.js";
+
+export default Cli.command({
+  description: "Run an evaluation module",
+  args: z.object({
+    file: z.string().describe("Evaluation module path, resolved from the current directory"),
+  }),
+  async run(c) {
+    let spec;
+    try {
+      spec = await loadEvalFile(c.args.file);
+    } catch (cause) {
+      const message = cause instanceof Error ? cause.message : String(cause);
+      return c.error({ code: "EVAL_FILE", message, exitCode: 2 });
+    }
+
+    try {
+      const report = await runEvaluation(c.format === "json" ? { ...spec, quiet: true } : spec);
+      return { name: report.name, ok: report.ok, passed: report.passed, total: report.total };
+    } catch (cause) {
+      if (cause instanceof EvalFailure) {
+        return c.error({ code: "EVAL_FAILED", message: cause.message, exitCode: 1 });
+      }
+      const message = cause instanceof Error ? cause.message : String(cause);
+      return c.error({ code: "EVAL_CONFIG", message, exitCode: 2 });
+    }
+  },
+});

src/evalFile.ts

@@ -0,0 +1,75 @@
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+
+import type { Evaluation } from "./evaluation.js";
+import { TRACE } from "./runs.js";
+
+/** A missing file, a failed import, or a module with no evaluation spec. */
+export class EvalFileError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "EvalFileError";
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+/** A task `toTraced` can run: a function, `workflow()`, or an agent with `send`. */
+function isRunnable(value: unknown): boolean {
+  if (typeof value === "function") return true;
+  if (!isRecord(value)) return false;
+  if (TRACE in value) return true;
+  return typeof value["send"] === "function";
+}
+
+function isData(value: unknown): boolean {
+  return Array.isArray(value) || typeof value === "string";
+}
+
+function isSpec(value: unknown): value is Evaluation<unknown, unknown, unknown> {
+  return (
+    isRecord(value) &&
+    typeof value["name"] === "string" &&
+    isData(value["data"]) &&
+    isRunnable(value["task"])
+  );
+}
+
+function isZeroArg(value: unknown): value is () => unknown {
+  return typeof value === "function" && value.length === 0;
+}
+
+async function fromFactory(
+  value: unknown,
+): Promise<Evaluation<unknown, unknown, unknown> | undefined> {
+  if (!isZeroArg(value)) return undefined;
+  const produced: unknown = await value();
+  if (isSpec(produced)) return produced;
+  throw new EvalFileError("evaluation function did not return a spec");
+}
+
+/**
+ * Load an evaluation module. Uses `evaluation` when it is a spec, else `default`
+ * when it is a spec, else a zero-arg function on either export.
+ */
+export async function loadEvalFile(file: string): Promise<Evaluation<unknown, unknown, unknown>> {
+  const href = pathToFileURL(resolve(file)).href;
+  let mod: Record<string, unknown>;
+  try {
+    mod = (await import(href)) as Record<string, unknown>;
+  } catch (cause) {
+    const detail = cause instanceof Error ? cause.message : String(cause);
+    throw new EvalFileError(`could not import ${file}: ${detail}`);
+  }
+
+  const evaluation = mod["evaluation"];
+  const fallback = mod["default"];
+  if (isSpec(evaluation)) return evaluation;
+  if (isSpec(fallback)) return fallback;
+
+  const produced = (await fromFactory(evaluation)) ?? (await fromFactory(fallback));
+  if (produced !== undefined) return produced;
+  throw new EvalFileError(`${file} has no evaluation spec export`);
+}