Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"url": "git+https://github.com/rendotdev/rig.git"
},
"bin": {
"rig": "dist/rig.mjs"
"rig": "dist/bin.mjs"
},
"files": [
"dist/**/*",
Expand Down
256 changes: 204 additions & 52 deletions scripts/lib/benchmark.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@ import { mkdir, mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";

type BenchmarkRuntime = "bun" | "node";

class BenchmarkCommandClass {
constructor(
readonly name: string,
readonly args: string[],
readonly env: Record<string, string | undefined> = {},
readonly p50BudgetMs: Partial<Record<BenchmarkRuntime, number>> = {},
) {}
}

Expand All @@ -25,13 +28,13 @@ class RigBenchmarkRunnerClass {
private readonly warmups = 3,
) {}

async run(command: BenchmarkCommandClass): Promise<BenchmarkResult> {
for (let index = 0; index < this.warmups; index++) this.spawn(command);
async run(command: BenchmarkCommandClass, runtime: BenchmarkRuntime): Promise<BenchmarkResult> {
for (let index = 0; index < this.warmups; index++) this.spawn(command, runtime);

const times: number[] = [];
for (let index = 0; index < this.iterations; index++) {
const started = performance.now();
this.spawn(command);
this.spawn(command, runtime);
times.push(performance.now() - started);
}

Expand All @@ -45,89 +48,156 @@ class RigBenchmarkRunnerClass {
};
}

spawn(command: BenchmarkCommandClass): void {
spawn(command: BenchmarkCommandClass, runtime: BenchmarkRuntime): void {
const executable = runtime === "bun" ? process.execPath : "node";
const result = Bun.spawnSync({
cmd: ["node", this.entrypoint, ...command.args],
cmd: [executable, this.entrypoint, ...command.args],
env: { ...process.env, ...command.env },
stdout: "pipe",
stderr: "pipe",
});

if (result.exitCode !== 0) {
throw new Error(`${command.name} failed: ${result.stderr.toString()}`);
throw new Error(`${command.name} failed under ${runtime}: ${result.stderr.toString()}`);
}
}
}

class BenchmarkReporterClass {
print(result: BenchmarkResult): void {
print(result: BenchmarkResult, runtime: BenchmarkRuntime, budgetMs?: number): void {
const budget = budgetMs === undefined ? "" : ` budget=${budgetMs.toFixed(0)}ms`;
console.log(
`${result.name.padEnd(22)} min=${result.min.toFixed(1)}ms p50=${result.p50.toFixed(1)}ms p95=${result.p95.toFixed(1)}ms mean=${result.mean.toFixed(1)}ms`,
`${result.name.padEnd(28)} min=${result.min.toFixed(1)}ms p50=${result.p50.toFixed(1)}ms p95=${result.p95.toFixed(1)}ms mean=${result.mean.toFixed(1)}ms${budget}`,
);
if (budgetMs !== undefined && result.p50 > budgetMs && process.env.RIG_BENCH_ENFORCE !== "0") {
throw new Error(
`${result.name} exceeded its ${runtime} p50 budget: ${result.p50.toFixed(1)}ms > ${budgetMs.toFixed(1)}ms`,
);
}
}
}

export class RigBenchmarkSuiteClass {
private readonly entrypoint = process.env.RIG_BENCH_ENTRY ?? "dist/rig.mjs";
private readonly entrypoint = process.env.RIG_BENCH_ENTRY ?? "dist/bin.mjs";
private readonly iterations = Number(process.env.RIG_BENCH_ITERATIONS ?? 20);
private readonly runner = new RigBenchmarkRunnerClass(this.entrypoint, this.iterations);
private readonly reporter = new BenchmarkReporterClass();

async run(): Promise<void> {
const home = await mkdtemp(join(tmpdir(), "rig-bench-home-"));
try {
const environment = this.environment(home);
this.runner.spawn(
new BenchmarkCommandClass("setup", ["create", "sample"], {
RIG_HOME: home,
RIG_AGENT_SYNC: "0",
}),
new BenchmarkCommandClass("setup", ["create", "sample"], environment),
"bun",
);
await this.createNestedBenchmarkTools(home);

const commands = [
new BenchmarkCommandClass("config path", ["config", "path"], {
RIG_HOME: home,
RIG_AGENT_SYNC: "0",
}),
new BenchmarkCommandClass("list", ["list"], {
RIG_HOME: home,
RIG_AGENT_SYNC: "0",
}),
new BenchmarkCommandClass("help command", ["help", "sample.example"], {
RIG_HOME: home,
RIG_AGENT_SYNC: "0",
}),
new BenchmarkCommandClass("run command", ["run", "sample.example", "Agent"], {
RIG_HOME: home,
RIG_AGENT_SYNC: "0",
}),
new BenchmarkCommandClass(
"nested run (50)",
["run", "caller.call", "--input", JSON.stringify({ count: 50 })],
{ RIG_HOME: home, RIG_AGENT_SYNC: "0" },
),
];

if (process.env.RIG_BENCH_TYPECHECK === "1") {
commands.push(
new BenchmarkCommandClass("typecheck", ["typecheck", "sample"], {
RIG_HOME: home,
RIG_AGENT_SYNC: "0",
}),
);
}
await Promise.all([
this.createNestedBenchmarkTools(home),
this.createCollectionBenchmarkTool(home),
this.createLoggingBenchmarkTool(home),
this.createSearchBenchmarkTools(home),
]);

const commands = this.commands(environment);
console.log(`Entrypoint: ${this.entrypoint}`);
console.log(`Iterations: ${this.iterations}`);
await commands.reduce(async (previous, command) => {
await previous;
this.reporter.print(await this.runner.run(command));
await this.runtimes().reduce(async (previousRuntime, runtime) => {
await previousRuntime;
console.log(`\nRuntime: ${runtime}`);
await commands.reduce(async (previousCommand, command) => {
await previousCommand;
this.reporter.print(
await this.runner.run(command, runtime),
runtime,
command.p50BudgetMs[runtime],
);
}, Promise.resolve());
}, Promise.resolve());
} finally {
await rm(home, { recursive: true, force: true });
}
}

private commands(environment: Record<string, string | undefined>): BenchmarkCommandClass[] {
const searchToolCount = this.searchToolCount();
const commands = [
new BenchmarkCommandClass("config path", ["config", "path"], environment, {
bun: 60,
node: 130,
}),
new BenchmarkCommandClass(`list (${searchToolCount + 5} tools)`, ["list"], environment, {
bun: 70 + searchToolCount * 0.25,
node: 150 + searchToolCount * 0.3,
}),
new BenchmarkCommandClass(
`find (${searchToolCount} tools)`,
["find", "resize local image width"],
environment,
{ bun: 80 + searchToolCount * 0.3, node: 160 + searchToolCount * 0.4 },
),
new BenchmarkCommandClass("help command", ["help", "sample.example"], environment, {
bun: 90,
node: 150,
}),
new BenchmarkCommandClass("run command", ["run", "sample.example", "Agent"], environment, {
bun: 100,
node: 180,
}),
new BenchmarkCommandClass(
"nested run (50)",
["run", "caller.call", "--input", JSON.stringify({ count: 50 })],
environment,
{ bun: 110, node: 190 },
),
new BenchmarkCommandClass(
"collection unused (1000)",
["run", "collection-bench.noop"],
environment,
{ bun: 100, node: 180 },
),
new BenchmarkCommandClass(
"collection count (1000)",
["run", "collection-bench.count"],
environment,
{ bun: 130, node: 230 },
),
new BenchmarkCommandClass(
"structured logs (1000)",
["run", "logging-bench.write", "1000"],
{ ...environment, RIG_LOG: "1" },
{ bun: 120, node: 200 },
),
];

if (process.env.RIG_BENCH_TYPECHECK === "1") {
commands.push(new BenchmarkCommandClass("typecheck", ["typecheck", "sample"], environment));
}
return commands;
}

private environment(home: string): Record<string, string | undefined> {
return {
RIG_HOME: home,
RIG_AGENT_SYNC: "0",
RIG_UPDATE_CHECK: "0",
RIG_LOG: "0",
};
}

private runtimes(): BenchmarkRuntime[] {
const requested = (process.env.RIG_BENCH_RUNTIMES ?? "bun,node")
.split(",")
.map((runtime) => runtime.trim())
.filter(Boolean);
const invalid = requested.find((runtime) => runtime !== "bun" && runtime !== "node");
if (invalid) throw new Error(`Unsupported benchmark runtime: ${invalid}`);
return [...new Set(requested)] as BenchmarkRuntime[];
}

private searchToolCount(): number {
return Number(process.env.RIG_BENCH_TOOLS ?? 100);
}

private async createNestedBenchmarkTools(home: string): Promise<void> {
const toolsDir = join(home, "rig", "tools");
const calleeDir = join(toolsDir, "callee");
Expand All @@ -140,7 +210,6 @@ export class RigBenchmarkSuiteClass {
Bun.write(
join(calleeDir, "index.rig.ts"),
`export default (rig) => rig.defineTool({
name: "callee",
description: "Nested benchmark callee.",
commands: {
read: rig.defineCommand({
Expand All @@ -156,7 +225,6 @@ export class RigBenchmarkSuiteClass {
Bun.write(
join(callerDir, "index.rig.ts"),
`export default (rig) => rig.defineTool({
name: "caller",
description: "Nested benchmark caller.",
commands: {
call: rig.defineCommand({
Expand All @@ -177,4 +245,88 @@ export class RigBenchmarkSuiteClass {
),
]);
}

private async createCollectionBenchmarkTool(home: string): Promise<void> {
const toolDir = join(home, "rig", "tools", "collection-bench");
await Bun.write(
join(toolDir, "index.rig.ts"),
`export default (rig) => rig.defineTool({
description: "Collection benchmark tool.",
collections: { documents: {} },
commands: {
noop: rig.defineCommand({
description: "Return without using collections.",
input: rig.z.object({}),
output: rig.z.number(),
run: () => 1,
}),
count: rig.defineCommand({
description: "Count collection entries.",
input: rig.z.object({}),
output: rig.z.number(),
run: (context) => context.collections.documents.count(),
}),
},
});
`,
);
await Promise.all(
Array.from({ length: 1_000 }, (_, index) =>
Bun.write(
join(toolDir, "documents", `${index}.md`),
`---\nvalue: ${index}\n---\n\nBenchmark document ${index}.\n`,
),
),
);
}

private async createLoggingBenchmarkTool(home: string): Promise<void> {
await Bun.write(
join(home, "rig", "tools", "logging-bench", "index.rig.ts"),
`export default (rig) => rig.defineTool({
description: "Logging benchmark tool.",
commands: {
write: rig.defineCommand({
description: "Write structured log records.",
input: rig.z.object({ count: rig.z.coerce.number() }),
output: rig.z.number(),
run: (context) => {
for (let index = 0; index < context.input.count; index++) {
context.log.info({ index }, "Benchmark log record.");
}
return context.input.count;
},
}),
},
});
`,
);
}

private async createSearchBenchmarkTools(home: string): Promise<void> {
await Promise.all(
Array.from({ length: this.searchToolCount() }, (_, index) => {
const name = `search-${index}`;
return Bun.write(
join(home, "rig", "tools", name, "index.rig.ts"),
`export default (rig) => rig.defineTool({
description: "Resize, sharpen, and inspect local image documents.",
commands: {
convert: rig.defineCommand({
description: "Convert image ${index} with width, quality, and output format options.",
input: rig.z.object({
input: rig.z.string().describe("Local image path"),
width: rig.z.number().optional(),
format: rig.z.string().optional(),
}),
output: rig.z.string(),
run: (context) => context.input.input,
}),
},
});
`,
);
}),
);
}
}
2 changes: 1 addition & 1 deletion scripts/lib/smoke.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { tmpdir } from "node:os";
import { join } from "node:path";

export class SmokeScriptClass {
private readonly cliPath = join(import.meta.dir, "..", "..", "dist", "rig.mjs");
private readonly cliPath = join(import.meta.dir, "..", "..", "dist", "bin.mjs");
private home = "";

async run(): Promise<void> {
Expand Down
3 changes: 2 additions & 1 deletion src/application/cli/cli-application.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import type { Command } from "commander";
import type { RigDirectoryMigrationResult } from "../../config/migration";
import type { PathOptions } from "../../config/paths";
import { rigErrors } from "../../errors/RigError";
import { RigLoggerFactoryClass } from "../../runtime/logger";
import { rigPackageRoot } from "../../runtime/package-root";
import { commandTargets } from "../../tools/identifiers";
import { RunPipelineContextServiceClass } from "./pipeline-context";
Expand Down Expand Up @@ -532,6 +531,7 @@ export class CliApplicationClass {
const registries = configStore.registryEntries(config);
const currentVersion = this.version();

const { RigLoggerFactoryClass } = await import("../../runtime/logger");
new RigLoggerFactoryClass(options)
.app("cli")
.info({ version: currentVersion, tools: tools.length }, "Default status rendered.");
Expand Down Expand Up @@ -590,6 +590,7 @@ export class CliApplicationClass {
const registries = configStore.registryEntries(config);
const tools = await new ToolDiscoveryServiceClass(options).discover();

const { RigLoggerFactoryClass } = await import("../../runtime/logger");
new RigLoggerFactoryClass(options)
.app("doctor")
.info({ registries: registries.length, tools: tools.length }, "Doctor check completed.");
Expand Down
8 changes: 8 additions & 0 deletions src/bin.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/usr/bin/env bun
import { CliCompositionRootClass } from "./application/cli/composition-root";
import { isCliEntrypoint } from "./application/cli/runtime-bootstrap";

/* v8 ignore next 3 */
if (isCliEntrypoint(import.meta.url)) {
await new CliCompositionRootClass({ metaUrl: import.meta.url, argv: process.argv }, {}).run();
}
Loading