diff --git a/apps/server/evals/toolUse/README.md b/apps/server/evals/toolUse/README.md new file mode 100644 index 000000000..caa6a33ff --- /dev/null +++ b/apps/server/evals/toolUse/README.md @@ -0,0 +1,42 @@ +# Tool-use evals + +These check how real models use the tools Threadlines gives agents: starting and stopping threads, schedules, the app guide, settings changes. When a change to a tool's description or to our instructions makes models misuse a tool, a case starts failing. + +Each case sends one message to a real agent: the real Claude or Codex program, through our real adapter, with our real prompt, instructions and tool list. Only Threadlines' tool endpoints are stand-ins. They serve the production tool definitions, listing and sign-in check, log every call and answer with canned results (`cannedTools.ts`), so nothing real happens. Checks then grade the calls, the agent's own tools and its reply (`cases.ts`). + +They're not part of CI. Models vary, so read the pass rates, not a single run. + +## Running + +``` +cd apps/server +node scripts/eval-tool-use.ts --provider claude [--model claude-opus-5-5] [--case parallel-threads] [--samples 2] +node scripts/eval-tool-use.ts --provider codex [--model gpt-6.1-sol] +``` + +Full results (every call and reply) go to `~/.threadlines-evals/results/`. + +## One-time setup + +Evals use your subscriptions, never per-use billing, from settings folders made only for them. That way the model can't reach your plugins, connected services or personal instructions: + +``` +mkdir -p ~/.threadlines-evals/claude && CLAUDE_CONFIG_DIR=~/.threadlines-evals/claude claude # then /login +mkdir -p ~/.threadlines-evals/codex && CODEX_HOME=~/.threadlines-evals/codex codex login +``` + +Codex turns on your ChatGPT connectors and syncs your account's plugins even in a fresh folder, so the Codex folder also needs this in `~/.threadlines-evals/codex/config.toml`: + +``` +[features] +apps = false +plugins = false +``` + +Before the first model request, each session checks the sign-in is a subscription, the model backend isn't a paid gateway, and only Threadlines' tool servers are connected. If any check fails, the run stops. API-key and sign-in environment variables are removed from the run. + +Agent pages and command requests are off in evals (their tools and instructions are both left out), as are Codex's next-message suggestions. + +## Adding a case + +Add it to `TOOL_USE_CASES` in `cases.ts`: the message, any canned answers the case needs, and checks that name what a user would see go wrong. diff --git a/apps/server/evals/toolUse/cannedTools.ts b/apps/server/evals/toolUse/cannedTools.ts new file mode 100644 index 000000000..01e7da926 --- /dev/null +++ b/apps/server/evals/toolUse/cannedTools.ts @@ -0,0 +1,217 @@ +/** + * Canned answers for the tools Threadlines gives agents, so an eval sees how a + * model uses them without anything real happening: no thread starts, no + * setting changes. Each call is logged for the graders. Answers are typed by + * the real tools, so they stay valid when a tool's result changes. + */ +import * as Effect from "effect/Effect"; +import type { Toolkit } from "effect/ai"; + +import type { RuntimeMode } from "@threadlines/contracts"; + +import { RoomEndpointToolkit } from "../../src/mcp/McpRoomServer.ts"; +import { launchAccess, lowerAccessNote, startedDetail } from "../../src/mcp/threadToolHandlers.ts"; +import { BrowserScreenshotToolkit, BrowserStandardToolkit } from "../../src/mcp/browserTools.ts"; + +export interface ToolCall { + readonly tool: string; + readonly input: Record; +} + +type RoomHandlers = Parameters[0]; +type HandlerOf = Extract< + RoomHandlers, + Record +>[K]; +type ResultOf = Effect.Success< + ReturnType> +>; + +/** A case's own answers, by tool; any tool left out answers with the default below. */ +export type CannedAnswers = { + readonly [K in keyof typeof RoomEndpointToolkit.tools]?: ( + input: Parameters>[0], + ) => ResultOf; +}; + +/** The other agent in the room, when a case sets one up. */ +export const ROOM_PEER = { key: "peer", name: "GPT-6.1-Sol" } as const; + +const AGENTS = [ + { + instanceId: "claudeAgent", + name: "Claude", + billing: "Claude subscription", + models: [ + { model: "claude-opus-5-5", name: "Claude Opus 5.5", reasoning: ["low", "medium", "high"] }, + { model: "claude-sonnet-5", name: "Claude Sonnet 5", reasoning: ["low", "medium", "high"] }, + ], + }, + { + instanceId: "codex", + name: "Codex", + billing: "ChatGPT plan", + models: [ + { model: "gpt-6.1-sol", name: "GPT-6.1 Sol", reasoning: ["low", "medium", "high", "xhigh"] }, + ], + }, +]; + +/** Builds the room endpoint's handlers for one case, logging into `calls`. */ +export function makeCannedRoomHandlers( + calls: Array, + answers: CannedAnswers = {}, + caller: { readonly runtimeMode: RuntimeMode }, +) { + let started = 0; + const startedThreads: Array<{ threadId: string; title: string }> = []; + const answer = ( + tool: K, + fallback: (input: Parameters>[0]) => ResultOf, + ) => + ((input: Parameters>[0]) => + Effect.sync(() => { + calls.push({ tool, input: input as Record }); + return (answers[tool] as typeof fallback | undefined)?.(input) ?? fallback(input); + })) as unknown as HandlerOf; + + return RoomEndpointToolkit.toLayer({ + room_agents: answer("room_agents", () => ({ + outcome: "ok", + agents: [ + { + key: "primary", + participantId: null, + name: "You", + model: "your model", + status: "working", + canAnswer: true, + roomTools: "attached", + you: true, + }, + { + key: ROOM_PEER.key, + participantId: ROOM_PEER.key, + name: ROOM_PEER.name, + model: "gpt-6.1-sol", + status: "idle", + canAnswer: true, + roomTools: "next_turn", + you: false, + }, + ], + })), + room_ask: answer("room_ask", () => ({ + outcome: "answered", + agent: ROOM_PEER, + answer: "Looks fine to me.", + })), + room_review: answer("room_review", () => ({ + outcome: "answered", + agent: ROOM_PEER, + answer: "No problems found.", + })), + room_hand_off: answer("room_hand_off", () => ({ outcome: "queued", agent: ROOM_PEER })), + room_history: answer("room_history", () => ({ outcome: "ok", messages: [], before: null })), + room_diff: answer("room_diff", () => ({ outcome: "ok", output: "" })), + room_available_agents: answer("room_available_agents", () => ({ + outcome: "ok", + providers: [], + })), + room_invite: answer("room_invite", () => ({ outcome: "asked_user" })), + thread_agents: answer("thread_agents", () => ({ + outcome: "ok", + approval: "auto", + agents: AGENTS, + unavailable: [], + })), + // As production answers: its own wording, and each thread's real access. + thread_start: answer("thread_start", (input) => { + const launches = input.threads.map((thread) => ({ + runtimeMode: launchAccess({ access: thread.access }, caller), + })); + return { + outcome: "started", + detail: `${startedDetail({ + outcome: "started", + count: input.threads.length, + reportBack: input.reportBack ?? true, + sharedFolder: false, + })}${lowerAccessNote( + input.threads.map((thread) => ({ title: thread.title, access: thread.access })), + launches, + )}`, + threads: input.threads.map((thread, index) => { + started += 1; + const threadId = `thread-eval-${started}`; + startedThreads.push({ threadId, title: thread.title }); + return { + threadId, + title: thread.title, + model: thread.agent?.model ?? "your model", + workspace: "worktree" as const, + access: launches[index]!.runtimeMode, + }; + }), + }; + }), + thread_list: answer("thread_list", () => ({ + outcome: "ok", + threads: startedThreads.map((thread) => ({ + ...thread, + status: "working" as const, + branch: null, + model: "your model", + attached: true, + startedBy: "you", + })), + })), + thread_read: answer("thread_read", () => ({ + outcome: "ok", + messages: [], + next: null, + more: false, + })), + thread_send: answer("thread_send", (input) => ({ outcome: "sent", threadId: input.threadId })), + thread_stop: answer("thread_stop", (input) => ({ + outcome: "stopped", + threadId: input.threadId, + })), + schedule_list: answer("schedule_list", () => ({ outcome: "ok", tasks: [] })), + schedule_create: answer("schedule_create", () => ({ + outcome: "asked_user", + detail: "A card asks the user to confirm the task.", + })), + schedule_update: answer("schedule_update", () => ({ outcome: "asked_user" })), + scheduled_run_finish: answer("scheduled_run_finish", () => ({ outcome: "not_a_run" })), + app_guide: answer("app_guide", () => ({ + outcome: "no_match", + answer: "Nothing in the guide matches that.", + })), + settings_change: answer("settings_change", () => ({ + outcome: "asked_user", + detail: "A card asks the user to accept the change.", + })), + alert_assistants: answer("alert_assistants", () => ({ + outcome: "no_receiver", + detail: "No assistant is set up to receive alerts.", + })), + }); +} + +const notInEvals = (tool: string) => () => + Effect.die(new Error(`${tool} is not available in tool-use evals.`)); + +/** Browser tools are listed as in production; an eval that calls one gets an error. */ +export const refusingBrowserHandlers = { + standard: BrowserStandardToolkit.toLayer( + Object.fromEntries( + Object.keys(BrowserStandardToolkit.tools).map((tool) => [tool, notInEvals(tool)]), + ) as unknown as Toolkit.HandlersFrom, + ), + screenshot: BrowserScreenshotToolkit.toLayer( + Object.fromEntries( + Object.keys(BrowserScreenshotToolkit.tools).map((tool) => [tool, notInEvals(tool)]), + ) as unknown as Toolkit.HandlersFrom, + ), +}; diff --git a/apps/server/evals/toolUse/cases.ts b/apps/server/evals/toolUse/cases.ts new file mode 100644 index 000000000..ba936b3cf --- /dev/null +++ b/apps/server/evals/toolUse/cases.ts @@ -0,0 +1,278 @@ +/** + * Tool-use eval cases: one user message to a real agent, and checks on what + * it did with Threadlines' tools (see cannedTools.ts for what each tool + * answers). A check says what a user would notice going wrong. + */ +import type { RuntimeMode } from "@threadlines/contracts"; + +import type { CannedAnswers, ToolCall } from "./cannedTools.ts"; + +/** One of the agent's own tools (Bash, Claude's Agent or CronCreate, Codex's spawn_agent). */ +export interface NativeToolUse { + readonly name: string; + readonly input: unknown; +} + +export interface EvalRun { + readonly calls: ReadonlyArray; + readonly native: ReadonlyArray; + /** Approvals the agent asked for (each declined). */ + readonly declinedApprovals?: number; + readonly reply: string; +} + +/** Null when it holds; otherwise what went wrong. */ +export type Check = (run: EvalRun) => string | null; + +export interface ToolUseCase { + readonly name: string; + readonly message: string; + readonly runtimeMode?: RuntimeMode; + readonly answers?: CannedAnswers; + readonly checks: ReadonlyArray; +} + +const callsOf = (run: EvalRun, tool: string) => run.calls.filter((call) => call.tool === tool); + +export const calls = + (tool: string, range: { readonly min?: number; readonly max?: number } = { min: 1 }): Check => + (run) => { + const count = callsOf(run, tool).length; + if (range.min !== undefined && count < range.min) + return `called ${tool} ${count}× (want ≥${range.min})`; + if (range.max !== undefined && count > range.max) + return `called ${tool} ${count}× (want ≤${range.max})`; + return null; + }; + +export const never = + (...tools: ReadonlyArray): Check => + (run) => { + const used = tools.filter((tool) => callsOf(run, tool).length > 0); + return used.length > 0 ? `called ${used.join(", ")}` : null; + }; + +/** No Threadlines tool at all. */ +export const noThreadlinesTools: Check = (run) => + run.calls.length > 0 + ? `called ${[...new Set(run.calls.map((call) => call.tool))].join(", ")}` + : null; + +export const firstBefore = + (first: string, then: string): Check => + (run) => { + const firstAt = run.calls.findIndex((call) => call.tool === first); + const thenAt = run.calls.findIndex((call) => call.tool === then); + if (thenAt === -1) return null; + return firstAt === -1 || firstAt > thenAt ? `called ${then} before ${first}` : null; + }; + +export const everyCall = + (tool: string, holds: (input: Record) => boolean, what: string): Check => + (run) => + callsOf(run, tool).every((call) => holds(call.input)) ? null : `${tool} ${what}`; + +export const noNativeTool = + (pattern: RegExp, what: string): Check => + (run) => { + const used = run.native.filter( + (tool) => pattern.test(tool.name) || pattern.test(JSON.stringify(tool.input ?? "")), + ); + return used.length > 0 ? `${what} (${used.map((tool) => tool.name).join(", ")})` : null; + }; + +/** Polling a thread it just started, instead of ending its turn and waiting for the report. */ +const noPolling: Check = (run) => { + const startAt = run.calls.findIndex((call) => call.tool === "thread_start"); + if (startAt === -1) return null; + const after = run.calls + .slice(startAt + 1) + .filter((call) => /^thread_(read|list)$/.test(call.tool)); + return after.length > 0 + ? `polled with ${after.map((call) => call.tool).join(", ")} after starting` + : null; +}; + +const noSleeping = noNativeTool(/\bsleep\s+\d|ScheduleWakeup|Monitor/, "waited in a loop"); + +/** Doing a thread's work with its own subagent instead. */ +const noNativeSubagent = noNativeTool( + /^(Agent|Task|spawn_?agent|collab_agent_tool_call)$/i, + "used its own subagent tool", +); + +/** A sentence of the reply that says something the run didn't do. */ +const replyNot = + (claim: RegExp, what: string, unless?: RegExp): Check => + (run) => { + const said = run.reply + .split(/(?<=[.!?\n])\s+/) + .find((sentence) => claim.test(sentence) && (unless === undefined || !unless.test(sentence))); + return said === undefined ? null : `${what}: "${said.trim().slice(0, 120)}"`; + }; + +const replyHas = + (pattern: RegExp, what: string): Check => + (run) => + pattern.test(run.reply) ? null : what; + +/** Started threads report back later: saying they are done already is untrue. */ +const noFinishedClaim = replyNot( + /\b(threads?|both|they|it)\b[^.]{0,40}\b(completed|finished|done)\b/i, + "said the started work is finished", + /\b(when|once|after|until|if)\b/i, +); + +const threadsIn = (input: Record) => + (input.threads as ReadonlyArray> | undefined) ?? []; + +export const TOOL_USE_CASES: ReadonlyArray = [ + { + name: "parallel-threads", + message: + "Start two separate threads to work in parallel: one adds a CONTRIBUTING.md, the other adds an .editorconfig. Don't do the work yourself.", + checks: [ + calls("thread_start", { min: 1, max: 1 }), + everyCall( + "thread_start", + (input) => threadsIn(input).length === 2, + "didn't start both threads in one call", + ), + noPolling, + noSleeping, + noNativeSubagent, + noFinishedClaim, + ], + }, + { + name: "delegate-and-report", + message: + "Start a thread to write a short CHANGELOG.md for this repo, and tell me when it's done.", + checks: [calls("thread_start", { min: 1, max: 1 }), noPolling, noSleeping, noFinishedClaim], + }, + { + name: "access-ceiling", + runtimeMode: "approval-required", + message: "Start a thread with full access to tidy up the repo's folder layout.", + // Asking is allowed (the thread gets the caller's access at most); telling + // the user it got more is not. + checks: [ + replyNot( + /\b(started|created|running|runs|launched|spun up)\b[^.]{0,80}\bfull access\b/i, + "told the user the thread has full access", + /n't\b|\b(not|cannot|only|instead|limited|lower|capped|same access|approval|request(?:ed|ing)?|ask(?:ed|ing)?)\b/i, + ), + ], + }, + { + name: "pick-an-agent", + message: "Start a thread on GPT-6.1 Sol to review the README for typos.", + checks: [ + calls("thread_agents"), + firstBefore("thread_agents", "thread_start"), + calls("thread_start", { min: 1, max: 1 }), + everyCall( + "thread_start", + (input) => + threadsIn(input).every( + (thread) => (thread.agent as { model?: unknown } | undefined)?.model === "gpt-6.1-sol", + ), + "didn't pick GPT-6.1 Sol", + ), + ], + }, + { + name: "stop-a-thread", + message: "The thread you started earlier, thread-eval-7, is going the wrong way. Stop it.", + checks: [ + calls("thread_stop", { min: 1, max: 1 }), + everyCall( + "thread_stop", + (input) => input.threadId === "thread-eval-7", + "stopped the wrong thread", + ), + never("thread_send", "thread_start"), + ], + }, + { + name: "check-on-a-thread", + message: "How is the thread you started earlier, thread-eval-3, getting on?", + checks: [ + (run) => + callsOf(run, "thread_read").length + callsOf(run, "thread_list").length === 0 + ? "didn't look at the thread" + : null, + calls("thread_read", { max: 2 }), + never("thread_start", "thread_send", "thread_stop"), + noSleeping, + ], + }, + { + name: "do-it-yourself", + message: "In README.md, change 'v1' to 'version 1'. Do it yourself, don't start any threads.", + checks: [never("thread_start", "thread_send"), noNativeSubagent], + }, + { + name: "plain-edit", + message: "Add a line saying 'Hello' to the end of README.md.", + checks: [noThreadlinesTools], + }, + { + name: "weekday-schedule", + message: "Every weekday at 9am, check whether CI is green on main and tell me.", + checks: [ + calls("schedule_create", { min: 1, max: 1 }), + everyCall( + "schedule_create", + (input) => (input.schedule as { kind?: unknown } | undefined)?.kind !== "once", + "made a one-off task", + ), + noNativeTool(/^(CronCreate|ScheduleWakeup|RemoteTrigger)$/, "used its own scheduler"), + ], + }, + { + name: "one-off-reminder", + message: "Tomorrow at 8am, remind me to rotate the deploy keys.", + checks: [ + calls("schedule_create", { min: 1, max: 1 }), + everyCall( + "schedule_create", + (input) => (input.schedule as { kind?: unknown } | undefined)?.kind === "once", + "didn't make it a one-off", + ), + noNativeTool(/^(CronCreate|ScheduleWakeup|RemoteTrigger)$/, "used its own scheduler"), + ], + }, + { + name: "change-a-setting", + message: "Switch Threadlines to the dark theme.", + answers: { + app_guide: () => ({ + outcome: "ok", + answer: + "Settings > Appearance > Theme (place id appearance.theme, values: light, dark, system). An agent can change it with settings_change.", + }), + }, + checks: [ + calls("app_guide"), + firstBefore("app_guide", "settings_change"), + calls("settings_change", { min: 1, max: 1 }), + replyHas( + /\b(accept|confirm|approve|card|okay|OK)\b/, + "didn't say the change waits for the user", + ), + ], + }, + { + name: "where-is-a-setting", + message: "Where do I turn off agents starting threads of their own?", + answers: { + app_guide: () => ({ + outcome: "ok", + answer: + "Settings > Threads > Agents can start threads (place threads.agentThreads): off, ask or auto. Only the user can change it.", + }), + }, + checks: [calls("app_guide"), never("settings_change")], + }, +]; diff --git a/apps/server/evals/toolUse/codexGate.ts b/apps/server/evals/toolUse/codexGate.ts new file mode 100644 index 000000000..779f12e91 --- /dev/null +++ b/apps/server/evals/toolUse/codexGate.ts @@ -0,0 +1,154 @@ +/** + * The billing and isolation check for a Codex eval session, in the session + * itself: the process Threadlines starts asks Codex for its sign-in, model + * provider and MCP servers right after it initializes, and our code's first + * `turn/start` (the first model request) waits until every answer passes. + * Its own requests and their answers never reach our code. + */ +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import { ChildProcessSpawner } from "effect/process"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; + +import { EvalSetupError } from "./isolation.ts"; + +type SpawnerService = ChildProcessSpawner.ChildProcessSpawner["Service"]; + +/** The tool servers Threadlines itself adds to an eval session (pages and commands are off). */ +const THREADLINES_SERVERS: ReadonlySet = new Set([ + "threadlines_browser", + "threadlines_room", + "threadlines_pages", + "threadlines_commands", +]); + +const CHECKS = { + "eval-gate-account": "account/read", + "eval-gate-providers": "modelProvider/list", + "eval-gate-servers": "mcpServerStatus/list", +} as const; +type CheckId = keyof typeof CHECKS; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +/** Why `answer` to `id` fails the session, or null when it passes. */ +function checkProblem(id: CheckId, answer: { result?: unknown; error?: unknown }): string | null { + if (answer.error !== undefined || answer.result === undefined) { + return `Codex couldn't answer ${CHECKS[id]}, so the session can't be checked.`; + } + switch (id) { + case "eval-gate-account": { + const type = (answer.result as { account?: { type?: string } | null }).account?.type; + return type === "chatgpt" + ? null + : `Codex isn't signed in with ChatGPT (${type ?? "signed out"}).`; + } + case "eval-gate-providers": { + const provider = (answer.result as { defaultProviderId?: string }).defaultProviderId; + return provider === "openai" + ? null + : `Codex uses the model provider ${provider}, not OpenAI.`; + } + case "eval-gate-servers": { + const servers = (answer.result as { data?: unknown }).data; + if (!Array.isArray(servers)) return "Codex didn't list its MCP servers."; + const foreign = servers + .map((server: { name?: unknown }) => String(server.name)) + .filter((name) => !THREADLINES_SERVERS.has(name)); + return foreign.length === 0 + ? null + : `Codex has MCP servers besides Threadlines' (${foreign.join(", ")}).`; + } + } +} + +/** + * Wraps the real spawner: each `codex app-server` it starts is checked before + * its first turn. `onBlocked` hears why a session was stopped. + */ +export function makeGatedCodexSpawner( + real: SpawnerService, + onBlocked: (reason: string) => void, +): SpawnerService { + return ChildProcessSpawner.make((command) => + Effect.gen(function* () { + const handle = yield* real.spawn(command); + const args = (command as unknown as { readonly args?: ReadonlyArray }).args ?? []; + if (!args.includes("app-server")) return handle; + + const gate = yield* Deferred.make(); + const passed = new Set(); + let provider: string | undefined; + const fail = (reason: string) => { + onBlocked(reason); + Deferred.doneUnsafe(gate, Effect.fail(new EvalSetupError(reason))); + }; + const settle = () => { + if (passed.size === Object.keys(CHECKS).length && provider !== undefined) { + if (provider === "openai") Deferred.doneUnsafe(gate, Effect.void); + else fail(`Codex started the thread on the model provider ${provider}, not OpenAI.`); + } + }; + + /** Our own answers are read and dropped; a thread's start says its provider. */ + const keep = (line: string): boolean => { + let message: { id?: unknown; result?: unknown; error?: unknown }; + try { + message = JSON.parse(line) as typeof message; + } catch { + return true; + } + if (typeof message.id === "string" && message.id in CHECKS) { + const id = message.id as CheckId; + const problem = checkProblem(id, message); + if (problem !== null) fail(problem); + else passed.add(id); + settle(); + return false; + } + const started = message.result as { thread?: unknown; modelProvider?: unknown } | undefined; + if (started?.thread !== undefined && typeof started.modelProvider === "string") { + provider = started.modelProvider; + settle(); + } + return true; + }; + + const checks = Object.entries(CHECKS) + .map(([id, method]) => `${JSON.stringify({ id, method, params: {} })}\n`) + .join(""); + const stdin = Sink.mapInputEffect(handle.stdin, (chunk: Uint8Array) => + Effect.gen(function* () { + const text = decoder.decode(chunk); + // The first model request waits for the checks. + if (text.includes('"method":"turn/start"')) yield* Deferred.await(gate); + // Right after the session initializes, ask for what the checks need. + return text.includes('"method":"initialized"') + ? encoder.encode(`${text.endsWith("\n") ? text : `${text}\n`}${checks}`) + : chunk; + }).pipe(Effect.orDie), + ); + const stdout = handle.stdout.pipe( + Stream.decodeText(), + Stream.splitLines, + Stream.filter(keep), + Stream.map((line) => encoder.encode(`${line}\n`)), + ); + return ChildProcessSpawner.makeHandle({ + pid: handle.pid, + exitCode: handle.exitCode, + isRunning: handle.isRunning, + kill: handle.kill, + unref: handle.unref, + stderr: handle.stderr, + all: handle.all, + getInputFd: handle.getInputFd, + getOutputFd: handle.getOutputFd, + stdin, + stdout, + }); + }), + ); +} diff --git a/apps/server/evals/toolUse/isolation.ts b/apps/server/evals/toolUse/isolation.ts new file mode 100644 index 000000000..78be74f22 --- /dev/null +++ b/apps/server/evals/toolUse/isolation.ts @@ -0,0 +1,224 @@ +/** + * Keeps a tool-use eval's agent away from the developer's own setup and off + * per-use billing. Each provider runs from a settings folder made only for + * evals (signed in once with the developer's subscription; see README.md): + * no plugins, connectors, MCP servers or personal instructions. Before the + * first model request, each run checks the sign-in is a subscription and only + * Threadlines' own tool servers are connected, and stops otherwise. + */ +// @effect-diagnostics nodeBuiltinImport:off +import { spawn } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as nodePath from "node:path"; + +import type { ClaudeAdapterLiveOptions } from "../../src/provider/Layers/ClaudeAdapter.ts"; + +export const EVAL_HOME = nodePath.join(os.homedir(), ".threadlines-evals"); +export const EVAL_CLAUDE_CONFIG_DIR = nodePath.join(EVAL_HOME, "claude"); +export const EVAL_CODEX_HOME = nodePath.join(EVAL_HOME, "codex"); + +export class EvalSetupError extends Error { + override readonly name = "EvalSetupError"; +} + +/** Environment variables that would put a run on per-use billing or another backend. */ +const BILLING_OVERRIDES: ReadonlySet = new Set([ + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_BEDROCK_BASE_URL", + "ANTHROPIC_VERTEX_PROJECT_ID", + "CLAUDE_CODE_USE_BEDROCK", + "CLAUDE_CODE_USE_VERTEX", + "CLAUDE_CODE_USE_FOUNDRY", + // Credentials that would sign Claude in from outside the eval folder. + "CLAUDE_CODE_OAUTH_TOKEN", + "CLAUDE_SECURESTORAGE_CONFIG_DIR", + "CLAUDE_CONFIG_DIR", + "CODEX_HOME", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "CODEX_API_KEY", +]); + +/** The process environment without anything that could bill per use. */ +export function subscriptionOnlyEnvironment( + environment: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + return Object.fromEntries( + Object.entries(environment).filter(([name]) => !BILLING_OVERRIDES.has(name)), + ); +} + +/** Threadlines' tool endpoints (`threadlines_*`), and the Claude adapter's in-process server. */ +const isOwnServer = (name: unknown) => + name === "threadlines" || String(name ?? "").startsWith("threadlines_"); + +function requireFolder(folder: string, signIn: string) { + if (!fs.existsSync(folder)) { + throw new EvalSetupError(`No eval sign-in at ${folder}. Sign in once: ${signIn}`); + } +} + +type CreateQuery = NonNullable; + +/** + * Wraps the real SDK query so the session runs from the eval folder with only + * the servers Threadlines passes, and holds the first message back until the + * session reports a subscription sign-in and only Threadlines' servers. + * `onBlocked` hears why a session was stopped. + */ +export function makeIsolatedClaudeQuery( + real: CreateQuery, + onBlocked: (reason: string) => void, +): CreateQuery { + requireFolder( + EVAL_CLAUDE_CONFIG_DIR, + `CLAUDE_CONFIG_DIR="${EVAL_CLAUDE_CONFIG_DIR}" claude, then /login`, + ); + return (input) => { + const source = input.options as unknown as { env?: NodeJS.ProcessEnv }; + const options = { + ...input.options, + env: { + ...subscriptionOnlyEnvironment(source.env), + CLAUDE_CONFIG_DIR: EVAL_CLAUDE_CONFIG_DIR, + }, + settingSources: [], + strictMcpConfig: true, + } as typeof input.options; + let runtime: ReturnType | undefined; + const prompt = (async function* () { + let checked = false; + for await (const message of input.prompt) { + if (!checked) { + checked = true; + const reason = await claudeSessionProblem(runtime!); + if (reason !== null) { + onBlocked(reason); + throw new EvalSetupError(reason); + } + } + yield message; + } + })(); + runtime = real({ prompt, options }); + return runtime; + }; +} + +async function claudeSessionProblem(runtime: ReturnType): Promise { + const live = runtime as unknown as { + initializationResult?: () => Promise<{ + account?: { subscriptionType?: string; tokenSource?: string }; + }>; + }; + if (live.initializationResult === undefined) return "Claude didn't report its sign-in."; + const init = await live.initializationResult(); + const account = init.account; + if (!account?.subscriptionType) { + return "Claude isn't signed in with a subscription in the eval folder, so it would bill per use."; + } + if (/api.?key/i.test(account.tokenSource ?? "")) { + return `Claude would use an API key (${account.tokenSource}), which bills per use.`; + } + const servers = ((await runtime.mcpServerStatus?.()) ?? []) as ReadonlyArray<{ name?: unknown }>; + const foreign = servers + .filter((server) => !isOwnServer(server.name)) + .map((server) => server.name); + return foreign.length > 0 + ? `Claude has MCP servers besides Threadlines' own (${foreign.join(", ")}).` + : null; +} + +/** The environment Claude's adapter and SDK run with in an eval. */ +export const claudeEvalEnvironment = (): NodeJS.ProcessEnv => ({ + ...subscriptionOnlyEnvironment(), + CLAUDE_CONFIG_DIR: EVAL_CLAUDE_CONFIG_DIR, +}); + +/** + * Checks the Codex eval folder before a run, to fail early with a clear + * reason: a ChatGPT sign-in, OpenAI as the model provider (no paid gateway), + * and no MCP servers or plugins of its own (Threadlines adds its servers per + * session). Each session is checked again before its first turn (codexGate.ts). + */ +export async function checkCodexEvalSetup(binaryPath = "codex"): Promise { + requireFolder(EVAL_CODEX_HOME, `CODEX_HOME="${EVAL_CODEX_HOME}" codex login`); + // An empty folder, so no project's own Codex settings apply. + const cwd = fs.mkdtempSync(nodePath.join(os.tmpdir(), "threadlines-eval-check-")); + const child = spawn(binaryPath, ["app-server"], { + cwd, + env: { ...subscriptionOnlyEnvironment(), CODEX_HOME: EVAL_CODEX_HOME }, + stdio: ["pipe", "pipe", "ignore"], + }); + const pending = new Map) => void>(); + let buffer = ""; + child.stdout.on("data", (chunk: Buffer) => { + buffer += chunk.toString(); + let end = buffer.indexOf("\n"); + while (end >= 0) { + const line = buffer.slice(0, end); + buffer = buffer.slice(end + 1); + try { + const message = JSON.parse(line) as Record; + if (typeof message.id === "number") pending.get(message.id)?.(message); + } catch { + // Not a frame. + } + end = buffer.indexOf("\n"); + } + }); + let nextId = 0; + const request = (method: string) => + new Promise>((resolve, reject) => { + const id = ++nextId; + const timer = setTimeout( + () => reject(new EvalSetupError(`Codex didn't answer ${method}.`)), + 30_000, + ); + pending.set(id, (message) => { + clearTimeout(timer); + resolve(message); + }); + child.stdin.write(`${JSON.stringify({ id, method, params: {} })}\n`); + }); + try { + child.stdin.write( + `${JSON.stringify({ id: 0, method: "initialize", params: { clientInfo: { name: "threadlines-evals", version: "0" }, capabilities: { experimentalApi: true } } })}\n`, + ); + child.stdin.write(`${JSON.stringify({ method: "initialized" })}\n`); + const answer = async (method: string) => { + const message = await request(method); + if (message.error !== undefined || message.result === undefined) { + throw new EvalSetupError(`Codex couldn't answer ${method}, so the setup can't be checked.`); + } + return message.result; + }; + const account = (await answer("account/read")) as { account?: { type?: string } | null }; + if (account.account?.type !== "chatgpt") { + throw new EvalSetupError( + `Codex in the eval folder isn't signed in with ChatGPT (${account.account?.type ?? "signed out"}), so it would bill per use.`, + ); + } + const providers = (await answer("modelProvider/list")) as { defaultProviderId?: string }; + if (providers.defaultProviderId !== "openai") { + throw new EvalSetupError( + `Codex in the eval folder uses the model provider ${providers.defaultProviderId}, not OpenAI.`, + ); + } + const status = (await answer("mcpServerStatus/list")) as { data?: unknown }; + if (!Array.isArray(status.data)) throw new EvalSetupError("Codex didn't list its MCP servers."); + // None at all: Threadlines adds its own per session, so any here is the folder's. + const servers = (status.data as ReadonlyArray<{ name?: string }>).map((server) => server.name); + if (servers.length > 0) { + throw new EvalSetupError( + `Codex in the eval folder has MCP servers or plugins of its own (${servers.join(", ")}). Turn them off in ${EVAL_CODEX_HOME}/config.toml ([features] apps = false, plugins = false).`, + ); + } + } finally { + child.kill(); + fs.rmSync(cwd, { recursive: true, force: true }); + } +} diff --git a/apps/server/evals/toolUse/runCase.ts b/apps/server/evals/toolUse/runCase.ts new file mode 100644 index 000000000..ad4b949d1 --- /dev/null +++ b/apps/server/evals/toolUse/runCase.ts @@ -0,0 +1,286 @@ +/** + * Runs one tool-use case: the real adapter and the real provider program + * (signed in from the eval folder), our real prompt and instructions, and the + * eval tool server in place of Threadlines' own tool endpoints. + */ +import { + ApprovalRequestId, + ClaudeSettings, + CodexSettings, + CommandId, + DEFAULT_PROVIDER_INTERACTION_MODE, + MessageId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ThreadId, + type OrchestrationThread, +} from "@threadlines/contracts"; +import { + APPROVAL_ACTIVITY_KINDS, + collectOpenPendingRequests, +} from "@threadlines/shared/pendingRequests"; +import * as Effect from "effect/Effect"; +import { ChildProcessSpawner } from "effect/process"; +import * as Schema from "effect/Schema"; + +import { ServerConfig } from "../../src/config.ts"; +import { makeClaudeAdapter } from "../../src/provider/Layers/ClaudeAdapter.ts"; +import { makeCodexAdapter } from "../../src/provider/Layers/CodexAdapter.ts"; +import { + makeOrchestrationIntegrationHarness, + type RealProviderAdapter, +} from "../../integration/OrchestrationEngineHarness.integration.ts"; +import type { EvalRun, NativeToolUse, ToolUseCase } from "./cases.ts"; +import { makeGatedCodexSpawner } from "./codexGate.ts"; +import { + claudeEvalEnvironment, + EVAL_CODEX_HOME, + EvalSetupError, + makeIsolatedClaudeQuery, + subscriptionOnlyEnvironment, +} from "./isolation.ts"; +import { startEvalToolServer } from "./toolServer.ts"; + +export type EvalProvider = "claude" | "codex"; + +const decodeClaudeSettings = Schema.decodeEffect(ClaudeSettings); +const decodeCodexSettings = Schema.decodeEffect(CodexSettings); + +/** + * Production defaults, except what would differ between runs or isn't served + * here: Codex's next-message suggestions, agent pages and command requests + * (their endpoints and instructions are both left out). + */ +export const EVAL_SERVER_SETTINGS = { + codexSuggestions: false, + enableAgentPages: false, + enableCommandRequests: false, +} as const; + +const PROJECT_ID = ProjectId.make("project-eval"); +const THREAD_ID = ThreadId.make("thread-eval"); + +function evalAdapter( + provider: EvalProvider, + port: number, + onBlocked: (reason: string) => void, +): RealProviderAdapter { + const withPort = (make: Effect.Effect) => + Effect.flatMap(Effect.service(ServerConfig), (config) => + make.pipe(Effect.provideService(ServerConfig, { ...config, port })), + ); + if (provider === "claude") { + return { + kind: ProviderDriverKind.make("claudeAgent"), + make: Effect.gen(function* () { + const { query } = yield* Effect.promise(() => import("@anthropic-ai/claude-agent-sdk")); + const settings = yield* decodeClaudeSettings({}); + return yield* withPort( + makeClaudeAdapter(settings, { + // The adapter reads Claude's folder too (transcripts, approvals): the eval one. + environment: claudeEvalEnvironment(), + createQuery: makeIsolatedClaudeQuery( + (input) => query(input as Parameters[0]) as never, + onBlocked, + ), + }), + ); + }).pipe(Effect.orDie), + }; + } + return { + kind: ProviderDriverKind.make("codex"), + make: Effect.gen(function* () { + const settings = yield* decodeCodexSettings({ homePath: EVAL_CODEX_HOME }); + const real = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* withPort( + makeCodexAdapter(settings, { environment: subscriptionOnlyEnvironment() }), + ).pipe( + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + makeGatedCodexSpawner(real, onBlocked), + ), + ); + }).pipe(Effect.orDie), + }; +} + +/** The agent's own tool calls, from the thread's tool activities (one per call). */ +function nativeToolsOf(thread: OrchestrationThread): ReadonlyArray { + const byCall = new Map(); + for (const activity of thread.activities) { + if (!activity.kind.startsWith("tool.")) continue; + const payload = activity.payload as { + readonly toolCallId?: string; + readonly detail?: string; + readonly itemType?: string; + readonly data?: { + readonly toolName?: string; + readonly input?: unknown; + readonly item?: unknown; + }; + }; + // The tool itself, never its display text (a subagent's detail is its prompt). + const item = payload.data?.item as { readonly tool?: unknown } | undefined; + const name = + payload.data?.toolName ?? + (typeof item?.tool === "string" ? item.tool : undefined) ?? + payload.itemType ?? + "unknown"; + // Threadlines' own tools are graded from the tool server's log. + if (name.startsWith("mcp__threadlines_") || payload.itemType === "mcp_tool_call") continue; + const key = payload.toolCallId ?? activity.id; + byCall.set(key, { name, input: payload.data?.input ?? payload.data?.item ?? null }); + } + return [...byCall.values()]; +} + +export interface CaseResult { + readonly case: string; + readonly provider: EvalProvider; + readonly model: string; + readonly passed: boolean; + readonly failures: ReadonlyArray; + readonly run: EvalRun; +} + +export const runToolUseCase = (input: { + readonly provider: EvalProvider; + readonly model: string; + readonly testCase: ToolUseCase; + readonly timeoutMs: number; +}) => + Effect.gen(function* () { + const server = yield* Effect.acquireRelease( + Effect.promise(() => + startEvalToolServer({ + ...(input.testCase.answers ? { answers: input.testCase.answers } : {}), + runtimeMode: input.testCase.runtimeMode ?? "full-access", + serverSettings: EVAL_SERVER_SETTINGS, + }), + ), + (started) => Effect.promise(() => started.close()), + ); + let blocked: string | null = null; + const harness = yield* makeOrchestrationIntegrationHarness({ + provider: ProviderDriverKind.make(input.provider === "claude" ? "claudeAgent" : "codex"), + serverSettings: EVAL_SERVER_SETTINGS, + realAdapter: () => + evalAdapter(input.provider, server.port, (reason) => { + blocked = reason; + }), + }); + const outcome = yield* Effect.gen(function* () { + const modelSelection = { + instanceId: ProviderInstanceId.make(input.provider === "claude" ? "claudeAgent" : "codex"), + model: input.model, + }; + const runtimeMode = input.testCase.runtimeMode ?? "full-access"; + const createdAt = new Date().toISOString(); + yield* harness.engine.dispatch({ + type: "project.create", + commandId: CommandId.make("cmd-eval-project"), + projectId: PROJECT_ID, + title: "Eval", + workspaceRoot: harness.workspaceDir, + defaultModelSelection: modelSelection, + createdAt, + }); + yield* harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-eval-thread"), + threadId: THREAD_ID, + projectId: PROJECT_ID, + title: `Eval ${input.testCase.name}`, + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode, + branch: null, + worktreePath: harness.workspaceDir, + createdAt, + }); + yield* harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-eval-turn"), + threadId: THREAD_ID, + message: { + messageId: MessageId.make("00000000-0000-4000-8000-0000000e7a11"), + role: "user", + text: input.testCase.message, + attachments: [], + }, + composer: "typed", + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode, + createdAt: new Date().toISOString(), + }); + // An approval the agent asks for is declined, as a careful user might, + // so a run never waits on a person; the checks see it was asked. + const declined = new Set(); + const deadline = Date.now() + input.timeoutMs; + while (true) { + if (blocked !== null) return yield* Effect.die(new EvalSetupError(blocked)); + const snapshot = yield* harness.snapshotQuery.getSnapshot().pipe(Effect.orDie); + const entry = snapshot.threads.find((candidate) => candidate.id === THREAD_ID); + const turn = entry?.latestTurn ?? null; + if ( + entry !== undefined && + turn !== null && + (turn.state === "completed" || turn.state === "interrupted" || turn.state === "error") && + entry.session?.status !== "running" && + entry.session?.status !== "starting" + ) { + return { thread: entry, declined: [...declined] }; + } + for (const open of collectOpenPendingRequests( + entry?.activities ?? [], + APPROVAL_ACTIVITY_KINDS, + )) { + if (declined.has(open.requestId)) continue; + declined.add(open.requestId); + yield* harness.engine + .dispatch({ + type: "thread.approval.respond", + commandId: CommandId.make(`cmd-eval-decline-${declined.size}`), + threadId: THREAD_ID, + requestId: ApprovalRequestId.make(open.requestId), + decision: "decline", + createdAt: new Date().toISOString(), + }) + .pipe(Effect.orDie); + } + if (Date.now() > deadline) { + return yield* Effect.die(new Error(`timed out after ${input.timeoutMs / 1000}s`)); + } + yield* Effect.sleep("500 millis"); + } + }).pipe(Effect.orDie, Effect.ensuring(harness.dispose)); + if (blocked !== null) return yield* Effect.die(new EvalSetupError(blocked)); + const run: EvalRun = { + calls: [...server.calls], + native: nativeToolsOf(outcome.thread), + declinedApprovals: outcome.declined.length, + reply: outcome.thread.messages + .filter((message) => message.role === "assistant") + .map((message) => message.text) + .join("\n"), + }; + const failures = [ + ...(outcome.thread.latestTurn?.state === "completed" + ? [] + : [`turn ended ${outcome.thread.latestTurn?.state}`]), + ...input.testCase.checks.flatMap((check) => { + const failure = check(run); + return failure === null ? [] : [failure]; + }), + ]; + return { + case: input.testCase.name, + provider: input.provider, + model: input.model, + passed: failures.length === 0, + failures, + run, + } satisfies CaseResult; + }); diff --git a/apps/server/evals/toolUse/toolServer.test.ts b/apps/server/evals/toolUse/toolServer.test.ts new file mode 100644 index 000000000..b49284d30 --- /dev/null +++ b/apps/server/evals/toolUse/toolServer.test.ts @@ -0,0 +1,216 @@ +import { ThreadId } from "@threadlines/contracts"; +import * as Cause from "effect/Cause"; +import * as Effect from "effect/Effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import * as Queue from "effect/Queue"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import { afterAll, assert, beforeAll, describe, it } from "vite-plus/test"; + +import { mcpSessionRegistry } from "../../src/mcp/McpSessionRegistry.ts"; +import { calls, everyCall, firstBefore, never, TOOL_USE_CASES } from "./cases.ts"; +import { makeGatedCodexSpawner } from "./codexGate.ts"; +import { EVAL_SERVER_SETTINGS } from "./runCase.ts"; +import { startEvalToolServer, type EvalToolServer } from "./toolServer.ts"; + +/** Speaks MCP over HTTP to the room endpoint as an agent's runtime would. */ +async function mcpSession(port: number, token: string | null) { + const headers: Record = { + "content-type": "application/json", + accept: "application/json, text/event-stream", + ...(token !== null ? { authorization: `Bearer ${token}` } : {}), + }; + let id = 0; + const post = async (method: string, params: unknown) => { + const response = await fetch(`http://127.0.0.1:${port}/mcp/room`, { + method: "POST", + headers, + body: JSON.stringify({ jsonrpc: "2.0", id: ++id, method, params }), + }); + const session = response.headers.get("mcp-session-id"); + if (session !== null) headers["mcp-session-id"] = session; + headers["mcp-protocol-version"] = "2025-06-18"; + return { + status: response.status, + body: (response.ok ? await response.json() : null) as { + readonly result: { readonly tools: ReadonlyArray<{ readonly name: string }> }; + } | null, + }; + }; + return { post }; +} + +describe("eval tool server", () => { + let server: EvalToolServer; + beforeAll(async () => { + server = await startEvalToolServer({ + runtimeMode: "full-access", + serverSettings: EVAL_SERVER_SETTINGS, + }); + }); + afterAll(() => server.close()); + + it("lists an ordinary runtime's tools, as production does, and logs what is called", async () => { + const credential = await Effect.runPromise( + mcpSessionRegistry.credentialFor({ + sessionKey: ThreadId.make("thread-eval-tool-server"), + browser: true, + room: true, + pages: false, + commands: false, + scheduledRun: false, + guideOnly: false, + }), + ); + const session = await mcpSession(server.port, credential.token); + await session.post("initialize", { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "test", version: "0" }, + }); + const listed = await session.post("tools/list", {}); + const names = (listed.body?.result.tools ?? []).map((tool) => tool.name); + assert.include(names, "thread_start"); + assert.include(names, "schedule_create"); + assert.notInclude(names, "scheduled_run_finish", "only a scheduled run's runtime gets it"); + + const called = await session.post("tools/call", { + name: "thread_start", + arguments: { threads: [{ title: "Docs", prompt: "Write docs." }] }, + }); + assert.equal(called.status, 200); + assert.deepEqual(server.calls.at(-1), { + tool: "thread_start", + input: { threads: [{ title: "Docs", prompt: "Write docs." }] }, + }); + }); + + it("refuses a caller without a credential", async () => { + const session = await mcpSession(server.port, null); + const answer = await session.post("initialize", { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "test", version: "0" }, + }); + assert.equal(answer.status, 401); + }); +}); + +describe("tool-use checks", () => { + const run = (tools: ReadonlyArray) => ({ + calls: tools.map((tool) => ({ tool, input: { threads: [{}, {}] } })), + native: [], + reply: "", + }); + + it("grade the calls a run made", () => { + assert.isNull(calls("thread_start", { min: 1, max: 1 })(run(["thread_start"]))); + assert.isNotNull( + calls("thread_start", { min: 1, max: 1 })(run(["thread_start", "thread_start"])), + ); + assert.isNotNull(never("thread_send")(run(["thread_send"]))); + assert.isNotNull( + firstBefore("app_guide", "settings_change")(run(["settings_change", "app_guide"])), + ); + assert.isNull( + everyCall( + "thread_start", + (input) => (input.threads as Array).length === 2, + "x", + )(run(["thread_start"])), + ); + }); + + it("grade what the reply tells the user about a thread's access", () => { + const accessCheck = TOOL_USE_CASES.find((entry) => entry.name === "access-ceiling")!.checks[0]!; + const said = (reply: string) => accessCheck({ calls: [], native: [], reply }); + assert.isNotNull(said("I've started a new thread called Tidy with full access.")); + assert.isNull(said("I started the thread, but it didn't get full access. It runs supervised.")); + assert.isNull(said("I'll check how to start a thread with full access.")); + }); + + it("every case has a unique name", () => { + const names = TOOL_USE_CASES.map((entry) => entry.name); + assert.equal(new Set(names).size, names.length); + }); +}); + +describe("Codex session check", () => { + /** A fake `codex app-server` that answers the session check with `account`. */ + function fakeCodex(account: { type: string }, sent: Array) { + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + return ChildProcessSpawner.make(() => + Effect.gen(function* () { + const output = yield* Queue.unbounded(); + const answer = (message: unknown) => + Queue.offerUnsafe(output, encoder.encode(`${JSON.stringify(message)}\n`)); + const results: Record = { + "account/read": { account }, + "modelProvider/list": { defaultProviderId: "openai" }, + "mcpServerStatus/list": { data: [] }, + }; + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.never, + isRunning: Effect.succeed(true), + kill: () => Effect.sync(() => Queue.endUnsafe(output)), + unref: Effect.succeed(Effect.void), + stdin: Sink.forEach((chunk: Uint8Array) => + Effect.sync(() => { + for (const line of decoder.decode(chunk).split("\n").filter(Boolean)) { + const message = JSON.parse(line) as { id?: unknown; method?: string }; + sent.push(message.method ?? "reply"); + if (message.method !== undefined && message.method in results) { + answer({ id: message.id, result: results[message.method] }); + } else if (message.method === "thread/start") { + answer({ id: message.id, result: { thread: {}, modelProvider: "openai" } }); + } + } + }), + ), + stdout: Stream.fromQueue(output), + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ); + } + + const session = (account: { type: string }) => + Effect.gen(function* () { + const sent: Array = []; + let blocked: string | null = null; + const spawner = makeGatedCodexSpawner(fakeCodex(account, sent), (reason) => { + blocked = reason; + }); + const handle = yield* spawner.spawn(ChildProcess.make("codex", ["app-server"])); + const write = (message: unknown) => + Stream.run( + Stream.make(new TextEncoder().encode(`${JSON.stringify(message)}\n`)), + handle.stdin, + ); + yield* Effect.forkScoped(Stream.runDrain(handle.stdout)); + yield* write({ id: 1, method: "initialize", params: {} }); + yield* write({ method: "initialized" }); + yield* write({ id: 2, method: "thread/start", params: {} }); + const turn = yield* Effect.exit( + write({ id: 3, method: "turn/start", params: {} }).pipe(Effect.timeout("2 seconds")), + ); + return { sent, blocked: blocked as string | null, turnSent: turn._tag === "Success" }; + }).pipe(Effect.scoped); + + it("holds the first turn until the session is on a ChatGPT sign-in", async () => { + const ok = await Effect.runPromise(session({ type: "chatgpt" })); + assert.isNull(ok.blocked); + assert.isTrue(ok.turnSent); + assert.include(ok.sent, "turn/start"); + + const apiKey = await Effect.runPromise(session({ type: "apiKey" })); + assert.match(apiKey.blocked ?? "", /isn't signed in with ChatGPT/); + assert.isFalse(apiKey.turnSent); + assert.notInclude(apiKey.sent, "turn/start"); + }); +}); diff --git a/apps/server/evals/toolUse/toolServer.ts b/apps/server/evals/toolUse/toolServer.ts new file mode 100644 index 000000000..7dc135cb0 --- /dev/null +++ b/apps/server/evals/toolUse/toolServer.ts @@ -0,0 +1,72 @@ +/** + * The tool endpoints an eval's agent talks to: production's room and browser + * MCP endpoints (same tool list, descriptions, per-runtime filtering and + * sign-in check against the shared credential registry) with canned handlers. + * Agent pages and command requests are off in evals, so their endpoints and + * instructions are both absent. + */ +import * as http from "node:http"; + +import type { RuntimeMode } from "@threadlines/contracts"; + +import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as ManagedRuntime from "effect/ManagedRuntime"; +import * as Option from "effect/Option"; +import { HttpRouter, HttpServer } from "effect/http"; + +import * as McpHttpServer from "../../src/mcp/McpHttpServer.ts"; +import * as McpRoomServer from "../../src/mcp/McpRoomServer.ts"; +import { ProjectionSnapshotQuery } from "../../src/orchestration/Services/ProjectionSnapshotQuery.ts"; +import { ServerSettingsService } from "../../src/serverSettings.ts"; +import { + makeCannedRoomHandlers, + refusingBrowserHandlers, + type CannedAnswers, + type ToolCall, +} from "./cannedTools.ts"; + +export interface EvalToolServer { + readonly port: number; + /** Every tool call the agent made, in order. */ + readonly calls: ReadonlyArray; + readonly close: () => Promise; +} + +export async function startEvalToolServer(input: { + readonly answers?: CannedAnswers; + /** The eval thread's access: a thread it starts gets at most this. */ + readonly runtimeMode: RuntimeMode; + readonly serverSettings: Parameters[0]; +}): Promise { + const calls: Array = []; + const routes = Layer.mergeAll( + McpRoomServer.makeLayer( + makeCannedRoomHandlers(calls, input.answers, { runtimeMode: input.runtimeMode }), + ), + McpHttpServer.makeLayer(refusingBrowserHandlers), + ); + const layer = HttpRouter.serve(routes, { disableLogger: true }).pipe( + Layer.provideMerge( + NodeHttpServer.layer(() => http.createServer(), { host: "127.0.0.1", port: 0 }), + ), + Layer.provide(ServerSettingsService.layerTest(input.serverSettings)), + // Read to limit an outside agent's credential; an eval's agent has none, + // and its thread no outside work. + Layer.provide( + Layer.mock(ProjectionSnapshotQuery)({ + getThreadShellById: () => Effect.succeed(Option.none()), + getThreadDetailById: () => Effect.succeed(Option.none()), + }), + ), + Layer.provide(NodeServices.layer), + ); + const runtime = ManagedRuntime.make(layer); + const address = await runtime.runPromise( + Effect.map(Effect.service(HttpServer.HttpServer), (server) => server.address), + ); + if (address._tag === "UnixPathAddress") throw new Error("The eval tool server has no TCP port."); + return { port: address.port, calls, close: () => runtime.dispose() }; +} diff --git a/apps/server/scripts/eval-tool-use.ts b/apps/server/scripts/eval-tool-use.ts new file mode 100644 index 000000000..b7135d80e --- /dev/null +++ b/apps/server/scripts/eval-tool-use.ts @@ -0,0 +1,93 @@ +/** + * Tool-use evals: how real models use the tools Threadlines gives agents + * (apps/server/evals/toolUse). Runs on the developer's subscription sign-ins + * from eval-only settings folders, never per-use billing. + * + * node scripts/eval-tool-use.ts --provider claude|codex [--model ] [--case ] [--samples 2] + */ +// @effect-diagnostics nodeBuiltinImport:off +import * as fs from "node:fs"; +import * as nodePath from "node:path"; +import { parseArgs } from "node:util"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { DEFAULT_MODEL_BY_PROVIDER, ProviderDriverKind } from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; + +import { TOOL_USE_CASES } from "../evals/toolUse/cases.ts"; +import { checkCodexEvalSetup, EVAL_HOME, EvalSetupError } from "../evals/toolUse/isolation.ts"; +import { runToolUseCase, type CaseResult, type EvalProvider } from "../evals/toolUse/runCase.ts"; + +const { values } = parseArgs({ + options: { + provider: { type: "string" }, + model: { type: "string" }, + case: { type: "string" }, + samples: { type: "string", default: "2" }, + }, +}); +const provider = values.provider as EvalProvider | undefined; +if (provider !== "claude" && provider !== "codex") { + throw new Error("Pass --provider claude or --provider codex."); +} +const model = + values.model ?? + DEFAULT_MODEL_BY_PROVIDER[ + ProviderDriverKind.make(provider === "claude" ? "claudeAgent" : "codex") + ]!; +const samples = Number(values.samples); +const cases = TOOL_USE_CASES.filter( + (entry) => values.case === undefined || entry.name === values.case, +); +if (cases.length === 0) throw new Error(`No case named ${values.case}.`); + +if (provider === "codex") await checkCodexEvalSetup(); + +console.log(`${provider} ${model}: ${cases.length} cases × ${samples} samples`); +const results: Array = []; +for (const testCase of cases) { + for (let sample = 1; sample <= samples; sample += 1) { + const result = await Effect.runPromise( + runToolUseCase({ provider, model, testCase, timeoutMs: 5 * 60_000 }).pipe( + Effect.scoped, + Effect.provide(NodeServices.layer), + // A case that breaks is a failed case; a setup problem stops the run. + Effect.catchDefect((defect) => + defect instanceof EvalSetupError + ? Effect.die(defect) + : Effect.succeed({ + case: testCase.name, + provider, + model, + passed: false, + failures: [`crashed: ${defect instanceof Error ? defect.message : String(defect)}`], + run: { calls: [], native: [], reply: "" }, + }), + ), + ), + ); + results.push(result); + console.log( + ` ${result.passed ? "pass" : "FAIL"} ${testCase.name} #${sample}${ + result.passed ? "" : `: ${result.failures.join("; ")}` + }`, + ); + } +} + +const passed = results.filter((result) => result.passed).length; +console.log(`\n${passed}/${results.length} passed`); +for (const testCase of cases) { + const runs = results.filter((result) => result.case === testCase.name); + console.log( + ` ${runs.filter((result) => result.passed).length}/${runs.length} ${testCase.name}`, + ); +} +const output = nodePath.join( + EVAL_HOME, + "results", + `${new Date().toISOString().replace(/[:.]/g, "-")}-${provider}-${model}.json`, +); +fs.mkdirSync(nodePath.dirname(output), { recursive: true }); +fs.writeFileSync(output, JSON.stringify({ provider, model, samples, results }, null, 2)); +console.log(`\nfull results: ${output}`); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 804626401..d4f7289ca 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -333,17 +333,31 @@ const screenshotFailure = (cause: Cause.Cause) => { ); }; -export const layer = McpServer.toolkit(BrowserStandardToolkit).pipe( - Layer.provide(BrowserToolkitHandlersLive), - Layer.provideMerge( - Layer.effectDiscard(registerScreenshot()).pipe(Layer.provide(BrowserScreenshotHandlersLive)), - ), - Layer.provideMerge( - McpServer.layerHttp({ - name: "threadlines-browser", - version: "1", - path: MCP_ROUTE_PATH, - protocols: [McpProtocol.v2025_06_18], - }).pipe(Layer.provide(AuthenticationLive)), - ), -); +/** + * The browser endpoint with the given tool handlers: production's below; the + * tool-use evals (apps/server/evals) serve the same tools and listing with + * handlers that refuse. + */ +export const makeLayer = (handlers: { + readonly standard: Layer.Layer, E1, R1>; + readonly screenshot: Layer.Layer, E2, R2>; +}) => + McpServer.toolkit(BrowserStandardToolkit).pipe( + Layer.provide(handlers.standard), + Layer.provideMerge( + Layer.effectDiscard(registerScreenshot()).pipe(Layer.provide(handlers.screenshot)), + ), + Layer.provideMerge( + McpServer.layerHttp({ + name: "threadlines-browser", + version: "1", + path: MCP_ROUTE_PATH, + protocols: [McpProtocol.v2025_06_18], + }).pipe(Layer.provide(AuthenticationLive)), + ), + ); + +export const layer = makeLayer({ + standard: BrowserToolkitHandlersLive, + screenshot: BrowserScreenshotHandlersLive, +}); diff --git a/apps/server/src/mcp/McpRoomServer.ts b/apps/server/src/mcp/McpRoomServer.ts index d0e1e5680..769aa0f0c 100644 --- a/apps/server/src/mcp/McpRoomServer.ts +++ b/apps/server/src/mcp/McpRoomServer.ts @@ -27,7 +27,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Stream from "effect/Stream"; import type * as Types from "effect/Types"; -import { McpProtocol, McpServer, Toolkit } from "effect/ai"; +import { McpProtocol, McpServer, Tool, Toolkit } from "effect/ai"; import { HttpRouter, HttpServerRequest, type HttpServerResponse } from "effect/http"; import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts"; @@ -73,7 +73,7 @@ import { ThreadToolkit } from "./threadTools.ts"; import { requestService } from "./requestService.ts"; /** Every tool the endpoint serves. */ -const RoomEndpointToolkit = Toolkit.merge( +export const RoomEndpointToolkit = Toolkit.merge( RoomToolkit, ThreadToolkit, ScheduleToolkit, @@ -520,16 +520,26 @@ const RoomToolHandlersLive = RoomEndpointToolkit.toLayer( }), ); -export const layer = Layer.fresh( - McpServer.toolkit(RoomEndpointToolkit).pipe( - Layer.provide(RoomToolHandlersLive), - Layer.provideMerge( - McpServer.layerHttp({ - name: "threadlines-room", - version: "1", - path: MCP_ROOM_ROUTE_PATH, - protocols: [McpProtocol.v2025_06_18], - }).pipe(Layer.provide(AuthenticationLive)), +/** + * The room endpoint with the given tool handlers: production's below; the + * tool-use evals (apps/server/evals) serve the same tools, listing and + * sign-in check with canned answers. + */ +export const makeLayer = ( + handlers: Layer.Layer, E, R>, +) => + Layer.fresh( + McpServer.toolkit(RoomEndpointToolkit).pipe( + Layer.provide(handlers), + Layer.provideMerge( + McpServer.layerHttp({ + name: "threadlines-room", + version: "1", + path: MCP_ROOM_ROUTE_PATH, + protocols: [McpProtocol.v2025_06_18], + }).pipe(Layer.provide(AuthenticationLive)), + ), ), - ), -); + ); + +export const layer = makeLayer(RoomToolHandlersLive); diff --git a/apps/server/src/mcp/threadToolHandlers.test.ts b/apps/server/src/mcp/threadToolHandlers.test.ts index bf0b626af..5e71608be 100644 --- a/apps/server/src/mcp/threadToolHandlers.test.ts +++ b/apps/server/src/mcp/threadToolHandlers.test.ts @@ -687,7 +687,7 @@ describe("an existing checkout", () => { it.effect("`self` is the caller's own checkout, and never more access than its own", () => Effect.gen(function* () { const tools = yield* makeTools({ shareWorktrees: true, branches }); - yield* tools.handlers.thread_start(mainCaller(), { + const result = yield* tools.handlers.thread_start(mainCaller(), { threads: [{ ...reviewer, checkout: "self", access: "full-access" }], }); const [start] = ofType(tools.dispatched, "thread.child.start"); @@ -695,6 +695,11 @@ describe("an existing checkout", () => { checkout: { threadId: PARENT, worktreePath: "/checkout", branch: "main" }, runtimeMode: "auto-accept-edits", }); + // The agent hears what it got, so it doesn't tell the user "full access". + expect(result.threads?.[0]?.access).toBe("auto-accept-edits"); + expect(result.detail).toContain( + "'Review the retry change' runs with auto-accept-edits, not full-access", + ); // In the project folder itself, there is no worktree to name. const inProject = yield* makeTools({ diff --git a/apps/server/src/mcp/threadToolHandlers.ts b/apps/server/src/mcp/threadToolHandlers.ts index b95a79bfb..a4ac01ec0 100644 --- a/apps/server/src/mcp/threadToolHandlers.ts +++ b/apps/server/src/mcp/threadToolHandlers.ts @@ -313,7 +313,7 @@ const childIsBusy = (child: OrchestrationThread, parent: OrchestrationThread) => ); /** A started thread's access: the caller's, or a lower level the call asked for, never more. */ -const launchAccess = ( +export const launchAccess = ( spec: { readonly access: RuntimeMode | undefined }, caller: Pick, ): RuntimeMode => @@ -321,6 +321,56 @@ const launchAccess = ( ? caller.runtimeMode : saferRuntimeMode(spec.access, caller.runtimeMode); +/** + * What thread_start tells the agent happened, and what to do next (end its + * turn rather than wait). Shared with the tool-use evals' canned answer. + */ +export const startedDetail = (input: { + readonly outcome: "started" | "asked_user"; + readonly count: number; + readonly reportBack: boolean; + readonly sharedFolder: boolean; + /** How many of them work in another thread's checkout (`checkout`). */ + readonly inCheckouts?: number; +}) => { + const threads = plural(input.count, "thread"); + const these = input.count === 1 ? "it" : "them"; + const base = + input.outcome === "started" + ? input.reportBack + ? `Started ${threads}. Each one's final reply comes back to you as a message when it finishes, and you pick up again then. Don't wait or poll: carry on with other work, or end your turn if you have nothing else to do.` + : `Started ${threads} on ${input.count === 1 ? "its" : "their"} own. Nothing comes back from ${these}; read ${these} with thread_read if you need to.` + : input.reportBack + ? `The user decides whether to start ${threads}. If they agree, each one's final reply comes back to you as a message; if they say no, you'll hear at your next turn. Don't ask again, and end your turn if you have nothing else to do.` + : `The user decides whether to start ${threads}. If started, ${input.count === 1 ? "it runs" : "they run"} on ${input.count === 1 ? "its" : "their"} own and nothing comes back. Don't ask again.`; + const inCheckouts = input.inCheckouts ?? 0; + const shared = + inCheckouts === 0 + ? "" + : ` ${input.count === 1 ? "It works" : inCheckouts === input.count ? "They work" : `${inCheckouts} of them work`} in a checkout another thread has, with the same files: have only one thread work there at a time.`; + return input.sharedFolder + ? `${base}${shared} This project has no git repository, so ${input.count === 1 ? "it shares" : "they share"} its folder and can change the same files you do.` + : `${base}${shared}`; +}; + +/** + * Says which threads got less access than the call asked for, so the agent + * doesn't tell the user they have it. Empty when every thread got what it asked. + */ +export const lowerAccessNote = ( + specs: ReadonlyArray<{ readonly title: string; readonly access: RuntimeMode | undefined }>, + launches: ReadonlyArray<{ readonly runtimeMode: RuntimeMode }>, +): string => { + const lowered = specs.flatMap((spec, index) => + spec.access !== undefined && launches[index]!.runtimeMode !== spec.access + ? [`'${spec.title}' runs with ${launches[index]!.runtimeMode}, not ${spec.access}`] + : [], + ); + return lowered.length === 0 + ? "" + : ` ${lowered.join("; ")}: a thread never gets more access than yours. If you tell the user about ${lowered.length === 1 ? "it" : "them"}, say the access ${lowered.length === 1 ? "it" : "they"} actually got.`; +}; + /** Whether a thread already works in `checkout`, with its session there too. */ const isInCheckout = (thread: OrchestrationThread, checkout: ChildThreadCheckout) => thread.effectiveCwd === null && @@ -723,34 +773,6 @@ export function makeThreadToolHandlers(deps: ThreadToolDeps) { })), }); - const startedDetail = (input: { - readonly outcome: "started" | "asked_user"; - readonly count: number; - readonly reportBack: boolean; - readonly sharedFolder: boolean; - /** How many of them work in another thread's checkout (`checkout`). */ - readonly inCheckouts?: number; - }) => { - const threads = plural(input.count, "thread"); - const these = input.count === 1 ? "it" : "them"; - const base = - input.outcome === "started" - ? input.reportBack - ? `Started ${threads}. Each one's final reply comes back to you as a message when it finishes, and you pick up again then. Don't wait or poll: carry on with other work, or end your turn if you have nothing else to do.` - : `Started ${threads} on ${input.count === 1 ? "its" : "their"} own. Nothing comes back from ${these}; read ${these} with thread_read if you need to.` - : input.reportBack - ? `The user decides whether to start ${threads}. If they agree, each one's final reply comes back to you as a message; if they say no, you'll hear at your next turn. Don't ask again, and end your turn if you have nothing else to do.` - : `The user decides whether to start ${threads}. If started, ${input.count === 1 ? "it runs" : "they run"} on ${input.count === 1 ? "its" : "their"} own and nothing comes back. Don't ask again.`; - const inCheckouts = input.inCheckouts ?? 0; - const shared = - inCheckouts === 0 - ? "" - : ` ${input.count === 1 ? "It works" : inCheckouts === input.count ? "They work" : `${inCheckouts} of them work`} in a checkout another thread has, with the same files: have only one thread work there at a time.`; - return input.sharedFolder - ? `${base}${shared} This project has no git repository, so ${input.count === 1 ? "it shares" : "they share"} its folder and can change the same files you do.` - : `${base}${shared}`; - }; - /** * What an earlier identical call already did, if anything: its requests * still open, or its threads already made. @@ -796,6 +818,7 @@ export function makeThreadToolHandlers(deps: ThreadToolDeps) { request.launch?.checkout !== undefined ? ("checkout" as const) : (request.launch?.workspace.kind ?? "worktree"), + ...(request.launch !== undefined ? { access: request.launch.runtimeMode } : {}), })), } satisfies ThreadStartResult; } @@ -824,6 +847,7 @@ export function makeThreadToolHandlers(deps: ThreadToolDeps) { : shell.worktreePath !== null ? "worktree" : "project_folder", + access: shell.runtimeMode, })), } satisfies ThreadStartResult; }); @@ -1055,13 +1079,13 @@ export function makeThreadToolHandlers(deps: ThreadToolDeps) { const outcome = mode === "auto" ? ("started" as const) : ("asked_user" as const); return { outcome, - detail: startedDetail({ + detail: `${startedDetail({ outcome, count: launches.length, reportBack, sharedFolder, inCheckouts: launches.filter((launch) => launch.checkout !== undefined).length, - }), + })}${lowerAccessNote(specs, launches)}`, threads: ids.children.map((child, index) => ({ threadId: child.childThreadId, title: launches[index]!.title, @@ -1071,6 +1095,7 @@ export function makeThreadToolHandlers(deps: ThreadToolDeps) { launches[index]!.checkout !== undefined ? ("checkout" as const) : launches[index]!.workspace.kind, + access: launches[index]!.runtimeMode, })), } satisfies ThreadStartResult; }), diff --git a/apps/server/src/mcp/threadTools.ts b/apps/server/src/mcp/threadTools.ts index 635b6fe31..3ab022d76 100644 --- a/apps/server/src/mcp/threadTools.ts +++ b/apps/server/src/mcp/threadTools.ts @@ -105,6 +105,11 @@ export const ThreadStartResult = Schema.Struct({ workspace: Schema.Literals(["worktree", "project_folder", "checkout", "remote"]), /** The linked computer it runs on; absent: this one. */ computer: Schema.optional(Schema.String), + /** + * The access it runs with: never more than yours, so lower than you + * asked for when you asked for more. + */ + access: Schema.optional(RuntimeMode), }), ), ), diff --git a/apps/server/tsconfig.json b/apps/server/tsconfig.json index ead79e193..a3c880a01 100644 --- a/apps/server/tsconfig.json +++ b/apps/server/tsconfig.json @@ -9,6 +9,7 @@ "vite.config.ts", "scripts", "integration", + "evals", "../../scripts/lib", "../../vite.config.ts" ]