From 873d773f104e9628c3b105dd8b4db1fd5acb6067 Mon Sep 17 00:00:00 2001 From: Badcuban <108198679+badcuban@users.noreply.github.com> Date: Mon, 16 Nov 2026 06:25:05 -0500 Subject: [PATCH] feat(server): tool-use evals, and thread_start says the access a thread got Nothing checked how real models use the tools Threadlines gives agents, so a tool description or instruction change that made them misuse a tool went unnoticed. apps/server/evals/toolUse runs real Claude Code and Codex through our real adapters, with our real prompt, instructions and tool list. The room and browser MCP endpoints are production's (McpRoomServer and McpHttpServer now take their handlers through makeLayer) with canned answers that log every call. Twelve cases grade the calls, the agent's own tools and its reply. Runs are local (scripts/eval-tool-use.ts), from eval-only sign-in folders with no plugins, connectors or personal instructions, and each session is checked for a subscription sign-in and only Threadlines' tool servers before its first model request. First finding, fixed: asked for a thread with more access than its own, an agent told the user the thread had it, because thread_start's result didn't say it was capped. Each started thread now reports its access, and the detail says which ones got less than asked. Second finding, left as is: Codex uses its own subagents when asked to start threads. --- apps/server/evals/toolUse/README.md | 42 +++ apps/server/evals/toolUse/cannedTools.ts | 217 +++++++++++++ apps/server/evals/toolUse/cases.ts | 278 +++++++++++++++++ apps/server/evals/toolUse/codexGate.ts | 154 ++++++++++ apps/server/evals/toolUse/isolation.ts | 224 ++++++++++++++ apps/server/evals/toolUse/runCase.ts | 286 ++++++++++++++++++ apps/server/evals/toolUse/toolServer.test.ts | 216 +++++++++++++ apps/server/evals/toolUse/toolServer.ts | 72 +++++ apps/server/scripts/eval-tool-use.ts | 93 ++++++ apps/server/src/mcp/McpHttpServer.ts | 42 ++- apps/server/src/mcp/McpRoomServer.ts | 38 ++- .../server/src/mcp/threadToolHandlers.test.ts | 7 +- apps/server/src/mcp/threadToolHandlers.ts | 87 ++++-- apps/server/src/mcp/threadTools.ts | 5 + apps/server/tsconfig.json | 1 + 15 files changed, 1702 insertions(+), 60 deletions(-) create mode 100644 apps/server/evals/toolUse/README.md create mode 100644 apps/server/evals/toolUse/cannedTools.ts create mode 100644 apps/server/evals/toolUse/cases.ts create mode 100644 apps/server/evals/toolUse/codexGate.ts create mode 100644 apps/server/evals/toolUse/isolation.ts create mode 100644 apps/server/evals/toolUse/runCase.ts create mode 100644 apps/server/evals/toolUse/toolServer.test.ts create mode 100644 apps/server/evals/toolUse/toolServer.ts create mode 100644 apps/server/scripts/eval-tool-use.ts diff --git a/apps/server/evals/toolUse/README.md b/apps/server/evals/toolUse/README.md new file mode 100644 index 000000000..caa6a33ff --- /dev/null +++ b/apps/server/evals/toolUse/README.md @@ -0,0 +1,42 @@ +# Tool-use evals + +These check how real models use the tools Threadlines gives agents: starting and stopping threads, schedules, the app guide, settings changes. When a change to a tool's description or to our instructions makes models misuse a tool, a case starts failing. + +Each case sends one message to a real agent: the real Claude or Codex program, through our real adapter, with our real prompt, instructions and tool list. Only Threadlines' tool endpoints are stand-ins. They serve the production tool definitions, listing and sign-in check, log every call and answer with canned results (`cannedTools.ts`), so nothing real happens. Checks then grade the calls, the agent's own tools and its reply (`cases.ts`). + +They're not part of CI. Models vary, so read the pass rates, not a single run. + +## Running + +``` +cd apps/server +node scripts/eval-tool-use.ts --provider claude [--model claude-opus-5-5] [--case parallel-threads] [--samples 2] +node scripts/eval-tool-use.ts --provider codex [--model gpt-6.1-sol] +``` + +Full results (every call and reply) go to `~/.threadlines-evals/results/`. + +## One-time setup + +Evals use your subscriptions, never per-use billing, from settings folders made only for them. That way the model can't reach your plugins, connected services or personal instructions: + +``` +mkdir -p ~/.threadlines-evals/claude && CLAUDE_CONFIG_DIR=~/.threadlines-evals/claude claude # then /login +mkdir -p ~/.threadlines-evals/codex && CODEX_HOME=~/.threadlines-evals/codex codex login +``` + +Codex turns on your ChatGPT connectors and syncs your account's plugins even in a fresh folder, so the Codex folder also needs this in `~/.threadlines-evals/codex/config.toml`: + +``` +[features] +apps = false +plugins = false +``` + +Before the first model request, each session checks the sign-in is a subscription, the model backend isn't a paid gateway, and only Threadlines' tool servers are connected. If any check fails, the run stops. API-key and sign-in environment variables are removed from the run. + +Agent pages and command requests are off in evals (their tools and instructions are both left out), as are Codex's next-message suggestions. + +## Adding a case + +Add it to `TOOL_USE_CASES` in `cases.ts`: the message, any canned answers the case needs, and checks that name what a user would see go wrong. diff --git a/apps/server/evals/toolUse/cannedTools.ts b/apps/server/evals/toolUse/cannedTools.ts new file mode 100644 index 000000000..01e7da926 --- /dev/null +++ b/apps/server/evals/toolUse/cannedTools.ts @@ -0,0 +1,217 @@ +/** + * Canned answers for the tools Threadlines gives agents, so an eval sees how a + * model uses them without anything real happening: no thread starts, no + * setting changes. Each call is logged for the graders. Answers are typed by + * the real tools, so they stay valid when a tool's result changes. + */ +import * as Effect from "effect/Effect"; +import type { Toolkit } from "effect/ai"; + +import type { RuntimeMode } from "@threadlines/contracts"; + +import { RoomEndpointToolkit } from "../../src/mcp/McpRoomServer.ts"; +import { launchAccess, lowerAccessNote, startedDetail } from "../../src/mcp/threadToolHandlers.ts"; +import { BrowserScreenshotToolkit, BrowserStandardToolkit } from "../../src/mcp/browserTools.ts"; + +export interface ToolCall { + readonly tool: string; + readonly input: Record; +} + +type RoomHandlers = Parameters[0]; +type HandlerOf = Extract< + RoomHandlers, + Record +>[K]; +type ResultOf = Effect.Success< + ReturnType> +>; + +/** A case's own answers, by tool; any tool left out answers with the default below. */ +export type CannedAnswers = { + readonly [K in keyof typeof RoomEndpointToolkit.tools]?: ( + input: Parameters>[0], + ) => ResultOf; +}; + +/** The other agent in the room, when a case sets one up. */ +export const ROOM_PEER = { key: "peer", name: "GPT-6.1-Sol" } as const; + +const AGENTS = [ + { + instanceId: "claudeAgent", + name: "Claude", + billing: "Claude subscription", + models: [ + { model: "claude-opus-5-5", name: "Claude Opus 5.5", reasoning: ["low", "medium", "high"] }, + { model: "claude-sonnet-5", name: "Claude Sonnet 5", reasoning: ["low", "medium", "high"] }, + ], + }, + { + instanceId: "codex", + name: "Codex", + billing: "ChatGPT plan", + models: [ + { model: "gpt-6.1-sol", name: "GPT-6.1 Sol", reasoning: ["low", "medium", "high", "xhigh"] }, + ], + }, +]; + +/** Builds the room endpoint's handlers for one case, logging into `calls`. */ +export function makeCannedRoomHandlers( + calls: Array, + answers: CannedAnswers = {}, + caller: { readonly runtimeMode: RuntimeMode }, +) { + let started = 0; + const startedThreads: Array<{ threadId: string; title: string }> = []; + const answer = ( + tool: K, + fallback: (input: Parameters>[0]) => ResultOf, + ) => + ((input: Parameters>[0]) => + Effect.sync(() => { + calls.push({ tool, input: input as Record }); + return (answers[tool] as typeof fallback | undefined)?.(input) ?? fallback(input); + })) as unknown as HandlerOf; + + return RoomEndpointToolkit.toLayer({ + room_agents: answer("room_agents", () => ({ + outcome: "ok", + agents: [ + { + key: "primary", + participantId: null, + name: "You", + model: "your model", + status: "working", + canAnswer: true, + roomTools: "attached", + you: true, + }, + { + key: ROOM_PEER.key, + participantId: ROOM_PEER.key, + name: ROOM_PEER.name, + model: "gpt-6.1-sol", + status: "idle", + canAnswer: true, + roomTools: "next_turn", + you: false, + }, + ], + })), + room_ask: answer("room_ask", () => ({ + outcome: "answered", + agent: ROOM_PEER, + answer: "Looks fine to me.", + })), + room_review: answer("room_review", () => ({ + outcome: "answered", + agent: ROOM_PEER, + answer: "No problems found.", + })), + room_hand_off: answer("room_hand_off", () => ({ outcome: "queued", agent: ROOM_PEER })), + room_history: answer("room_history", () => ({ outcome: "ok", messages: [], before: null })), + room_diff: answer("room_diff", () => ({ outcome: "ok", output: "" })), + room_available_agents: answer("room_available_agents", () => ({ + outcome: "ok", + providers: [], + })), + room_invite: answer("room_invite", () => ({ outcome: "asked_user" })), + thread_agents: answer("thread_agents", () => ({ + outcome: "ok", + approval: "auto", + agents: AGENTS, + unavailable: [], + })), + // As production answers: its own wording, and each thread's real access. + thread_start: answer("thread_start", (input) => { + const launches = input.threads.map((thread) => ({ + runtimeMode: launchAccess({ access: thread.access }, caller), + })); + return { + outcome: "started", + detail: `${startedDetail({ + outcome: "started", + count: input.threads.length, + reportBack: input.reportBack ?? true, + sharedFolder: false, + })}${lowerAccessNote( + input.threads.map((thread) => ({ title: thread.title, access: thread.access })), + launches, + )}`, + threads: input.threads.map((thread, index) => { + started += 1; + const threadId = `thread-eval-${started}`; + startedThreads.push({ threadId, title: thread.title }); + return { + threadId, + title: thread.title, + model: thread.agent?.model ?? "your model", + workspace: "worktree" as const, + access: launches[index]!.runtimeMode, + }; + }), + }; + }), + thread_list: answer("thread_list", () => ({ + outcome: "ok", + threads: startedThreads.map((thread) => ({ + ...thread, + status: "working" as const, + branch: null, + model: "your model", + attached: true, + startedBy: "you", + })), + })), + thread_read: answer("thread_read", () => ({ + outcome: "ok", + messages: [], + next: null, + more: false, + })), + thread_send: answer("thread_send", (input) => ({ outcome: "sent", threadId: input.threadId })), + thread_stop: answer("thread_stop", (input) => ({ + outcome: "stopped", + threadId: input.threadId, + })), + schedule_list: answer("schedule_list", () => ({ outcome: "ok", tasks: [] })), + schedule_create: answer("schedule_create", () => ({ + outcome: "asked_user", + detail: "A card asks the user to confirm the task.", + })), + schedule_update: answer("schedule_update", () => ({ outcome: "asked_user" })), + scheduled_run_finish: answer("scheduled_run_finish", () => ({ outcome: "not_a_run" })), + app_guide: answer("app_guide", () => ({ + outcome: "no_match", + answer: "Nothing in the guide matches that.", + })), + settings_change: answer("settings_change", () => ({ + outcome: "asked_user", + detail: "A card asks the user to accept the change.", + })), + alert_assistants: answer("alert_assistants", () => ({ + outcome: "no_receiver", + detail: "No assistant is set up to receive alerts.", + })), + }); +} + +const notInEvals = (tool: string) => () => + Effect.die(new Error(`${tool} is not available in tool-use evals.`)); + +/** Browser tools are listed as in production; an eval that calls one gets an error. */ +export const refusingBrowserHandlers = { + standard: BrowserStandardToolkit.toLayer( + Object.fromEntries( + Object.keys(BrowserStandardToolkit.tools).map((tool) => [tool, notInEvals(tool)]), + ) as unknown as Toolkit.HandlersFrom, + ), + screenshot: BrowserScreenshotToolkit.toLayer( + Object.fromEntries( + Object.keys(BrowserScreenshotToolkit.tools).map((tool) => [tool, notInEvals(tool)]), + ) as unknown as Toolkit.HandlersFrom, + ), +}; diff --git a/apps/server/evals/toolUse/cases.ts b/apps/server/evals/toolUse/cases.ts new file mode 100644 index 000000000..ba936b3cf --- /dev/null +++ b/apps/server/evals/toolUse/cases.ts @@ -0,0 +1,278 @@ +/** + * Tool-use eval cases: one user message to a real agent, and checks on what + * it did with Threadlines' tools (see cannedTools.ts for what each tool + * answers). A check says what a user would notice going wrong. + */ +import type { RuntimeMode } from "@threadlines/contracts"; + +import type { CannedAnswers, ToolCall } from "./cannedTools.ts"; + +/** One of the agent's own tools (Bash, Claude's Agent or CronCreate, Codex's spawn_agent). */ +export interface NativeToolUse { + readonly name: string; + readonly input: unknown; +} + +export interface EvalRun { + readonly calls: ReadonlyArray; + readonly native: ReadonlyArray; + /** Approvals the agent asked for (each declined). */ + readonly declinedApprovals?: number; + readonly reply: string; +} + +/** Null when it holds; otherwise what went wrong. */ +export type Check = (run: EvalRun) => string | null; + +export interface ToolUseCase { + readonly name: string; + readonly message: string; + readonly runtimeMode?: RuntimeMode; + readonly answers?: CannedAnswers; + readonly checks: ReadonlyArray; +} + +const callsOf = (run: EvalRun, tool: string) => run.calls.filter((call) => call.tool === tool); + +export const calls = + (tool: string, range: { readonly min?: number; readonly max?: number } = { min: 1 }): Check => + (run) => { + const count = callsOf(run, tool).length; + if (range.min !== undefined && count < range.min) + return `called ${tool} ${count}× (want ≥${range.min})`; + if (range.max !== undefined && count > range.max) + return `called ${tool} ${count}× (want ≤${range.max})`; + return null; + }; + +export const never = + (...tools: ReadonlyArray): Check => + (run) => { + const used = tools.filter((tool) => callsOf(run, tool).length > 0); + return used.length > 0 ? `called ${used.join(", ")}` : null; + }; + +/** No Threadlines tool at all. */ +export const noThreadlinesTools: Check = (run) => + run.calls.length > 0 + ? `called ${[...new Set(run.calls.map((call) => call.tool))].join(", ")}` + : null; + +export const firstBefore = + (first: string, then: string): Check => + (run) => { + const firstAt = run.calls.findIndex((call) => call.tool === first); + const thenAt = run.calls.findIndex((call) => call.tool === then); + if (thenAt === -1) return null; + return firstAt === -1 || firstAt > thenAt ? `called ${then} before ${first}` : null; + }; + +export const everyCall = + (tool: string, holds: (input: Record) => boolean, what: string): Check => + (run) => + callsOf(run, tool).every((call) => holds(call.input)) ? null : `${tool} ${what}`; + +export const noNativeTool = + (pattern: RegExp, what: string): Check => + (run) => { + const used = run.native.filter( + (tool) => pattern.test(tool.name) || pattern.test(JSON.stringify(tool.input ?? "")), + ); + return used.length > 0 ? `${what} (${used.map((tool) => tool.name).join(", ")})` : null; + }; + +/** Polling a thread it just started, instead of ending its turn and waiting for the report. */ +const noPolling: Check = (run) => { + const startAt = run.calls.findIndex((call) => call.tool === "thread_start"); + if (startAt === -1) return null; + const after = run.calls + .slice(startAt + 1) + .filter((call) => /^thread_(read|list)$/.test(call.tool)); + return after.length > 0 + ? `polled with ${after.map((call) => call.tool).join(", ")} after starting` + : null; +}; + +const noSleeping = noNativeTool(/\bsleep\s+\d|ScheduleWakeup|Monitor/, "waited in a loop"); + +/** Doing a thread's work with its own subagent instead. */ +const noNativeSubagent = noNativeTool( + /^(Agent|Task|spawn_?agent|collab_agent_tool_call)$/i, + "used its own subagent tool", +); + +/** A sentence of the reply that says something the run didn't do. */ +const replyNot = + (claim: RegExp, what: string, unless?: RegExp): Check => + (run) => { + const said = run.reply + .split(/(?<=[.!?\n])\s+/) + .find((sentence) => claim.test(sentence) && (unless === undefined || !unless.test(sentence))); + return said === undefined ? null : `${what}: "${said.trim().slice(0, 120)}"`; + }; + +const replyHas = + (pattern: RegExp, what: string): Check => + (run) => + pattern.test(run.reply) ? null : what; + +/** Started threads report back later: saying they are done already is untrue. */ +const noFinishedClaim = replyNot( + /\b(threads?|both|they|it)\b[^.]{0,40}\b(completed|finished|done)\b/i, + "said the started work is finished", + /\b(when|once|after|until|if)\b/i, +); + +const threadsIn = (input: Record) => + (input.threads as ReadonlyArray> | undefined) ?? []; + +export const TOOL_USE_CASES: ReadonlyArray = [ + { + name: "parallel-threads", + message: + "Start two separate threads to work in parallel: one adds a CONTRIBUTING.md, the other adds an .editorconfig. Don't do the work yourself.", + checks: [ + calls("thread_start", { min: 1, max: 1 }), + everyCall( + "thread_start", + (input) => threadsIn(input).length === 2, + "didn't start both threads in one call", + ), + noPolling, + noSleeping, + noNativeSubagent, + noFinishedClaim, + ], + }, + { + name: "delegate-and-report", + message: + "Start a thread to write a short CHANGELOG.md for this repo, and tell me when it's done.", + checks: [calls("thread_start", { min: 1, max: 1 }), noPolling, noSleeping, noFinishedClaim], + }, + { + name: "access-ceiling", + runtimeMode: "approval-required", + message: "Start a thread with full access to tidy up the repo's folder layout.", + // Asking is allowed (the thread gets the caller's access at most); telling + // the user it got more is not. + checks: [ + replyNot( + /\b(started|created|running|runs|launched|spun up)\b[^.]{0,80}\bfull access\b/i, + "told the user the thread has full access", + /n't\b|\b(not|cannot|only|instead|limited|lower|capped|same access|approval|request(?:ed|ing)?|ask(?:ed|ing)?)\b/i, + ), + ], + }, + { + name: "pick-an-agent", + message: "Start a thread on GPT-6.1 Sol to review the README for typos.", + checks: [ + calls("thread_agents"), + firstBefore("thread_agents", "thread_start"), + calls("thread_start", { min: 1, max: 1 }), + everyCall( + "thread_start", + (input) => + threadsIn(input).every( + (thread) => (thread.agent as { model?: unknown } | undefined)?.model === "gpt-6.1-sol", + ), + "didn't pick GPT-6.1 Sol", + ), + ], + }, + { + name: "stop-a-thread", + message: "The thread you started earlier, thread-eval-7, is going the wrong way. Stop it.", + checks: [ + calls("thread_stop", { min: 1, max: 1 }), + everyCall( + "thread_stop", + (input) => input.threadId === "thread-eval-7", + "stopped the wrong thread", + ), + never("thread_send", "thread_start"), + ], + }, + { + name: "check-on-a-thread", + message: "How is the thread you started earlier, thread-eval-3, getting on?", + checks: [ + (run) => + callsOf(run, "thread_read").length + callsOf(run, "thread_list").length === 0 + ? "didn't look at the thread" + : null, + calls("thread_read", { max: 2 }), + never("thread_start", "thread_send", "thread_stop"), + noSleeping, + ], + }, + { + name: "do-it-yourself", + message: "In README.md, change 'v1' to 'version 1'. Do it yourself, don't start any threads.", + checks: [never("thread_start", "thread_send"), noNativeSubagent], + }, + { + name: "plain-edit", + message: "Add a line saying 'Hello' to the end of README.md.", + checks: [noThreadlinesTools], + }, + { + name: "weekday-schedule", + message: "Every weekday at 9am, check whether CI is green on main and tell me.", + checks: [ + calls("schedule_create", { min: 1, max: 1 }), + everyCall( + "schedule_create", + (input) => (input.schedule as { kind?: unknown } | undefined)?.kind !== "once", + "made a one-off task", + ), + noNativeTool(/^(CronCreate|ScheduleWakeup|RemoteTrigger)$/, "used its own scheduler"), + ], + }, + { + name: "one-off-reminder", + message: "Tomorrow at 8am, remind me to rotate the deploy keys.", + checks: [ + calls("schedule_create", { min: 1, max: 1 }), + everyCall( + "schedule_create", + (input) => (input.schedule as { kind?: unknown } | undefined)?.kind === "once", + "didn't make it a one-off", + ), + noNativeTool(/^(CronCreate|ScheduleWakeup|RemoteTrigger)$/, "used its own scheduler"), + ], + }, + { + name: "change-a-setting", + message: "Switch Threadlines to the dark theme.", + answers: { + app_guide: () => ({ + outcome: "ok", + answer: + "Settings > Appearance > Theme (place id appearance.theme, values: light, dark, system). An agent can change it with settings_change.", + }), + }, + checks: [ + calls("app_guide"), + firstBefore("app_guide", "settings_change"), + calls("settings_change", { min: 1, max: 1 }), + replyHas( + /\b(accept|confirm|approve|card|okay|OK)\b/, + "didn't say the change waits for the user", + ), + ], + }, + { + name: "where-is-a-setting", + message: "Where do I turn off agents starting threads of their own?", + answers: { + app_guide: () => ({ + outcome: "ok", + answer: + "Settings > Threads > Agents can start threads (place threads.agentThreads): off, ask or auto. Only the user can change it.", + }), + }, + checks: [calls("app_guide"), never("settings_change")], + }, +]; diff --git a/apps/server/evals/toolUse/codexGate.ts b/apps/server/evals/toolUse/codexGate.ts new file mode 100644 index 000000000..779f12e91 --- /dev/null +++ b/apps/server/evals/toolUse/codexGate.ts @@ -0,0 +1,154 @@ +/** + * The billing and isolation check for a Codex eval session, in the session + * itself: the process Threadlines starts asks Codex for its sign-in, model + * provider and MCP servers right after it initializes, and our code's first + * `turn/start` (the first model request) waits until every answer passes. + * Its own requests and their answers never reach our code. + */ +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import { ChildProcessSpawner } from "effect/process"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; + +import { EvalSetupError } from "./isolation.ts"; + +type SpawnerService = ChildProcessSpawner.ChildProcessSpawner["Service"]; + +/** The tool servers Threadlines itself adds to an eval session (pages and commands are off). */ +const THREADLINES_SERVERS: ReadonlySet = new Set([ + "threadlines_browser", + "threadlines_room", + "threadlines_pages", + "threadlines_commands", +]); + +const CHECKS = { + "eval-gate-account": "account/read", + "eval-gate-providers": "modelProvider/list", + "eval-gate-servers": "mcpServerStatus/list", +} as const; +type CheckId = keyof typeof CHECKS; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +/** Why `answer` to `id` fails the session, or null when it passes. */ +function checkProblem(id: CheckId, answer: { result?: unknown; error?: unknown }): string | null { + if (answer.error !== undefined || answer.result === undefined) { + return `Codex couldn't answer ${CHECKS[id]}, so the session can't be checked.`; + } + switch (id) { + case "eval-gate-account": { + const type = (answer.result as { account?: { type?: string } | null }).account?.type; + return type === "chatgpt" + ? null + : `Codex isn't signed in with ChatGPT (${type ?? "signed out"}).`; + } + case "eval-gate-providers": { + const provider = (answer.result as { defaultProviderId?: string }).defaultProviderId; + return provider === "openai" + ? null + : `Codex uses the model provider ${provider}, not OpenAI.`; + } + case "eval-gate-servers": { + const servers = (answer.result as { data?: unknown }).data; + if (!Array.isArray(servers)) return "Codex didn't list its MCP servers."; + const foreign = servers + .map((server: { name?: unknown }) => String(server.name)) + .filter((name) => !THREADLINES_SERVERS.has(name)); + return foreign.length === 0 + ? null + : `Codex has MCP servers besides Threadlines' (${foreign.join(", ")}).`; + } + } +} + +/** + * Wraps the real spawner: each `codex app-server` it starts is checked before + * its first turn. `onBlocked` hears why a session was stopped. + */ +export function makeGatedCodexSpawner( + real: SpawnerService, + onBlocked: (reason: string) => void, +): SpawnerService { + return ChildProcessSpawner.make((command) => + Effect.gen(function* () { + const handle = yield* real.spawn(command); + const args = (command as unknown as { readonly args?: ReadonlyArray }).args ?? []; + if (!args.includes("app-server")) return handle; + + const gate = yield* Deferred.make(); + const passed = new Set(); + let provider: string | undefined; + const fail = (reason: string) => { + onBlocked(reason); + Deferred.doneUnsafe(gate, Effect.fail(new EvalSetupError(reason))); + }; + const settle = () => { + if (passed.size === Object.keys(CHECKS).length && provider !== undefined) { + if (provider === "openai") Deferred.doneUnsafe(gate, Effect.void); + else fail(`Codex started the thread on the model provider ${provider}, not OpenAI.`); + } + }; + + /** Our own answers are read and dropped; a thread's start says its provider. */ + const keep = (line: string): boolean => { + let message: { id?: unknown; result?: unknown; error?: unknown }; + try { + message = JSON.parse(line) as typeof message; + } catch { + return true; + } + if (typeof message.id === "string" && message.id in CHECKS) { + const id = message.id as CheckId; + const problem = checkProblem(id, message); + if (problem !== null) fail(problem); + else passed.add(id); + settle(); + return false; + } + const started = message.result as { thread?: unknown; modelProvider?: unknown } | undefined; + if (started?.thread !== undefined && typeof started.modelProvider === "string") { + provider = started.modelProvider; + settle(); + } + return true; + }; + + const checks = Object.entries(CHECKS) + .map(([id, method]) => `${JSON.stringify({ id, method, params: {} })}\n`) + .join(""); + const stdin = Sink.mapInputEffect(handle.stdin, (chunk: Uint8Array) => + Effect.gen(function* () { + const text = decoder.decode(chunk); + // The first model request waits for the checks. + if (text.includes('"method":"turn/start"')) yield* Deferred.await(gate); + // Right after the session initializes, ask for what the checks need. + return text.includes('"method":"initialized"') + ? encoder.encode(`${text.endsWith("\n") ? text : `${text}\n`}${checks}`) + : chunk; + }).pipe(Effect.orDie), + ); + const stdout = handle.stdout.pipe( + Stream.decodeText(), + Stream.splitLines, + Stream.filter(keep), + Stream.map((line) => encoder.encode(`${line}\n`)), + ); + return ChildProcessSpawner.makeHandle({ + pid: handle.pid, + exitCode: handle.exitCode, + isRunning: handle.isRunning, + kill: handle.kill, + unref: handle.unref, + stderr: handle.stderr, + all: handle.all, + getInputFd: handle.getInputFd, + getOutputFd: handle.getOutputFd, + stdin, + stdout, + }); + }), + ); +} diff --git a/apps/server/evals/toolUse/isolation.ts b/apps/server/evals/toolUse/isolation.ts new file mode 100644 index 000000000..78be74f22 --- /dev/null +++ b/apps/server/evals/toolUse/isolation.ts @@ -0,0 +1,224 @@ +/** + * Keeps a tool-use eval's agent away from the developer's own setup and off + * per-use billing. Each provider runs from a settings folder made only for + * evals (signed in once with the developer's subscription; see README.md): + * no plugins, connectors, MCP servers or personal instructions. Before the + * first model request, each run checks the sign-in is a subscription and only + * Threadlines' own tool servers are connected, and stops otherwise. + */ +// @effect-diagnostics nodeBuiltinImport:off +import { spawn } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as nodePath from "node:path"; + +import type { ClaudeAdapterLiveOptions } from "../../src/provider/Layers/ClaudeAdapter.ts"; + +export const EVAL_HOME = nodePath.join(os.homedir(), ".threadlines-evals"); +export const EVAL_CLAUDE_CONFIG_DIR = nodePath.join(EVAL_HOME, "claude"); +export const EVAL_CODEX_HOME = nodePath.join(EVAL_HOME, "codex"); + +export class EvalSetupError extends Error { + override readonly name = "EvalSetupError"; +} + +/** Environment variables that would put a run on per-use billing or another backend. */ +const BILLING_OVERRIDES: ReadonlySet = new Set([ + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_BEDROCK_BASE_URL", + "ANTHROPIC_VERTEX_PROJECT_ID", + "CLAUDE_CODE_USE_BEDROCK", + "CLAUDE_CODE_USE_VERTEX", + "CLAUDE_CODE_USE_FOUNDRY", + // Credentials that would sign Claude in from outside the eval folder. + "CLAUDE_CODE_OAUTH_TOKEN", + "CLAUDE_SECURESTORAGE_CONFIG_DIR", + "CLAUDE_CONFIG_DIR", + "CODEX_HOME", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "CODEX_API_KEY", +]); + +/** The process environment without anything that could bill per use. */ +export function subscriptionOnlyEnvironment( + environment: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv { + return Object.fromEntries( + Object.entries(environment).filter(([name]) => !BILLING_OVERRIDES.has(name)), + ); +} + +/** Threadlines' tool endpoints (`threadlines_*`), and the Claude adapter's in-process server. */ +const isOwnServer = (name: unknown) => + name === "threadlines" || String(name ?? "").startsWith("threadlines_"); + +function requireFolder(folder: string, signIn: string) { + if (!fs.existsSync(folder)) { + throw new EvalSetupError(`No eval sign-in at ${folder}. Sign in once: ${signIn}`); + } +} + +type CreateQuery = NonNullable; + +/** + * Wraps the real SDK query so the session runs from the eval folder with only + * the servers Threadlines passes, and holds the first message back until the + * session reports a subscription sign-in and only Threadlines' servers. + * `onBlocked` hears why a session was stopped. + */ +export function makeIsolatedClaudeQuery( + real: CreateQuery, + onBlocked: (reason: string) => void, +): CreateQuery { + requireFolder( + EVAL_CLAUDE_CONFIG_DIR, + `CLAUDE_CONFIG_DIR="${EVAL_CLAUDE_CONFIG_DIR}" claude, then /login`, + ); + return (input) => { + const source = input.options as unknown as { env?: NodeJS.ProcessEnv }; + const options = { + ...input.options, + env: { + ...subscriptionOnlyEnvironment(source.env), + CLAUDE_CONFIG_DIR: EVAL_CLAUDE_CONFIG_DIR, + }, + settingSources: [], + strictMcpConfig: true, + } as typeof input.options; + let runtime: ReturnType | undefined; + const prompt = (async function* () { + let checked = false; + for await (const message of input.prompt) { + if (!checked) { + checked = true; + const reason = await claudeSessionProblem(runtime!); + if (reason !== null) { + onBlocked(reason); + throw new EvalSetupError(reason); + } + } + yield message; + } + })(); + runtime = real({ prompt, options }); + return runtime; + }; +} + +async function claudeSessionProblem(runtime: ReturnType): Promise { + const live = runtime as unknown as { + initializationResult?: () => Promise<{ + account?: { subscriptionType?: string; tokenSource?: string }; + }>; + }; + if (live.initializationResult === undefined) return "Claude didn't report its sign-in."; + const init = await live.initializationResult(); + const account = init.account; + if (!account?.subscriptionType) { + return "Claude isn't signed in with a subscription in the eval folder, so it would bill per use."; + } + if (/api.?key/i.test(account.tokenSource ?? "")) { + return `Claude would use an API key (${account.tokenSource}), which bills per use.`; + } + const servers = ((await runtime.mcpServerStatus?.()) ?? []) as ReadonlyArray<{ name?: unknown }>; + const foreign = servers + .filter((server) => !isOwnServer(server.name)) + .map((server) => server.name); + return foreign.length > 0 + ? `Claude has MCP servers besides Threadlines' own (${foreign.join(", ")}).` + : null; +} + +/** The environment Claude's adapter and SDK run with in an eval. */ +export const claudeEvalEnvironment = (): NodeJS.ProcessEnv => ({ + ...subscriptionOnlyEnvironment(), + CLAUDE_CONFIG_DIR: EVAL_CLAUDE_CONFIG_DIR, +}); + +/** + * Checks the Codex eval folder before a run, to fail early with a clear + * reason: a ChatGPT sign-in, OpenAI as the model provider (no paid gateway), + * and no MCP servers or plugins of its own (Threadlines adds its servers per + * session). Each session is checked again before its first turn (codexGate.ts). + */ +export async function checkCodexEvalSetup(binaryPath = "codex"): Promise { + requireFolder(EVAL_CODEX_HOME, `CODEX_HOME="${EVAL_CODEX_HOME}" codex login`); + // An empty folder, so no project's own Codex settings apply. + const cwd = fs.mkdtempSync(nodePath.join(os.tmpdir(), "threadlines-eval-check-")); + const child = spawn(binaryPath, ["app-server"], { + cwd, + env: { ...subscriptionOnlyEnvironment(), CODEX_HOME: EVAL_CODEX_HOME }, + stdio: ["pipe", "pipe", "ignore"], + }); + const pending = new Map) => void>(); + let buffer = ""; + child.stdout.on("data", (chunk: Buffer) => { + buffer += chunk.toString(); + let end = buffer.indexOf("\n"); + while (end >= 0) { + const line = buffer.slice(0, end); + buffer = buffer.slice(end + 1); + try { + const message = JSON.parse(line) as Record; + if (typeof message.id === "number") pending.get(message.id)?.(message); + } catch { + // Not a frame. + } + end = buffer.indexOf("\n"); + } + }); + let nextId = 0; + const request = (method: string) => + new Promise>((resolve, reject) => { + const id = ++nextId; + const timer = setTimeout( + () => reject(new EvalSetupError(`Codex didn't answer ${method}.`)), + 30_000, + ); + pending.set(id, (message) => { + clearTimeout(timer); + resolve(message); + }); + child.stdin.write(`${JSON.stringify({ id, method, params: {} })}\n`); + }); + try { + child.stdin.write( + `${JSON.stringify({ id: 0, method: "initialize", params: { clientInfo: { name: "threadlines-evals", version: "0" }, capabilities: { experimentalApi: true } } })}\n`, + ); + child.stdin.write(`${JSON.stringify({ method: "initialized" })}\n`); + const answer = async (method: string) => { + const message = await request(method); + if (message.error !== undefined || message.result === undefined) { + throw new EvalSetupError(`Codex couldn't answer ${method}, so the setup can't be checked.`); + } + return message.result; + }; + const account = (await answer("account/read")) as { account?: { type?: string } | null }; + if (account.account?.type !== "chatgpt") { + throw new EvalSetupError( + `Codex in the eval folder isn't signed in with ChatGPT (${account.account?.type ?? "signed out"}), so it would bill per use.`, + ); + } + const providers = (await answer("modelProvider/list")) as { defaultProviderId?: string }; + if (providers.defaultProviderId !== "openai") { + throw new EvalSetupError( + `Codex in the eval folder uses the model provider ${providers.defaultProviderId}, not OpenAI.`, + ); + } + const status = (await answer("mcpServerStatus/list")) as { data?: unknown }; + if (!Array.isArray(status.data)) throw new EvalSetupError("Codex didn't list its MCP servers."); + // None at all: Threadlines adds its own per session, so any here is the folder's. + const servers = (status.data as ReadonlyArray<{ name?: string }>).map((server) => server.name); + if (servers.length > 0) { + throw new EvalSetupError( + `Codex in the eval folder has MCP servers or plugins of its own (${servers.join(", ")}). Turn them off in ${EVAL_CODEX_HOME}/config.toml ([features] apps = false, plugins = false).`, + ); + } + } finally { + child.kill(); + fs.rmSync(cwd, { recursive: true, force: true }); + } +} diff --git a/apps/server/evals/toolUse/runCase.ts b/apps/server/evals/toolUse/runCase.ts new file mode 100644 index 000000000..ad4b949d1 --- /dev/null +++ b/apps/server/evals/toolUse/runCase.ts @@ -0,0 +1,286 @@ +/** + * Runs one tool-use case: the real adapter and the real provider program + * (signed in from the eval folder), our real prompt and instructions, and the + * eval tool server in place of Threadlines' own tool endpoints. + */ +import { + ApprovalRequestId, + ClaudeSettings, + CodexSettings, + CommandId, + DEFAULT_PROVIDER_INTERACTION_MODE, + MessageId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ThreadId, + type OrchestrationThread, +} from "@threadlines/contracts"; +import { + APPROVAL_ACTIVITY_KINDS, + collectOpenPendingRequests, +} from "@threadlines/shared/pendingRequests"; +import * as Effect from "effect/Effect"; +import { ChildProcessSpawner } from "effect/process"; +import * as Schema from "effect/Schema"; + +import { ServerConfig } from "../../src/config.ts"; +import { makeClaudeAdapter } from "../../src/provider/Layers/ClaudeAdapter.ts"; +import { makeCodexAdapter } from "../../src/provider/Layers/CodexAdapter.ts"; +import { + makeOrchestrationIntegrationHarness, + type RealProviderAdapter, +} from "../../integration/OrchestrationEngineHarness.integration.ts"; +import type { EvalRun, NativeToolUse, ToolUseCase } from "./cases.ts"; +import { makeGatedCodexSpawner } from "./codexGate.ts"; +import { + claudeEvalEnvironment, + EVAL_CODEX_HOME, + EvalSetupError, + makeIsolatedClaudeQuery, + subscriptionOnlyEnvironment, +} from "./isolation.ts"; +import { startEvalToolServer } from "./toolServer.ts"; + +export type EvalProvider = "claude" | "codex"; + +const decodeClaudeSettings = Schema.decodeEffect(ClaudeSettings); +const decodeCodexSettings = Schema.decodeEffect(CodexSettings); + +/** + * Production defaults, except what would differ between runs or isn't served + * here: Codex's next-message suggestions, agent pages and command requests + * (their endpoints and instructions are both left out). + */ +export const EVAL_SERVER_SETTINGS = { + codexSuggestions: false, + enableAgentPages: false, + enableCommandRequests: false, +} as const; + +const PROJECT_ID = ProjectId.make("project-eval"); +const THREAD_ID = ThreadId.make("thread-eval"); + +function evalAdapter( + provider: EvalProvider, + port: number, + onBlocked: (reason: string) => void, +): RealProviderAdapter { + const withPort = (make: Effect.Effect) => + Effect.flatMap(Effect.service(ServerConfig), (config) => + make.pipe(Effect.provideService(ServerConfig, { ...config, port })), + ); + if (provider === "claude") { + return { + kind: ProviderDriverKind.make("claudeAgent"), + make: Effect.gen(function* () { + const { query } = yield* Effect.promise(() => import("@anthropic-ai/claude-agent-sdk")); + const settings = yield* decodeClaudeSettings({}); + return yield* withPort( + makeClaudeAdapter(settings, { + // The adapter reads Claude's folder too (transcripts, approvals): the eval one. + environment: claudeEvalEnvironment(), + createQuery: makeIsolatedClaudeQuery( + (input) => query(input as Parameters[0]) as never, + onBlocked, + ), + }), + ); + }).pipe(Effect.orDie), + }; + } + return { + kind: ProviderDriverKind.make("codex"), + make: Effect.gen(function* () { + const settings = yield* decodeCodexSettings({ homePath: EVAL_CODEX_HOME }); + const real = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* withPort( + makeCodexAdapter(settings, { environment: subscriptionOnlyEnvironment() }), + ).pipe( + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + makeGatedCodexSpawner(real, onBlocked), + ), + ); + }).pipe(Effect.orDie), + }; +} + +/** The agent's own tool calls, from the thread's tool activities (one per call). */ +function nativeToolsOf(thread: OrchestrationThread): ReadonlyArray { + const byCall = new Map(); + for (const activity of thread.activities) { + if (!activity.kind.startsWith("tool.")) continue; + const payload = activity.payload as { + readonly toolCallId?: string; + readonly detail?: string; + readonly itemType?: string; + readonly data?: { + readonly toolName?: string; + readonly input?: unknown; + readonly item?: unknown; + }; + }; + // The tool itself, never its display text (a subagent's detail is its prompt). + const item = payload.data?.item as { readonly tool?: unknown } | undefined; + const name = + payload.data?.toolName ?? + (typeof item?.tool === "string" ? item.tool : undefined) ?? + payload.itemType ?? + "unknown"; + // Threadlines' own tools are graded from the tool server's log. + if (name.startsWith("mcp__threadlines_") || payload.itemType === "mcp_tool_call") continue; + const key = payload.toolCallId ?? activity.id; + byCall.set(key, { name, input: payload.data?.input ?? payload.data?.item ?? null }); + } + return [...byCall.values()]; +} + +export interface CaseResult { + readonly case: string; + readonly provider: EvalProvider; + readonly model: string; + readonly passed: boolean; + readonly failures: ReadonlyArray; + readonly run: EvalRun; +} + +export const runToolUseCase = (input: { + readonly provider: EvalProvider; + readonly model: string; + readonly testCase: ToolUseCase; + readonly timeoutMs: number; +}) => + Effect.gen(function* () { + const server = yield* Effect.acquireRelease( + Effect.promise(() => + startEvalToolServer({ + ...(input.testCase.answers ? { answers: input.testCase.answers } : {}), + runtimeMode: input.testCase.runtimeMode ?? "full-access", + serverSettings: EVAL_SERVER_SETTINGS, + }), + ), + (started) => Effect.promise(() => started.close()), + ); + let blocked: string | null = null; + const harness = yield* makeOrchestrationIntegrationHarness({ + provider: ProviderDriverKind.make(input.provider === "claude" ? "claudeAgent" : "codex"), + serverSettings: EVAL_SERVER_SETTINGS, + realAdapter: () => + evalAdapter(input.provider, server.port, (reason) => { + blocked = reason; + }), + }); + const outcome = yield* Effect.gen(function* () { + const modelSelection = { + instanceId: ProviderInstanceId.make(input.provider === "claude" ? "claudeAgent" : "codex"), + model: input.model, + }; + const runtimeMode = input.testCase.runtimeMode ?? "full-access"; + const createdAt = new Date().toISOString(); + yield* harness.engine.dispatch({ + type: "project.create", + commandId: CommandId.make("cmd-eval-project"), + projectId: PROJECT_ID, + title: "Eval", + workspaceRoot: harness.workspaceDir, + defaultModelSelection: modelSelection, + createdAt, + }); + yield* harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-eval-thread"), + threadId: THREAD_ID, + projectId: PROJECT_ID, + title: `Eval ${input.testCase.name}`, + modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode, + branch: null, + worktreePath: harness.workspaceDir, + createdAt, + }); + yield* harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-eval-turn"), + threadId: THREAD_ID, + message: { + messageId: MessageId.make("00000000-0000-4000-8000-0000000e7a11"), + role: "user", + text: input.testCase.message, + attachments: [], + }, + composer: "typed", + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode, + createdAt: new Date().toISOString(), + }); + // An approval the agent asks for is declined, as a careful user might, + // so a run never waits on a person; the checks see it was asked. + const declined = new Set(); + const deadline = Date.now() + input.timeoutMs; + while (true) { + if (blocked !== null) return yield* Effect.die(new EvalSetupError(blocked)); + const snapshot = yield* harness.snapshotQuery.getSnapshot().pipe(Effect.orDie); + const entry = snapshot.threads.find((candidate) => candidate.id === THREAD_ID); + const turn = entry?.latestTurn ?? null; + if ( + entry !== undefined && + turn !== null && + (turn.state === "completed" || turn.state === "interrupted" || turn.state === "error") && + entry.session?.status !== "running" && + entry.session?.status !== "starting" + ) { + return { thread: entry, declined: [...declined] }; + } + for (const open of collectOpenPendingRequests( + entry?.activities ?? [], + APPROVAL_ACTIVITY_KINDS, + )) { + if (declined.has(open.requestId)) continue; + declined.add(open.requestId); + yield* harness.engine + .dispatch({ + type: "thread.approval.respond", + commandId: CommandId.make(`cmd-eval-decline-${declined.size}`), + threadId: THREAD_ID, + requestId: ApprovalRequestId.make(open.requestId), + decision: "decline", + createdAt: new Date().toISOString(), + }) + .pipe(Effect.orDie); + } + if (Date.now() > deadline) { + return yield* Effect.die(new Error(`timed out after ${input.timeoutMs / 1000}s`)); + } + yield* Effect.sleep("500 millis"); + } + }).pipe(Effect.orDie, Effect.ensuring(harness.dispose)); + if (blocked !== null) return yield* Effect.die(new EvalSetupError(blocked)); + const run: EvalRun = { + calls: [...server.calls], + native: nativeToolsOf(outcome.thread), + declinedApprovals: outcome.declined.length, + reply: outcome.thread.messages + .filter((message) => message.role === "assistant") + .map((message) => message.text) + .join("\n"), + }; + const failures = [ + ...(outcome.thread.latestTurn?.state === "completed" + ? [] + : [`turn ended ${outcome.thread.latestTurn?.state}`]), + ...input.testCase.checks.flatMap((check) => { + const failure = check(run); + return failure === null ? [] : [failure]; + }), + ]; + return { + case: input.testCase.name, + provider: input.provider, + model: input.model, + passed: failures.length === 0, + failures, + run, + } satisfies CaseResult; + }); diff --git a/apps/server/evals/toolUse/toolServer.test.ts b/apps/server/evals/toolUse/toolServer.test.ts new file mode 100644 index 000000000..b49284d30 --- /dev/null +++ b/apps/server/evals/toolUse/toolServer.test.ts @@ -0,0 +1,216 @@ +import { ThreadId } from "@threadlines/contracts"; +import * as Cause from "effect/Cause"; +import * as Effect from "effect/Effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/process"; +import * as Queue from "effect/Queue"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import { afterAll, assert, beforeAll, describe, it } from "vite-plus/test"; + +import { mcpSessionRegistry } from "../../src/mcp/McpSessionRegistry.ts"; +import { calls, everyCall, firstBefore, never, TOOL_USE_CASES } from "./cases.ts"; +import { makeGatedCodexSpawner } from "./codexGate.ts"; +import { EVAL_SERVER_SETTINGS } from "./runCase.ts"; +import { startEvalToolServer, type EvalToolServer } from "./toolServer.ts"; + +/** Speaks MCP over HTTP to the room endpoint as an agent's runtime would. */ +async function mcpSession(port: number, token: string | null) { + const headers: Record = { + "content-type": "application/json", + accept: "application/json, text/event-stream", + ...(token !== null ? { authorization: `Bearer ${token}` } : {}), + }; + let id = 0; + const post = async (method: string, params: unknown) => { + const response = await fetch(`http://127.0.0.1:${port}/mcp/room`, { + method: "POST", + headers, + body: JSON.stringify({ jsonrpc: "2.0", id: ++id, method, params }), + }); + const session = response.headers.get("mcp-session-id"); + if (session !== null) headers["mcp-session-id"] = session; + headers["mcp-protocol-version"] = "2025-06-18"; + return { + status: response.status, + body: (response.ok ? await response.json() : null) as { + readonly result: { readonly tools: ReadonlyArray<{ readonly name: string }> }; + } | null, + }; + }; + return { post }; +} + +describe("eval tool server", () => { + let server: EvalToolServer; + beforeAll(async () => { + server = await startEvalToolServer({ + runtimeMode: "full-access", + serverSettings: EVAL_SERVER_SETTINGS, + }); + }); + afterAll(() => server.close()); + + it("lists an ordinary runtime's tools, as production does, and logs what is called", async () => { + const credential = await Effect.runPromise( + mcpSessionRegistry.credentialFor({ + sessionKey: ThreadId.make("thread-eval-tool-server"), + browser: true, + room: true, + pages: false, + commands: false, + scheduledRun: false, + guideOnly: false, + }), + ); + const session = await mcpSession(server.port, credential.token); + await session.post("initialize", { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "test", version: "0" }, + }); + const listed = await session.post("tools/list", {}); + const names = (listed.body?.result.tools ?? []).map((tool) => tool.name); + assert.include(names, "thread_start"); + assert.include(names, "schedule_create"); + assert.notInclude(names, "scheduled_run_finish", "only a scheduled run's runtime gets it"); + + const called = await session.post("tools/call", { + name: "thread_start", + arguments: { threads: [{ title: "Docs", prompt: "Write docs." }] }, + }); + assert.equal(called.status, 200); + assert.deepEqual(server.calls.at(-1), { + tool: "thread_start", + input: { threads: [{ title: "Docs", prompt: "Write docs." }] }, + }); + }); + + it("refuses a caller without a credential", async () => { + const session = await mcpSession(server.port, null); + const answer = await session.post("initialize", { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "test", version: "0" }, + }); + assert.equal(answer.status, 401); + }); +}); + +describe("tool-use checks", () => { + const run = (tools: ReadonlyArray) => ({ + calls: tools.map((tool) => ({ tool, input: { threads: [{}, {}] } })), + native: [], + reply: "", + }); + + it("grade the calls a run made", () => { + assert.isNull(calls("thread_start", { min: 1, max: 1 })(run(["thread_start"]))); + assert.isNotNull( + calls("thread_start", { min: 1, max: 1 })(run(["thread_start", "thread_start"])), + ); + assert.isNotNull(never("thread_send")(run(["thread_send"]))); + assert.isNotNull( + firstBefore("app_guide", "settings_change")(run(["settings_change", "app_guide"])), + ); + assert.isNull( + everyCall( + "thread_start", + (input) => (input.threads as Array).length === 2, + "x", + )(run(["thread_start"])), + ); + }); + + it("grade what the reply tells the user about a thread's access", () => { + const accessCheck = TOOL_USE_CASES.find((entry) => entry.name === "access-ceiling")!.checks[0]!; + const said = (reply: string) => accessCheck({ calls: [], native: [], reply }); + assert.isNotNull(said("I've started a new thread called Tidy with full access.")); + assert.isNull(said("I started the thread, but it didn't get full access. It runs supervised.")); + assert.isNull(said("I'll check how to start a thread with full access.")); + }); + + it("every case has a unique name", () => { + const names = TOOL_USE_CASES.map((entry) => entry.name); + assert.equal(new Set(names).size, names.length); + }); +}); + +describe("Codex session check", () => { + /** A fake `codex app-server` that answers the session check with `account`. */ + function fakeCodex(account: { type: string }, sent: Array) { + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + return ChildProcessSpawner.make(() => + Effect.gen(function* () { + const output = yield* Queue.unbounded(); + const answer = (message: unknown) => + Queue.offerUnsafe(output, encoder.encode(`${JSON.stringify(message)}\n`)); + const results: Record = { + "account/read": { account }, + "modelProvider/list": { defaultProviderId: "openai" }, + "mcpServerStatus/list": { data: [] }, + }; + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + exitCode: Effect.never, + isRunning: Effect.succeed(true), + kill: () => Effect.sync(() => Queue.endUnsafe(output)), + unref: Effect.succeed(Effect.void), + stdin: Sink.forEach((chunk: Uint8Array) => + Effect.sync(() => { + for (const line of decoder.decode(chunk).split("\n").filter(Boolean)) { + const message = JSON.parse(line) as { id?: unknown; method?: string }; + sent.push(message.method ?? "reply"); + if (message.method !== undefined && message.method in results) { + answer({ id: message.id, result: results[message.method] }); + } else if (message.method === "thread/start") { + answer({ id: message.id, result: { thread: {}, modelProvider: "openai" } }); + } + } + }), + ), + stdout: Stream.fromQueue(output), + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ); + } + + const session = (account: { type: string }) => + Effect.gen(function* () { + const sent: Array = []; + let blocked: string | null = null; + const spawner = makeGatedCodexSpawner(fakeCodex(account, sent), (reason) => { + blocked = reason; + }); + const handle = yield* spawner.spawn(ChildProcess.make("codex", ["app-server"])); + const write = (message: unknown) => + Stream.run( + Stream.make(new TextEncoder().encode(`${JSON.stringify(message)}\n`)), + handle.stdin, + ); + yield* Effect.forkScoped(Stream.runDrain(handle.stdout)); + yield* write({ id: 1, method: "initialize", params: {} }); + yield* write({ method: "initialized" }); + yield* write({ id: 2, method: "thread/start", params: {} }); + const turn = yield* Effect.exit( + write({ id: 3, method: "turn/start", params: {} }).pipe(Effect.timeout("2 seconds")), + ); + return { sent, blocked: blocked as string | null, turnSent: turn._tag === "Success" }; + }).pipe(Effect.scoped); + + it("holds the first turn until the session is on a ChatGPT sign-in", async () => { + const ok = await Effect.runPromise(session({ type: "chatgpt" })); + assert.isNull(ok.blocked); + assert.isTrue(ok.turnSent); + assert.include(ok.sent, "turn/start"); + + const apiKey = await Effect.runPromise(session({ type: "apiKey" })); + assert.match(apiKey.blocked ?? "", /isn't signed in with ChatGPT/); + assert.isFalse(apiKey.turnSent); + assert.notInclude(apiKey.sent, "turn/start"); + }); +}); diff --git a/apps/server/evals/toolUse/toolServer.ts b/apps/server/evals/toolUse/toolServer.ts new file mode 100644 index 000000000..7dc135cb0 --- /dev/null +++ b/apps/server/evals/toolUse/toolServer.ts @@ -0,0 +1,72 @@ +/** + * The tool endpoints an eval's agent talks to: production's room and browser + * MCP endpoints (same tool list, descriptions, per-runtime filtering and + * sign-in check against the shared credential registry) with canned handlers. + * Agent pages and command requests are off in evals, so their endpoints and + * instructions are both absent. + */ +import * as http from "node:http"; + +import type { RuntimeMode } from "@threadlines/contracts"; + +import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as ManagedRuntime from "effect/ManagedRuntime"; +import * as Option from "effect/Option"; +import { HttpRouter, HttpServer } from "effect/http"; + +import * as McpHttpServer from "../../src/mcp/McpHttpServer.ts"; +import * as McpRoomServer from "../../src/mcp/McpRoomServer.ts"; +import { ProjectionSnapshotQuery } from "../../src/orchestration/Services/ProjectionSnapshotQuery.ts"; +import { ServerSettingsService } from "../../src/serverSettings.ts"; +import { + makeCannedRoomHandlers, + refusingBrowserHandlers, + type CannedAnswers, + type ToolCall, +} from "./cannedTools.ts"; + +export interface EvalToolServer { + readonly port: number; + /** Every tool call the agent made, in order. */ + readonly calls: ReadonlyArray; + readonly close: () => Promise; +} + +export async function startEvalToolServer(input: { + readonly answers?: CannedAnswers; + /** The eval thread's access: a thread it starts gets at most this. */ + readonly runtimeMode: RuntimeMode; + readonly serverSettings: Parameters[0]; +}): Promise { + const calls: Array = []; + const routes = Layer.mergeAll( + McpRoomServer.makeLayer( + makeCannedRoomHandlers(calls, input.answers, { runtimeMode: input.runtimeMode }), + ), + McpHttpServer.makeLayer(refusingBrowserHandlers), + ); + const layer = HttpRouter.serve(routes, { disableLogger: true }).pipe( + Layer.provideMerge( + NodeHttpServer.layer(() => http.createServer(), { host: "127.0.0.1", port: 0 }), + ), + Layer.provide(ServerSettingsService.layerTest(input.serverSettings)), + // Read to limit an outside agent's credential; an eval's agent has none, + // and its thread no outside work. + Layer.provide( + Layer.mock(ProjectionSnapshotQuery)({ + getThreadShellById: () => Effect.succeed(Option.none()), + getThreadDetailById: () => Effect.succeed(Option.none()), + }), + ), + Layer.provide(NodeServices.layer), + ); + const runtime = ManagedRuntime.make(layer); + const address = await runtime.runPromise( + Effect.map(Effect.service(HttpServer.HttpServer), (server) => server.address), + ); + if (address._tag === "UnixPathAddress") throw new Error("The eval tool server has no TCP port."); + return { port: address.port, calls, close: () => runtime.dispose() }; +} diff --git a/apps/server/scripts/eval-tool-use.ts b/apps/server/scripts/eval-tool-use.ts new file mode 100644 index 000000000..b7135d80e --- /dev/null +++ b/apps/server/scripts/eval-tool-use.ts @@ -0,0 +1,93 @@ +/** + * Tool-use evals: how real models use the tools Threadlines gives agents + * (apps/server/evals/toolUse). Runs on the developer's subscription sign-ins + * from eval-only settings folders, never per-use billing. + * + * node scripts/eval-tool-use.ts --provider claude|codex [--model ] [--case ] [--samples 2] + */ +// @effect-diagnostics nodeBuiltinImport:off +import * as fs from "node:fs"; +import * as nodePath from "node:path"; +import { parseArgs } from "node:util"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { DEFAULT_MODEL_BY_PROVIDER, ProviderDriverKind } from "@threadlines/contracts"; +import * as Effect from "effect/Effect"; + +import { TOOL_USE_CASES } from "../evals/toolUse/cases.ts"; +import { checkCodexEvalSetup, EVAL_HOME, EvalSetupError } from "../evals/toolUse/isolation.ts"; +import { runToolUseCase, type CaseResult, type EvalProvider } from "../evals/toolUse/runCase.ts"; + +const { values } = parseArgs({ + options: { + provider: { type: "string" }, + model: { type: "string" }, + case: { type: "string" }, + samples: { type: "string", default: "2" }, + }, +}); +const provider = values.provider as EvalProvider | undefined; +if (provider !== "claude" && provider !== "codex") { + throw new Error("Pass --provider claude or --provider codex."); +} +const model = + values.model ?? + DEFAULT_MODEL_BY_PROVIDER[ + ProviderDriverKind.make(provider === "claude" ? "claudeAgent" : "codex") + ]!; +const samples = Number(values.samples); +const cases = TOOL_USE_CASES.filter( + (entry) => values.case === undefined || entry.name === values.case, +); +if (cases.length === 0) throw new Error(`No case named ${values.case}.`); + +if (provider === "codex") await checkCodexEvalSetup(); + +console.log(`${provider} ${model}: ${cases.length} cases × ${samples} samples`); +const results: Array = []; +for (const testCase of cases) { + for (let sample = 1; sample <= samples; sample += 1) { + const result = await Effect.runPromise( + runToolUseCase({ provider, model, testCase, timeoutMs: 5 * 60_000 }).pipe( + Effect.scoped, + Effect.provide(NodeServices.layer), + // A case that breaks is a failed case; a setup problem stops the run. + Effect.catchDefect((defect) => + defect instanceof EvalSetupError + ? Effect.die(defect) + : Effect.succeed({ + case: testCase.name, + provider, + model, + passed: false, + failures: [`crashed: ${defect instanceof Error ? defect.message : String(defect)}`], + run: { calls: [], native: [], reply: "" }, + }), + ), + ), + ); + results.push(result); + console.log( + ` ${result.passed ? "pass" : "FAIL"} ${testCase.name} #${sample}${ + result.passed ? "" : `: ${result.failures.join("; ")}` + }`, + ); + } +} + +const passed = results.filter((result) => result.passed).length; +console.log(`\n${passed}/${results.length} passed`); +for (const testCase of cases) { + const runs = results.filter((result) => result.case === testCase.name); + console.log( + ` ${runs.filter((result) => result.passed).length}/${runs.length} ${testCase.name}`, + ); +} +const output = nodePath.join( + EVAL_HOME, + "results", + `${new Date().toISOString().replace(/[:.]/g, "-")}-${provider}-${model}.json`, +); +fs.mkdirSync(nodePath.dirname(output), { recursive: true }); +fs.writeFileSync(output, JSON.stringify({ provider, model, samples, results }, null, 2)); +console.log(`\nfull results: ${output}`); diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 804626401..d4f7289ca 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -333,17 +333,31 @@ const screenshotFailure = (cause: Cause.Cause) => { ); }; -export const layer = McpServer.toolkit(BrowserStandardToolkit).pipe( - Layer.provide(BrowserToolkitHandlersLive), - Layer.provideMerge( - Layer.effectDiscard(registerScreenshot()).pipe(Layer.provide(BrowserScreenshotHandlersLive)), - ), - Layer.provideMerge( - McpServer.layerHttp({ - name: "threadlines-browser", - version: "1", - path: MCP_ROUTE_PATH, - protocols: [McpProtocol.v2025_06_18], - }).pipe(Layer.provide(AuthenticationLive)), - ), -); +/** + * The browser endpoint with the given tool handlers: production's below; the + * tool-use evals (apps/server/evals) serve the same tools and listing with + * handlers that refuse. + */ +export const makeLayer = (handlers: { + readonly standard: Layer.Layer, E1, R1>; + readonly screenshot: Layer.Layer, E2, R2>; +}) => + McpServer.toolkit(BrowserStandardToolkit).pipe( + Layer.provide(handlers.standard), + Layer.provideMerge( + Layer.effectDiscard(registerScreenshot()).pipe(Layer.provide(handlers.screenshot)), + ), + Layer.provideMerge( + McpServer.layerHttp({ + name: "threadlines-browser", + version: "1", + path: MCP_ROUTE_PATH, + protocols: [McpProtocol.v2025_06_18], + }).pipe(Layer.provide(AuthenticationLive)), + ), + ); + +export const layer = makeLayer({ + standard: BrowserToolkitHandlersLive, + screenshot: BrowserScreenshotHandlersLive, +}); diff --git a/apps/server/src/mcp/McpRoomServer.ts b/apps/server/src/mcp/McpRoomServer.ts index d0e1e5680..769aa0f0c 100644 --- a/apps/server/src/mcp/McpRoomServer.ts +++ b/apps/server/src/mcp/McpRoomServer.ts @@ -27,7 +27,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Stream from "effect/Stream"; import type * as Types from "effect/Types"; -import { McpProtocol, McpServer, Toolkit } from "effect/ai"; +import { McpProtocol, McpServer, Tool, Toolkit } from "effect/ai"; import { HttpRouter, HttpServerRequest, type HttpServerResponse } from "effect/http"; import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts"; @@ -73,7 +73,7 @@ import { ThreadToolkit } from "./threadTools.ts"; import { requestService } from "./requestService.ts"; /** Every tool the endpoint serves. */ -const RoomEndpointToolkit = Toolkit.merge( +export const RoomEndpointToolkit = Toolkit.merge( RoomToolkit, ThreadToolkit, ScheduleToolkit, @@ -520,16 +520,26 @@ const RoomToolHandlersLive = RoomEndpointToolkit.toLayer( }), ); -export const layer = Layer.fresh( - McpServer.toolkit(RoomEndpointToolkit).pipe( - Layer.provide(RoomToolHandlersLive), - Layer.provideMerge( - McpServer.layerHttp({ - name: "threadlines-room", - version: "1", - path: MCP_ROOM_ROUTE_PATH, - protocols: [McpProtocol.v2025_06_18], - }).pipe(Layer.provide(AuthenticationLive)), +/** + * The room endpoint with the given tool handlers: production's below; the + * tool-use evals (apps/server/evals) serve the same tools, listing and + * sign-in check with canned answers. + */ +export const makeLayer = ( + handlers: Layer.Layer, E, R>, +) => + Layer.fresh( + McpServer.toolkit(RoomEndpointToolkit).pipe( + Layer.provide(handlers), + Layer.provideMerge( + McpServer.layerHttp({ + name: "threadlines-room", + version: "1", + path: MCP_ROOM_ROUTE_PATH, + protocols: [McpProtocol.v2025_06_18], + }).pipe(Layer.provide(AuthenticationLive)), + ), ), - ), -); + ); + +export const layer = makeLayer(RoomToolHandlersLive); diff --git a/apps/server/src/mcp/threadToolHandlers.test.ts b/apps/server/src/mcp/threadToolHandlers.test.ts index bf0b626af..5e71608be 100644 --- a/apps/server/src/mcp/threadToolHandlers.test.ts +++ b/apps/server/src/mcp/threadToolHandlers.test.ts @@ -687,7 +687,7 @@ describe("an existing checkout", () => { it.effect("`self` is the caller's own checkout, and never more access than its own", () => Effect.gen(function* () { const tools = yield* makeTools({ shareWorktrees: true, branches }); - yield* tools.handlers.thread_start(mainCaller(), { + const result = yield* tools.handlers.thread_start(mainCaller(), { threads: [{ ...reviewer, checkout: "self", access: "full-access" }], }); const [start] = ofType(tools.dispatched, "thread.child.start"); @@ -695,6 +695,11 @@ describe("an existing checkout", () => { checkout: { threadId: PARENT, worktreePath: "/checkout", branch: "main" }, runtimeMode: "auto-accept-edits", }); + // The agent hears what it got, so it doesn't tell the user "full access". + expect(result.threads?.[0]?.access).toBe("auto-accept-edits"); + expect(result.detail).toContain( + "'Review the retry change' runs with auto-accept-edits, not full-access", + ); // In the project folder itself, there is no worktree to name. const inProject = yield* makeTools({ diff --git a/apps/server/src/mcp/threadToolHandlers.ts b/apps/server/src/mcp/threadToolHandlers.ts index b95a79bfb..a4ac01ec0 100644 --- a/apps/server/src/mcp/threadToolHandlers.ts +++ b/apps/server/src/mcp/threadToolHandlers.ts @@ -313,7 +313,7 @@ const childIsBusy = (child: OrchestrationThread, parent: OrchestrationThread) => ); /** A started thread's access: the caller's, or a lower level the call asked for, never more. */ -const launchAccess = ( +export const launchAccess = ( spec: { readonly access: RuntimeMode | undefined }, caller: Pick, ): RuntimeMode => @@ -321,6 +321,56 @@ const launchAccess = ( ? caller.runtimeMode : saferRuntimeMode(spec.access, caller.runtimeMode); +/** + * What thread_start tells the agent happened, and what to do next (end its + * turn rather than wait). Shared with the tool-use evals' canned answer. + */ +export const startedDetail = (input: { + readonly outcome: "started" | "asked_user"; + readonly count: number; + readonly reportBack: boolean; + readonly sharedFolder: boolean; + /** How many of them work in another thread's checkout (`checkout`). */ + readonly inCheckouts?: number; +}) => { + const threads = plural(input.count, "thread"); + const these = input.count === 1 ? "it" : "them"; + const base = + input.outcome === "started" + ? input.reportBack + ? `Started ${threads}. Each one's final reply comes back to you as a message when it finishes, and you pick up again then. Don't wait or poll: carry on with other work, or end your turn if you have nothing else to do.` + : `Started ${threads} on ${input.count === 1 ? "its" : "their"} own. Nothing comes back from ${these}; read ${these} with thread_read if you need to.` + : input.reportBack + ? `The user decides whether to start ${threads}. If they agree, each one's final reply comes back to you as a message; if they say no, you'll hear at your next turn. Don't ask again, and end your turn if you have nothing else to do.` + : `The user decides whether to start ${threads}. If started, ${input.count === 1 ? "it runs" : "they run"} on ${input.count === 1 ? "its" : "their"} own and nothing comes back. Don't ask again.`; + const inCheckouts = input.inCheckouts ?? 0; + const shared = + inCheckouts === 0 + ? "" + : ` ${input.count === 1 ? "It works" : inCheckouts === input.count ? "They work" : `${inCheckouts} of them work`} in a checkout another thread has, with the same files: have only one thread work there at a time.`; + return input.sharedFolder + ? `${base}${shared} This project has no git repository, so ${input.count === 1 ? "it shares" : "they share"} its folder and can change the same files you do.` + : `${base}${shared}`; +}; + +/** + * Says which threads got less access than the call asked for, so the agent + * doesn't tell the user they have it. Empty when every thread got what it asked. + */ +export const lowerAccessNote = ( + specs: ReadonlyArray<{ readonly title: string; readonly access: RuntimeMode | undefined }>, + launches: ReadonlyArray<{ readonly runtimeMode: RuntimeMode }>, +): string => { + const lowered = specs.flatMap((spec, index) => + spec.access !== undefined && launches[index]!.runtimeMode !== spec.access + ? [`'${spec.title}' runs with ${launches[index]!.runtimeMode}, not ${spec.access}`] + : [], + ); + return lowered.length === 0 + ? "" + : ` ${lowered.join("; ")}: a thread never gets more access than yours. If you tell the user about ${lowered.length === 1 ? "it" : "them"}, say the access ${lowered.length === 1 ? "it" : "they"} actually got.`; +}; + /** Whether a thread already works in `checkout`, with its session there too. */ const isInCheckout = (thread: OrchestrationThread, checkout: ChildThreadCheckout) => thread.effectiveCwd === null && @@ -723,34 +773,6 @@ export function makeThreadToolHandlers(deps: ThreadToolDeps) { })), }); - const startedDetail = (input: { - readonly outcome: "started" | "asked_user"; - readonly count: number; - readonly reportBack: boolean; - readonly sharedFolder: boolean; - /** How many of them work in another thread's checkout (`checkout`). */ - readonly inCheckouts?: number; - }) => { - const threads = plural(input.count, "thread"); - const these = input.count === 1 ? "it" : "them"; - const base = - input.outcome === "started" - ? input.reportBack - ? `Started ${threads}. Each one's final reply comes back to you as a message when it finishes, and you pick up again then. Don't wait or poll: carry on with other work, or end your turn if you have nothing else to do.` - : `Started ${threads} on ${input.count === 1 ? "its" : "their"} own. Nothing comes back from ${these}; read ${these} with thread_read if you need to.` - : input.reportBack - ? `The user decides whether to start ${threads}. If they agree, each one's final reply comes back to you as a message; if they say no, you'll hear at your next turn. Don't ask again, and end your turn if you have nothing else to do.` - : `The user decides whether to start ${threads}. If started, ${input.count === 1 ? "it runs" : "they run"} on ${input.count === 1 ? "its" : "their"} own and nothing comes back. Don't ask again.`; - const inCheckouts = input.inCheckouts ?? 0; - const shared = - inCheckouts === 0 - ? "" - : ` ${input.count === 1 ? "It works" : inCheckouts === input.count ? "They work" : `${inCheckouts} of them work`} in a checkout another thread has, with the same files: have only one thread work there at a time.`; - return input.sharedFolder - ? `${base}${shared} This project has no git repository, so ${input.count === 1 ? "it shares" : "they share"} its folder and can change the same files you do.` - : `${base}${shared}`; - }; - /** * What an earlier identical call already did, if anything: its requests * still open, or its threads already made. @@ -796,6 +818,7 @@ export function makeThreadToolHandlers(deps: ThreadToolDeps) { request.launch?.checkout !== undefined ? ("checkout" as const) : (request.launch?.workspace.kind ?? "worktree"), + ...(request.launch !== undefined ? { access: request.launch.runtimeMode } : {}), })), } satisfies ThreadStartResult; } @@ -824,6 +847,7 @@ export function makeThreadToolHandlers(deps: ThreadToolDeps) { : shell.worktreePath !== null ? "worktree" : "project_folder", + access: shell.runtimeMode, })), } satisfies ThreadStartResult; }); @@ -1055,13 +1079,13 @@ export function makeThreadToolHandlers(deps: ThreadToolDeps) { const outcome = mode === "auto" ? ("started" as const) : ("asked_user" as const); return { outcome, - detail: startedDetail({ + detail: `${startedDetail({ outcome, count: launches.length, reportBack, sharedFolder, inCheckouts: launches.filter((launch) => launch.checkout !== undefined).length, - }), + })}${lowerAccessNote(specs, launches)}`, threads: ids.children.map((child, index) => ({ threadId: child.childThreadId, title: launches[index]!.title, @@ -1071,6 +1095,7 @@ export function makeThreadToolHandlers(deps: ThreadToolDeps) { launches[index]!.checkout !== undefined ? ("checkout" as const) : launches[index]!.workspace.kind, + access: launches[index]!.runtimeMode, })), } satisfies ThreadStartResult; }), diff --git a/apps/server/src/mcp/threadTools.ts b/apps/server/src/mcp/threadTools.ts index 635b6fe31..3ab022d76 100644 --- a/apps/server/src/mcp/threadTools.ts +++ b/apps/server/src/mcp/threadTools.ts @@ -105,6 +105,11 @@ export const ThreadStartResult = Schema.Struct({ workspace: Schema.Literals(["worktree", "project_folder", "checkout", "remote"]), /** The linked computer it runs on; absent: this one. */ computer: Schema.optional(Schema.String), + /** + * The access it runs with: never more than yours, so lower than you + * asked for when you asked for more. + */ + access: Schema.optional(RuntimeMode), }), ), ), diff --git a/apps/server/tsconfig.json b/apps/server/tsconfig.json index ead79e193..a3c880a01 100644 --- a/apps/server/tsconfig.json +++ b/apps/server/tsconfig.json @@ -9,6 +9,7 @@ "vite.config.ts", "scripts", "integration", + "evals", "../../scripts/lib", "../../vite.config.ts" ]