diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml new file mode 100644 index 00000000..4e57ba9f --- /dev/null +++ b/.github/workflows/sandbox.yml @@ -0,0 +1,42 @@ +name: Sandbox + +# The runner sandbox's boundary test (`just runner-sandbox`) runs nightly on a +# GitHub-hosted runner, outside pull requests and the merge queue, so landing +# never waits on it. GitHub reports a failed scheduled run only to the account +# that last changed the cron schedule (with Actions notifications on), not to +# the repository's watchers; that is the maintainer, whose account lands the +# workflow, so they see a boundary failure. +on: + schedule: + - cron: "23 7 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: sandbox-${{ github.ref }} + cancel-in-progress: true + +jobs: + runner-sandbox: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f + with: + determinate: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install dependencies + run: nix develop --command just install + # The test runs bubblewrap from the Nix store, which Ubuntu's AppArmor would + # refuse the unprivileged user namespace it needs. + - name: Allow unprivileged user namespaces + run: | + if [ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]; then + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + fi + - name: Check the runner sandbox's boundary + run: nix develop --command just runner-sandbox diff --git a/README.md b/README.md index 1adc2bd7..1caba7e7 100644 --- a/README.md +++ b/README.md @@ -1194,7 +1194,11 @@ language servers and formatters are off. The CLI itself, Claude Code or commands start from a minimal environment, so its login, the opencode server's password, and the kix tools' token stay out of their reach. This is an OS sandbox, not a VM, and it narrows rather than removes the risk of running -agent commands on your machine. +agent commands on your machine. On Linux, `just runner-sandbox` in this +repository runs a scripted model through the pinned opencode and +sandbox-runtime in a checkout that tries to read credentials, write outside its +sandbox, plant opencode config, and reach the network, and fails if any attempt +gets through. CI runs it nightly, outside the merge queue. On Linux the sandbox needs `bwrap` and `socat` (the flake's `runner` package pins both) and unprivileged user namespaces, including the nested one Claude diff --git a/flake.nix b/flake.nix index 610f5670..c84ae3c5 100644 --- a/flake.nix +++ b/flake.nix @@ -105,6 +105,13 @@ # `flock` (the land daemon's repository lock) and `script` (the tests' terminal), which # self-hosted NixOS runners don't otherwise provide. ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isLinux [ pkgs.util-linux ]; + # The runner package's opencode and sandbox-runtime, for the sandbox boundary test + # (test/live/runner-sandbox.test.ts), which runs where srt does. Named as the runner + # finds them rather than put on PATH, where they would shadow the user's own opencode. + env = pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { + QUEST_RUNNER_OPENCODE = pkgs.lib.getExe opencode; + QUEST_RUNNER_SRT = pkgs.lib.getExe pkgs.sandbox-runtime; + }; }; formatter = pkgs.nixfmt; checks = { diff --git a/justfile b/justfile index ca2f2a67..ce6e2eb3 100644 --- a/justfile +++ b/justfile @@ -44,7 +44,13 @@ _test: # Codex and Claude Code CLIs. Uses each CLI's login and spends its usage; run it # after upgrading either CLI. Arguments go to vitest, such as `-t codex`. review-sandbox *args: - QUEST_LIVE=1 bun run vitest run {{args}} + QUEST_LIVE=1 bun run vitest run test/live/review-sandbox {{args}} + +# Check that a runner job's commands cannot escape its sandbox, with the pinned +# opencode and sandbox-runtime and a scripted model, on Linux. Needs no login; +# CI runs it nightly (.github/workflows/sandbox.yml). Run it after bumping either. +runner-sandbox *args: + QUEST_LIVE=1 bun run vitest run test/live/runner-sandbox {{args}} # Apply formatters without changing program behavior. fix: diff --git a/quest/m0/kix/sandbox/README.md b/quest/m0/kix/sandbox/README.md index 3270a659..53f19d0e 100644 --- a/quest/m0/kix/sandbox/README.md +++ b/quest/m0/kix/sandbox/README.md @@ -67,6 +67,14 @@ Shipped in [moq-dev/quest#212](https://github.com/moq-dev/quest/pull/212) opencode 1.19 still imports the plugins a checkout's own `opencode.json` or `.opencode/` names, even with project config disabled, so the runner keeps the agent from creating them and refuses a checkout that holds them. + `test/live/runner-sandbox.test.ts` checks that boundary with the pinned + opencode and srt and a scripted model. It runs in `just runner-sandbox` + and a nightly GitHub-hosted workflow (`.github/workflows/sandbox.yml`), + not in `just test` or the merge queue, whose container can't mount + `/proc` for bwrap, so a bump that weakens the boundary fails the next + nightly run. Decided 2026-11-03 (user, /quest-complete), after the merge + queue dropped [moq-dev/quest#404](https://github.com/moq-dev/quest/pull/404): + its own CI, nightly, so landing never blocks on it. - Decided 2026-10-24 (user) in `/quest-plan`: split the remaining work into the children below, ranked so the earliest unblock the most. Dropped (user): read-only registry access, which Claude Code's sandbox cannot @@ -78,6 +86,5 @@ Shipped in [moq-dev/quest#212](https://github.com/moq-dev/quest/pull/212) ## Required - [No Unix sockets in opencode jobs](/quest/m0/kix/sandbox/seccomp.md) - srt's seccomp helper is packaged, so commands can't reach the Nix daemon, and the runner refuses to start without it -- [Boundary test in CI](/quest/m0/kix/sandbox/live-test.md) - the opencode sandbox's hostile-repository test runs on Linux CI with a scripted model - [Trust rechecked on queue](/quest/m0/kix/sandbox/trust-on-queue.md) - a job's head trust is reread whenever it queues, so the approval decision shows without a free runner - [Claude Code boundary test](/quest/m0/kix/sandbox/claude-boundary.md) - the same hostile-repository test for Claude Code jobs, with a fake model set in the account's settings diff --git a/quest/m0/kix/sandbox/claude-boundary.md b/quest/m0/kix/sandbox/claude-boundary.md index 6032b882..5960ba29 100644 --- a/quest/m0/kix/sandbox/claude-boundary.md +++ b/quest/m0/kix/sandbox/claude-boundary.md @@ -1,4 +1,4 @@ -# [M] The Claude Code sandbox's boundary test runs in CI +# [M] The Claude Code sandbox's boundary test runs nightly in CI ## Goal @@ -6,9 +6,9 @@ Claude Code jobs get the same checked-in hostile-repository boundary test that opencode jobs got in [moq-dev/quest#404](https://github.com/moq-dev/quest/pull/404): a scripted fake model, set as the account's endpoint in its home, drives the pinned -Claude Code through the runner's launch path on Linux CI, and every escape -attempt is refused. -A Claude Code bump that weakens the boundary fails CI before it lands. +Claude Code through the runner's launch path in the nightly sandbox workflow +on Linux, and every escape attempt is refused. +A Claude Code bump that weakens the boundary fails the next nightly run. ## Plan @@ -28,7 +28,8 @@ Evidence, gathered 2026-11-02: Work: -- Reuse #404's attempts and layout: denied reads by path and symlink, +- Reuse the attempts and layout of opencode's test, + `test/live/runner-sandbox.test.ts`: denied reads by path and symlink, writes outside the checkout and job temp directory, planted project settings and hooks, the network directly and through the proxy, and the Unix-socket probe from the seccomp quest. @@ -37,7 +38,7 @@ Work: `TMPDIR=/claude-`, which the runner's `TMPDIR` (`src/runner/claude.ts`) puts inside `allowWrite`; the test confirms it instead of trusting the read. -- Point the account at the fake model in its home, as #404's `signIn` +- Point the account at the fake model in its home, as that test's `signIn` configures opencode's provider: `ANTHROPIC_BASE_URL` in the `env` of `$CLAUDE_CONFIG_DIR/settings.json`, which `--setting-sources user` loads. An ambient `ANTHROPIC_BASE_URL` never reaches Claude Code, since @@ -46,10 +47,17 @@ Work: settings' `env` also reaches the commands, the fake key there is no secret, so the unreadable-secret attempt reads a planted credential file in the home instead. -- Like #404, each shell attempt first succeeds outside the sandbox, and the - test fails rather than skips on Linux when the sandbox can't start. +- Like opencode's test, each shell attempt first succeeds outside the + sandbox, and the test fails rather than skips on Linux when the sandbox + can't start. +- Run it where opencode's test runs: name it + `test/live/runner-sandbox-claude.test.ts` (or extend + `runner-sandbox.test.ts`), so `just runner-sandbox` and the nightly + `.github/workflows/sandbox.yml` pick it up, and keep it out of + `just test` and the merge queue. Decided 2026-11-03 (user, + /quest-complete) for opencode's test: its own CI, nightly, so landing + never blocks on it. ## Required - [No Unix sockets in opencode jobs](/quest/m0/kix/sandbox/seccomp.md) - the Unix-socket probe this test reuses -- [Boundary test in CI](/quest/m0/kix/sandbox/live-test.md) - #404's attempts and layout this test reuses diff --git a/quest/m0/kix/sandbox/live-test.md b/quest/m0/kix/sandbox/live-test.md deleted file mode 100644 index 4db1e813..00000000 --- a/quest/m0/kix/sandbox/live-test.md +++ /dev/null @@ -1,33 +0,0 @@ -# [S] The opencode sandbox's boundary test runs in CI - -## Goal - -The opencode sandbox's hostile-repository boundary test is checked in as -`test/live/runner-sandbox.test.ts`, drives opencode and srt with a scripted -model, and runs on Linux in this repository's checks, on `main` and in the -merge group. An srt or opencode bump that weakens the boundary fails CI -before it lands. - -## Plan - -Decided 2026-11-02 (user, /quest-plan of -[moq-dev/quest#338](https://github.com/moq-dev/quest/pull/338)'s suggested -follow-ups): - -- How it runs: - - ✅ CI on Linux: a scripted model needs no login and spends no usage - - A manual recipe, like `just review-sandbox` -- Milestone: ✅ m0 · m1 - -#338 ran the test by hand against srt 0.0.79. Merging main moved srt to -0.0.81, which the maintainer accepted from the release notes and a direct -srt run, without rerunning the full test. The checked-in test covers what -that run did: a checkout that tries to read a denied file, write outside the -checkout and attempt directory, create `.opencode/plugin`, `opencode.json` or -`opencode.jsonc`, and reach the network. Each attempt is refused, and nothing -is left behind in the checkout. Follow `test/live/review-sandbox.test.ts` -for the layout. Since this test needs no login, it runs without -`QUEST_LIVE`, skipping only where srt can't run (macOS and Windows). - -[opencode 2.x](/quest/m0/opencode-sdk.md) reruns the boundary test on 2.x; -with this quest done, that rerun is this test. diff --git a/quest/m0/opencode-sdk.md b/quest/m0/opencode-sdk.md index 955b199e..9411b64d 100644 --- a/quest/m0/opencode-sdk.md +++ b/quest/m0/opencode-sdk.md @@ -52,12 +52,11 @@ What the change does: - Recheck each of the sandbox's rules on 2.x: the `shell` wrapper through srt, the commands' environment allowlist (`keptEnvironment`), `OPENCODE_DISABLE_PROJECT_CONFIG`, `external_directory` and `webfetch` - denies, the plugin's realpath check, and `lsp`/`formatter` off. Rerun the - sandbox's hostile-repository boundary test against 2.x. + denies, the plugin's realpath check, and `lsp`/`formatter` off. The + sandbox's boundary test, `test/live/runner-sandbox.test.ts`, runs the + pinned opencode only in `just runner-sandbox` and nightly CI, so run it + on the 2.x pin before landing; extend it to any rule 2.x changes, such as + its tool names or plugin hooks. - Run the runner's live opencode checks that the [Kix decision UI](/quest/m0/kix/README.md) rehearsal lists, if a runner is available. - -## Required - -- [Boundary test in CI](/quest/m0/kix/sandbox/live-test.md) - the boundary test this quest reruns on 2.x is checked in and runs in CI diff --git a/src/runner/opencode.ts b/src/runner/opencode.ts index abe468b2..335f3d54 100644 --- a/src/runner/opencode.ts +++ b/src/runner/opencode.ts @@ -199,12 +199,16 @@ export default { * environment, which holds the server password, the tools' token in * `OPENCODE_CONFIG_CONTENT`, and any provider key opencode exported, then * sources the private HOME and the devShell inside the sandbox before the - * command, as Claude Code does with `CLAUDE_ENV_FILE`. + * command, as Claude Code does with `CLAUDE_ENV_FILE`. srt replaces the + * command's TMPDIR with its own `CLAUDE_CODE_TMPDIR`, or `/tmp/claude` + * without it, so that names the attempt's directory too. */ export function shellScript(srt: string, settings: string, sandbox: Sandbox, env: NodeJS.ProcessEnv): string { - const kept = Object.entries({ ...keptEnvironment(env), TMPDIR: sandbox.tmp }).map(([name, value]) => - quote(`${name}=${value}`), - ); + const kept = Object.entries({ + ...keptEnvironment(env), + TMPDIR: sandbox.tmp, + CLAUDE_CODE_TMPDIR: sandbox.tmp, + }).map(([name, value]) => quote(`${name}=${value}`)); const inner = `. ${quote(sandbox.env)} && quest_command=$QUEST_COMMAND && unset QUEST_COMMAND && eval "$quest_command"`; return `#!/bin/sh if [ "$#" -ne 2 ] || [ "$1" != -c ]; then diff --git a/test/live/runner-sandbox.test.ts b/test/live/runner-sandbox.test.ts new file mode 100644 index 00000000..b18803d2 --- /dev/null +++ b/test/live/runner-sandbox.test.ts @@ -0,0 +1,466 @@ +// The opencode sandbox's boundary, with the real opencode and sandbox-runtime +// (`srt`): a scripted model drives one runner job through `Opencode.launch` in +// a checkout that tries to escape, and each attempt is refused. +// +// `test/runner.test.ts` checks only the settings the runner hands opencode and +// srt; whether they confine anything is those programs' behavior, which can +// change in any release. The scripted model needs no login and spends no +// usage, so `just runner-sandbox` runs it, and CI does nightly; a host that +// can't run bwrap, such as the merge queue's container, keeps it out of +// `just test`. srt runs on Linux here; its macOS Seatbelt run waits on +// /quest/m1/kix/sandbox/README.md. +// +// Each attempt first runs unconfined with the same paths, so an attempt that +// would no longer escape anyway fails as a broken control instead of passing +// as refused. +import { spawn, spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; + +import { requireProgram } from "../../src/cli/program"; +import { accountDirectories, Opencode, PROJECT_CONFIG, projectConfig } from "../../src/runner/opencode"; +import { binaryPath } from "../../src/runner/cli"; +import { closeSandbox, openSandbox, quote, type Sandbox } from "../../src/runner/sandbox"; + +/** What the scripted provider's key is, which no command may read. */ +const KEY = "quest-boundary-provider-key"; +/** The kix tools' bearer token, which no command may see. */ +const TOOLS_TOKEN = "quest-boundary-tools-token"; + +interface Paths { + /** The job's checkout: a worktree of `repository`. */ + checkout: string; + /** The bare clone, which commands read but must not write, such as a hook git would run on the host. */ + repository: string; + /** The runner's state, which commands can neither read nor write. */ + runner: string; + /** The account's home under it, holding its opencode credentials. */ + account: string; + /** A shared temporary directory, outside both the checkout and the attempt's directory. */ + outside: string; + /** The scripted model's server, which commands must not reach. */ + url: string; +} + +/** One escape the checkout tries, as a command for opencode's bash tool, and how to tell it worked. */ +interface Attempt { + name: string; + command(paths: Paths): string; + /** Whether the attempt got through, from its output and the host's state. */ + escaped(paths: Paths, output: string): boolean; + /** Undo what a successful control left behind. */ + clean?(paths: Paths): void; +} + +const auth = (paths: Paths) => join(accountDirectories(paths.account).XDG_DATA_HOME, "opencode", "auth.json"); + +/** Commands avoid `cat`, `touch`, `mkdir`, and the like, which opencode's own path check refuses before srt sees them. */ +const ATTEMPTS: Attempt[] = [ + { + name: "reads the account's credentials by absolute path", + command: (paths) => `head -c 4096 ${quote(auth(paths))}`, + escaped: (_paths, output) => output.includes(KEY), + }, + { + name: "reads them through a symlink the checkout commits", + command: () => "head -c 4096 leak", + escaped: (_paths, output) => output.includes(KEY), + }, + { + name: "writes the runner's state", + command: (paths) => `printf x > ${quote(join(paths.runner, "escaped"))}`, + escaped: (paths) => existsSync(join(paths.runner, "escaped")), + clean: (paths) => rmSync(join(paths.runner, "escaped"), { force: true }), + }, + { + name: "writes a shared temporary directory", + command: (paths) => `printf x > ${quote(join(paths.outside, "escaped"))}`, + escaped: (paths) => existsSync(join(paths.outside, "escaped")), + clean: (paths) => rmSync(join(paths.outside, "escaped"), { force: true }), + }, + { + name: "plants a hook in the bare clone, which git would run on the host", + command: (paths) => `printf '#!/bin/sh\\n' > ${quote(join(paths.repository, "hooks", "pre-commit"))}`, + escaped: (paths) => existsSync(join(paths.repository, "hooks", "pre-commit")), + clean: (paths) => rmSync(join(paths.repository, "hooks", "pre-commit"), { force: true }), + }, + { + name: "creates .opencode/plugin, which the server would import", + command: () => "install -d .opencode/plugin && printf 'export default {}\\n' > .opencode/plugin/p.js", + escaped: (paths) => existsSync(join(paths.checkout, ".opencode", "plugin")), + clean: (paths) => rmSync(join(paths.checkout, ".opencode"), { recursive: true, force: true }), + }, + { + name: "writes opencode.json", + command: () => `printf '{"plugin":["./p.js"]}' > opencode.json`, + escaped: (paths) => projectConfig(paths.checkout).includes("opencode.json"), + clean: (paths) => rmSync(join(paths.checkout, "opencode.json"), { force: true }), + }, + { + name: "links opencode.jsonc to a file it can write", + command: () => "ln -s allowed.txt opencode.jsonc", + escaped: (paths) => projectConfig(paths.checkout).includes("opencode.jsonc"), + clean: (paths) => rmSync(join(paths.checkout, "opencode.jsonc"), { force: true }), + }, + { + // srt exempts loopback from its proxy, so this one asks the proxy for it. + name: "reaches a host off the allowlist through srt's proxy", + command: (paths) => `env -u NO_PROXY -u no_proxy ${fetchCommand(paths, "proxy")}`, + escaped: () => escapes.includes("/escape/proxy"), + clean: () => (escapes.length = 0), + }, + { + name: "reaches it directly, bypassing the proxy", + command: (paths) => + `env ${PROXIES.map((name) => `-u ${name} -u ${name.toLowerCase()}`).join(" ")} ${fetchCommand(paths, "direct")}`, + escaped: () => escapes.includes("/escape/direct"), + clean: () => (escapes.length = 0), + }, +]; + +const PROXIES = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"]; + +/** A command fetching the scripted model's `/escape/`, printing how it went. */ +function fetchCommand(paths: Paths, path: string): string { + const script = `console.log(await fetch(${JSON.stringify(`${paths.url}/escape/${path}`)}).then((r) => r.status, String))`; + return `${quote(bun)} -e ${quote(script)}`; +} + +/** + * An escape through opencode's own file tools, which run in the unconfined + * server: the runner's plugin must refuse them, so a release that stops + * calling its hook fails here. + */ +interface FileAttempt { + name: string; + call: Call; + escaped(paths: Paths, output: string): boolean; +} + +const FILE_ATTEMPTS: FileAttempt[] = [ + { + name: "the read tool follows the committed symlink to the credentials", + call: { tool: "read", args: { filePath: "leak" } }, + escaped: (_paths, output) => output.includes(KEY), + }, + { + name: "the write tool writes .opencode/opencode.json", + call: { tool: "write", args: { filePath: ".opencode/opencode.json", content: `{"plugin":["./p.js"]}` } }, + escaped: (paths) => existsSync(join(paths.checkout, ".opencode", "opencode.json")), + }, +]; + +/** One tool call the scripted model makes. */ +interface Call { + tool: string; + args: Record; +} + +const bash = (command: string): Call => ({ tool: "bash", args: { command, description: "Run a step" } }); + +/** A command inside the sandbox that must work, so the refusals above are not every command failing. */ +const ALLOWED = bash(`printf allowed > allowed.txt && printf ok > "$TMPDIR/allowed" && echo ALLOWED-$((6 * 7))`); +/** The command's environment, which must hold none of the server's secrets. */ +const ENVIRONMENT = bash("env"); + +let scratch: string; +let paths: Paths; +let server: Server; +let bun: string; +/** The paths commands fetched from the scripted model's server. */ +const escapes: string[] = []; +/** Every request the agent sent the model, which carries each command's output back. */ +const requests: ChatRequest[] = []; +/** The tool calls the scripted model makes, one per step, in order. */ +let script: Call[] = []; +/** Tools the script calls that opencode did not offer. */ +const missing = new Set(); + +const linux = process.platform === "linux"; + +describe.skipIf(!linux)("opencode's sandbox", () => { + beforeAll(async () => { + bun = requireProgram("bun", process.env); + scratch = realpathSync(mkdtempSync(join(tmpdir(), "quest-boundary-"))); + const runner = join(scratch, "runner"); + const account = join(runner, "accounts", "scripted"); + const outside = join(scratch, "outside"); + mkdirSync(outside, { recursive: true }); + mkdirSync(account, { recursive: true }); + + const repository = join(runner, "repos", "fixture.git"); + const seed = join(scratch, "seed"); + git(scratch, "init", "-q", "-b", "main", seed); + symlinkSync(join(accountDirectories(account).XDG_DATA_HOME, "opencode", "auth.json"), join(seed, "leak")); + writeFileSync(join(seed, "README.md"), "fixture\n"); + git(seed, "add", "."); + git(seed, "commit", "-qm", "base"); + git(scratch, "clone", "-q", "--bare", seed, repository); + const checkout = join(runner, "checkouts", "job"); + git(repository, "worktree", "add", "-q", "--detach", checkout, "main"); + + server = await listen(model); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("the scripted model has no port"); + paths = { + checkout, + repository, + runner, + account, + outside, + url: `http://127.0.0.1:${address.port}`, + }; + signIn(account, `${paths.url}/v1`); + }); + + afterAll(async () => { + if (server) await new Promise((resolve) => server.close(resolve)); + if (scratch) rmSync(scratch, { recursive: true, force: true }); + }); + + // Each attempt with the same paths and no sandbox, so a refusal below means the sandbox refused it. + test("each attempt escapes when run unconfined", async () => { + for (const attempt of ATTEMPTS) { + // Not spawnSync: the network attempt needs this process to answer it. + const output = await shell(attempt.command(paths), paths.checkout); + expect(attempt.escaped(paths, output), `${attempt.name}: the control did not escape\n${output}`).toBe(true); + attempt.clean?.(paths); + } + expect(projectConfig(paths.checkout)).toEqual([]); + }); + + test("an opencode job's commands cannot escape", { timeout: 5 * 60_000 }, async () => { + const srt = binaryPath("srt", process.env); + const opencode = binaryPath("opencode", process.env); + const sandbox = openSandbox({ + cwd: paths.checkout, + repository: paths.repository, + runnerHome: paths.runner, + devShell: null, + trusted: false, + }); + script = [ + ALLOWED, + ENVIRONMENT, + ...ATTEMPTS.map((attempt) => bash(attempt.command(paths))), + ...FILE_ATTEMPTS.map((attempt) => attempt.call), + ]; + let output = ""; + let exit: Awaited>; + try { + exit = await run(new Opencode(opencode, srt, process.env), sandbox, (line) => (output += line)); + } finally { + closeSandbox(sandbox); + } + const results = toolResults(); + const context = `exit ${JSON.stringify(exit)}\n${output.slice(-4000)}`; + expect([...missing], "opencode no longer offers a tool the script calls").toEqual([]); + expect(exit.failed, context).toBe(false); + expect(exit.reply).toBe("done"); + expect(results, `the model saw ${results.length} of ${script.length} results\n${context}`).toHaveLength( + script.length, + ); + + const [allowed, environment, ...rest] = results; + const attempts = rest.slice(0, ATTEMPTS.length); + const files = rest.slice(ATTEMPTS.length); + expect(allowed, "a permitted command failed, so every refusal would pass vacuously").toContain("ALLOWED-42"); + expect(readFileSync(join(paths.checkout, "allowed.txt"), "utf8")).toBe("allowed"); + for (const secret of [KEY, TOOLS_TOKEN, "OPENCODE_", "XDG_DATA_HOME"]) + expect(environment, `a command's environment holds ${secret}`).not.toContain(secret); + expect(environment).toContain(`HOME=${join(sandbox.tmp, "home")}`); + expect(environment, "srt replaced the attempt's TMPDIR").toContain(`TMPDIR=${sandbox.tmp}\n`); + ATTEMPTS.forEach((attempt, index) => + expect(attempt.escaped(paths, attempts[index]!), `${attempt.name}\n${attempts[index]}`).toBe(false), + ); + FILE_ATTEMPTS.forEach((attempt, index) => { + expect(attempt.escaped(paths, files[index]!), `${attempt.name}\n${files[index]}`).toBe(false); + expect(files[index], `${attempt.name}: not the runner's plugin refusing it`).toMatch( + /outside the checkout|opencode's own config/, + ); + }); + + for (const secret of [KEY, TOOLS_TOKEN]) + expect(JSON.stringify(requests), `${secret} reached the model`).not.toContain(secret); + // Nothing left behind: no opencode config, not even srt's empty mount points, and no stray file. + for (const name of PROJECT_CONFIG) expect(existsSync(join(paths.checkout, name)), name).toBe(false); + expect(git(paths.checkout, "status", "--porcelain", "--ignored")).toBe("?? allowed.txt\n"); + }); +}); + +/** How long the job may run before it is stopped, within the test's timeout and the stop's grace. */ +const RUN_BOUND = 4 * 60_000; + +/** + * One turn of a job on the account, with the kix tools at a port nothing + * listens on. A turn that runs past `RUN_BOUND` is stopped and awaited, so a + * hung opencode or srt never outlives the test. + */ +async function run(adapter: Opencode, sandbox: Sandbox, output: (line: string) => void) { + const running = adapter.launch({ + cwd: paths.checkout, + home: paths.account, + session: "boundary", + resume: false, + prompt: "Run the commands you are given.", + system: "", + tools: { url: "http://127.0.0.1:9/mcp", token: TOOLS_TOKEN }, + sandbox, + env: process.env, + output, + }); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + running.stop(); + }, RUN_BOUND); + const exit = await running.done.finally(() => clearTimeout(timer)); + if (timedOut) throw new Error(`the job ran past ${RUN_BOUND / 60_000} minutes, so it was stopped`); + return exit; +} + +/** + * Sign the account in to the scripted provider: an OpenAI-compatible API on + * localhost, with an API key no command may read, picked as the account's model. + */ +function signIn(account: string, baseURL: string) { + const dirs = accountDirectories(account); + mkdirSync(join(dirs.XDG_DATA_HOME, "opencode"), { recursive: true }); + writeFileSync( + join(dirs.XDG_DATA_HOME, "opencode", "auth.json"), + JSON.stringify({ scripted: { type: "api", key: KEY } }), + ); + mkdirSync(join(dirs.XDG_CONFIG_HOME, "opencode"), { recursive: true }); + writeFileSync( + join(dirs.XDG_CONFIG_HOME, "opencode", "opencode.json"), + JSON.stringify({ + model: "scripted/boundary", + provider: { + scripted: { + npm: "@ai-sdk/openai-compatible", + name: "Scripted", + options: { baseURL }, + models: { boundary: { name: "Boundary", tool_call: true } }, + }, + }, + }), + ); +} + +interface ChatMessage { + role: string; + content?: unknown; + tool_calls?: unknown[]; +} + +interface ChatRequest { + messages: ChatMessage[]; + tools?: { function: { name: string } }[]; +} + +/** The output of each command, in order, as the agent reported it back to the model. */ +function toolResults(): string[] { + const last = requests.filter((request) => request.tools?.length).at(-1); + return (last?.messages ?? []).filter((m) => m.role === "tool").map((m) => text(m.content)); +} + +function text(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) + return content.map((part) => (part as { text?: string }).text ?? JSON.stringify(part)).join(""); + return JSON.stringify(content); +} + +/** + * The scripted model: a request whose conversation holds k tool results makes + * the script's call k, then replies "done". A request without tools, + * such as opencode's title, gets a plain reply. Anything else the server sees + * is a command reaching it. + */ +function model(req: IncomingMessage, res: ServerResponse) { + if (req.method !== "POST" || !req.url?.endsWith("/chat/completions")) { + escapes.push(req.url ?? ""); + res.writeHead(200).end("reached"); + return; + } + let body = ""; + req.setEncoding("utf8").on("data", (chunk: string) => (body += chunk)); + req.on("end", () => { + const request = JSON.parse(body) as ChatRequest; + requests.push(request); + res.writeHead(200, { "content-type": "text/event-stream" }); + const send = (delta: object, finish: string | null = null) => + res.write( + `data: ${JSON.stringify({ + id: "chatcmpl-boundary", + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: "boundary", + choices: [{ index: 0, delta, finish_reason: finish }], + })}\n\n`, + ); + const step = request.messages.filter((m) => m.role === "tool").length; + // opencode's title request offers no tools. + const call = request.tools?.length ? script[step] : undefined; + if (call && !request.tools?.some((t) => t.function.name === call.tool)) missing.add(call.tool); + if (call === undefined || missing.size) { + send({ role: "assistant", content: request.tools?.length ? "done" : "Boundary test" }); + send({}, "stop"); + } else { + send({ + role: "assistant", + tool_calls: [ + { + index: 0, + id: `call_${step}`, + type: "function", + function: { name: call.tool, arguments: JSON.stringify(call.args) }, + }, + ], + }); + send({}, "tool_calls"); + } + res.end("data: [DONE]\n\n"); + }); +} + +/** A command's stdout and stderr, run unconfined with `sh -c` in `cwd`. */ +function shell(command: string, cwd: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn("/bin/sh", ["-c", command], { cwd, stdio: ["ignore", "pipe", "pipe"] }); + let output = ""; + child.stdout.setEncoding("utf8").on("data", (chunk: string) => (output += chunk)); + child.stderr.setEncoding("utf8").on("data", (chunk: string) => (output += chunk)); + child.once("error", reject); + child.once("close", () => resolve(output)); + }); +} + +function listen(handler: (req: IncomingMessage, res: ServerResponse) => void): Promise { + return new Promise((resolve, reject) => { + const created = createServer(handler); + created.once("error", reject); + created.listen(0, "127.0.0.1", () => resolve(created)); + }); +} + +function git(cwd: string, ...args: string[]): string { + const run = spawnSync("git", ["-c", "user.name=Fixture", "-c", "user.email=fixture@localhost", ...args], { + cwd, + encoding: "utf8", + }); + if (run.status !== 0) throw new Error(`git ${args.join(" ")} failed: ${run.stderr}`); + return run.stdout; +} diff --git a/test/runner.test.ts b/test/runner.test.ts index e98a456f..219832d3 100644 --- a/test/runner.test.ts +++ b/test/runner.test.ts @@ -1105,6 +1105,7 @@ describe("account add", () => { ...process.env, PATH: `${bin}:${process.env.PATH}`, QUEST_RUNNER_HOME: join(dir, "home"), + QUEST_RUNNER_OPENCODE: join(bin, "opencode"), QUEST_RUNNER_SRT: join(bin, "srt"), }, }; @@ -1939,6 +1940,8 @@ describe("quest runner", () => { expect(srt.env).not.toContain("OPENCODE_CONFIG_CONTENT"); expect(srt.env).not.toContain("AWS_SECRET_ACCESS_KEY"); expect(srt.env).not.toContain("XDG_DATA_HOME"); + // srt hands commands this as their TMPDIR. + expect(srt.env).toContain("CLAUDE_CODE_TMPDIR"); expect(shell.shell).toBe(`1|${tmp}/home|none|none|${tmp}`); expect(first.stdin).toContain("kix adds for this job: Extra rule."); expect(first.system).toContain("checkpoint"); diff --git a/vitest.config.ts b/vitest.config.ts index bc28023c..b48676ed 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,8 +1,9 @@ import { configDefaults, defineConfig } from "vitest/config"; -// `test/live` runs the real agent CLIs on the user's logins, so it runs only -// with QUEST_LIVE=1, as `just review-sandbox` sets, and alone. A reviewer turn -// takes minutes; its timeout only bounds a hung CLI. +// `test/live` runs the real agent CLIs, so it runs only with QUEST_LIVE=1, and +// alone: `just review-sandbox` on the user's logins, and `just runner-sandbox` +// with a scripted model, nightly in CI. A reviewer turn takes minutes; its +// timeout only bounds a hung CLI. const live = process.env.QUEST_LIVE === "1"; export default defineConfig({