From 0390b4f08fd80554074a08c9b59a81669e95377b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 18:54:18 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m0/kix/sandbox/live-test.md Quest-Claim: 98724b86fb4fa995 From c9c42e8266daa079119ffc53715c861e83c77e4d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 19:13:22 -0800 Subject: [PATCH 2/5] test(runner): check the opencode sandbox's boundary in CI test/live/runner-sandbox.test.ts drives the pinned opencode 1.19 and srt through Opencode.launch with a scripted OpenAI-compatible model on localhost. Its checkout tries to read the account's credentials, write the runner's state, a shared temporary directory, and the bare clone's hooks, plant opencode config through commands and file tools, and reach the network through srt's proxy and directly. Each attempt first escapes when run unconfined, so a refusal means the sandbox refused it. It needs no login, so it runs in `just test` on Linux, on main and in the merge group. The test found that srt replaces a command's TMPDIR with its own CLAUDE_CODE_TMPDIR, or a nonexistent /tmp/claude, so opencode jobs had no writable TMPDIR. The shell now hands srt the attempt's directory. The Linux dev shell names the runner's opencode and srt in QUEST_RUNNER_OPENCODE and QUEST_RUNNER_SRT, and the hosted check job lifts Ubuntu's AppArmor restriction on unprivileged user namespaces for bwrap. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/check.yml | 7 + README.md | 5 +- flake.nix | 7 + quest/m0/kix/sandbox/README.md | 4 +- quest/m0/kix/sandbox/live-test.md | 33 --- quest/m0/opencode-sdk.md | 10 +- src/runner/opencode.ts | 12 +- test/live/runner-sandbox.test.ts | 449 ++++++++++++++++++++++++++++++ test/runner.test.ts | 3 + vitest.config.ts | 10 +- 10 files changed, 491 insertions(+), 49 deletions(-) delete mode 100644 quest/m0/kix/sandbox/live-test.md create mode 100644 test/live/runner-sandbox.test.ts diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 13ff19e5..2ad07061 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -36,6 +36,13 @@ jobs: sudo loginctl enable-linger "$USER" sudo systemctl start "user@$(id -u).service" echo "XDG_RUNTIME_DIR=/run/user/$(id -u)" >> "$GITHUB_ENV" + # The sandbox boundary test runs bubblewrap from the Nix store, which Ubuntu's + # AppArmor would refuse the unprivileged user namespace it needs. + - name: Allow unprivileged user namespaces + run: | + if [ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]; then + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + fi - name: Check run: nix develop --command just check - name: Test diff --git a/README.md b/README.md index 31c9c53c..f40964ac 100644 --- a/README.md +++ b/README.md @@ -1088,7 +1088,10 @@ language servers and formatters are off. The CLI itself, Claude Code or commands start from a minimal environment, so its login, the opencode server's password, and the kix tools' token stay out of their reach. This is an OS sandbox, not a VM, and it narrows rather than removes the risk of running -agent commands on your machine. +agent commands on your machine. On Linux, `just test` in this repository runs +a scripted model through the pinned opencode and sandbox-runtime in a checkout +that tries to read credentials, write outside its sandbox, plant opencode +config, and reach the network, and fails if any attempt gets through. On Linux the sandbox needs `bwrap` and `socat` (the flake's `runner` package pins both) and unprivileged user namespaces; on Ubuntu 23.10 and later, diff --git a/flake.nix b/flake.nix index 95c4a90c..5beb6ecc 100644 --- a/flake.nix +++ b/flake.nix @@ -102,6 +102,13 @@ # `flock` (the land daemon's repository lock) and `script` (the tests' terminal), which # self-hosted NixOS runners don't otherwise provide. ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isLinux [ pkgs.util-linux ]; + # The runner package's opencode and sandbox-runtime, for the sandbox boundary test + # (test/live/runner-sandbox.test.ts), which runs where srt does. Named as the runner + # finds them rather than put on PATH, where they would shadow the user's own opencode. + env = pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { + QUEST_RUNNER_OPENCODE = pkgs.lib.getExe opencode; + QUEST_RUNNER_SRT = pkgs.lib.getExe pkgs.sandbox-runtime; + }; }; formatter = pkgs.nixfmt; checks = { diff --git a/quest/m0/kix/sandbox/README.md b/quest/m0/kix/sandbox/README.md index 712d87b1..946d95be 100644 --- a/quest/m0/kix/sandbox/README.md +++ b/quest/m0/kix/sandbox/README.md @@ -67,6 +67,9 @@ Shipped in [moq-dev/quest#212](https://github.com/moq-dev/quest/pull/212) opencode 1.19 still imports the plugins a checkout's own `opencode.json` or `.opencode/` names, even with project config disabled, so the runner keeps the agent from creating them and refuses a checkout that holds them. + `test/live/runner-sandbox.test.ts` checks that boundary with the pinned + opencode and srt and a scripted model on every Linux `just test`, so CI + catches a bump that weakens it. - Decided 2026-10-24 (user) in `/quest-plan`: split the remaining work into the children below, ranked so the earliest unblock the most. Dropped (user): read-only registry access, which Claude Code's sandbox cannot @@ -78,4 +81,3 @@ Shipped in [moq-dev/quest#212](https://github.com/moq-dev/quest/pull/212) ## Required - [Head approval](/quest/m0/kix/sandbox/head-approval.md) - a maintainer approves an exact head, such as one kix pushed, for host runners -- [Boundary test in CI](/quest/m0/kix/sandbox/live-test.md) - the opencode sandbox's hostile-repository test runs on Linux CI with a scripted model diff --git a/quest/m0/kix/sandbox/live-test.md b/quest/m0/kix/sandbox/live-test.md deleted file mode 100644 index 4db1e813..00000000 --- a/quest/m0/kix/sandbox/live-test.md +++ /dev/null @@ -1,33 +0,0 @@ -# [S] The opencode sandbox's boundary test runs in CI - -## Goal - -The opencode sandbox's hostile-repository boundary test is checked in as -`test/live/runner-sandbox.test.ts`, drives opencode and srt with a scripted -model, and runs on Linux in this repository's checks, on `main` and in the -merge group. An srt or opencode bump that weakens the boundary fails CI -before it lands. - -## Plan - -Decided 2026-11-02 (user, /quest-plan of -[moq-dev/quest#338](https://github.com/moq-dev/quest/pull/338)'s suggested -follow-ups): - -- How it runs: - - ✅ CI on Linux: a scripted model needs no login and spends no usage - - A manual recipe, like `just review-sandbox` -- Milestone: ✅ m0 · m1 - -#338 ran the test by hand against srt 0.0.79. Merging main moved srt to -0.0.81, which the maintainer accepted from the release notes and a direct -srt run, without rerunning the full test. The checked-in test covers what -that run did: a checkout that tries to read a denied file, write outside the -checkout and attempt directory, create `.opencode/plugin`, `opencode.json` or -`opencode.jsonc`, and reach the network. Each attempt is refused, and nothing -is left behind in the checkout. Follow `test/live/review-sandbox.test.ts` -for the layout. Since this test needs no login, it runs without -`QUEST_LIVE`, skipping only where srt can't run (macOS and Windows). - -[opencode 2.x](/quest/m0/opencode-sdk.md) reruns the boundary test on 2.x; -with this quest done, that rerun is this test. diff --git a/quest/m0/opencode-sdk.md b/quest/m0/opencode-sdk.md index 955b199e..b900dc46 100644 --- a/quest/m0/opencode-sdk.md +++ b/quest/m0/opencode-sdk.md @@ -52,12 +52,10 @@ What the change does: - Recheck each of the sandbox's rules on 2.x: the `shell` wrapper through srt, the commands' environment allowlist (`keptEnvironment`), `OPENCODE_DISABLE_PROJECT_CONFIG`, `external_directory` and `webfetch` - denies, the plugin's realpath check, and `lsp`/`formatter` off. Rerun the - sandbox's hostile-repository boundary test against 2.x. + denies, the plugin's realpath check, and `lsp`/`formatter` off. The + sandbox's boundary test, `test/live/runner-sandbox.test.ts`, runs the + pinned opencode in `just test`, so it reruns on 2.x with the pin; extend + it to any rule 2.x changes, such as its tool names or plugin hooks. - Run the runner's live opencode checks that the [Kix decision UI](/quest/m0/kix/README.md) rehearsal lists, if a runner is available. - -## Required - -- [Boundary test in CI](/quest/m0/kix/sandbox/live-test.md) - the boundary test this quest reruns on 2.x is checked in and runs in CI diff --git a/src/runner/opencode.ts b/src/runner/opencode.ts index abe468b2..335f3d54 100644 --- a/src/runner/opencode.ts +++ b/src/runner/opencode.ts @@ -199,12 +199,16 @@ export default { * environment, which holds the server password, the tools' token in * `OPENCODE_CONFIG_CONTENT`, and any provider key opencode exported, then * sources the private HOME and the devShell inside the sandbox before the - * command, as Claude Code does with `CLAUDE_ENV_FILE`. + * command, as Claude Code does with `CLAUDE_ENV_FILE`. srt replaces the + * command's TMPDIR with its own `CLAUDE_CODE_TMPDIR`, or `/tmp/claude` + * without it, so that names the attempt's directory too. */ export function shellScript(srt: string, settings: string, sandbox: Sandbox, env: NodeJS.ProcessEnv): string { - const kept = Object.entries({ ...keptEnvironment(env), TMPDIR: sandbox.tmp }).map(([name, value]) => - quote(`${name}=${value}`), - ); + const kept = Object.entries({ + ...keptEnvironment(env), + TMPDIR: sandbox.tmp, + CLAUDE_CODE_TMPDIR: sandbox.tmp, + }).map(([name, value]) => quote(`${name}=${value}`)); const inner = `. ${quote(sandbox.env)} && quest_command=$QUEST_COMMAND && unset QUEST_COMMAND && eval "$quest_command"`; return `#!/bin/sh if [ "$#" -ne 2 ] || [ "$1" != -c ]; then diff --git a/test/live/runner-sandbox.test.ts b/test/live/runner-sandbox.test.ts new file mode 100644 index 00000000..dea33648 --- /dev/null +++ b/test/live/runner-sandbox.test.ts @@ -0,0 +1,449 @@ +// The opencode sandbox's boundary, with the real opencode and sandbox-runtime +// (`srt`): a scripted model drives one runner job through `Opencode.launch` in +// a checkout that tries to escape, and each attempt is refused. +// +// `test/runner.test.ts` checks only the settings the runner hands opencode and +// srt; whether they confine anything is those programs' behavior, which can +// change in any release. The scripted model needs no login and spends no +// usage, so this runs in `just test` and CI. srt runs on Linux here; its macOS +// Seatbelt run waits on /quest/m1/kix/sandbox/README.md. +// +// Each attempt first runs unconfined with the same paths, so an attempt that +// would no longer escape anyway fails as a broken control instead of passing +// as refused. +import { spawn, spawnSync } from "node:child_process"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; + +import { requireProgram } from "../../src/cli/program"; +import { accountDirectories, Opencode, PROJECT_CONFIG, projectConfig } from "../../src/runner/opencode"; +import { binaryPath } from "../../src/runner/cli"; +import { closeSandbox, openSandbox, quote, type Sandbox } from "../../src/runner/sandbox"; + +/** What the scripted provider's key is, which no command may read. */ +const KEY = "quest-boundary-provider-key"; +/** The kix tools' bearer token, which no command may see. */ +const TOOLS_TOKEN = "quest-boundary-tools-token"; + +interface Paths { + /** The job's checkout: a worktree of `repository`. */ + checkout: string; + /** The bare clone, which commands read but must not write, such as a hook git would run on the host. */ + repository: string; + /** The runner's state, which commands can neither read nor write. */ + runner: string; + /** The account's home under it, holding its opencode credentials. */ + account: string; + /** A shared temporary directory, outside both the checkout and the attempt's directory. */ + outside: string; + /** The scripted model's server, which commands must not reach. */ + url: string; +} + +/** One escape the checkout tries, as a command for opencode's bash tool, and how to tell it worked. */ +interface Attempt { + name: string; + command(paths: Paths): string; + /** Whether the attempt got through, from its output and the host's state. */ + escaped(paths: Paths, output: string): boolean; + /** Undo what a successful control left behind. */ + clean?(paths: Paths): void; +} + +const auth = (paths: Paths) => join(accountDirectories(paths.account).XDG_DATA_HOME, "opencode", "auth.json"); + +/** Commands avoid `cat`, `touch`, `mkdir`, and the like, which opencode's own path check refuses before srt sees them. */ +const ATTEMPTS: Attempt[] = [ + { + name: "reads the account's credentials by absolute path", + command: (paths) => `head -c 4096 ${quote(auth(paths))}`, + escaped: (_paths, output) => output.includes(KEY), + }, + { + name: "reads them through a symlink the checkout commits", + command: () => "head -c 4096 leak", + escaped: (_paths, output) => output.includes(KEY), + }, + { + name: "writes the runner's state", + command: (paths) => `printf x > ${quote(join(paths.runner, "escaped"))}`, + escaped: (paths) => existsSync(join(paths.runner, "escaped")), + clean: (paths) => rmSync(join(paths.runner, "escaped"), { force: true }), + }, + { + name: "writes a shared temporary directory", + command: (paths) => `printf x > ${quote(join(paths.outside, "escaped"))}`, + escaped: (paths) => existsSync(join(paths.outside, "escaped")), + clean: (paths) => rmSync(join(paths.outside, "escaped"), { force: true }), + }, + { + name: "plants a hook in the bare clone, which git would run on the host", + command: (paths) => `printf '#!/bin/sh\\n' > ${quote(join(paths.repository, "hooks", "pre-commit"))}`, + escaped: (paths) => existsSync(join(paths.repository, "hooks", "pre-commit")), + clean: (paths) => rmSync(join(paths.repository, "hooks", "pre-commit"), { force: true }), + }, + { + name: "creates .opencode/plugin, which the server would import", + command: () => "install -d .opencode/plugin && printf 'export default {}\\n' > .opencode/plugin/p.js", + escaped: (paths) => existsSync(join(paths.checkout, ".opencode", "plugin")), + clean: (paths) => rmSync(join(paths.checkout, ".opencode"), { recursive: true, force: true }), + }, + { + name: "writes opencode.json", + command: () => `printf '{"plugin":["./p.js"]}' > opencode.json`, + escaped: (paths) => projectConfig(paths.checkout).includes("opencode.json"), + clean: (paths) => rmSync(join(paths.checkout, "opencode.json"), { force: true }), + }, + { + name: "links opencode.jsonc to a file it can write", + command: () => "ln -s allowed.txt opencode.jsonc", + escaped: (paths) => projectConfig(paths.checkout).includes("opencode.jsonc"), + clean: (paths) => rmSync(join(paths.checkout, "opencode.jsonc"), { force: true }), + }, + { + // srt exempts loopback from its proxy, so this one asks the proxy for it. + name: "reaches a host off the allowlist through srt's proxy", + command: (paths) => `env -u NO_PROXY -u no_proxy ${fetchCommand(paths, "proxy")}`, + escaped: () => escapes.includes("/escape/proxy"), + clean: () => (escapes.length = 0), + }, + { + name: "reaches it directly, bypassing the proxy", + command: (paths) => + `env ${PROXIES.map((name) => `-u ${name} -u ${name.toLowerCase()}`).join(" ")} ${fetchCommand(paths, "direct")}`, + escaped: () => escapes.includes("/escape/direct"), + clean: () => (escapes.length = 0), + }, +]; + +const PROXIES = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"]; + +/** A command fetching the scripted model's `/escape/`, printing how it went. */ +function fetchCommand(paths: Paths, path: string): string { + const script = `console.log(await fetch(${JSON.stringify(`${paths.url}/escape/${path}`)}).then((r) => r.status, String))`; + return `${quote(bun)} -e ${quote(script)}`; +} + +/** + * An escape through opencode's own file tools, which run in the unconfined + * server: the runner's plugin must refuse them, so a release that stops + * calling its hook fails here. + */ +interface FileAttempt { + name: string; + call: Call; + escaped(paths: Paths, output: string): boolean; +} + +const FILE_ATTEMPTS: FileAttempt[] = [ + { + name: "the read tool follows the committed symlink to the credentials", + call: { tool: "read", args: { filePath: "leak" } }, + escaped: (_paths, output) => output.includes(KEY), + }, + { + name: "the write tool writes .opencode/opencode.json", + call: { tool: "write", args: { filePath: ".opencode/opencode.json", content: `{"plugin":["./p.js"]}` } }, + escaped: (paths) => existsSync(join(paths.checkout, ".opencode", "opencode.json")), + }, +]; + +/** One tool call the scripted model makes. */ +interface Call { + tool: string; + args: Record; +} + +const bash = (command: string): Call => ({ tool: "bash", args: { command, description: "Run a step" } }); + +/** A command inside the sandbox that must work, so the refusals above are not every command failing. */ +const ALLOWED = bash(`printf allowed > allowed.txt && printf ok > "$TMPDIR/allowed" && echo ALLOWED-$((6 * 7))`); +/** The command's environment, which must hold none of the server's secrets. */ +const ENVIRONMENT = bash("env"); + +let scratch: string; +let paths: Paths; +let server: Server; +let bun: string; +/** The paths commands fetched from the scripted model's server. */ +const escapes: string[] = []; +/** Every request the agent sent the model, which carries each command's output back. */ +const requests: ChatRequest[] = []; +/** The tool calls the scripted model makes, one per step, in order. */ +let script: Call[] = []; +/** Tools the script calls that opencode did not offer. */ +const missing = new Set(); + +const linux = process.platform === "linux"; + +describe.skipIf(!linux)("opencode's sandbox", () => { + beforeAll(async () => { + bun = requireProgram("bun", process.env); + scratch = realpathSync(mkdtempSync(join(tmpdir(), "quest-boundary-"))); + const runner = join(scratch, "runner"); + const account = join(runner, "accounts", "scripted"); + const outside = join(scratch, "outside"); + mkdirSync(outside, { recursive: true }); + mkdirSync(account, { recursive: true }); + + const repository = join(runner, "repos", "fixture.git"); + const seed = join(scratch, "seed"); + git(scratch, "init", "-q", "-b", "main", seed); + symlinkSync(join(accountDirectories(account).XDG_DATA_HOME, "opencode", "auth.json"), join(seed, "leak")); + writeFileSync(join(seed, "README.md"), "fixture\n"); + git(seed, "add", "."); + git(seed, "commit", "-qm", "base"); + git(scratch, "clone", "-q", "--bare", seed, repository); + const checkout = join(runner, "checkouts", "job"); + git(repository, "worktree", "add", "-q", "--detach", checkout, "main"); + + server = await listen(model); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("the scripted model has no port"); + paths = { + checkout, + repository, + runner, + account, + outside, + url: `http://127.0.0.1:${address.port}`, + }; + signIn(account, `${paths.url}/v1`); + }); + + afterAll(async () => { + if (server) await new Promise((resolve) => server.close(resolve)); + if (scratch) rmSync(scratch, { recursive: true, force: true }); + }); + + // Each attempt with the same paths and no sandbox, so a refusal below means the sandbox refused it. + test("each attempt escapes when run unconfined", async () => { + for (const attempt of ATTEMPTS) { + // Not spawnSync: the network attempt needs this process to answer it. + const output = await shell(attempt.command(paths), paths.checkout); + expect(attempt.escaped(paths, output), `${attempt.name}: the control did not escape\n${output}`).toBe(true); + attempt.clean?.(paths); + } + expect(projectConfig(paths.checkout)).toEqual([]); + }); + + test("an opencode job's commands cannot escape", { timeout: 5 * 60_000 }, async () => { + const srt = binaryPath("srt", process.env); + const opencode = binaryPath("opencode", process.env); + const sandbox = openSandbox({ + cwd: paths.checkout, + repository: paths.repository, + runnerHome: paths.runner, + devShell: null, + trusted: false, + }); + script = [ + ALLOWED, + ENVIRONMENT, + ...ATTEMPTS.map((attempt) => bash(attempt.command(paths))), + ...FILE_ATTEMPTS.map((attempt) => attempt.call), + ]; + let output = ""; + let exit: Awaited>; + try { + exit = await run(new Opencode(opencode, srt, process.env), sandbox, (line) => (output += line)); + } finally { + closeSandbox(sandbox); + } + const results = toolResults(); + const context = `exit ${JSON.stringify(exit)}\n${output.slice(-4000)}`; + expect([...missing], "opencode no longer offers a tool the script calls").toEqual([]); + expect(exit.failed, context).toBe(false); + expect(exit.reply).toBe("done"); + expect(results, `the model saw ${results.length} of ${script.length} results\n${context}`).toHaveLength( + script.length, + ); + + const [allowed, environment, ...rest] = results; + const attempts = rest.slice(0, ATTEMPTS.length); + const files = rest.slice(ATTEMPTS.length); + expect(allowed, "a permitted command failed, so every refusal would pass vacuously").toContain("ALLOWED-42"); + expect(readFileSync(join(paths.checkout, "allowed.txt"), "utf8")).toBe("allowed"); + for (const secret of [KEY, TOOLS_TOKEN, "OPENCODE_", "XDG_DATA_HOME"]) + expect(environment, `a command's environment holds ${secret}`).not.toContain(secret); + expect(environment).toContain(`HOME=${join(sandbox.tmp, "home")}`); + expect(environment, "srt replaced the attempt's TMPDIR").toContain(`TMPDIR=${sandbox.tmp}\n`); + ATTEMPTS.forEach((attempt, index) => + expect(attempt.escaped(paths, attempts[index]!), `${attempt.name}\n${attempts[index]}`).toBe(false), + ); + FILE_ATTEMPTS.forEach((attempt, index) => { + expect(attempt.escaped(paths, files[index]!), `${attempt.name}\n${files[index]}`).toBe(false); + expect(files[index], `${attempt.name}: not the runner's plugin refusing it`).toMatch( + /outside the checkout|opencode's own config/, + ); + }); + + for (const secret of [KEY, TOOLS_TOKEN]) + expect(JSON.stringify(requests), `${secret} reached the model`).not.toContain(secret); + // Nothing left behind: no opencode config, not even srt's empty mount points, and no stray file. + for (const name of PROJECT_CONFIG) expect(existsSync(join(paths.checkout, name)), name).toBe(false); + expect(git(paths.checkout, "status", "--porcelain", "--ignored")).toBe("?? allowed.txt\n"); + }); +}); + +/** One turn of a job on the account, with the kix tools at a port nothing listens on. */ +function run(adapter: Opencode, sandbox: Sandbox, output: (line: string) => void) { + return adapter.launch({ + cwd: paths.checkout, + home: paths.account, + session: "boundary", + resume: false, + prompt: "Run the commands you are given.", + system: "", + tools: { url: "http://127.0.0.1:9/mcp", token: TOOLS_TOKEN }, + sandbox, + env: process.env, + output, + }).done; +} + +/** + * Sign the account in to the scripted provider: an OpenAI-compatible API on + * localhost, with an API key no command may read, picked as the account's model. + */ +function signIn(account: string, baseURL: string) { + const dirs = accountDirectories(account); + mkdirSync(join(dirs.XDG_DATA_HOME, "opencode"), { recursive: true }); + writeFileSync( + join(dirs.XDG_DATA_HOME, "opencode", "auth.json"), + JSON.stringify({ scripted: { type: "api", key: KEY } }), + ); + mkdirSync(join(dirs.XDG_CONFIG_HOME, "opencode"), { recursive: true }); + writeFileSync( + join(dirs.XDG_CONFIG_HOME, "opencode", "opencode.json"), + JSON.stringify({ + model: "scripted/boundary", + provider: { + scripted: { + npm: "@ai-sdk/openai-compatible", + name: "Scripted", + options: { baseURL }, + models: { boundary: { name: "Boundary", tool_call: true } }, + }, + }, + }), + ); +} + +interface ChatMessage { + role: string; + content?: unknown; + tool_calls?: unknown[]; +} + +interface ChatRequest { + messages: ChatMessage[]; + tools?: { function: { name: string } }[]; +} + +/** The output of each command, in order, as the agent reported it back to the model. */ +function toolResults(): string[] { + const last = requests.filter((request) => request.tools?.length).at(-1); + return (last?.messages ?? []).filter((m) => m.role === "tool").map((m) => text(m.content)); +} + +function text(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) + return content.map((part) => (part as { text?: string }).text ?? JSON.stringify(part)).join(""); + return JSON.stringify(content); +} + +/** + * The scripted model: a request whose conversation holds k tool results makes + * the script's call k, then replies "done". A request without tools, + * such as opencode's title, gets a plain reply. Anything else the server sees + * is a command reaching it. + */ +function model(req: IncomingMessage, res: ServerResponse) { + if (req.method !== "POST" || !req.url?.endsWith("/chat/completions")) { + escapes.push(req.url ?? ""); + res.writeHead(200).end("reached"); + return; + } + let body = ""; + req.setEncoding("utf8").on("data", (chunk: string) => (body += chunk)); + req.on("end", () => { + const request = JSON.parse(body) as ChatRequest; + requests.push(request); + res.writeHead(200, { "content-type": "text/event-stream" }); + const send = (delta: object, finish: string | null = null) => + res.write( + `data: ${JSON.stringify({ + id: "chatcmpl-boundary", + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model: "boundary", + choices: [{ index: 0, delta, finish_reason: finish }], + })}\n\n`, + ); + const step = request.messages.filter((m) => m.role === "tool").length; + // opencode's title request offers no tools. + const call = request.tools?.length ? script[step] : undefined; + if (call && !request.tools?.some((t) => t.function.name === call.tool)) missing.add(call.tool); + if (call === undefined || missing.size) { + send({ role: "assistant", content: request.tools?.length ? "done" : "Boundary test" }); + send({}, "stop"); + } else { + send({ + role: "assistant", + tool_calls: [ + { + index: 0, + id: `call_${step}`, + type: "function", + function: { name: call.tool, arguments: JSON.stringify(call.args) }, + }, + ], + }); + send({}, "tool_calls"); + } + res.end("data: [DONE]\n\n"); + }); +} + +/** A command's stdout and stderr, run unconfined with `sh -c` in `cwd`. */ +function shell(command: string, cwd: string): Promise { + return new Promise((resolve, reject) => { + const child = spawn("/bin/sh", ["-c", command], { cwd, stdio: ["ignore", "pipe", "pipe"] }); + let output = ""; + child.stdout.setEncoding("utf8").on("data", (chunk: string) => (output += chunk)); + child.stderr.setEncoding("utf8").on("data", (chunk: string) => (output += chunk)); + child.once("error", reject); + child.once("close", () => resolve(output)); + }); +} + +function listen(handler: (req: IncomingMessage, res: ServerResponse) => void): Promise { + return new Promise((resolve, reject) => { + const created = createServer(handler); + created.once("error", reject); + created.listen(0, "127.0.0.1", () => resolve(created)); + }); +} + +function git(cwd: string, ...args: string[]): string { + const run = spawnSync("git", ["-c", "user.name=Fixture", "-c", "user.email=fixture@localhost", ...args], { + cwd, + encoding: "utf8", + }); + if (run.status !== 0) throw new Error(`git ${args.join(" ")} failed: ${run.stderr}`); + return run.stdout; +} diff --git a/test/runner.test.ts b/test/runner.test.ts index ae25751c..7de5e718 100644 --- a/test/runner.test.ts +++ b/test/runner.test.ts @@ -1095,6 +1095,7 @@ describe("account add", () => { ...process.env, PATH: `${bin}:${process.env.PATH}`, QUEST_RUNNER_HOME: join(dir, "home"), + QUEST_RUNNER_OPENCODE: join(bin, "opencode"), QUEST_RUNNER_SRT: join(bin, "srt"), }, }; @@ -1852,6 +1853,8 @@ describe("quest runner", () => { expect(srt.env).not.toContain("OPENCODE_CONFIG_CONTENT"); expect(srt.env).not.toContain("AWS_SECRET_ACCESS_KEY"); expect(srt.env).not.toContain("XDG_DATA_HOME"); + // srt hands commands this as their TMPDIR. + expect(srt.env).toContain("CLAUDE_CODE_TMPDIR"); expect(shell.shell).toBe(`1|${tmp}/home|none|none|${tmp}`); expect(first.stdin).toContain("kix adds for this job: Extra rule."); expect(first.system).toContain("checkpoint"); diff --git a/vitest.config.ts b/vitest.config.ts index bc28023c..177d082b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,9 +1,11 @@ import { configDefaults, defineConfig } from "vitest/config"; -// `test/live` runs the real agent CLIs on the user's logins, so it runs only -// with QUEST_LIVE=1, as `just review-sandbox` sets, and alone. A reviewer turn -// takes minutes; its timeout only bounds a hung CLI. +// `test/live` runs the real agent CLIs. Those on the user's logins, listed +// here, run only with QUEST_LIVE=1, as `just review-sandbox` sets, and alone. +// A reviewer turn takes minutes; its timeout only bounds a hung CLI. The rest, +// such as the runner's sandbox test with its scripted model, run everywhere. const live = process.env.QUEST_LIVE === "1"; +const LOGINS = ["test/live/review-sandbox.test.ts"]; export default defineConfig({ plugins: [ @@ -22,6 +24,6 @@ export default defineConfig({ env: { GIT_CONFIG_GLOBAL: "/dev/null" }, ...(live ? { include: ["test/live/**/*.test.ts"], testTimeout: 30 * 60_000, hookTimeout: 60_000 } - : { include: ["test/**/*.test.ts"], exclude: [...configDefaults.exclude, "test/live/**"] }), + : { include: ["test/**/*.test.ts"], exclude: [...configDefaults.exclude, ...LOGINS] }), }, }); From 4aea676b44bc8adbdee634c5db0c44207616a6b3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 19:16:26 -0800 Subject: [PATCH 3/5] test(runner): stop a boundary job that runs past its bound A hung opencode or srt would otherwise outlive the test's timeout, with its detached server and children left running on the runner. Co-Authored-By: Claude Opus 5.5 --- test/live/runner-sandbox.test.ts | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/test/live/runner-sandbox.test.ts b/test/live/runner-sandbox.test.ts index dea33648..c7e4a7e8 100644 --- a/test/live/runner-sandbox.test.ts +++ b/test/live/runner-sandbox.test.ts @@ -298,9 +298,16 @@ describe.skipIf(!linux)("opencode's sandbox", () => { }); }); -/** One turn of a job on the account, with the kix tools at a port nothing listens on. */ -function run(adapter: Opencode, sandbox: Sandbox, output: (line: string) => void) { - return adapter.launch({ +/** How long the job may run before it is stopped, within the test's timeout and the stop's grace. */ +const RUN_BOUND = 4 * 60_000; + +/** + * One turn of a job on the account, with the kix tools at a port nothing + * listens on. A turn that runs past `RUN_BOUND` is stopped and awaited, so a + * hung opencode or srt never outlives the test. + */ +async function run(adapter: Opencode, sandbox: Sandbox, output: (line: string) => void) { + const running = adapter.launch({ cwd: paths.checkout, home: paths.account, session: "boundary", @@ -311,7 +318,15 @@ function run(adapter: Opencode, sandbox: Sandbox, output: (line: string) => void sandbox, env: process.env, output, - }).done; + }); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + running.stop(); + }, RUN_BOUND); + const exit = await running.done.finally(() => clearTimeout(timer)); + if (timedOut) throw new Error(`the job ran past ${RUN_BOUND / 60_000} minutes, so it was stopped`); + return exit; } /** From 61d17d440a392542857785f9be49b4427f8c4365 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 11:49:51 -0800 Subject: [PATCH 4/5] test(runner): run the sandbox boundary test nightly, outside the merge queue The merge queue's container can't mount /proc for bwrap, so the test failed there as designed. Per the user's 2026-11-03 decision it leaves `just test`, gets `just runner-sandbox`, and runs in its own nightly and dispatchable GitHub-hosted workflow that carries the AppArmor step. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/check.yml | 7 ----- .github/workflows/sandbox.yml | 39 +++++++++++++++++++++++++ README.md | 9 +++--- justfile | 8 ++++- quest/m0/kix/sandbox/README.md | 9 ++++-- quest/m0/kix/sandbox/claude-boundary.md | 15 +++++++--- quest/m0/opencode-sdk.md | 5 ++-- test/live/runner-sandbox.test.ts | 6 ++-- vitest.config.ts | 11 ++++--- 9 files changed, 81 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/sandbox.yml diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2ad07061..13ff19e5 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -36,13 +36,6 @@ jobs: sudo loginctl enable-linger "$USER" sudo systemctl start "user@$(id -u).service" echo "XDG_RUNTIME_DIR=/run/user/$(id -u)" >> "$GITHUB_ENV" - # The sandbox boundary test runs bubblewrap from the Nix store, which Ubuntu's - # AppArmor would refuse the unprivileged user namespace it needs. - - name: Allow unprivileged user namespaces - run: | - if [ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]; then - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - fi - name: Check run: nix develop --command just check - name: Test diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml new file mode 100644 index 00000000..6a0d840f --- /dev/null +++ b/.github/workflows/sandbox.yml @@ -0,0 +1,39 @@ +name: Sandbox + +# The runner sandbox's boundary test (`just runner-sandbox`) runs nightly on a +# GitHub-hosted runner, outside pull requests and the merge queue, so landing +# never waits on it; a failed scheduled run notifies the repository's watchers. +on: + schedule: + - cron: "23 7 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: sandbox-${{ github.ref }} + cancel-in-progress: true + +jobs: + runner-sandbox: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f + with: + determinate: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install dependencies + run: nix develop --command just install + # The test runs bubblewrap from the Nix store, which Ubuntu's AppArmor would + # refuse the unprivileged user namespace it needs. + - name: Allow unprivileged user namespaces + run: | + if [ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]; then + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + fi + - name: Check the runner sandbox's boundary + run: nix develop --command just runner-sandbox diff --git a/README.md b/README.md index 9154b817..1caba7e7 100644 --- a/README.md +++ b/README.md @@ -1194,10 +1194,11 @@ language servers and formatters are off. The CLI itself, Claude Code or commands start from a minimal environment, so its login, the opencode server's password, and the kix tools' token stay out of their reach. This is an OS sandbox, not a VM, and it narrows rather than removes the risk of running -agent commands on your machine. On Linux, `just test` in this repository runs -a scripted model through the pinned opencode and sandbox-runtime in a checkout -that tries to read credentials, write outside its sandbox, plant opencode -config, and reach the network, and fails if any attempt gets through. +agent commands on your machine. On Linux, `just runner-sandbox` in this +repository runs a scripted model through the pinned opencode and +sandbox-runtime in a checkout that tries to read credentials, write outside its +sandbox, plant opencode config, and reach the network, and fails if any attempt +gets through. CI runs it nightly, outside the merge queue. On Linux the sandbox needs `bwrap` and `socat` (the flake's `runner` package pins both) and unprivileged user namespaces, including the nested one Claude diff --git a/justfile b/justfile index ca2f2a67..ce6e2eb3 100644 --- a/justfile +++ b/justfile @@ -44,7 +44,13 @@ _test: # Codex and Claude Code CLIs. Uses each CLI's login and spends its usage; run it # after upgrading either CLI. Arguments go to vitest, such as `-t codex`. review-sandbox *args: - QUEST_LIVE=1 bun run vitest run {{args}} + QUEST_LIVE=1 bun run vitest run test/live/review-sandbox {{args}} + +# Check that a runner job's commands cannot escape its sandbox, with the pinned +# opencode and sandbox-runtime and a scripted model, on Linux. Needs no login; +# CI runs it nightly (.github/workflows/sandbox.yml). Run it after bumping either. +runner-sandbox *args: + QUEST_LIVE=1 bun run vitest run test/live/runner-sandbox {{args}} # Apply formatters without changing program behavior. fix: diff --git a/quest/m0/kix/sandbox/README.md b/quest/m0/kix/sandbox/README.md index 670f2690..53f19d0e 100644 --- a/quest/m0/kix/sandbox/README.md +++ b/quest/m0/kix/sandbox/README.md @@ -68,8 +68,13 @@ Shipped in [moq-dev/quest#212](https://github.com/moq-dev/quest/pull/212) or `.opencode/` names, even with project config disabled, so the runner keeps the agent from creating them and refuses a checkout that holds them. `test/live/runner-sandbox.test.ts` checks that boundary with the pinned - opencode and srt and a scripted model on every Linux `just test`, so CI - catches a bump that weakens it. + opencode and srt and a scripted model. It runs in `just runner-sandbox` + and a nightly GitHub-hosted workflow (`.github/workflows/sandbox.yml`), + not in `just test` or the merge queue, whose container can't mount + `/proc` for bwrap, so a bump that weakens the boundary fails the next + nightly run. Decided 2026-11-03 (user, /quest-complete), after the merge + queue dropped [moq-dev/quest#404](https://github.com/moq-dev/quest/pull/404): + its own CI, nightly, so landing never blocks on it. - Decided 2026-10-24 (user) in `/quest-plan`: split the remaining work into the children below, ranked so the earliest unblock the most. Dropped (user): read-only registry access, which Claude Code's sandbox cannot diff --git a/quest/m0/kix/sandbox/claude-boundary.md b/quest/m0/kix/sandbox/claude-boundary.md index 9f82360c..5960ba29 100644 --- a/quest/m0/kix/sandbox/claude-boundary.md +++ b/quest/m0/kix/sandbox/claude-boundary.md @@ -1,4 +1,4 @@ -# [M] The Claude Code sandbox's boundary test runs in CI +# [M] The Claude Code sandbox's boundary test runs nightly in CI ## Goal @@ -6,9 +6,9 @@ Claude Code jobs get the same checked-in hostile-repository boundary test that opencode jobs got in [moq-dev/quest#404](https://github.com/moq-dev/quest/pull/404): a scripted fake model, set as the account's endpoint in its home, drives the pinned -Claude Code through the runner's launch path on Linux CI, and every escape -attempt is refused. -A Claude Code bump that weakens the boundary fails CI before it lands. +Claude Code through the runner's launch path in the nightly sandbox workflow +on Linux, and every escape attempt is refused. +A Claude Code bump that weakens the boundary fails the next nightly run. ## Plan @@ -50,6 +50,13 @@ Work: - Like opencode's test, each shell attempt first succeeds outside the sandbox, and the test fails rather than skips on Linux when the sandbox can't start. +- Run it where opencode's test runs: name it + `test/live/runner-sandbox-claude.test.ts` (or extend + `runner-sandbox.test.ts`), so `just runner-sandbox` and the nightly + `.github/workflows/sandbox.yml` pick it up, and keep it out of + `just test` and the merge queue. Decided 2026-11-03 (user, + /quest-complete) for opencode's test: its own CI, nightly, so landing + never blocks on it. ## Required diff --git a/quest/m0/opencode-sdk.md b/quest/m0/opencode-sdk.md index b900dc46..9411b64d 100644 --- a/quest/m0/opencode-sdk.md +++ b/quest/m0/opencode-sdk.md @@ -54,8 +54,9 @@ What the change does: `OPENCODE_DISABLE_PROJECT_CONFIG`, `external_directory` and `webfetch` denies, the plugin's realpath check, and `lsp`/`formatter` off. The sandbox's boundary test, `test/live/runner-sandbox.test.ts`, runs the - pinned opencode in `just test`, so it reruns on 2.x with the pin; extend - it to any rule 2.x changes, such as its tool names or plugin hooks. + pinned opencode only in `just runner-sandbox` and nightly CI, so run it + on the 2.x pin before landing; extend it to any rule 2.x changes, such as + its tool names or plugin hooks. - Run the runner's live opencode checks that the [Kix decision UI](/quest/m0/kix/README.md) rehearsal lists, if a runner is available. diff --git a/test/live/runner-sandbox.test.ts b/test/live/runner-sandbox.test.ts index c7e4a7e8..b18803d2 100644 --- a/test/live/runner-sandbox.test.ts +++ b/test/live/runner-sandbox.test.ts @@ -5,8 +5,10 @@ // `test/runner.test.ts` checks only the settings the runner hands opencode and // srt; whether they confine anything is those programs' behavior, which can // change in any release. The scripted model needs no login and spends no -// usage, so this runs in `just test` and CI. srt runs on Linux here; its macOS -// Seatbelt run waits on /quest/m1/kix/sandbox/README.md. +// usage, so `just runner-sandbox` runs it, and CI does nightly; a host that +// can't run bwrap, such as the merge queue's container, keeps it out of +// `just test`. srt runs on Linux here; its macOS Seatbelt run waits on +// /quest/m1/kix/sandbox/README.md. // // Each attempt first runs unconfined with the same paths, so an attempt that // would no longer escape anyway fails as a broken control instead of passing diff --git a/vitest.config.ts b/vitest.config.ts index 177d082b..b48676ed 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,11 +1,10 @@ import { configDefaults, defineConfig } from "vitest/config"; -// `test/live` runs the real agent CLIs. Those on the user's logins, listed -// here, run only with QUEST_LIVE=1, as `just review-sandbox` sets, and alone. -// A reviewer turn takes minutes; its timeout only bounds a hung CLI. The rest, -// such as the runner's sandbox test with its scripted model, run everywhere. +// `test/live` runs the real agent CLIs, so it runs only with QUEST_LIVE=1, and +// alone: `just review-sandbox` on the user's logins, and `just runner-sandbox` +// with a scripted model, nightly in CI. A reviewer turn takes minutes; its +// timeout only bounds a hung CLI. const live = process.env.QUEST_LIVE === "1"; -const LOGINS = ["test/live/review-sandbox.test.ts"]; export default defineConfig({ plugins: [ @@ -24,6 +23,6 @@ export default defineConfig({ env: { GIT_CONFIG_GLOBAL: "/dev/null" }, ...(live ? { include: ["test/live/**/*.test.ts"], testTimeout: 30 * 60_000, hookTimeout: 60_000 } - : { include: ["test/**/*.test.ts"], exclude: [...configDefaults.exclude, ...LOGINS] }), + : { include: ["test/**/*.test.ts"], exclude: [...configDefaults.exclude, "test/live/**"] }), }, }); From b34e4f58b0c5517f3118cc81fa2f164215109e6c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 3 Nov 2026 12:07:00 -0800 Subject: [PATCH 5/5] ci(sandbox): say who GitHub notifies of a failed nightly run GitHub reports a failed scheduled run only to the account that last changed the cron schedule, not to the repository's watchers. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/sandbox.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml index 6a0d840f..4e57ba9f 100644 --- a/.github/workflows/sandbox.yml +++ b/.github/workflows/sandbox.yml @@ -2,7 +2,10 @@ name: Sandbox # The runner sandbox's boundary test (`just runner-sandbox`) runs nightly on a # GitHub-hosted runner, outside pull requests and the merge queue, so landing -# never waits on it; a failed scheduled run notifies the repository's watchers. +# never waits on it. GitHub reports a failed scheduled run only to the account +# that last changed the cron schedule (with Actions notifications on), not to +# the repository's watchers; that is the maintainer, whose account lands the +# workflow, so they see a boundary failure. on: schedule: - cron: "23 7 * * *"