diff --git a/README.md b/README.md index 31c9c53c..82b0cdac 100644 --- a/README.md +++ b/README.md @@ -1091,8 +1091,33 @@ sandbox, not a VM, and it narrows rather than removes the risk of running agent commands on your machine. On Linux the sandbox needs `bwrap` and `socat` (the flake's `runner` package -pins both) and unprivileged user namespaces; on Ubuntu 23.10 and later, -AppArmor allows them only for programs with a profile granting `userns`. +pins both) and unprivileged user namespaces, including the nested one Claude +Code's seccomp helper makes inside the sandbox. Before taking jobs, `quest runner` +runs a trivial command in nested sandbox namespaces and refuses to start if +bubblewrap cannot make them. On Ubuntu 23.10 and later, AppArmor allows them +only to programs whose profile grants `userns`, so for the runner package's +bubblewrap the refusal prints this profile, filled in with its Nix store path, +and the commands that install it as `/etc/apparmor.d/quest-bwrap`: + +``` +abi , +include + +profile quest-bwrap /nix/store/HASH-bubblewrap-VERSION/bin/bwrap flags=(unconfined) { + userns, +} +``` + +It is the form Ubuntu gives other programs that sandbox themselves. Ubuntu's +own `bwrap-userns-restrict` profile for `/usr/bin/bwrap` takes capabilities from +the commands bubblewrap runs, which breaks Claude Code's seccomp helper, so a +system bwrap there is refused, and a second profile for it would disable both; +use the runner package instead. The job's commands still get no user namespace +of their own: Claude Code's seccomp helper refuses them one, and on opencode +accounts bubblewrap does. The profile names one exact store path, since anyone +who can build with Nix can add a store path matching a glob, so a runner update +that changes bubblewrap refuses to start until you install the profile it prints. + Merge jobs retain existing PR submodule pointers when merging the base. Their `wait` tool wakes on GitHub webhook hints or reconnects and rereads the PR; a hint never approves work. If someone moves the head, the maintainer can diff --git a/quest/m1/kix/sandbox/README.md b/quest/m1/kix/sandbox/README.md index ef1681d3..f66fc35d 100644 --- a/quest/m1/kix/sandbox/README.md +++ b/quest/m1/kix/sandbox/README.md @@ -17,5 +17,4 @@ design decisions for the whole sandbox stay in the m0 epic's Plan. ## Required - [macOS sandbox](/quest/m1/kix/sandbox/macos.md) - the boundary verified under Seatbelt, then the answer-approval gate removed -- [Ubuntu AppArmor](/quest/m1/kix/sandbox/apparmor.md) - the sandbox starts on stock Ubuntu, or the runner refuses at startup - [Untrusted devShells](/quest/m1/kix/sandbox/vm.md) - untrusted PR heads build their devShell in a per-job VM diff --git a/quest/m1/kix/sandbox/apparmor.md b/quest/m1/kix/sandbox/apparmor.md deleted file mode 100644 index af5c5b26..00000000 --- a/quest/m1/kix/sandbox/apparmor.md +++ /dev/null @@ -1,19 +0,0 @@ -# [S] Start the sandbox on stock Ubuntu - -## Goal - -A runner started from an ordinary terminal on Ubuntu 23.10 or later runs -sandboxed jobs, or refuses to start with an explanation, instead of failing -each job. - -## Plan - -Found on [moq-dev/quest#212](https://github.com/moq-dev/quest/pull/212): -Ubuntu's AppArmor restriction on unprivileged user namespaces made the system -`/usr/bin/bwrap` fail. The Nix-pinned bwrap worked only because that process -tree ran under the Claude desktop app's profile, which allows them. - -- Document an AppArmor profile for the runner's pinned bwrap. -- At startup, when a Claude Code account needs the sandbox, run a trivial - sandboxed command and refuse with the profile's install hint if it fails. -- Test on a stock Ubuntu image, from a normal login shell. diff --git a/src/runner/cli.ts b/src/runner/cli.ts index d3e1c564..f15bbf55 100644 --- a/src/runner/cli.ts +++ b/src/runner/cli.ts @@ -14,6 +14,7 @@ import { Opencode } from "./opencode"; import { type Account, type Login, RunnerHome, defaultHome } from "./home"; import type { Job } from "./protocol"; import { Runner } from "./runner"; +import { sandboxRefusal } from "./sandbox"; import { which } from "../cli/program"; export const DEFAULT_URL = "https://kix.quest"; @@ -69,19 +70,27 @@ export function adapterFor(name: AdapterName, env: NodeJS.ProcessEnv): Adapter { /** * The adapters a running runner launches jobs with, each resolved once, the * first time an account needs it. On Linux the sandbox needs bubblewrap and - * socat, so they are checked then; macOS has Seatbelt built in. Claude Code - * runs them itself, and sandbox-runtime for opencode. + * socat, and bubblewrap must be able to make its namespaces, so they are + * checked then; macOS has Seatbelt built in. Claude Code runs them itself, and + * sandbox-runtime for opencode. */ export function runnerAdapters(env: NodeJS.ProcessEnv): (name: AdapterName) => Adapter { const adapters = new Map(); + let sandboxed = process.platform !== "linux"; return (name) => { let adapter = adapters.get(name); if (adapter) return adapter; - const missing = process.platform === "linux" ? ["bwrap", "socat"].filter((tool) => !which(tool, env)) : []; - if (missing.length) - throw new RunnerInputError( - `${missing.join(" and ")} ${missing.length === 1 ? "is" : "are"} not installed; jobs run their commands in an OS sandbox (install bubblewrap and socat, or use the flake's runner package)`, - ); + if (!sandboxed) { + const [bwrap, socat] = [which("bwrap", env), which("socat", env)]; + const missing = [bwrap ? null : "bwrap", socat ? null : "socat"].filter((tool) => tool !== null); + if (missing.length) + throw new RunnerInputError( + `${missing.join(" and ")} ${missing.length === 1 ? "is" : "are"} not installed; jobs run their commands in an OS sandbox (install bubblewrap and socat, or use the flake's runner package)`, + ); + const refusal = sandboxRefusal(bwrap!); + if (refusal) throw new RunnerInputError(refusal); + sandboxed = true; + } adapters.set(name, (adapter = adapterFor(name, env))); return adapter; }; diff --git a/src/runner/sandbox.ts b/src/runner/sandbox.ts index e81ab25e..3bb4fd42 100644 --- a/src/runner/sandbox.ts +++ b/src/runner/sandbox.ts @@ -7,7 +7,8 @@ // reach no network host. The project's devShell is evaluated by the runner from the // job's revision, never the checkout an agent edited, and its environment, // shellHook included, is sourced inside the sandbox before each command. -import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; @@ -114,6 +115,89 @@ export function runtimeSettings(sandbox: Sandbox, denyWrite: string[] = []): Rec }; } +/** + * An AppArmor profile that lets `bwrap` and the commands it runs make user + * namespaces on a system restricting unprivileged ones, as Ubuntu 23.10 and + * later do: the form Ubuntu gives other programs that sandbox themselves. + * Ubuntu's own `bwrap-userns-restrict` takes capabilities from the commands + * bwrap runs, which breaks the seccomp helper Claude Code runs inside it, since + * that makes a nested user namespace for its capabilities. The job's commands + * still get none of their own: the helper's filter refuses them one, and + * bubblewrap does where sandbox-runtime runs no helper. It attaches to one exact + * path rather than a glob, since anyone who can build with Nix can put a + * matching name in the store. + */ +export function bwrapProfile(bwrap: string): string { + return `abi ,\ninclude \n\nprofile quest-bwrap ${bwrap} flags=(unconfined) {\n userns,\n}\n`; +} + +/** Whether AppArmor keeps unprivileged user namespaces to programs whose profile allows them. */ +function restrictsUserNamespaces(): boolean { + try { + return readFileSync("/proc/sys/kernel/apparmor_restrict_unprivileged_userns", "utf8").trim() === "1"; + } catch { + return false; + } +} + +/** Namespaces as Claude Code's sandbox and sandbox-runtime make them, around `command`. */ +function namespaces(command: string[]): string[] { + return [ + "--new-session", + "--die-with-parent", + "--unshare-user", + "--unshare-pid", + "--unshare-net", + "--ro-bind", + "/", + "/", + "--dev", + "/dev", + "--proc", + "/proc", + ...command, + ]; +} + +/** + * Why `bwrap` cannot make a sandbox here, or null when it can, so a runner + * refuses to start rather than fail every job. It runs a trivial command in + * the sandbox's namespaces, nested in a second bwrap, since Claude Code's + * seccomp helper makes a nested user namespace inside the sandbox. + */ +export function sandboxRefusal(bwrap: string, restricted: boolean = restrictsUserNamespaces()): string | null { + const probe = spawnSync(bwrap, namespaces([bwrap, ...namespaces(["true"])]), { + stdio: ["ignore", "ignore", "pipe"], + encoding: "utf8", + timeout: 10_000, + }); + if (!probe.error && probe.status === 0) return null; + const said = + probe.error?.message ?? (probe.stderr.trim().split("\n").at(-1) || `exit ${probe.status ?? probe.signal}`); + return refusalReason(bwrap, said, restricted); +} + +/** + * The refusal for a `bwrap` that failed the probe saying `said`. Where AppArmor + * restricts user namespaces, it carries the profile to install for a bwrap in + * the Nix store, such as the runner package's. A system bwrap gets none, since + * AppArmor attaches neither of two profiles for one path, and the distribution + * may already ship one for it, as Ubuntu does for `/usr/bin/bwrap`. + */ +export function refusalReason(bwrap: string, said: string, restricted: boolean): string { + const reason = `${bwrap} cannot make the sandbox's namespaces (${said})`; + if (!restricted) + return `${reason}; jobs run their commands in a bubblewrap sandbox, which needs unprivileged user namespaces`; + const apparmor = `${reason}. AppArmor allows unprivileged user namespaces only to programs whose profile grants them`; + if (!bwrap.startsWith("/nix/store/")) + return `${apparmor}; run the flake's runner package (\`nix run github:moq-dev/quest#runner -- runner\`), which prints the profile for its own bubblewrap`; + return ( + `${apparmor}; install this profile for the runner's bubblewrap, then start the runner again:\n\n` + + `sudo tee /etc/apparmor.d/quest-bwrap >/dev/null <<'EOF'\n${bwrapProfile(bwrap)}EOF\n` + + "sudo apparmor_parser -r /etc/apparmor.d/quest-bwrap" + ); +} + export function closeSandbox(sandbox: Sandbox): void { rmSync(sandbox.tmp, { recursive: true, force: true }); } diff --git a/test/runner.test.ts b/test/runner.test.ts index ae25751c..6f9597f9 100644 --- a/test/runner.test.ts +++ b/test/runner.test.ts @@ -36,7 +36,15 @@ import { FLUSH_BYTES, OutputUploader, split } from "../src/runner/output"; import { formatAnswer, resumePrompt, startPrompt, systemPrompt } from "../src/runner/prompt"; import type { Job } from "../src/runner/protocol"; import { Runner } from "../src/runner/runner"; -import { REGISTRIES, type Sandbox, closeSandbox, openSandbox } from "../src/runner/sandbox"; +import { + REGISTRIES, + type Sandbox, + bwrapProfile, + closeSandbox, + openSandbox, + refusalReason, + sandboxRefusal, +} from "../src/runner/sandbox"; import { Changes, fingerprint, WAIT_LIMIT, waitForChange } from "../src/runner/wait"; import { Workspace } from "../src/runner/workspace"; import { initRepository, template, tempDir } from "./fixture"; @@ -1141,6 +1149,46 @@ describe("runner adapters", () => { install("srt"); expect(adapter("opencode")).toBeInstanceOf(Opencode); }); + + test.skipIf(process.platform !== "linux")("refuse to start when bubblewrap cannot make the sandbox", () => { + const bin = join(tempDir(), "bin"); + mkdirSync(bin); + for (const [name, script] of [ + ["claude", ""], + ["socat", ""], + // What the Nix-pinned bwrap says under Ubuntu's restriction on unprivileged user namespaces. + ["bwrap", "echo 'bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted' >&2\nexit 1\n"], + ] as const) { + writeFileSync(join(bin, name), `#!/bin/sh\n${script}`); + chmodSync(join(bin, name), 0o755); + } + const bwrap = join(realpathSync(bin), "bwrap"); + expect(() => runnerAdapters({ PATH: bin })("claude")).toThrow( + `${bwrap} cannot make the sandbox's namespaces (bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted)`, + ); + expect(sandboxRefusal(join(bin, "claude"))).toBeNull(); + // Under AppArmor's restriction, the refusal carries the profile for the runner package's exact bwrap. + const said = "bwrap: loopback: Failed RTM_NEWADDR: Operation not permitted"; + const pinned = "/nix/store/mvj6y8k3xs35cj7kq937xi44q4hl3r93-bubblewrap-0.12.0/bin/bwrap"; + const restricted = refusalReason(pinned, said, true); + expect(restricted).toContain( + `sudo tee /etc/apparmor.d/quest-bwrap >/dev/null <<'EOF'\n${bwrapProfile(pinned)}EOF\n`, + ); + expect(restricted).toMatch(/\nsudo apparmor_parser -r \/etc\/apparmor.d\/quest-bwrap$/); + expect(bwrapProfile(pinned)).toContain(`profile quest-bwrap ${pinned} flags=(unconfined) {\n userns,\n}`); + // A system bwrap may have the distribution's profile, which a second one would disable. + const system = refusalReason("/usr/bin/bwrap", said, true); + expect(system).toContain("nix run github:moq-dev/quest#runner -- runner"); + expect(system).not.toContain("apparmor_parser"); + const unrestricted = refusalReason(pinned, said, false); + expect(unrestricted).toContain("needs unprivileged user namespaces"); + expect(unrestricted).not.toContain("AppArmor"); + }); + + test("document the AppArmor profile the runner prints", () => { + const readme = readFileSync(resolve(import.meta.dirname, "../README.md"), "utf8"); + expect(readme).toContain(bwrapProfile("/nix/store/HASH-bubblewrap-VERSION/bin/bwrap")); + }); }); describe("login", () => {