diff --git a/CHANGELOG.md b/CHANGELOG.md index 9799351c02..7d796aec37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,17 @@ file is the human-readable summary of what changed and why. ### Desktop / Web UI +- **Windows sandboxes are now firewalled** — on macOS and Linux the sandbox + guest has no network device at all, but a WSL guest shares Windows' network, + so a Windows sandbox could previously reach the internet directly. The + sandbox's processes are now confined by a firewall inside the guest: they + can reach the sandbox's own server and the desktop app's model-traffic + relay, and nothing else — connections anywhere else fail immediately, and + other WSL distributions on the machine are unaffected. The firewall is + mandatory: a sandbox that cannot set it up refuses to start rather than run + open. **Requires the updated sandbox image** — after updating, use a + sandbox's **Reset** button to pick it up; a sandbox still on the old image + fails to start with "nft is missing from the guest image". - **Chat now works inside Windows sandboxes** — every model request from a WSL sandbox failed with `Cannot connect to API: other side closed` and retried forever. The guest-side bridge that relays model traffic to the desktop app diff --git a/packages/codev-sandbox/image/Dockerfile b/packages/codev-sandbox/image/Dockerfile index 88f01ed900..34dd9af7c2 100644 --- a/packages/codev-sandbox/image/Dockerfile +++ b/packages/codev-sandbox/image/Dockerfile @@ -7,7 +7,7 @@ ARG DEBIAN_FRONTEND=noninteractive # Base toolchain: node/git/socat/ca-certs always; python + build tools + the # CLI set per the sandbox plan. iproute2 is needed by the vz init (loopback). RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates curl wget git unzip zip jq ripgrep sqlite3 socat iproute2 busybox-static \ + ca-certificates curl wget git unzip zip jq ripgrep sqlite3 socat iproute2 nftables busybox-static \ procps less vim-tiny openssh-client \ python3 python3-pip python3-venv python3-dev \ build-essential pkg-config \ diff --git a/packages/codev-sandbox/script/e2e-wsl.ts b/packages/codev-sandbox/script/e2e-wsl.ts index 18ebe611c2..57ee926311 100644 --- a/packages/codev-sandbox/script/e2e-wsl.ts +++ b/packages/codev-sandbox/script/e2e-wsl.ts @@ -10,7 +10,18 @@ // // --hold keeps the distribution running so you can poke at it; --keep skips the // unregister so it survives the run (clean up with `wsl --unregister codev-e2e`). +// +// --egress serves a local broker upstream and probes the egress firewall from +// inside the running guest: the broker path must work from the sandbox's +// cgroup, direct egress from that cgroup must be rejected, and the same +// direct egress from outside the cgroup must still work (proving the rules +// are scoped to the sandbox, not the shared WSL network namespace). +// --egress-control overrides the external endpoint the direct-egress +// probes dial (default: the host serving --image, which is loopback, so pass +// a real external host:443 for a meaningful control on locked-down networks). import { createHash } from "node:crypto" +import { spawn } from "node:child_process" +import { lookup } from "node:dns/promises" import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import { basename, join, resolve } from "node:path" @@ -24,6 +35,7 @@ const image = resolve(requireArg("--image")) const bundle = resolve(requireArg("--bundle")) const hold = process.argv.includes("--hold") const keep = process.argv.includes("--keep") +const egress = process.argv.includes("--egress") const work = await mkdtemp(join(tmpdir(), "codev-wsl-e2e-")) const projectDir = argValue("--project") ? resolve(argValue("--project")!) : join(work, "project") @@ -38,6 +50,7 @@ for await (const chunk of payload.stream()) hash.update(chunk) const sha256 = hash.digest("hex") const server = Bun.serve({ port: 0, fetch: () => new Response(Bun.file(image)) }) +const upstream = egress ? Bun.serve({ port: 0, fetch: () => Response.json({ marker: "broker-upstream" }) }) : null const provider = createWslProvider({ dataDir: join(work, "data"), @@ -48,6 +61,14 @@ const provider = createWslProvider({ sha256, bytes: payload.size, }), + ...(upstream + ? { + resolveEgress: async () => ({ + routes: [{ prefix: "/probe", upstream: `http://127.0.0.1:${upstream.port}` }], + guestConfig: `{}`, + }), + } + : {}), startTimeoutMs: 180_000, }) @@ -86,6 +107,40 @@ const pathsOk = await fetch(`${sandbox.url}/path`, { .catch(() => false) console.log(`path api: ${pathsOk ? "OK" : "FAILED"}`) +// The firewall confines the boot cgroup, so meaningful probes must run inside +// it (as the agent would) and the scoping control outside it (as a neighbour +// distro would). +let firewallOk = true +if (egress) { + const distro = `codev-${config.name}` + const enterCgroup = `echo $$ > /sys/fs/cgroup/net_cls/${distro}/cgroup.procs || exit 9` + const connectProbe = (host: string, port: number) => + `node -e "const s=require('net').connect({host:'${host}',port:${port},timeout:7000});` + + `s.on('connect',()=>{console.log('CONNECTED');process.exit(0)});` + + `s.on('timeout',()=>{console.log('TIMEOUT');process.exit(2)});` + + `s.on('error',(e)=>{console.log('BLOCKED:'+e.code);process.exit(3)})"` + + const broker = await runGuestScript([ + enterCgroup, + `setpriv --reuid=codev --regid=codev --init-groups node -e "fetch('http://127.0.0.1:8788/probe/x').then((r)=>r.text()).then((t)=>console.log('BROKER:'+t)).catch((e)=>{console.log('BROKER-FAILED:'+e.message);process.exit(4)})"`, + ]) + const brokerOk = broker.includes(`BROKER:{"marker":"broker-upstream"}`) + console.log(`broker via cgroup: ${brokerOk ? "OK" : `FAILED (${broker.trim()})`}`) + + const control = argValue("--egress-control") ?? `${await hostAddress()}:443` + const [controlHost, controlPort] = control.split(":") + const inside = await runGuestScript([enterCgroup, connectProbe(controlHost!, Number(controlPort ?? 443))]) + const insideBlocked = inside.includes("BLOCKED:") + console.log(`direct egress inside cgroup: ${insideBlocked ? `blocked OK (${inside.trim()})` : `NOT BLOCKED (${inside.trim()})`}`) + + const outside = await runGuestScript([connectProbe(controlHost!, Number(controlPort ?? 443))]) + const outsideOk = outside.includes("CONNECTED") + console.log(`direct egress outside cgroup: ${outsideOk ? "OK" : `FAILED (${outside.trim()})`}`) + + firewallOk = brokerOk && insideBlocked && outsideOk + console.log(`firewall: ${firewallOk ? "OK" : "FAILED"}`) +} + if (hold) { console.log("holding the distribution up; Ctrl-C to stop") await new Promise(() => {}) @@ -97,15 +152,39 @@ await Promise.race([exited, new Promise((resolve) => setTimeout(resolve, 10_000) if (keep) console.log(`keeping the distribution; remove it with: wsl --unregister codev-${config.name}`) if (!keep) await provider.destroy(config) await cleanup() -if (!healthy || !pathsOk) process.exit(1) +if (!healthy || !pathsOk || !firewallOk) process.exit(1) console.log("e2e OK") process.exit(0) async function cleanup() { server.stop(true) + upstream?.stop(true) await rm(work, { recursive: true, force: true }) } +// Runs a root shell script inside the distro over stdin — the same channel the +// provider boots through, and for the same reason: wsl.exe corrupts quoted +// spaces in argv. +function runGuestScript(lines: string[]) { + return new Promise((done) => { + const child = spawn("wsl.exe", ["-d", `codev-${config.name}`, "-u", "root", "--", "/bin/sh"], { + stdio: ["pipe", "pipe", "pipe"], + }) + const output: Buffer[] = [] + child.stdout.on("data", (chunk: Buffer) => output.push(chunk)) + child.stderr.on("data", (chunk: Buffer) => output.push(chunk)) + child.once("close", () => done(Buffer.concat(output).toString("utf8"))) + child.stdin.end(`${lines.join("\n")}\n`) + }) +} + +// Default direct-egress control endpoint: the host that served the image is +// the one address guaranteed reachable from this network. +async function hostAddress() { + const url = new URL(argValue("--image-host") ?? "https://netmind.viettel.vn") + return (await lookup(url.hostname)).address +} + function argValue(flag: string) { const index = process.argv.indexOf(flag) return index >= 0 ? process.argv[index + 1] : undefined diff --git a/packages/codev-sandbox/src/wsl/firewall.test.ts b/packages/codev-sandbox/src/wsl/firewall.test.ts new file mode 100644 index 0000000000..40756fcb4c --- /dev/null +++ b/packages/codev-sandbox/src/wsl/firewall.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from "bun:test" +import { wslFirewallClassid, wslFirewallName, wslFirewallScript } from "./firewall" + +test("wslFirewallName strips characters nft identifiers cannot carry", () => { + expect(wslFirewallName("codev-my.project_1")).toBe("codev_my_project_1") +}) + +test("wslFirewallClassid namespaces the server port", () => { + expect(wslFirewallClassid(54791)).toBe(0x434f0000 + 54791) + // Distinct ports (one per running sandbox) never collide. + expect(wslFirewallClassid(1)).not.toBe(wslFirewallClassid(2)) +}) + +test("firewall script fails closed at every step", () => { + const script = wslFirewallScript({ distro: "codev-e2e", serverPort: 50000, brokerPort: null }).join("\n") + expect(script).toContain(`command -v nft >/dev/null 2>&1 || { echo "codev sandbox: nft is missing`) + expect(script).toContain("mount -t cgroup -o net_cls net_cls /sys/fs/cgroup/net_cls || exit 1") + expect(script).toContain("/sys/fs/cgroup/net_cls/codev-e2e/net_cls.classid || exit 1") + expect(script).toContain("nft -f - < /sys/fs/cgroup/net_cls/codev-e2e/cgroup.procs || exit 1") +}) + +test("firewall rules confine only the sandbox classid and keep loopback", () => { + const classid = wslFirewallClassid(50000) + const script = wslFirewallScript({ distro: "codev-e2e", serverPort: 50000, brokerPort: null }).join("\n") + expect(script).toContain(`table inet codev_e2e {`) + expect(script).toContain(`meta cgroup ${classid} oifname "lo" accept`) + expect(script).toContain(`meta cgroup ${classid} reject`) + // Without a broker there must be no hole to the host. + expect(script).not.toContain("tcp dport") +}) + +test("a configured broker gets exactly one egress hole", () => { + const classid = wslFirewallClassid(50000) + const script = wslFirewallScript({ distro: "codev-e2e", serverPort: 50000, brokerPort: 8790 }).join("\n") + expect(script).toContain(`meta cgroup ${classid} ip daddr $HOST_IP tcp dport 8790 accept`) + // The reject rule still closes everything else. + expect(script).toContain(`meta cgroup ${classid} reject`) +}) + +test("the reset idiom replaces a leftover table from a previous boot", () => { + const lines = wslFirewallScript({ distro: "codev-e2e", serverPort: 50000, brokerPort: null }) + const empty = lines.indexOf("table inet codev_e2e {}") + const del = lines.indexOf("delete table inet codev_e2e") + const full = lines.indexOf("table inet codev_e2e {") + expect(empty).toBeGreaterThan(-1) + expect(del).toBe(empty + 1) + expect(full).toBe(del + 1) +}) diff --git a/packages/codev-sandbox/src/wsl/firewall.ts b/packages/codev-sandbox/src/wsl/firewall.ts new file mode 100644 index 0000000000..0f26f9dcef --- /dev/null +++ b/packages/codev-sandbox/src/wsl/firewall.ts @@ -0,0 +1,90 @@ +// Egress firewall for WSL sandboxes. +// +// The vz and bwrap guests are NIC-less by construction, but every WSL2 distro +// shares one utility VM, one network namespace, and one IP — verified live: +// two distros report the same /proc/self/ns/net inode. That shape rules out +// the obvious host-side design: Hyper-V firewall rules target a VMCreatorId, +// which is all of WSL at once (there is no per-distribution targeting), so a +// creator-wide outbound block would also cut off the user's own distros and +// Docker Desktop. The only enforcement point that can distinguish this +// sandbox's traffic from its neighbours' is the shared guest kernel itself: +// nftables rules that match a marker carried by the sandbox's own processes. +// +// The marker is a cgroup v1 net_cls classid, not the cgroup2 socket match: +// Microsoft's WSL kernel ships without CONFIG_NFT_SOCKET (`socket cgroupv2` +// rules fail to load, observed live), but with CONFIG_CGROUP_NET_CLASSID=y. +// The boot script (root, before privileges drop) mounts net_cls, gives this +// sandbox's group a classid derived from its reserved server port — unique +// among concurrently running sandboxes by construction — and moves itself in, +// so the server and every tool it spawns inherit the marker. Egress policy: +// loopback always (the codev server and the broker stub live on 127.0.0.1), +// the host-side broker endpoint when one is configured, reject everything +// else. Reject rather than drop so a blocked connect fails immediately, +// matching the NIC-less backends' behavior instead of hanging until timeout. +// +// Fail closed: every step ends the boot on failure, so a guest image without +// nftables, or a kernel without these matches, refuses to start the sandbox +// instead of starting it unfirewalled. +// +// Residual exposure, accepted and deliberate: the VM's loopback interface is +// shared by every distro, and the blanket loopback allow means the sandbox +// can reach 127.0.0.1 services owned by neighbour distros. It cannot be +// closed by port rules without breaking the agent's own localhost workflows +// (dev servers it starts and then dials), and the output hook cannot see the +// receiving socket's owner. vz/bwrap guests do not share their loopback, so +// this is WSL-only. + +// nft identifiers share the distro name, which is already restricted to +// [A-Za-z0-9._-]; nft identifiers cannot carry dots or dashes. +export function wslFirewallName(distro: string) { + return distro.replace(/[^A-Za-z0-9_]/g, "_") +} + +// "CO" in the classid's upper half namespaces codev sandboxes; the lower half +// is the sandbox's host-reserved server port, unique per running sandbox. +export function wslFirewallClassid(serverPort: number) { + return (0x434f << 16) + serverPort +} + +// Shell lines for the boot script. `hostIpVar` names the shell variable that +// holds the broker host address at runtime (never interpolated here — the +// script computes it after the guest network is up). The script reaches the +// guest over stdin, so quoted spaces are safe. +export function wslFirewallScript(options: { + distro: string + serverPort: number + brokerPort: number | null + hostIpVar?: string +}) { + const table = wslFirewallName(options.distro) + const classid = wslFirewallClassid(options.serverPort) + const cgroup = `/sys/fs/cgroup/net_cls/${options.distro}` + const hostIp = `$${options.hostIpVar ?? "HOST_IP"}` + return [ + `command -v nft >/dev/null 2>&1 || { echo "codev sandbox: nft is missing from the guest image" >&2; exit 1; }`, + // The mount is per-distro (each distro has its own mount namespace); the + // classid and the rules live in the shared kernel, scoped by the marker. + `mkdir -p /sys/fs/cgroup/net_cls || exit 1`, + `mountpoint -q /sys/fs/cgroup/net_cls || mount -t cgroup -o net_cls net_cls /sys/fs/cgroup/net_cls || exit 1`, + `mkdir -p ${cgroup} || exit 1`, + `echo ${classid} > ${cgroup}/net_cls.classid || exit 1`, + // Declare-empty + delete + declare-full is nft's idempotent-reset idiom: + // one transaction, no error when the table does not exist yet, and a + // leftover table from a previous run of this sandbox is replaced whole. + `nft -f - < ${cgroup}/cgroup.procs || exit 1`, + ] +} diff --git a/packages/codev-sandbox/src/wsl/provider.test.ts b/packages/codev-sandbox/src/wsl/provider.test.ts index 12ae8213c8..cf52850120 100644 --- a/packages/codev-sandbox/src/wsl/provider.test.ts +++ b/packages/codev-sandbox/src/wsl/provider.test.ts @@ -237,6 +237,35 @@ test("configures automount at import and runs the server unprivileged", async () await rm(dir, { recursive: true, force: true }) }) +test("boot confines the server behind the cgroup firewall before socat starts", async () => { + const dir = await mkdtemp(join(tmpdir(), "wsl-test-")) + const provider = createWslProvider( + await preparedOptions(dir, { + resolveEgress: async () => ({ + routes: [{ prefix: "/probe", upstream: "http://127.0.0.1:9" }], + guestConfig: `{}`, + }), + }), + ) + await provider.create(config) + const sandbox = await provider.start(config) + + const state = JSON.parse(await Bun.file(join(dir, "state.json")).text()) + const script = state.bootScript as string + // Fail closed: no nft, no sandbox. + expect(script).toContain("command -v nft >/dev/null 2>&1 ||") + // The broker hole matches the socat bridge's target port. + const brokerPort = Number(script.match(/TCP:\$HOST_IP:(\d+)/)?.[1]) + expect(script).toContain(`tcp dport ${brokerPort} accept`) + // Confinement happens before the bridge exists, and the server inherits it. + expect(script.indexOf("cgroup.procs")).toBeLessThan(script.indexOf("socat TCP-LISTEN")) + expect(script.indexOf("cgroup.procs")).toBeLessThan(script.indexOf("setpriv")) + + sandbox.stop() + await provider.destroy(config) + await rm(dir, { recursive: true, force: true }) +}) + test("routeless egress still injects the guest config but starts no broker", async () => { const dir = await mkdtemp(join(tmpdir(), "wsl-test-")) const provider = createWslProvider( diff --git a/packages/codev-sandbox/src/wsl/provider.ts b/packages/codev-sandbox/src/wsl/provider.ts index eff8b3de44..63d29ac218 100644 --- a/packages/codev-sandbox/src/wsl/provider.ts +++ b/packages/codev-sandbox/src/wsl/provider.ts @@ -11,6 +11,7 @@ import { } from "../boot" import { createBrokerServer, type SandboxEgress } from "../broker" import { readImageMarker, stageAsset, writeImageMarker } from "../image" +import { wslFirewallScript } from "./firewall" import type { SandboxAsset } from "../manifest" import { windowsBuildNumber, windowsSandboxAvailability } from "../policy" import type { SandboxAvailability, SandboxConfig, SandboxJobOptions, SandboxProvider } from "../provider" @@ -224,9 +225,15 @@ export function createWslProvider(options: WslProviderOptions): SandboxProvider ` [ -n "$HOST_IP" ] && break`, ` sleep 0.2`, `done`, - `socat TCP-LISTEN:8788,bind=127.0.0.1,fork,reuseaddr TCP:$HOST_IP:${brokerPort} &`, ] : []), + // The firewall confines this boot's process tree before anything that + // can reach the network starts; the socat bridge below is inside the + // confined cgroup and rides the broker allow rule. + ...wslFirewallScript({ distro, serverPort: port, brokerPort }), + ...(brokerPort !== null + ? [`socat TCP-LISTEN:8788,bind=127.0.0.1,fork,reuseaddr TCP:$HOST_IP:${brokerPort} &`] + : []), `cd ${shellQuote(await guestPath(distro, config.projectDir))}`, // Root did the bind mounts and the socat bridge; the agent itself runs // as the image's unprivileged codev user, which owns the automounted