From e692cde80b16042169c640f5a5dd90adacc49141 Mon Sep 17 00:00:00 2001 From: minhn4 Date: Wed, 12 Aug 2026 08:18:00 +0700 Subject: [PATCH] feat(sandbox): add the Virtualization.framework provider (sandbox Phase 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit macOS backend for the sandbox: a small Swift helper (codev-vmm) plus the vz SandboxProvider that drives it. - native/vzhelper: SwiftPM executable (macOS 12+) that boots one Linux VM via VZLinuxBootLoader (kernel + initrd + virtio disk), mounts virtio-fs shares (workspace rw, bundle ro, guest-config ro), routes the guest serial console to stderr, and attaches NO network device — the guest's only egress is vsock, bridged by the helper to host unix sockets (host-to-guest server path, guest-to-host broker path). Protocol: VmConfig JSON in, newline-delimited JSON events on stdout, stop via stdin/EOF/SIGTERM. Ad-hoc signed in dev with the com.apple.security.virtualization entitlement (build:vzhelper). - vz/protocol.ts: typed wire protocol + line-buffered event reader - vz/provider.ts: availability hard-blocks below macOS 12 per plan and requires the helper binary; create unpacks the vz image tarball and clones the instance disk copy-on-write (FICLONE/APFS clonefile); start writes guest boot.json (credentials + serve args via the config share), gates on the helper's running event then the health poll, and reuses the Phase 1 TCP-to-unix forwarder; reset re-clones the pristine disk without re-downloading. Provider tests drive a fake helper process speaking the real protocol (spawn, events, unix-socket bridge, health, stop). The real binary compiles and its error protocol is smoke-verified; actually booting a VM needs the guest image from the image pipeline (parallel track). Co-Authored-By: Claude Fable 5 --- packages/codev-sandbox/README.md | 33 +- .../codev-sandbox/native/vzhelper/.gitignore | 2 + .../native/vzhelper/Package.swift | 10 + .../native/vzhelper/Sources/main.swift | 311 ++++++++++++++++++ .../native/vzhelper/vzhelper.entitlements | 8 + packages/codev-sandbox/package.json | 3 +- .../codev-sandbox/script/build-vzhelper.ts | 19 ++ packages/codev-sandbox/src/index.ts | 2 + .../codev-sandbox/src/vz/protocol.test.ts | 34 ++ packages/codev-sandbox/src/vz/protocol.ts | 70 ++++ .../codev-sandbox/src/vz/provider.test.ts | 246 ++++++++++++++ packages/codev-sandbox/src/vz/provider.ts | 251 ++++++++++++++ 12 files changed, 984 insertions(+), 5 deletions(-) create mode 100644 packages/codev-sandbox/native/vzhelper/.gitignore create mode 100644 packages/codev-sandbox/native/vzhelper/Package.swift create mode 100644 packages/codev-sandbox/native/vzhelper/Sources/main.swift create mode 100644 packages/codev-sandbox/native/vzhelper/vzhelper.entitlements create mode 100644 packages/codev-sandbox/script/build-vzhelper.ts create mode 100644 packages/codev-sandbox/src/vz/protocol.test.ts create mode 100644 packages/codev-sandbox/src/vz/protocol.ts create mode 100644 packages/codev-sandbox/src/vz/provider.test.ts create mode 100644 packages/codev-sandbox/src/vz/provider.ts diff --git a/packages/codev-sandbox/README.md b/packages/codev-sandbox/README.md index d406a23e80..bf9ad670ea 100644 --- a/packages/codev-sandbox/README.md +++ b/packages/codev-sandbox/README.md @@ -40,7 +40,32 @@ bwrap can't run (macOS, restricted CI runners). Packaging note for later phases: the .deb should ship an AppArmor profile permitting unprivileged user namespaces on Ubuntu 24.04+. -Later phases add the macOS (Virtualization.framework) and Windows (managed -WSL distro) providers, the desktop main-process wiring, and the `packages/app` -UI. Runtime code sticks to Node APIs (no Bun-only calls) so it can run inside -the Electron main process. +Phase 2 adds the macOS backend, `vz/` + `native/vzhelper`: + +- `native/vzhelper` — `codev-vmm`, a small Swift helper (SwiftPM, macOS 12+) + that boots one Linux VM via Virtualization.framework: VZLinuxBootLoader + (kernel + initrd + virtio disk), virtio-fs shares, guest serial console on + stderr, and **no network device** — the guest's only egress is vsock, which + the helper bridges to host unix sockets (host-to-guest for the server path, + guest-to-host for the broker path). Build with `bun run build:vzhelper` + (`--universal` for arm64+x86_64); dev builds are ad-hoc signed with the + `com.apple.security.virtualization` entitlement, release packaging re-signs + with the app identity. +- `vz/protocol.ts` — the helper wire protocol: VmConfig JSON in, + newline-delimited state/error events out. +- `vz/provider.ts` — the `SandboxProvider`: availability gates on macOS 12+ + (hard block per plan) and helper presence; `create` unpacks the vz image + tarball (`vmlinux` + `initrd` + `rootfs.img`) and clones the instance disk + copy-on-write (APFS clonefile); `start` writes the guest `boot.json` + (credentials + serve args, shared read-only via the `config` virtio-fs tag), + waits for the helper's `running` event, then reuses the Phase 1 TCP→unix + forwarder and health gate; `reset` re-clones the pristine disk. + +Guest image contract for vz (image pipeline): the init must mount the +`workspace`/`bundle`/`config` virtio-fs tags, read `/…/config/boot.json`, run +the server bundle with its env and args, and bridge vsock port 4096 to the +server's TCP port and TCP port 8788 to vsock (e.g. via socat) for the broker. + +Still to come: the Windows managed-WSL provider, the desktop main-process +wiring, and the `packages/app` UI. Runtime code sticks to Node APIs (no +Bun-only calls) so it can run inside the Electron main process. diff --git a/packages/codev-sandbox/native/vzhelper/.gitignore b/packages/codev-sandbox/native/vzhelper/.gitignore new file mode 100644 index 0000000000..e11dfecddf --- /dev/null +++ b/packages/codev-sandbox/native/vzhelper/.gitignore @@ -0,0 +1,2 @@ +.build/ +dist/ diff --git a/packages/codev-sandbox/native/vzhelper/Package.swift b/packages/codev-sandbox/native/vzhelper/Package.swift new file mode 100644 index 0000000000..3ec421f490 --- /dev/null +++ b/packages/codev-sandbox/native/vzhelper/Package.swift @@ -0,0 +1,10 @@ +// swift-tools-version:5.9 +import PackageDescription + +let package = Package( + name: "codev-vmm", + platforms: [.macOS(.v12)], + targets: [ + .executableTarget(name: "codev-vmm", path: "Sources") + ] +) diff --git a/packages/codev-sandbox/native/vzhelper/Sources/main.swift b/packages/codev-sandbox/native/vzhelper/Sources/main.swift new file mode 100644 index 0000000000..88ac029ee0 --- /dev/null +++ b/packages/codev-sandbox/native/vzhelper/Sources/main.swift @@ -0,0 +1,311 @@ +// codev-vmm: boots one NIC-less Linux VM via Virtualization.framework and +// bridges its vsock ports to host unix sockets. +// +// Protocol (matches src/vz/protocol.ts): +// argv[1] path to a VmConfig JSON file +// stdout one JSON event per line: {"event":"state","state":...} | {"event":"error","message":...} +// stderr guest serial console + diagnostics +// stdin {"cmd":"stop"} requests a stop; EOF (parent died) also stops +// +// The VM deliberately gets no network device: its only egress is vsock, which +// this helper exposes as unix sockets so the TS side can gate every byte. + +import Foundation +import Virtualization + +struct ShareConfig: Codable { + let tag: String + let path: String + let readOnly: Bool +} + +struct ForwardConfig: Codable { + let direction: String // "host-to-guest" | "guest-to-host" + let vsockPort: UInt32 + let socketPath: String +} + +struct VmConfig: Codable { + let kernel: String + let initrd: String + let disk: String + let cmdline: String + let cpus: Int + let memoryMiB: UInt64 + let shares: [ShareConfig] + let forwards: [ForwardConfig] +} + +let stdoutLock = NSLock() + +func emit(_ payload: [String: String]) { + guard let data = try? JSONSerialization.data(withJSONObject: payload) else { return } + stdoutLock.lock() + FileHandle.standardOutput.write(data) + FileHandle.standardOutput.write(Data([0x0A])) + stdoutLock.unlock() +} + +func fail(_ message: String) -> Never { + emit(["event": "error", "message": message]) + exit(1) +} + +// MARK: - Byte relay between two file descriptors + +final class Relay { + private let a: Int32 + private let b: Int32 + // Retains the VZVirtioSocketConnection whose fd we are copying, so the + // framework does not invalidate it mid-stream. + private let retained: AnyObject? + private var remaining = 2 + private let lock = NSLock() + + init(_ a: Int32, _ b: Int32, retaining: AnyObject? = nil) { + self.a = a + self.b = b + self.retained = retaining + stream(a, b) + stream(b, a) + } + + private func stream(_ source: Int32, _ sink: Int32) { + DispatchQueue.global().async { [self] in + var buffer = [UInt8](repeating: 0, count: 65536) + loop: while true { + let count = read(source, &buffer, buffer.count) + if count <= 0 { break } + var offset = 0 + while offset < count { + let written = buffer.withUnsafeBytes { raw in + write(sink, raw.baseAddress!.advanced(by: offset), count - offset) + } + if written <= 0 { break loop } + offset += written + } + } + shutdown(sink, SHUT_WR) + lock.lock() + remaining -= 1 + let done = remaining == 0 + lock.unlock() + if done { + close(a) + close(b) + } + } + } +} + +// MARK: - Unix sockets + +func withUnixAddress(_ path: String, _ body: (UnsafePointer, socklen_t) -> T) -> T? { + var address = sockaddr_un() + address.sun_family = sa_family_t(AF_UNIX) + let bytes = path.utf8CString + guard bytes.count <= MemoryLayout.size(ofValue: address.sun_path) else { return nil } + withUnsafeMutableBytes(of: &address.sun_path) { destination in + bytes.withUnsafeBytes { source in destination.copyBytes(from: source) } + } + return withUnsafePointer(to: &address) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { rebound in + body(rebound, socklen_t(MemoryLayout.size)) + } + } +} + +func makeUnixListener(_ path: String) -> Int32 { + unlink(path) + let fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { fail("socket() failed for \(path): \(String(cString: strerror(errno)))") } + let bound = withUnixAddress(path) { address, size in bind(fd, address, size) } + guard bound == 0 else { fail("bind() failed for \(path): \(String(cString: strerror(errno)))") } + guard listen(fd, 16) == 0 else { fail("listen() failed for \(path): \(String(cString: strerror(errno)))") } + return fd +} + +func connectUnix(_ path: String) -> Int32? { + let fd = socket(AF_UNIX, SOCK_STREAM, 0) + guard fd >= 0 else { return nil } + let connected = withUnixAddress(path) { address, size in connect(fd, address, size) } + guard connected == 0 else { + close(fd) + return nil + } + return fd +} + +// MARK: - vsock forwarding + +// host-to-guest: accept on a host unix socket, dial the guest vsock port. +func forwardHostToGuest(_ socketDevice: VZVirtioSocketDevice, listenerFd: Int32, vsockPort: UInt32) { + DispatchQueue.global().async { + while true { + let client = accept(listenerFd, nil, nil) + if client < 0 { break } + DispatchQueue.main.async { + socketDevice.connect(toPort: vsockPort) { result in + switch result { + case .success(let connection): + _ = Relay(client, dup(connection.fileDescriptor), retaining: connection) + case .failure(let error): + FileHandle.standardError.write(Data("vsock connect \(vsockPort) failed: \(error)\n".utf8)) + close(client) + } + } + } + } + } +} + +// guest-to-host: the guest dials a vsock port, we dial the host unix socket. +final class GuestDialDelegate: NSObject, VZVirtioSocketListenerDelegate { + private let socketPath: String + + init(socketPath: String) { + self.socketPath = socketPath + } + + func listener( + _ listener: VZVirtioSocketListener, + shouldAcceptNewConnection connection: VZVirtioSocketConnection, + from socketDevice: VZVirtioSocketDevice + ) -> Bool { + guard let fd = connectUnix(socketPath) else { + FileHandle.standardError.write(Data("dial \(socketPath) failed: \(String(cString: strerror(errno)))\n".utf8)) + return false + } + _ = Relay(dup(connection.fileDescriptor), fd, retaining: connection) + return true + } +} + +// MARK: - VM configuration + +func buildConfiguration(_ config: VmConfig) throws -> VZVirtualMachineConfiguration { + let machine = VZVirtualMachineConfiguration() + let boot = VZLinuxBootLoader(kernelURL: URL(fileURLWithPath: config.kernel)) + boot.initialRamdiskURL = URL(fileURLWithPath: config.initrd) + boot.commandLine = config.cmdline + machine.bootLoader = boot + machine.cpuCount = min( + max(config.cpus, VZVirtualMachineConfiguration.minimumAllowedCPUCount), + VZVirtualMachineConfiguration.maximumAllowedCPUCount + ) + machine.memorySize = min( + max(config.memoryMiB * 1024 * 1024, VZVirtualMachineConfiguration.minimumAllowedMemorySize), + VZVirtualMachineConfiguration.maximumAllowedMemorySize + ) + let disk = try VZDiskImageStorageDeviceAttachment(url: URL(fileURLWithPath: config.disk), readOnly: false) + machine.storageDevices = [VZVirtioBlockDeviceConfiguration(attachment: disk)] + machine.entropyDevices = [VZVirtioEntropyDeviceConfiguration()] + machine.memoryBalloonDevices = [VZVirtioTraditionalMemoryBalloonDeviceConfiguration()] + // Deliberately no network device: the guest's only egress is vsock. + machine.socketDevices = [VZVirtioSocketDeviceConfiguration()] + machine.directorySharingDevices = config.shares.map { share in + let device = VZVirtioFileSystemDeviceConfiguration(tag: share.tag) + device.share = VZSingleDirectoryShare( + directory: VZSharedDirectory(url: URL(fileURLWithPath: share.path), readOnly: share.readOnly) + ) + return device + } + // Guest serial console goes to stderr so boot problems are diagnosable from + // the provider's captured stderr tail. + let serial = VZVirtioConsoleDeviceSerialPortConfiguration() + serial.attachment = VZFileHandleSerialPortAttachment( + fileHandleForReading: nil, + fileHandleForWriting: FileHandle.standardError + ) + machine.serialPorts = [serial] + try machine.validate() + return machine +} + +final class MachineDelegate: NSObject, VZVirtualMachineDelegate { + func guestDidStop(_ virtualMachine: VZVirtualMachine) { + emit(["event": "state", "state": "stopped"]) + exit(0) + } + + func virtualMachine(_ virtualMachine: VZVirtualMachine, didStopWithError error: Error) { + fail("virtual machine stopped: \(error.localizedDescription)") + } +} + +// MARK: - Main + +var retained: [AnyObject] = [] + +let arguments = CommandLine.arguments +guard arguments.count == 2 else { fail("usage: codev-vmm ") } +guard let configData = FileManager.default.contents(atPath: arguments[1]) else { + fail("cannot read config: \(arguments[1])") +} +guard let config = try? JSONDecoder().decode(VmConfig.self, from: configData) else { + fail("invalid config: \(arguments[1])") +} + +emit(["event": "state", "state": "starting"]) + +let machineConfiguration: VZVirtualMachineConfiguration +do { + machineConfiguration = try buildConfiguration(config) +} catch { + fail("invalid virtual machine configuration: \(error.localizedDescription)") +} + +let vm = VZVirtualMachine(configuration: machineConfiguration, queue: .main) +let machineDelegate = MachineDelegate() +retained.append(machineDelegate) +vm.delegate = machineDelegate + +func requestStop() { + DispatchQueue.main.async { + vm.stop { _ in + emit(["event": "state", "state": "stopped"]) + exit(0) + } + } +} + +// Stop on stdin command or EOF (parent process died), and on SIGTERM. +DispatchQueue.global().async { + while let line = readLine() { + if line.contains("stop") { requestStop() } + } + requestStop() +} +signal(SIGTERM, SIG_IGN) +let sigterm = DispatchSource.makeSignalSource(signal: SIGTERM, queue: .main) +sigterm.setEventHandler { requestStop() } +sigterm.resume() +retained.append(sigterm) + +DispatchQueue.main.async { + vm.start { result in + switch result { + case .success: + guard let socketDevice = vm.socketDevices.first as? VZVirtioSocketDevice else { + fail("virtual machine has no vsock device") + } + for forward in config.forwards { + if forward.direction == "host-to-guest" { + forwardHostToGuest(socketDevice, listenerFd: makeUnixListener(forward.socketPath), vsockPort: forward.vsockPort) + continue + } + let delegate = GuestDialDelegate(socketPath: forward.socketPath) + let listener = VZVirtioSocketListener() + listener.delegate = delegate + retained.append(delegate) + retained.append(listener) + socketDevice.setSocketListener(listener, forPort: forward.vsockPort) + } + emit(["event": "state", "state": "running"]) + case .failure(let error): + fail("virtual machine failed to start: \(error.localizedDescription)") + } + } +} + +RunLoop.main.run() diff --git a/packages/codev-sandbox/native/vzhelper/vzhelper.entitlements b/packages/codev-sandbox/native/vzhelper/vzhelper.entitlements new file mode 100644 index 0000000000..d7d0d6e8b6 --- /dev/null +++ b/packages/codev-sandbox/native/vzhelper/vzhelper.entitlements @@ -0,0 +1,8 @@ + + + + + com.apple.security.virtualization + + + diff --git a/packages/codev-sandbox/package.json b/packages/codev-sandbox/package.json index a3cf21f42d..8ebd893f40 100644 --- a/packages/codev-sandbox/package.json +++ b/packages/codev-sandbox/package.json @@ -6,7 +6,8 @@ "type": "module", "scripts": { "typecheck": "tsc --noEmit", - "test": "bun test" + "test": "bun test", + "build:vzhelper": "bun script/build-vzhelper.ts" }, "devDependencies": { "@types/bun": "catalog:" diff --git a/packages/codev-sandbox/script/build-vzhelper.ts b/packages/codev-sandbox/script/build-vzhelper.ts new file mode 100644 index 0000000000..b9d8166802 --- /dev/null +++ b/packages/codev-sandbox/script/build-vzhelper.ts @@ -0,0 +1,19 @@ +#!/usr/bin/env bun +// Builds the codev-vmm Virtualization.framework helper and ad-hoc signs it +// with the virtualization entitlement so it can run VMs in development. +// Release packaging re-signs it with the desktop app's real identity. +// Pass --universal to build arm64 + x86_64 (requires both toolchain slices). +import { $ } from "bun" +import { fileURLToPath } from "node:url" + +const root = fileURLToPath(new URL("../native/vzhelper/", import.meta.url)) +const universal = process.argv.includes("--universal") +const archFlags = universal ? ["--arch", "arm64", "--arch", "x86_64"] : [] +const productDir = universal ? ".build/apple/Products/Release" : ".build/release" + +await $`swift build -c release --package-path ${root} ${archFlags}` +const binary = `${root}${productDir}/codev-vmm` +await $`codesign --force --sign - --entitlements ${root}vzhelper.entitlements ${binary}` +await $`mkdir -p ${root}dist` +await $`cp ${binary} ${root}dist/codev-vmm` +console.log(`built ${root}dist/codev-vmm${universal ? " (universal)" : ""}`) diff --git a/packages/codev-sandbox/src/index.ts b/packages/codev-sandbox/src/index.ts index 92420de425..150f6ac108 100644 --- a/packages/codev-sandbox/src/index.ts +++ b/packages/codev-sandbox/src/index.ts @@ -10,3 +10,5 @@ export * from "./bwrap/args" export * from "./bwrap/bridge" export * from "./bwrap/rootfs" export * from "./bwrap/provider" +export * from "./vz/protocol" +export * from "./vz/provider" diff --git a/packages/codev-sandbox/src/vz/protocol.test.ts b/packages/codev-sandbox/src/vz/protocol.test.ts new file mode 100644 index 0000000000..d509f88717 --- /dev/null +++ b/packages/codev-sandbox/src/vz/protocol.test.ts @@ -0,0 +1,34 @@ +import { expect, test } from "bun:test" +import { createVzHelperEventReader, parseVzHelperEvent } from "./protocol" + +test("parses helper state and error events", () => { + expect(parseVzHelperEvent('{"event":"state","state":"running"}')).toEqual({ event: "state", state: "running" }) + expect(parseVzHelperEvent('{"event":"error","message":"boom"}')).toEqual({ event: "error", message: "boom" }) +}) + +test("ignores lines that are not well-formed events", () => { + expect(parseVzHelperEvent("")).toBe(null) + expect(parseVzHelperEvent("guest kernel noise")).toBe(null) + expect(parseVzHelperEvent('{"event":"state","state":"warping"}')).toBe(null) + expect(parseVzHelperEvent('{"event":"error"}')).toBe(null) + expect(parseVzHelperEvent("{not json")).toBe(null) + expect(parseVzHelperEvent("[1,2]")).toBe(null) +}) + +test("line-buffers chunks that split or batch events", () => { + const seen: unknown[] = [] + const read = createVzHelperEventReader((event) => seen.push(event)) + read('{"event":"state","sta') + expect(seen).toEqual([]) + read('te":"starting"}\n{"event":"state","state":"running"}\nnoise\n{"event":"err') + expect(seen).toEqual([ + { event: "state", state: "starting" }, + { event: "state", state: "running" }, + ]) + read('or","message":"late"}\n') + expect(seen).toEqual([ + { event: "state", state: "starting" }, + { event: "state", state: "running" }, + { event: "error", message: "late" }, + ]) +}) diff --git a/packages/codev-sandbox/src/vz/protocol.ts b/packages/codev-sandbox/src/vz/protocol.ts new file mode 100644 index 0000000000..faed51777a --- /dev/null +++ b/packages/codev-sandbox/src/vz/protocol.ts @@ -0,0 +1,70 @@ +// Wire protocol between the TS provider and the codev-vmm Swift helper +// (native/vzhelper): a VmConfig JSON file in, newline-delimited JSON events +// out on stdout. + +export type VzShare = { + tag: string + path: string + readOnly: boolean +} + +export type VzForward = { + direction: "host-to-guest" | "guest-to-host" + vsockPort: number + socketPath: string +} + +export type VzVmConfig = { + kernel: string + initrd: string + disk: string + cmdline: string + cpus: number + memoryMiB: number + shares: VzShare[] + forwards: VzForward[] +} + +export type VzHelperEvent = + | { event: "state"; state: "starting" | "running" | "stopped" } + | { event: "error"; message: string } + +const HELPER_STATES = new Set(["starting", "running", "stopped"]) + +export function parseVzHelperEvent(line: string): VzHelperEvent | null { + const trimmed = line.trim() + if (!trimmed.startsWith("{")) return null + const parsed = parseJsonObject(trimmed) + if (!parsed) return null + if (parsed.event === "state" && typeof parsed.state === "string" && HELPER_STATES.has(parsed.state)) { + return { event: "state", state: parsed.state as "starting" | "running" | "stopped" } + } + if (parsed.event === "error" && typeof parsed.message === "string") { + return { event: "error", message: parsed.message } + } + return null +} + +// Line-buffers helper stdout: chunks may split or batch events arbitrarily. +export function createVzHelperEventReader(onEvent: (event: VzHelperEvent) => void) { + let buffer = "" + return (chunk: string | Uint8Array) => { + buffer += chunk + const lines = buffer.split("\n") + buffer = lines.pop() ?? "" + for (const line of lines) { + const event = parseVzHelperEvent(line) + if (event) onEvent(event) + } + } +} + +function parseJsonObject(text: string): Record | null { + try { + const value: unknown = JSON.parse(text) + if (!value || typeof value !== "object" || Array.isArray(value)) return null + return value as Record + } catch { + return null + } +} diff --git a/packages/codev-sandbox/src/vz/provider.test.ts b/packages/codev-sandbox/src/vz/provider.test.ts new file mode 100644 index 0000000000..8a0b242df7 --- /dev/null +++ b/packages/codev-sandbox/src/vz/provider.test.ts @@ -0,0 +1,246 @@ +import { expect, test } from "bun:test" +import { spawn } from "node:child_process" +import { createHash } from "node:crypto" +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import type { SandboxConfig } from "../provider" +import { runCommand } from "../bwrap/rootfs" +import { createVzProvider, vzSandboxDirs, type VzProviderOptions } from "./provider" + +const config: SandboxConfig = { id: "sandbox:default", name: "default", backend: "vz", projectDir: "/tmp" } + +// A fake codev-vmm speaking the real protocol: reads the VmConfig, listens on +// the host-to-guest unix socket like the helper would, answers health checks, +// and exits on the stop command. +const FAKE_HELPER = `import { createServer } from "node:net" +import { readFileSync } from "node:fs" +const config = JSON.parse(readFileSync(process.argv[2], "utf8")) +const serverForward = config.forwards.find((forward) => forward.direction === "host-to-guest") +process.stdout.write(JSON.stringify({ event: "state", state: "starting" }) + "\\n") +process.stderr.write("guest console noise\\n") +const listener = createServer((socket) => { + socket.on("data", () => socket.end("HTTP/1.1 200 OK\\r\\ncontent-length: 2\\r\\nconnection: close\\r\\n\\r\\nok")) +}) +listener.listen(serverForward.socketPath, () => { + process.stdout.write(JSON.stringify({ event: "state", state: "running" }) + "\\n") +}) +process.stdin.on("data", (data) => { + if (String(data).includes("stop")) { + process.stdout.write(JSON.stringify({ event: "state", state: "stopped" }) + "\\n") + process.exit(0) + } +}) +` + +const FAILING_HELPER = `process.stdout.write(JSON.stringify({ event: "error", message: "virtual machine failed to start: no entitlement" }) + "\\n") +process.stderr.write("codesign says no\\n") +setTimeout(() => process.exit(1), 5000) +` + +const SILENT_HELPER = `process.stdout.write(JSON.stringify({ event: "state", state: "starting" }) + "\\n") +setInterval(() => {}, 1000) +` + +// Runs .mjs fake helpers through the current runtime, since helperPath +// normally points at a native binary. +const runViaExec = ((command: string, args: string[], opts: unknown) => + spawn(process.execPath, [command, ...args], opts as never)) as unknown as typeof spawn + +async function preparedOptions(dir: string, helperScript: string, overrides?: Partial) { + const helperPath = join(dir, "fake-helper.mjs") + await writeFile(helperPath, helperScript) + const dirs = vzSandboxDirs(join(dir, "data"), config.name) + await mkdir(dirs.image, { recursive: true }) + await writeFile(join(dirs.image, "vmlinux"), "kernel") + await writeFile(join(dirs.image, "initrd"), "initrd") + await writeFile(join(dirs.image, "rootfs.img"), "pristine-disk") + await writeFile(dirs.disk, "pristine-disk") + const projectDir = join(dir, "project") + await mkdir(projectDir, { recursive: true }) + const options: VzProviderOptions = { + dataDir: join(dir, "data"), + cacheDir: join(dir, "cache"), + bundleDir: join(dir, "bundle"), + helperPath, + resolveImage: async () => { + throw new Error("not used") + }, + platform: "darwin", + spawnImpl: runViaExec, + ...overrides, + } + return { options, projectDir, dirs } +} + +test("availability gates on platform, macOS version, and helper presence", async () => { + const dir = await mkdtemp(join(tmpdir(), "codev-vz-")) + const helperPath = join(dir, "codev-vmm") + const base = { + dataDir: dir, + cacheDir: dir, + bundleDir: dir, + helperPath, + resolveImage: async (): Promise => { + throw new Error("not used") + }, + } + expect(await createVzProvider({ ...base, platform: "linux" }).availability()).toEqual({ + supported: false, + message: "Virtualization framework sandboxes require macOS", + }) + const older = await createVzProvider({ + ...base, + platform: "darwin", + readMacosVersion: async () => "11.7.10", + }).availability() + expect(older.supported).toBe(false) + if (!older.supported) expect(older.message).toContain("requires macOS 12") + const missingHelper = await createVzProvider({ + ...base, + platform: "darwin", + readMacosVersion: async () => "15.3", + }).availability() + expect(missingHelper.supported).toBe(false) + if (!missingHelper.supported) expect(missingHelper.message).toContain("helper is not installed") + await writeFile(helperPath, "binary") + expect( + await createVzProvider({ ...base, platform: "darwin", readMacosVersion: async () => "15.3" }).availability(), + ).toEqual({ supported: true, warning: null }) + await rm(dir, { recursive: true, force: true }) +}) + +test("starts the VM via the helper protocol and serves health through the vsock bridge", async () => { + const dir = await mkdtemp(join(tmpdir(), "codev-vz-")) + const prepared = await preparedOptions(dir, FAKE_HELPER) + const provider = createVzProvider(prepared.options) + + const sandbox = await provider.start({ ...config, projectDir: prepared.projectDir }) + expect(sandbox.url).toStartWith("http://127.0.0.1:") + expect(sandbox.username).toBe("codev") + + const vmConfig = JSON.parse(await readFile(join(prepared.dirs.runtime, "vm.json"), "utf8")) + expect(vmConfig.cmdline).toContain("root=/dev/vda") + expect(vmConfig.shares).toEqual([ + { tag: "workspace", path: prepared.projectDir, readOnly: false }, + { tag: "bundle", path: prepared.options.bundleDir, readOnly: true }, + { tag: "config", path: prepared.dirs.guestConfig, readOnly: true }, + ]) + expect(vmConfig.forwards.map((forward: { direction: string }) => forward.direction)).toEqual([ + "host-to-guest", + "guest-to-host", + ]) + const boot = JSON.parse(await readFile(join(prepared.dirs.guestConfig, "boot.json"), "utf8")) + expect(boot.env.CODEV_SERVER_PASSWORD).toBe(sandbox.password) + expect(boot.serveArgs).toEqual(["serve", "--hostname", "127.0.0.1", "--port", "4096"]) + + let exited = false + sandbox.onExit(() => { + exited = true + }) + sandbox.stop() + const until = Date.now() + 3000 + while (!exited && Date.now() < until) await Bun.sleep(10) + expect(exited).toBe(true) + + await rm(dir, { recursive: true, force: true }) +}) + +test("surfaces helper error events with the stderr tail", async () => { + const dir = await mkdtemp(join(tmpdir(), "codev-vz-")) + const prepared = await preparedOptions(dir, FAILING_HELPER) + const provider = createVzProvider(prepared.options) + expect(provider.start({ ...config, projectDir: prepared.projectDir })).rejects.toThrow( + /no entitlement.*codesign says no/s, + ) + await Bun.sleep(20) + await rm(dir, { recursive: true, force: true }) +}) + +test("times out when the helper never reaches running", async () => { + const dir = await mkdtemp(join(tmpdir(), "codev-vz-")) + const prepared = await preparedOptions(dir, SILENT_HELPER, { startTimeoutMs: 300 }) + const provider = createVzProvider(prepared.options) + await expect(provider.start({ ...config, projectDir: prepared.projectDir })).rejects.toThrow( + "did not start within 300ms", + ) + await rm(dir, { recursive: true, force: true }) +}) + +test("start requires create to have run first", async () => { + const dir = await mkdtemp(join(tmpdir(), "codev-vz-")) + const prepared = await preparedOptions(dir, FAKE_HELPER) + await rm(prepared.dirs.disk, { force: true }) + const provider = createVzProvider(prepared.options) + expect(provider.start({ ...config, projectDir: prepared.projectDir })).rejects.toThrow("missing disk") + await rm(dir, { recursive: true, force: true }) +}) + +test("create unpacks the image, validates its contents, and clones the instance disk", async () => { + const dir = await mkdtemp(join(tmpdir(), "codev-vz-")) + const tree = join(dir, "tree") + await mkdir(tree, { recursive: true }) + await writeFile(join(tree, "vmlinux"), "kernel") + await writeFile(join(tree, "initrd"), "initrd") + await writeFile(join(tree, "rootfs.img"), "pristine-disk") + const tarPath = join(dir, "vz.tar") + expect((await runCommand("tar", ["-cf", tarPath, "-C", tree, "vmlinux", "initrd", "rootfs.img"])).code).toBe(0) + const payload = await Bun.file(tarPath).bytes() + const server = Bun.serve({ port: 0, fetch: () => new Response(payload) }) + + const options: VzProviderOptions = { + dataDir: join(dir, "data"), + cacheDir: join(dir, "cache"), + bundleDir: join(dir, "bundle"), + helperPath: join(dir, "codev-vmm"), + platform: "darwin", + resolveImage: async () => ({ + url: `http://127.0.0.1:${server.port}/vz.tar`, + sha256: createHash("sha256").update(payload).digest("hex"), + bytes: payload.byteLength, + }), + } + const provider = createVzProvider(options) + const dirs = vzSandboxDirs(options.dataDir, config.name) + + await provider.create(config) + expect(await readFile(dirs.disk, "utf8")).toBe("pristine-disk") + + await writeFile(dirs.disk, "guest state scribbled all over the disk") + await provider.reset(config) + expect(await readFile(dirs.disk, "utf8")).toBe("pristine-disk") + + await provider.destroy(config) + expect(await stat(dirs.base).catch(() => null)).toBe(null) + + server.stop(true) + await rm(dir, { recursive: true, force: true }) +}) + +test("create rejects an image tarball missing required files", async () => { + const dir = await mkdtemp(join(tmpdir(), "codev-vz-")) + const tree = join(dir, "tree") + await mkdir(tree, { recursive: true }) + await writeFile(join(tree, "vmlinux"), "kernel") + const tarPath = join(dir, "vz.tar") + expect((await runCommand("tar", ["-cf", tarPath, "-C", tree, "vmlinux"])).code).toBe(0) + const payload = await Bun.file(tarPath).bytes() + const server = Bun.serve({ port: 0, fetch: () => new Response(payload) }) + + const provider = createVzProvider({ + dataDir: join(dir, "data"), + cacheDir: join(dir, "cache"), + bundleDir: join(dir, "bundle"), + helperPath: join(dir, "codev-vmm"), + platform: "darwin", + resolveImage: async () => ({ + url: `http://127.0.0.1:${server.port}/vz.tar`, + sha256: createHash("sha256").update(payload).digest("hex"), + bytes: payload.byteLength, + }), + }) + expect(provider.create(config)).rejects.toThrow("vz image is missing initrd, rootfs.img") + await Bun.sleep(20) + server.stop(true) + await rm(dir, { recursive: true, force: true }) +}) diff --git a/packages/codev-sandbox/src/vz/provider.ts b/packages/codev-sandbox/src/vz/provider.ts new file mode 100644 index 0000000000..eb9fc5eda3 --- /dev/null +++ b/packages/codev-sandbox/src/vz/provider.ts @@ -0,0 +1,251 @@ +import { spawn } from "node:child_process" +import { constants, copyFile, mkdir, rm, stat, writeFile } from "node:fs/promises" +import { join } from "node:path" +import { + checkSandboxHealth, + generateSandboxCredentials, + pollSandboxHealth, + sandboxBootEnv, + sandboxServeArgs, +} from "../boot" +import { forwardTcpToUnix } from "../bwrap/bridge" +import { runCommand, unpackRootfs } from "../bwrap/rootfs" +import { stageAsset } from "../image" +import type { SandboxAsset } from "../manifest" +import { macosSandboxAvailability } from "../policy" +import type { SandboxAvailability, SandboxConfig, SandboxProvider } from "../provider" +import { createVzHelperEventReader, type VzHelperEvent, type VzVmConfig } from "./protocol" + +// Contents of the vz image tarball (built by the image pipeline). +export const VZ_KERNEL_FILE = "vmlinux" +export const VZ_INITRD_FILE = "initrd" +export const VZ_DISK_FILE = "rootfs.img" + +// Guest agent contract: the image's init bridges vsock SERVER_PORT to the +// codev server's TCP port and dials vsock BROKER_PORT for egress, reading +// credentials and serve args from the "config" virtio-fs share's boot.json. +export const VZ_SERVER_VSOCK_PORT = 4096 +export const VZ_BROKER_VSOCK_PORT = 8788 +export const VZ_GUEST_HOME = "/home/codev" +const VZ_CMDLINE = "console=hvc0 root=/dev/vda rw" + +export type VzProviderOptions = { + dataDir: string + cacheDir: string + // Host path of the server bundle (dist/node), shared read-only into the + // guest so version parity with the desktop app is automatic. + bundleDir: string + // The signed codev-vmm binary (native/vzhelper). + helperPath: string + resolveImage: (opts?: { signal?: AbortSignal }) => Promise + cpus?: number + memoryMiB?: number + platform?: NodeJS.Platform + readMacosVersion?: () => Promise + spawnImpl?: typeof spawn + fetchImpl?: typeof fetch + startTimeoutMs?: number +} + +export function vzSandboxDirs(dataDir: string, name: string) { + const base = join(dataDir, name) + return { + base, + image: join(base, "image"), + disk: join(base, "disk.img"), + runtime: join(base, "runtime"), + guestConfig: join(base, "guest-config"), + } +} + +export function createVzProvider(options: VzProviderOptions): SandboxProvider { + const platform = options.platform ?? process.platform + const spawnImpl = options.spawnImpl ?? spawn + const fetchImpl = options.fetchImpl ?? fetch + const startTimeoutMs = options.startTimeoutMs ?? 60_000 + const readMacosVersion = options.readMacosVersion ?? defaultMacosVersion + + const prepare = async (config: SandboxConfig, opts?: { signal?: AbortSignal }) => { + const asset = await options.resolveImage(opts) + const image = await stageAsset(options.cacheDir, asset, { fetchImpl, signal: opts?.signal }) + const dirs = vzSandboxDirs(options.dataDir, config.name) + await unpackRootfs(image, dirs.image, opts) + const missing = [] + for (const file of [VZ_KERNEL_FILE, VZ_INITRD_FILE, VZ_DISK_FILE]) { + const found = await stat(join(dirs.image, file)).catch(() => null) + if (!found?.isFile()) missing.push(file) + } + if (missing.length) throw new Error(`vz image is missing ${missing.join(", ")}`) + // The pristine unpacked disk stays untouched; the instance runs a + // copy-on-write clone (APFS clonefile via FICLONE, plain copy elsewhere), + // so reset is just re-cloning. + await copyFile(join(dirs.image, VZ_DISK_FILE), dirs.disk, constants.COPYFILE_FICLONE) + } + + return { + backend: "vz", + + async availability(): Promise { + if (platform !== "darwin") + return { supported: false, message: "Virtualization framework sandboxes require macOS" } + const version = await readMacosVersion() + const policy = macosSandboxAvailability(version ?? "") + if (!policy.supported) return policy + const helper = await stat(options.helperPath).catch(() => null) + if (!helper?.isFile()) + return { supported: false, message: `sandbox VM helper is not installed (expected at ${options.helperPath})` } + return policy + }, + + async create(config, opts) { + await prepare(config, opts) + }, + + async start(config) { + const dirs = vzSandboxDirs(options.dataDir, config.name) + const project = await stat(config.projectDir).catch(() => null) + if (!project?.isDirectory()) throw new Error(`project directory does not exist: ${config.projectDir}`) + const disk = await stat(dirs.disk).catch(() => null) + if (!disk?.isFile()) throw new Error(`sandbox has not been created: missing disk at ${dirs.disk}`) + await rm(dirs.runtime, { recursive: true, force: true }) + await mkdir(dirs.runtime, { recursive: true }) + await mkdir(dirs.guestConfig, { recursive: true }) + + const credentials = generateSandboxCredentials() + await writeFile( + join(dirs.guestConfig, "boot.json"), + JSON.stringify({ + env: sandboxBootEnv(credentials, VZ_GUEST_HOME), + serveArgs: sandboxServeArgs({ hostname: "127.0.0.1", port: VZ_SERVER_VSOCK_PORT }), + }), + ) + const serverSocket = join(dirs.runtime, "server.sock") + const vmConfig: VzVmConfig = { + kernel: join(dirs.image, VZ_KERNEL_FILE), + initrd: join(dirs.image, VZ_INITRD_FILE), + disk: dirs.disk, + cmdline: VZ_CMDLINE, + cpus: options.cpus ?? 4, + memoryMiB: options.memoryMiB ?? 4096, + shares: [ + { tag: "workspace", path: config.projectDir, readOnly: false }, + { tag: "bundle", path: options.bundleDir, readOnly: true }, + { tag: "config", path: dirs.guestConfig, readOnly: true }, + ], + forwards: [ + { direction: "host-to-guest", vsockPort: VZ_SERVER_VSOCK_PORT, socketPath: serverSocket }, + { direction: "guest-to-host", vsockPort: VZ_BROKER_VSOCK_PORT, socketPath: join(dirs.runtime, "broker.sock") }, + ], + } + const configPath = join(dirs.runtime, "vm.json") + await writeFile(configPath, JSON.stringify(vmConfig)) + + const child = spawnImpl(options.helperPath, [configPath], { stdio: ["pipe", "pipe", "pipe"] }) + let stderrTail = "" + child.stderr?.on("data", (chunk) => { + stderrTail = `${stderrTail}${chunk}`.slice(-2000) + }) + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => { + child.once("exit", (code, signal) => resolve({ code, signal })) + }) + const handlers: { error?: (message: string) => void; running?: () => void } = {} + const helperError = new Promise((resolve) => { + handlers.error = resolve + }) + const running = new Promise<"running">((resolve) => { + handlers.running = () => resolve("running") + }) + child.stdout?.on( + "data", + createVzHelperEventReader((event: VzHelperEvent) => { + if (event.event === "error") handlers.error?.(event.message) + if (event.event === "state" && event.state === "running") handlers.running?.() + }), + ) + + // Helper events arrive on stdout and can outrun the stderr pipe; give + // stderr a beat to flush so failures carry the diagnostic tail. + const failure = async (detail: string) => { + child.kill() + await new Promise((resolve) => setTimeout(resolve, 100)) + const tail = stderrTail.trim() + return new Error(`${detail}${tail ? `: ${tail}` : ""}`) + } + let timeoutHandle: ReturnType | undefined + const timeout = new Promise<"timeout">((resolve) => { + timeoutHandle = setTimeout(() => resolve("timeout"), startTimeoutMs) + }) + + const bootOutcome = await Promise.race([running, helperError, exited, timeout]) + if (bootOutcome !== "running") { + clearTimeout(timeoutHandle) + if (bootOutcome === "timeout") throw await failure(`sandbox VM did not start within ${startTimeoutMs}ms`) + if (typeof bootOutcome === "string") throw await failure(`sandbox VM failed: ${bootOutcome}`) + throw await failure( + `sandbox VM helper exited during startup (code=${bootOutcome.code ?? "null"} signal=${bootOutcome.signal ?? "null"})`, + ) + } + + const forwarder = await forwardTcpToUnix(serverSocket) + const url = `http://127.0.0.1:${forwarder.port}` + const abort = new AbortController() + const healthOutcome = await Promise.race([ + pollSandboxHealth(() => checkSandboxHealth(url, credentials, fetchImpl), abort.signal).then( + () => "healthy" as const, + ), + helperError, + exited, + timeout, + ]) + abort.abort() + clearTimeout(timeoutHandle) + if (healthOutcome !== "healthy") { + await forwarder.close() + if (healthOutcome === "timeout") + throw await failure(`sandbox server did not become healthy within ${startTimeoutMs}ms`) + if (typeof healthOutcome === "string") throw await failure(`sandbox VM failed: ${healthOutcome}`) + throw await failure( + `sandbox VM helper exited during startup (code=${healthOutcome.code ?? "null"} signal=${healthOutcome.signal ?? "null"})`, + ) + } + + let stopKill: ReturnType | undefined + void exited.then(() => clearTimeout(stopKill)) + return { + url, + username: credentials.username, + password: credentials.password, + stop: () => { + // Ask the helper for a guest stop; escalate to kill if it hangs. + child.stdin?.write('{"cmd":"stop"}\n') + stopKill = setTimeout(() => child.kill(), 3000) + void forwarder.close() + }, + onExit: (cb) => { + void exited.then((exit) => { + void forwarder.close() + cb(exit.code, exit.signal) + }) + }, + } + }, + + async destroy(config) { + await rm(vzSandboxDirs(options.dataDir, config.name).base, { recursive: true, force: true }) + }, + + async reset(config) { + const dirs = vzSandboxDirs(options.dataDir, config.name) + // Guest state lives in the instance disk; re-cloning the pristine image + // disk resets it without re-downloading anything. + await rm(dirs.disk, { force: true }) + await copyFile(join(dirs.image, VZ_DISK_FILE), dirs.disk, constants.COPYFILE_FICLONE) + }, + } +} + +async function defaultMacosVersion() { + const result = await runCommand("sw_vers", ["-productVersion"]).catch(() => null) + if (!result || result.code !== 0) return null + return result.stdout.trim() +}