From b55f4b0b0eca1b059eec1e05d06b5be85d7ff03d Mon Sep 17 00:00:00 2001 From: minhn4 Date: Wed, 12 Aug 2026 10:01:16 +0700 Subject: [PATCH 1/2] feat(sandbox): add the guest image build and publish pipeline (sandbox Phase 3) Build the sandbox guest images with Docker so the pipeline runs the same on macOS and Linux. Per arch it produces a bwrap rootfs tarball, a vz bundle of vmlinux plus an ext4 rootfs, and a manifest.json matching src/manifest.ts. - image/: one Debian guest consumed by every backend, a minimal virtio-only kernel, a tools image for ext4 conversion, and the system-wide codev-office install (skipped with --no-office) - script/build-image.ts: per-arch build, digest-pinned base, sparse-aware tarring, sha256 manifest assembly - script/publish-image.ts: uploads versioned artifacts first, manifest.json last, so clients never see a version before its assets exist - script/e2e-vz.ts: boots a locally built vz image end-to-end through codev-vmm and the vsock health check Co-Authored-By: Claude Opus 5 (1M context) --- packages/codev-sandbox/README.md | 38 +++- packages/codev-sandbox/image/Dockerfile | 38 ++++ packages/codev-sandbox/image/guest/codev-init | 44 +++++ .../codev-sandbox/image/kernel/Dockerfile | 22 +++ .../codev-sandbox/image/kernel/codev.config | 23 +++ .../codev-sandbox/image/office/bundle/.keep | 0 .../image/office/install-office.sh | 77 ++++++++ packages/codev-sandbox/image/tools/Dockerfile | 6 + .../native/vzhelper/Sources/main.swift | 5 +- packages/codev-sandbox/script/build-image.ts | 170 ++++++++++++++++++ packages/codev-sandbox/script/e2e-vz.ts | 88 +++++++++ .../codev-sandbox/script/publish-image.ts | 56 ++++++ packages/codev-sandbox/src/bwrap/provider.ts | 4 + packages/codev-sandbox/src/const.ts | 4 +- .../codev-sandbox/src/vz/provider.test.ts | 2 +- packages/codev-sandbox/src/vz/provider.ts | 9 +- 16 files changed, 577 insertions(+), 9 deletions(-) create mode 100644 packages/codev-sandbox/image/Dockerfile create mode 100644 packages/codev-sandbox/image/guest/codev-init create mode 100644 packages/codev-sandbox/image/kernel/Dockerfile create mode 100644 packages/codev-sandbox/image/kernel/codev.config create mode 100644 packages/codev-sandbox/image/office/bundle/.keep create mode 100644 packages/codev-sandbox/image/office/install-office.sh create mode 100644 packages/codev-sandbox/image/tools/Dockerfile create mode 100644 packages/codev-sandbox/script/build-image.ts create mode 100644 packages/codev-sandbox/script/e2e-vz.ts create mode 100644 packages/codev-sandbox/script/publish-image.ts diff --git a/packages/codev-sandbox/README.md b/packages/codev-sandbox/README.md index bf9ad670ea..37b411fafa 100644 --- a/packages/codev-sandbox/README.md +++ b/packages/codev-sandbox/README.md @@ -66,6 +66,38 @@ Guest image contract for vz (image pipeline): the init must mount the the server bundle with its env and args, and bridge vsock port 4096 to the server's TCP port and TCP port 8788 to vsock (e.g. via socat) for the broker. -Still to come: the Windows managed-WSL provider, the desktop main-process -wiring, and the `packages/app` UI. Runtime code sticks to Node APIs (no -Bun-only calls) so it can run inside the Electron main process. +The image pipeline lives in `image/` and builds every backend's guest from +one Debian rootfs, fully containerized (any machine with Docker): + +- `image/Dockerfile` — base toolchain (node 22, python3 + pip/venv, + build-essential, git, ripgrep/jq/sqlite3/curl/unzip, socat, iproute2) plus + the optional office layer: `image/office/install-office.sh` translates the + codev-office setup script system-wide — pandoc + LibreOffice + fonts from + Debian, .NET 8 SDK at `/opt/dotnet`, the pinned Python package set, the + bundle's pure-JS node-module trees on `NODE_PATH`, and the office skills at + `/opt/codev-skills` with an `/etc/skel` codev.json pointing at them. Fresh + sandbox homes get skel (vz at image build; bwrap seeded by the boot + script), and images declare extra env in `/etc/codev-guest.env`. +- `image/guest/codev-init` — the vz guest's PID 1: mounts the virtio-fs tags, + runs the socat vsock bridges (4096 server / 8788 broker, host CID 2), and + execs the server bundle per `boot.json`; powers off when the server dies. +- `image/kernel/` — a minimal pinned kernel (6.6 LTS) with virtio + blk/fs/vsock/console + ext4 compiled in: no initrd, no modules, no network + drivers. Cached by Docker layers after the first build. +- `script/build-image.ts` — orchestrates buildx → rootfs tar → ext4 disk + (`mkfs.ext4 -d`, no root needed) → kernel → `vz-.tar.gz` / + `bwrap-.tar.gz` → `manifest.json` (validated against + `src/manifest.ts` before writing). `--office-bundle` stages the + arch-independent parts of codev-office-ubuntu.zip; binaries are fetched + arch-correct inside the build. Build each arch on a native host — x64 under + emulation works but is slow. +- `script/publish-image.ts` — uploads artifacts first, `manifest.json` last + (clients never see a version whose artifacts aren't in place), via `mc` + when `CODEV_STORAGE_TARGET` is set, otherwise prints the commands. +- `script/e2e-vz.ts` — boots a locally built image through `codev-vmm` and + waits for the authenticated health check: the full-stack smoke test. + +Still to come: the Windows managed-WSL provider (consumes the same rootfs tar +via `wsl --import`), the desktop main-process wiring, and the `packages/app` +UI. Runtime code sticks to Node APIs (no Bun-only calls) so it can run inside +the Electron main process. diff --git a/packages/codev-sandbox/image/Dockerfile b/packages/codev-sandbox/image/Dockerfile new file mode 100644 index 0000000000..c2ebec943d --- /dev/null +++ b/packages/codev-sandbox/image/Dockerfile @@ -0,0 +1,38 @@ +# Guest image for the codev sandbox: one Debian rootfs consumed by all +# backends (bwrap tar, vz ext4 disk, later wsl --import tar). +# Built by script/build-image.ts via docker buildx; TARGETARCH is amd64|arm64. + +FROM debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 AS base +ARG DEBIAN_FRONTEND=noninteractive +# Base toolchain: node/git/socat/ca-certs always; python + build tools + the +# CLI set per the sandbox plan. iproute2 is needed by the vz init (loopback). +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl wget git unzip zip jq ripgrep sqlite3 socat iproute2 \ + procps less vim-tiny openssh-client \ + python3 python3-pip python3-venv python3-dev \ + build-essential pkg-config \ + && rm -rf /var/lib/apt/lists/* +ARG TARGETARCH +ARG NODE_VERSION=v22.15.0 +RUN set -eux; \ + case "$TARGETARCH" in amd64) NODE_ARCH=x64 ;; arm64) NODE_ARCH=arm64 ;; *) echo "unsupported arch: $TARGETARCH" >&2; exit 1 ;; esac; \ + curl -fsSL "https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-${NODE_ARCH}.tar.xz" \ + | tar -xJ -C /usr/local --strip-components=1; \ + node --version +RUN useradd --create-home --shell /bin/bash --uid 1000 codev + +# Office layer: the codev-office skill dependencies (dotnet/pandoc/libreoffice/ +# python packages/node module trees/skills), system-wide instead of per-user +# because sandbox homes are fresh. Skipped when OFFICE=0. +FROM base AS office +ARG OFFICE=1 +COPY office/install-office.sh /tmp/install-office.sh +COPY office/bundle/ /tmp/office-bundle/ +RUN if [ "$OFFICE" = "1" ]; then bash /tmp/install-office.sh /tmp/office-bundle; fi \ + && rm -rf /tmp/office-bundle /tmp/install-office.sh + +# Guest layer: the vz init (PID 1) and the mount points every backend expects. +# Harmless in the bwrap/wsl tars. +FROM office AS guest +COPY guest/codev-init /sbin/codev-init +RUN chmod 755 /sbin/codev-init && mkdir -p /workspace /opt/codev /codev-config diff --git a/packages/codev-sandbox/image/guest/codev-init b/packages/codev-sandbox/image/guest/codev-init new file mode 100644 index 0000000000..9d0464bb7c --- /dev/null +++ b/packages/codev-sandbox/image/guest/codev-init @@ -0,0 +1,44 @@ +#!/bin/bash +# PID 1 of the vz sandbox guest (kernel cmdline: init=/sbin/codev-init). +# Mounts the virtio-fs shares, bridges vsock to loopback TCP, and runs the +# codev server per /codev-config/boot.json. The guest has no NIC: socat over +# vsock (host CID 2) is the only way bytes leave this machine. +set -u + +mount -t proc proc /proc +mount -t sysfs sys /sys +mountpoint -q /dev || mount -t devtmpfs dev /dev +mkdir -p /dev/pts /dev/shm +mount -t devpts devpts /dev/pts 2>/dev/null || true +mount -t tmpfs tmpfs /dev/shm 2>/dev/null || true +mount -t tmpfs tmpfs /tmp +mount -t virtiofs workspace /workspace +mount -t virtiofs bundle /opt/codev +mount -t virtiofs config /codev-config +ip link set lo up + +[ -f /etc/codev-guest.env ] && . /etc/codev-guest.env + +# Server path: host connects vsock:4096 -> guest loopback TCP. Broker path: +# guest loopback TCP:8788 -> host vsock:8788 (the egress broker). +socat VSOCK-LISTEN:4096,fork,reuseaddr TCP:127.0.0.1:4096 & +socat TCP-LISTEN:8788,bind=127.0.0.1,fork,reuseaddr VSOCK-CONNECT:2:8788 & + +eval "$(node -e ' +const boot = require("/codev-config/boot.json") +for (const [key, value] of Object.entries(boot.env)) console.log("export " + key + "=" + JSON.stringify(value)) +console.log("CODEV_SERVE_ARGS=" + JSON.stringify(boot.serveArgs.join(" "))) +')" + +export HOME=/home/codev +cd /workspace +node /opt/codev/node.js $CODEV_SERVE_ARGS & +SERVER=$! + +# Stay PID 1: reap orphans, power off when the server dies so the helper's +# guestDidStop fires and the provider sees a clean exit. +while :; do + wait -n 2>/dev/null + kill -0 "$SERVER" 2>/dev/null || break +done +poweroff -f diff --git a/packages/codev-sandbox/image/kernel/Dockerfile b/packages/codev-sandbox/image/kernel/Dockerfile new file mode 100644 index 0000000000..52c43469f4 --- /dev/null +++ b/packages/codev-sandbox/image/kernel/Dockerfile @@ -0,0 +1,22 @@ +# Minimal sandbox guest kernel: virtio blk/fs/vsock/console + ext4 compiled +# in, no modules, no initrd, no network drivers. Built per-arch by +# script/build-image.ts; docker layer caching makes rebuilds cheap unless +# KERNEL_VERSION or the fragment changes. +FROM debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 AS build +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential bc bison flex libssl-dev libelf-dev python3 curl xz-utils kmod cpio ca-certificates \ + && rm -rf /var/lib/apt/lists/* +ARG KERNEL_VERSION=6.6.100 +WORKDIR /src +RUN curl -fsSL "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${KERNEL_VERSION}.tar.xz" | tar -xJ --strip-components=1 +COPY codev.config /src/codev.config +RUN make defconfig \ + && scripts/kconfig/merge_config.sh -m .config codev.config \ + && make olddefconfig +RUN make -j"$(nproc)" $(case "$(uname -m)" in aarch64) echo Image ;; x86_64) echo bzImage ;; esac) +# Normalize the artifact name; the vz provider expects "vmlinux". +RUN mkdir /out && cp "$(case "$(uname -m)" in aarch64) echo arch/arm64/boot/Image ;; x86_64) echo arch/x86/boot/bzImage ;; esac)" /out/vmlinux + +FROM scratch AS artifact +COPY --from=build /out/vmlinux /vmlinux diff --git a/packages/codev-sandbox/image/kernel/codev.config b/packages/codev-sandbox/image/kernel/codev.config new file mode 100644 index 0000000000..7b5088df29 --- /dev/null +++ b/packages/codev-sandbox/image/kernel/codev.config @@ -0,0 +1,23 @@ +# Merged over defconfig: everything the vz guest needs, compiled in (=y) so +# no initrd or module loading is required. No network device drivers — the +# guest's only transport is vsock. +CONFIG_VIRTIO=y +CONFIG_VIRTIO_MENU=y +CONFIG_VIRTIO_PCI=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_VIRTIO_CONSOLE=y +CONFIG_HW_RANDOM_VIRTIO=y +CONFIG_VIRTIO_BALLOON=y +CONFIG_VSOCKETS=y +CONFIG_VIRTIO_VSOCKETS=y +CONFIG_FUSE_FS=y +CONFIG_VIRTIO_FS=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_TMPFS=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_UNIX=y +CONFIG_INET=y +CONFIG_DAX=y diff --git a/packages/codev-sandbox/image/office/bundle/.keep b/packages/codev-sandbox/image/office/bundle/.keep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/codev-sandbox/image/office/install-office.sh b/packages/codev-sandbox/image/office/install-office.sh new file mode 100644 index 0000000000..f071fafb12 --- /dev/null +++ b/packages/codev-sandbox/image/office/install-office.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# System-wide translation of codev-office-ubuntu-setup.sh for the sandbox +# guest image. Runs inside the Docker build as root; $1 is the staged bundle +# dir (skill zips + pure-JS node-module zips — arch-independent parts only; +# dotnet/pandoc/libreoffice/wheels are fetched arch-correct right here, since +# the BUILD machine has internet even though the sealed guest will not). +set -euxo pipefail +BUNDLE="${1:-/tmp/office-bundle}" +ARCH="$(dpkg --print-architecture)" + +# pandoc + LibreOffice + fonts from Debian (available on amd64 and arm64). +apt-get update +apt-get install -y --no-install-recommends \ + pandoc libreoffice-writer libreoffice-calc libreoffice-impress \ + fonts-dejavu fonts-liberation2 +rm -rf /var/lib/apt/lists/* + +# .NET SDK 8 into /opt/dotnet (the docx/xlsx skills drive OpenXML with it). +case "$ARCH" in + amd64) DOTNET_ARCH=x64 ;; + arm64) DOTNET_ARCH=arm64 ;; +esac +curl -fsSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh +bash /tmp/dotnet-install.sh --channel 8.0 --install-dir /opt/dotnet --architecture "$DOTNET_ARCH" +rm /tmp/dotnet-install.sh +ln -sf /opt/dotnet/dotnet /usr/local/bin/dotnet +dotnet --list-sdks + +# Python package set mirrored from the office bundle's wheel list, installed +# into the system python at build time (the guest never runs pip online). +pip3 install --break-system-packages --no-cache-dir \ + pandas openpyxl 'markitdown[pptx]' reportlab pypdf matplotlib + +# Prebuilt pure-JS node module trees from the bundle, merged into one tree on +# NODE_PATH (mirrors the setup script's ~/.node_modules require() fallback). +if compgen -G "$BUNDLE/*-node-modules.zip" >/dev/null; then + mkdir -p /opt/codev-office/node_modules /tmp/modules-staging + for mod_zip in "$BUNDLE"/*-node-modules.zip; do + unzip -q "$mod_zip" -d /tmp/modules-staging/current + if [ -d /tmp/modules-staging/current/node_modules ]; then + cp -R /tmp/modules-staging/current/node_modules/. /opt/codev-office/node_modules/ + fi + rm -rf /tmp/modules-staging/current + done +fi + +# Skills into a system path; /etc/skel seeds each fresh sandbox home with the +# config pointing at them (vz homes get skel at image build, bwrap homes are +# seeded by the provider boot script). +if compgen -G "$BUNDLE/*-skill.zip" >/dev/null; then + mkdir -p /opt/codev-skills + for skill_zip in "$BUNDLE"/*-skill.zip; do + name="$(basename "$skill_zip" .zip)" + name="${name%-skill}" + unzip -q "$skill_zip" -d "/opt/codev-skills/$name" + done + mkdir -p /etc/skel/.config/codev + cat > /etc/skel/.config/codev/codev.json <<'JSON' +{ + "skills": { + "paths": ["/opt/codev-skills"] + } +} +JSON +fi + +# Env the guest boot paths source (vz init and the bwrap boot script). +cat > /etc/codev-guest.env <<'ENV' +export DOTNET_ROOT=/opt/dotnet +export NODE_PATH=/opt/codev-office/node_modules +export DOTNET_CLI_TELEMETRY_OPTOUT=1 +ENV + +# The image build already populated /home/codev before this layer; re-apply +# skel so baked homes (vz disk) get the skills config too. +cp -Rn /etc/skel/. /home/codev/ 2>/dev/null || true +chown -R codev:codev /home/codev diff --git a/packages/codev-sandbox/image/tools/Dockerfile b/packages/codev-sandbox/image/tools/Dockerfile new file mode 100644 index 0000000000..e6eb78b1c8 --- /dev/null +++ b/packages/codev-sandbox/image/tools/Dockerfile @@ -0,0 +1,6 @@ +# Small helper image for rootfs->ext4 conversion and sparse-aware tarring; +# runs as root inside the container so file ownership survives, and gives the +# pipeline GNU tar + mkfs.ext4 -d regardless of the host OS. +FROM debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241 +RUN apt-get update && apt-get install -y --no-install-recommends e2fsprogs tar gzip \ + && rm -rf /var/lib/apt/lists/* diff --git a/packages/codev-sandbox/native/vzhelper/Sources/main.swift b/packages/codev-sandbox/native/vzhelper/Sources/main.swift index 88ac029ee0..2799ecf494 100644 --- a/packages/codev-sandbox/native/vzhelper/Sources/main.swift +++ b/packages/codev-sandbox/native/vzhelper/Sources/main.swift @@ -186,7 +186,10 @@ final class GuestDialDelegate: NSObject, VZVirtioSocketListenerDelegate { func buildConfiguration(_ config: VmConfig) throws -> VZVirtualMachineConfiguration { let machine = VZVirtualMachineConfiguration() let boot = VZLinuxBootLoader(kernelURL: URL(fileURLWithPath: config.kernel)) - boot.initialRamdiskURL = URL(fileURLWithPath: config.initrd) + // Empty initrd means the kernel boots without one (virtio compiled in). + if !config.initrd.isEmpty { + boot.initialRamdiskURL = URL(fileURLWithPath: config.initrd) + } boot.commandLine = config.cmdline machine.bootLoader = boot machine.cpuCount = min( diff --git a/packages/codev-sandbox/script/build-image.ts b/packages/codev-sandbox/script/build-image.ts new file mode 100644 index 0000000000..7583b56ef8 --- /dev/null +++ b/packages/codev-sandbox/script/build-image.ts @@ -0,0 +1,170 @@ +#!/usr/bin/env bun +// Builds the sandbox guest images with Docker (fully containerized: works the +// same on macOS and Linux). Produces, per arch: +// bwrap-.tar.gz rootfs tarball (bubblewrap now, wsl --import later) +// vz-.tar.gz vmlinux + rootfs.img for Virtualization.framework +// plus a manifest.json matching src/manifest.ts. +// +// Usage: +// bun script/build-image.ts --version 2026.08.1 +// [--arch arm64] [--arch x64] default: host arch only (x64 under +// emulation is slow; run on a native +// host or CI for the other arch) +// [--office-bundle path.zip] codev-office bundle; its skill zips +// and pure-JS node-module zips are +// baked in (other parts are fetched +// arch-correct during the build) +// [--no-office] base toolchain only +// [--skip-kernel] reuse an existing vmlinux- +// [--out dir] default: dist-images/ +import { $ } from "bun" +import { createHash } from "node:crypto" +import { cp, mkdir, readdir, rm, stat, writeFile } from "node:fs/promises" +import { arch as hostArch } from "node:os" +import { join } from "node:path" +import { fileURLToPath } from "node:url" +import { SANDBOX_IMAGES_BASE_URL } from "../src/const" +import { parseSandboxManifest, type SandboxAsset } from "../src/manifest" +import type { SandboxArch } from "../src/provider" + +const imageDir = fileURLToPath(new URL("../image/", import.meta.url)) +const DISK_SIZE = "8G" + +const version = argValue("--version") +if (!version) { + console.error("--version is required (e.g. 2026.08.1)") + process.exit(1) +} +const archs = collectArchs() +const officeBundle = argValue("--office-bundle") +const office = process.argv.includes("--no-office") ? "0" : "1" +const skipKernel = process.argv.includes("--skip-kernel") +const outDir = argValue("--out") ?? join(process.cwd(), "dist-images") +await mkdir(outDir, { recursive: true }) + +// Slow corporate registries can exceed buildkit's metadata deadline; the +// FROMs are digest-pinned and this prepull (with retries) primes the local +// store so buildx never has to resolve remotely. +const BASE_IMAGE = "debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241" +for (let attempt = 1; ; attempt++) { + const pulled = await $`docker pull ${BASE_IMAGE}`.quiet().nothrow() + if (pulled.exitCode === 0) break + if (attempt >= 3) { + console.error("cannot pull the pinned debian base image; check registry access") + process.exit(1) + } + console.log(`base image pull failed (attempt ${attempt}); retrying...`) +} + +await stageOfficeBundle() + +const assets: Record = {} +for (const arch of archs) { + const docker = arch === "x64" ? "amd64" : "arm64" + console.log(`\n=== ${arch} (linux/${docker}) ===`) + + console.log("building guest image...") + await $`docker buildx build --platform linux/${docker} --build-arg OFFICE=${office} -t codev-sandbox-guest:${arch} --load ${imageDir}` + console.log("building tools image...") + await $`docker buildx build --platform linux/${docker} -t codev-sandbox-tools:${arch} --load ${join(imageDir, "tools")}` + + console.log("exporting rootfs tarball...") + const cid = (await $`docker create --platform linux/${docker} codev-sandbox-guest:${arch}`.text()).trim() + const bwrapTar = join(outDir, `bwrap-${arch}.tar`) + await $`docker export ${cid} -o ${bwrapTar}` + await $`docker rm ${cid}`.quiet() + await $`docker run --rm -v ${outDir}:/out codev-sandbox-tools:${arch} gzip -f -6 /out/bwrap-${arch}.tar` + assets[`bwrap-${arch}`] = await describeAsset(join(outDir, `bwrap-${arch}.tar.gz`)) + + console.log("building ext4 disk...") + await $`docker run --rm -v ${outDir}:/out codev-sandbox-tools:${arch} sh -ec ${[ + `mkdir /r`, + `tar -xzf /out/bwrap-${arch}.tar.gz -C /r`, + `truncate -s ${DISK_SIZE} /out/rootfs-${arch}.img`, + `mkfs.ext4 -F -q -d /r /out/rootfs-${arch}.img`, + ].join(" && ")}` + + const kernel = join(outDir, `vmlinux-${arch}`) + if (!skipKernel) { + console.log("building kernel (cached after the first run)...") + await $`docker buildx build --platform linux/${docker} --target build -t codev-sandbox-kernel:${arch} --load ${join(imageDir, "kernel")}` + const kernelCid = (await $`docker create --platform linux/${docker} codev-sandbox-kernel:${arch} true`.text()).trim() + await $`docker cp ${kernelCid}:/out/vmlinux ${kernel}` + await $`docker rm ${kernelCid}`.quiet() + } + if (!(await stat(kernel).catch(() => null))) { + console.error(`missing ${kernel}; run without --skip-kernel`) + process.exit(1) + } + + console.log("assembling vz tarball...") + const vzStage = join(outDir, `vz-${arch}`) + await rm(vzStage, { recursive: true, force: true }) + await mkdir(vzStage, { recursive: true }) + await cp(kernel, join(vzStage, "vmlinux")) + await $`docker run --rm -v ${outDir}:/out codev-sandbox-tools:${arch} sh -ec ${[ + `mv /out/rootfs-${arch}.img /out/vz-${arch}/rootfs.img`, + `tar -Sczf /out/vz-${arch}.tar.gz -C /out/vz-${arch} vmlinux rootfs.img`, + ].join(" && ")}` + await rm(vzStage, { recursive: true, force: true }) + assets[`vz-${arch}`] = await describeAsset(join(outDir, `vz-${arch}.tar.gz`)) +} + +const manifest = parseSandboxManifest({ + schema: 1, + imageVersion: version, + images: assets, + wslInstaller: null, +}) +await writeFile(join(outDir, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`) + +console.log(`\nwrote ${join(outDir, "manifest.json")}`) +for (const [key, asset] of Object.entries(assets)) { + console.log(` ${key}: ${(asset.bytes / 1024 / 1024).toFixed(0)} MiB sha256 ${asset.sha256.slice(0, 12)}…`) +} +console.log(`publish with: bun script/publish-image.ts --version ${version} --out ${outDir}`) + +function argValue(flag: string) { + const index = process.argv.indexOf(flag) + return index >= 0 ? process.argv[index + 1] : undefined +} + +function collectArchs(): SandboxArch[] { + const flags = process.argv.flatMap((value, index) => (value === "--arch" ? [process.argv[index + 1]] : [])) + if (flags.length) return flags.filter((value): value is SandboxArch => value === "arm64" || value === "x64") + return [hostArch() === "arm64" ? "arm64" : "x64"] +} + +// Only the arch-independent parts of the office bundle enter the build +// context: skill zips and prebuilt pure-JS node-module trees. Binaries +// (dotnet, pandoc, wheels) are fetched per-arch inside the build instead. +async function stageOfficeBundle() { + const stage = join(imageDir, "office", "bundle") + await rm(stage, { recursive: true, force: true }) + await mkdir(stage, { recursive: true }) + await writeFile(join(stage, ".keep"), "") + if (!officeBundle) return + const scratch = join(outDir, "office-bundle-stage") + await rm(scratch, { recursive: true, force: true }) + await mkdir(scratch, { recursive: true }) + await $`unzip -q -o ${officeBundle} -d ${scratch}` + const dirs = await readdir(scratch) + const root = dirs.length === 1 ? join(scratch, dirs[0]!) : scratch + const wanted = (await readdir(root)).filter( + (name) => name.endsWith("-skill.zip") || name.endsWith("-node-modules.zip"), + ) + for (const name of wanted) await cp(join(root, name), join(stage, name)) + await rm(scratch, { recursive: true, force: true }) + console.log(`staged from office bundle: ${wanted.join(", ") || "(nothing matched)"}`) +} + +async function describeAsset(path: string): Promise { + const file = Bun.file(path) + const hash = createHash("sha256") + for await (const chunk of file.stream()) hash.update(chunk) + return { + url: `${SANDBOX_IMAGES_BASE_URL}/${version}/${path.split("/").pop()}`, + sha256: hash.digest("hex"), + bytes: file.size, + } +} diff --git a/packages/codev-sandbox/script/e2e-vz.ts b/packages/codev-sandbox/script/e2e-vz.ts new file mode 100644 index 0000000000..db155cf9a6 --- /dev/null +++ b/packages/codev-sandbox/script/e2e-vz.ts @@ -0,0 +1,88 @@ +#!/usr/bin/env bun +// Boots a real sandbox VM end-to-end on macOS: serves a locally built vz +// image over loopback HTTP, creates the sandbox (download → verify → unpack → +// disk clone), starts it through codev-vmm, and waits for the codev server's +// authenticated health check through the vsock bridge. Dev utility. +// +// Usage: +// bun script/e2e-vz.ts --image dist-images/vz-arm64.tar.gz \ +// --bundle ../opencode/dist/node [--project dir] [--hold] +import { createHash } from "node:crypto" +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { fileURLToPath } from "node:url" +import { checkSandboxHealth } from "../src/boot" +import type { SandboxConfig } from "../src/provider" +import { createVzProvider } from "../src/vz/provider" + +const image = requireArg("--image") +const bundle = requireArg("--bundle") +const hold = process.argv.includes("--hold") + +const work = await mkdtemp(join(tmpdir(), "codev-vz-e2e-")) +const projectDir = argValue("--project") ?? join(work, "project") +await mkdir(projectDir, { recursive: true }) +await writeFile(join(projectDir, "hello.txt"), "hello from the host project\n") + +const payload = Bun.file(image) +const hash = createHash("sha256") +for await (const chunk of payload.stream()) hash.update(chunk) +const server = Bun.serve({ port: 0, fetch: () => new Response(Bun.file(image)) }) + +const provider = createVzProvider({ + dataDir: join(work, "data"), + cacheDir: join(work, "cache"), + bundleDir: bundle, + helperPath: fileURLToPath(new URL("../native/vzhelper/dist/codev-vmm", import.meta.url)), + resolveImage: async () => ({ + url: `http://127.0.0.1:${server.port}/${image.split("/").pop()}`, + sha256: hash.digest("hex"), + bytes: payload.size, + }), + startTimeoutMs: 180_000, +}) + +console.log("availability:", await provider.availability()) +const config: SandboxConfig = { id: "sandbox:e2e", name: "e2e", backend: "vz", projectDir } + +console.time("create") +await provider.create(config) +console.timeEnd("create") + +console.time("start") +const sandbox = await provider.start(config) +console.timeEnd("start") +console.log(`server: ${sandbox.url} (user ${sandbox.username})`) + +const healthy = await checkSandboxHealth(sandbox.url, { username: sandbox.username ?? "", password: sandbox.password }) +console.log(`health: ${healthy ? "OK" : "FAILED"}`) + +if (hold) { + console.log("holding the VM up; Ctrl-C to stop") + await new Promise(() => {}) +} + +const exited = new Promise((resolve) => sandbox.onExit(() => resolve())) +sandbox.stop() +await Promise.race([exited, new Promise((resolve) => setTimeout(resolve, 10_000))]) +await provider.destroy(config) +server.stop(true) +await rm(work, { recursive: true, force: true }) +if (!healthy) process.exit(1) +console.log("e2e OK") +process.exit(0) + +function argValue(flag: string) { + const index = process.argv.indexOf(flag) + return index >= 0 ? process.argv[index + 1] : undefined +} + +function requireArg(flag: string) { + const value = argValue(flag) + if (!value) { + console.error(`${flag} is required`) + process.exit(1) + } + return value +} diff --git a/packages/codev-sandbox/script/publish-image.ts b/packages/codev-sandbox/script/publish-image.ts new file mode 100644 index 0000000000..c0b841b12e --- /dev/null +++ b/packages/codev-sandbox/script/publish-image.ts @@ -0,0 +1,56 @@ +#!/usr/bin/env bun +// Publishes built sandbox images to the codev-storage bucket. +// Usage: bun script/publish-image.ts --version 2026.08.1 [--out dist-images] +// +// Uses the MinIO client when CODEV_STORAGE_TARGET is set (an `mc` alias/bucket +// prefix, e.g. "codev/codev-storage"); otherwise prints the exact commands so +// the upload can run wherever the storage credentials live. The versioned +// artifacts upload first and manifest.json last — clients only see a version +// once every artifact it references is in place. +import { $ } from "bun" +import { readFile, stat } from "node:fs/promises" +import { join } from "node:path" +import { parseSandboxManifest } from "../src/manifest" + +const version = argValue("--version") +if (!version) { + console.error("--version is required") + process.exit(1) +} +const outDir = argValue("--out") ?? join(process.cwd(), "dist-images") + +const manifest = parseSandboxManifest(JSON.parse(await readFile(join(outDir, "manifest.json"), "utf8"))) +if (manifest.imageVersion !== version) { + console.error(`manifest.json is for ${manifest.imageVersion}, not ${version}; rebuild first`) + process.exit(1) +} +const files = Object.values(manifest.images).flatMap((asset) => (asset ? [asset.url.split("/").pop()!] : [])) +for (const name of files) { + const file = await stat(join(outDir, name)).catch(() => null) + if (!file?.isFile()) { + console.error(`missing artifact ${name} in ${outDir}`) + process.exit(1) + } +} + +const target = process.env.CODEV_STORAGE_TARGET +const uploads = [ + ...files.map((name) => [join(outDir, name), `codev-code/sandbox/${version}/${name}`] as const), + [join(outDir, "manifest.json"), "codev-code/sandbox/manifest.json"] as const, +] +if (!target) { + console.log("CODEV_STORAGE_TARGET is not set (mc alias/bucket, e.g. codev/codev-storage).") + console.log("Run these where the storage credentials live:\n") + for (const [local, remote] of uploads) console.log(` mc cp ${local} /${remote}`) + process.exit(0) +} +for (const [local, remote] of uploads) { + console.log(`uploading ${remote}...`) + await $`mc cp ${local} ${target}/${remote}` +} +console.log(`published sandbox image ${version}`) + +function argValue(flag: string) { + const index = process.argv.indexOf(flag) + return index >= 0 ? process.argv[index + 1] : undefined +} diff --git a/packages/codev-sandbox/src/bwrap/provider.ts b/packages/codev-sandbox/src/bwrap/provider.ts index d9010fab6f..166254d50c 100644 --- a/packages/codev-sandbox/src/bwrap/provider.ts +++ b/packages/codev-sandbox/src/bwrap/provider.ts @@ -99,6 +99,10 @@ export function createBwrapProvider(options: BwrapProviderOptions): SandboxProvi const credentials = generateSandboxCredentials() const bootScript = [ + // Fresh homes get the image's skel (skills config); images may also + // declare extra env (DOTNET_ROOT, NODE_PATH, ...) in codev-guest.env. + `cp -Rn /etc/skel/. "$HOME"/ 2>/dev/null || true`, + `[ -f /etc/codev-guest.env ] && . /etc/codev-guest.env`, `${guestNode} ${GUEST_BRIDGE}/forward.mjs unix-to-tcp ${GUEST_RUNTIME}/server.sock ${GUEST_SERVER_PORT} &`, `${guestNode} ${GUEST_BRIDGE}/forward.mjs tcp-to-unix ${GUEST_BROKER_PORT} ${GUEST_RUNTIME}/broker.sock &`, `exec ${guestNode} ${GUEST_BUNDLE}/node.js ${sandboxServeArgs({ hostname: "127.0.0.1", port: GUEST_SERVER_PORT }).join(" ")}`, diff --git a/packages/codev-sandbox/src/const.ts b/packages/codev-sandbox/src/const.ts index 7f4b9901b3..d2e1c724ac 100644 --- a/packages/codev-sandbox/src/const.ts +++ b/packages/codev-sandbox/src/const.ts @@ -3,7 +3,9 @@ // user-visible at download time. const STORAGE_BASE = atob("aHR0cHM6Ly9uZXRtaW5kLnZpZXR0ZWwudm4vY29kZXYtc3RvcmFnZQ==") -export const SANDBOX_MANIFEST_URL = `${STORAGE_BASE}/codev-code/sandbox/manifest.json` +export const SANDBOX_IMAGES_BASE_URL = `${STORAGE_BASE}/codev-code/sandbox` + +export const SANDBOX_MANIFEST_URL = `${SANDBOX_IMAGES_BASE_URL}/manifest.json` // WSL MSI installers mirrored for corporate networks where only the storage // host is whitelisted; the Microsoft `wsl --install` path stays as fallback. diff --git a/packages/codev-sandbox/src/vz/provider.test.ts b/packages/codev-sandbox/src/vz/provider.test.ts index 8a0b242df7..e674e00134 100644 --- a/packages/codev-sandbox/src/vz/provider.test.ts +++ b/packages/codev-sandbox/src/vz/provider.test.ts @@ -239,7 +239,7 @@ test("create rejects an image tarball missing required files", async () => { bytes: payload.byteLength, }), }) - expect(provider.create(config)).rejects.toThrow("vz image is missing initrd, rootfs.img") + expect(provider.create(config)).rejects.toThrow("vz image is missing rootfs.img") await Bun.sleep(20) server.stop(true) await rm(dir, { recursive: true, force: true }) diff --git a/packages/codev-sandbox/src/vz/provider.ts b/packages/codev-sandbox/src/vz/provider.ts index eb9fc5eda3..135fbd5712 100644 --- a/packages/codev-sandbox/src/vz/provider.ts +++ b/packages/codev-sandbox/src/vz/provider.ts @@ -27,7 +27,7 @@ export const VZ_DISK_FILE = "rootfs.img" export const VZ_SERVER_VSOCK_PORT = 4096 export const VZ_BROKER_VSOCK_PORT = 8788 export const VZ_GUEST_HOME = "/home/codev" -const VZ_CMDLINE = "console=hvc0 root=/dev/vda rw" +const VZ_CMDLINE = "console=hvc0 root=/dev/vda rw init=/sbin/codev-init" export type VzProviderOptions = { dataDir: string @@ -70,8 +70,10 @@ export function createVzProvider(options: VzProviderOptions): SandboxProvider { const image = await stageAsset(options.cacheDir, asset, { fetchImpl, signal: opts?.signal }) const dirs = vzSandboxDirs(options.dataDir, config.name) await unpackRootfs(image, dirs.image, opts) + // initrd is optional: the pipeline's custom kernel has virtio + ext4 + // compiled in and boots without one. const missing = [] - for (const file of [VZ_KERNEL_FILE, VZ_INITRD_FILE, VZ_DISK_FILE]) { + for (const file of [VZ_KERNEL_FILE, VZ_DISK_FILE]) { const found = await stat(join(dirs.image, file)).catch(() => null) if (!found?.isFile()) missing.push(file) } @@ -120,9 +122,10 @@ export function createVzProvider(options: VzProviderOptions): SandboxProvider { }), ) const serverSocket = join(dirs.runtime, "server.sock") + const initrd = await stat(join(dirs.image, VZ_INITRD_FILE)).catch(() => null) const vmConfig: VzVmConfig = { kernel: join(dirs.image, VZ_KERNEL_FILE), - initrd: join(dirs.image, VZ_INITRD_FILE), + initrd: initrd?.isFile() ? join(dirs.image, VZ_INITRD_FILE) : "", disk: dirs.disk, cmdline: VZ_CMDLINE, cpus: options.cpus ?? 4, From 95d403ff80e1c35d614b84c32c1a36a5c78d1cfd Mon Sep 17 00:00:00 2001 From: minhn4 Date: Wed, 12 Aug 2026 10:08:10 +0700 Subject: [PATCH 2/2] fix(sandbox): guard partial image publishes and retry registry-facing builds manifest.json is deliberately unversioned because it is the pointer every client reads, so publishing replaces it wholesale. Nothing stopped a single-arch or dev-versioned build from retiring the working manifest for the platforms it omits; publish-image.ts now refuses both unless --force. build-image.ts prepulls the digest-pinned base, but buildkit still resolves the FROM remotely and a real run died on "load metadata ... DeadlineExceeded" even with the image local. Route the pull and the three buildx builds through one bounded retry; buildkit keeps completed layers, so a retry resumes. Also ignore the office bundle staging dir contents, which are build inputs. Co-Authored-By: Claude Opus 5 (1M context) --- .../image/office/bundle/.gitignore | 5 ++ packages/codev-sandbox/script/build-image.ts | 48 +++++++++++++------ .../codev-sandbox/script/publish-image.ts | 22 ++++++++- 3 files changed, 60 insertions(+), 15 deletions(-) create mode 100644 packages/codev-sandbox/image/office/bundle/.gitignore diff --git a/packages/codev-sandbox/image/office/bundle/.gitignore b/packages/codev-sandbox/image/office/bundle/.gitignore new file mode 100644 index 0000000000..d96415764a --- /dev/null +++ b/packages/codev-sandbox/image/office/bundle/.gitignore @@ -0,0 +1,5 @@ +# Staging dir: build-image.ts copies the office bundle's skill and node-module +# zips here per run. They are build inputs, not sources. +* +!.gitignore +!.keep diff --git a/packages/codev-sandbox/script/build-image.ts b/packages/codev-sandbox/script/build-image.ts index 7583b56ef8..d7b7b76f8a 100644 --- a/packages/codev-sandbox/script/build-image.ts +++ b/packages/codev-sandbox/script/build-image.ts @@ -43,18 +43,12 @@ const outDir = argValue("--out") ?? join(process.cwd(), "dist-images") await mkdir(outDir, { recursive: true }) // Slow corporate registries can exceed buildkit's metadata deadline; the -// FROMs are digest-pinned and this prepull (with retries) primes the local -// store so buildx never has to resolve remotely. +// FROMs are digest-pinned and this prepull primes the local store. It reduces +// but does not remove remote resolution: buildkit still resolves the FROM and +// can fail with "load metadata ... DeadlineExceeded" even once the image is +// local, so every registry-facing step goes through retry() below. const BASE_IMAGE = "debian:bookworm-slim@sha256:abd67ffcfa541b485a3dff59865ab629aa048a6c613e639d36e7456b0b229241" -for (let attempt = 1; ; attempt++) { - const pulled = await $`docker pull ${BASE_IMAGE}`.quiet().nothrow() - if (pulled.exitCode === 0) break - if (attempt >= 3) { - console.error("cannot pull the pinned debian base image; check registry access") - process.exit(1) - } - console.log(`base image pull failed (attempt ${attempt}); retrying...`) -} +await retry("base image pull", () => $`docker pull ${BASE_IMAGE}`.quiet().nothrow()) await stageOfficeBundle() @@ -64,9 +58,16 @@ for (const arch of archs) { console.log(`\n=== ${arch} (linux/${docker}) ===`) console.log("building guest image...") - await $`docker buildx build --platform linux/${docker} --build-arg OFFICE=${office} -t codev-sandbox-guest:${arch} --load ${imageDir}` + await retry( + "guest image build", + () => + $`docker buildx build --platform linux/${docker} --build-arg OFFICE=${office} -t codev-sandbox-guest:${arch} --load ${imageDir}`.nothrow(), + ) console.log("building tools image...") - await $`docker buildx build --platform linux/${docker} -t codev-sandbox-tools:${arch} --load ${join(imageDir, "tools")}` + await retry( + "tools image build", + () => $`docker buildx build --platform linux/${docker} -t codev-sandbox-tools:${arch} --load ${join(imageDir, "tools")}`.nothrow(), + ) console.log("exporting rootfs tarball...") const cid = (await $`docker create --platform linux/${docker} codev-sandbox-guest:${arch}`.text()).trim() @@ -87,7 +88,11 @@ for (const arch of archs) { const kernel = join(outDir, `vmlinux-${arch}`) if (!skipKernel) { console.log("building kernel (cached after the first run)...") - await $`docker buildx build --platform linux/${docker} --target build -t codev-sandbox-kernel:${arch} --load ${join(imageDir, "kernel")}` + await retry( + "kernel build", + () => + $`docker buildx build --platform linux/${docker} --target build -t codev-sandbox-kernel:${arch} --load ${join(imageDir, "kernel")}`.nothrow(), + ) const kernelCid = (await $`docker create --platform linux/${docker} codev-sandbox-kernel:${arch} true`.text()).trim() await $`docker cp ${kernelCid}:/out/vmlinux ${kernel}` await $`docker rm ${kernelCid}`.quiet() @@ -124,6 +129,21 @@ for (const [key, asset] of Object.entries(assets)) { } console.log(`publish with: bun script/publish-image.ts --version ${version} --out ${outDir}`) +// Bounded retry for the registry-facing steps. Buildkit's metadata resolve is +// the flaky one on a slow link; buildx keeps completed layers cached, so a +// retry resumes the build instead of restarting it. +async function retry(label: string, run: () => Promise<{ exitCode: number }>) { + for (let attempt = 1; ; attempt++) { + const result = await run() + if (result.exitCode === 0) return + if (attempt >= 3) { + console.error(`${label} failed after ${attempt} attempts; check registry access`) + process.exit(1) + } + console.log(`${label} failed (attempt ${attempt}); retrying...`) + } +} + function argValue(flag: string) { const index = process.argv.indexOf(flag) return index >= 0 ? process.argv[index + 1] : undefined diff --git a/packages/codev-sandbox/script/publish-image.ts b/packages/codev-sandbox/script/publish-image.ts index c0b841b12e..e84e3e111f 100644 --- a/packages/codev-sandbox/script/publish-image.ts +++ b/packages/codev-sandbox/script/publish-image.ts @@ -1,6 +1,7 @@ #!/usr/bin/env bun // Publishes built sandbox images to the codev-storage bucket. // Usage: bun script/publish-image.ts --version 2026.08.1 [--out dist-images] +// [--force] // // Uses the MinIO client when CODEV_STORAGE_TARGET is set (an `mc` alias/bucket // prefix, e.g. "codev/codev-storage"); otherwise prints the exact commands so @@ -10,7 +11,7 @@ import { $ } from "bun" import { readFile, stat } from "node:fs/promises" import { join } from "node:path" -import { parseSandboxManifest } from "../src/manifest" +import { manifestImageKey, parseSandboxManifest } from "../src/manifest" const version = argValue("--version") if (!version) { @@ -24,6 +25,25 @@ if (manifest.imageVersion !== version) { console.error(`manifest.json is for ${manifest.imageVersion}, not ${version}; rebuild first`) process.exit(1) } +// manifest.json is deliberately unversioned: it is the pointer every client +// reads (SANDBOX_MANIFEST_URL), and publishing replaces it wholesale. A +// partial or dev-versioned upload therefore does not just add a version, it +// retires the working one for whichever platform it omits. +const force = process.argv.includes("--force") +const missing = (["bwrap", "vz"] as const) + .flatMap((backend) => (["arm64", "x64"] as const).map((arch) => manifestImageKey(backend, arch))) + .filter((key) => !manifest.images[key]) +if (missing.length && !force) { + console.error(`manifest.json has no ${missing.join(", ")}; those platforms would fail on every sandbox launch`) + console.error("build the missing arch first, or pass --force to publish a partial manifest") + process.exit(1) +} +if (!/^\d+\.\d+\.\d+$/.test(version) && !force) { + console.error(`refusing to publish ${version}: it is not a release version, and every client would be pointed at it`) + console.error("use a release version like 2026.08.1, or pass --force") + process.exit(1) +} + const files = Object.values(manifest.images).flatMap((asset) => (asset ? [asset.url.split("/").pop()!] : [])) for (const name of files) { const file = await stat(join(outDir, name)).catch(() => null)