Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions crates/openshell-core/src/driver_mounts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,21 @@ pub enum SelinuxLabel {
/// directory and by drivers whose workspace remains fixed.
pub const DEFAULT_WORKSPACE_ROOT: &str = "/sandbox";

/// Paths reserved by VM guest initialization.
pub const VM_RESERVED_GUEST_PATHS: &[&str] = &[
"/.openshell",
"/.openshell-bootstrap",
"/overlay",
"/lower",
"/newroot",
"/image-cache",
"/srv",
"/proc",
"/sys",
"/dev",
"/tmp",
];

/// Validate a non-empty driver mount source.
pub fn validate_mount_source(source: &str, field: &str) -> Result<(), String> {
if source.is_empty() {
Expand Down Expand Up @@ -120,6 +135,38 @@ pub fn resolve_oci_workspace_root(working_dir: &str) -> Result<String, String> {
Ok(workspace_root)
}

/// Resolve a VM workload's image-declared working directory.
pub fn resolve_vm_image_workspace_root(working_dir: &str) -> Result<String, String> {
let root = resolve_oci_workspace_root(working_dir)?;
for reserved in VM_RESERVED_GUEST_PATHS.iter().copied().chain(["/run"]) {
if paths_overlap(Path::new(&root), Path::new(reserved)) {
return Err(format!(
"VM workspace '{root}' conflicts with guest path '{reserved}'"
));
}
}
Ok(root)
}

/// Resolve a VM workspace that guest init will assign to the sandbox user.
pub fn resolve_vm_managed_workspace_root(root: &str) -> Result<String, String> {
if root.is_empty() || root == "/" || root.contains(':') {
return Err("VM workspace_root must be a non-root absolute path without ':'".to_string());
}
let root = resolve_vm_image_workspace_root(root)?;
if matches!(root.as_str(), "/home" | "/var")
|| [
"/etc", "/root", "/boot", "/bin", "/sbin", "/lib", "/lib32", "/lib64", "/libx32",
"/usr",
]
.iter()
.any(|reserved| Path::new(&root).starts_with(reserved))
{
return Err(format!("VM workspace '{root}' is a protected system path"));
}
Ok(root)
}

fn normalize_absolute_container_path(value: &str, field: &str) -> Result<String, String> {
if value.is_empty() {
return Err(format!("{field} must not be empty"));
Expand Down
8 changes: 6 additions & 2 deletions crates/openshell-driver-vm/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,8 @@ Select the VM driver with `--compute-driver vm`, `OPENSHELL_COMPUTE_DRIVER=vm`,
| `vcpus` | `2` | vCPUs per sandbox. |
| `mem_mib` | `2048` | Memory per sandbox, in MiB. |
| `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. |
| `workspace_mode` | `"managed"` | `managed` prepares the configured guest directory for the sandbox user. `image` uses the image's OCI `WORKDIR`, falling back to the managed `/sandbox` for missing, empty, `/`, or `/sandbox` values. Rootfs tar archives have no OCI `WORKDIR`. |
| `workspace_root` | `"/sandbox"` | Absolute path in the guest used by `managed` mode. |
| `krun_log_level` | `1` | libkrun verbosity (0-5). |
| `guest_tls_ca` | unset | Historical key name for the host supervisor's gateway CA certificate. Required when `grpc_endpoint` uses `https://`; never copied into the guest. |
| `https_proxy` | unset | Corporate forward proxy (`http://host:port` or `https://host:port`) that host control chains policy-approved TLS CONNECT egress through. Host-loopback proxy URLs work because control runs on the gateway host. |
Expand All @@ -182,10 +184,12 @@ Select the VM driver with `--compute-driver vm`, `OPENSHELL_COMPUTE_DRIVER=vm`,
| `proxy_connect_by_hostname` | unset | Send hostnames rather than validated IPs in CONNECT. Last resort for proxies whose ACLs reject IP CONNECT targets. |
| `proxy_ca_bundle` | unset | Gateway-host PEM CA bundle trusted for the corporate proxy and TLS-intercepted server certificates. The driver validates it and stages it at a fixed non-secret guest path in the protected overlay. Requires `https_proxy`. |
| `provider_spiffe_workload_api_tcp_endpoint` | unset | Explicit guest-reachable `tcp:IP:port` SPIFFE Workload API listener for provider token exchange. It requires `provider_spiffe_allow_guest_tcp = true`; a host UNIX socket is never silently exposed to a VM guest. |
| `enable_bind_mounts` | `false` | Allow per-sandbox `driver_config.vm.mounts` host directory shares (libkrun virtiofs only). The driver writes a `tag\ttarget\tmode` manifest to `/.openshell/mounts.manifest` in the overlay, and guest init mounts each share after `/sandbox` ownership fixups. Rejected while resource admission is enabled. |
| `enable_bind_mounts` | `false` | Allow per-sandbox `driver_config.vm.mounts` host directory shares (libkrun virtiofs only). The driver writes a `tag\ttarget\tmode` manifest to `/.openshell/mounts.manifest` in the overlay, and guest init mounts each share after workspace setup. Rejected while resource admission is enabled. |

The proxy settings are operator-owned and deployment-level: they are not accepted through `template.driver_config.vm`, and the driver passes them only to native host control. Every present-but-invalid value is fatal at gateway or sandbox startup rather than degrading to a direct dial.

In `managed` mode, OpenShell assigns ownership of a custom workspace directory to the sandbox user and preserves existing files and their ownership. In `image` mode, the directory must already exist without symlink components and be writable by the resolved sandbox user. Existing VM image preparation still normalizes the compatibility `/sandbox` tree. The driver records the selected workspace at creation and reuses it with the sandbox's writable overlay across restarts. It also retains the prepared workload image for a custom workspace or `image` mode.

See [`openshell-gateway --help`](../openshell-server/src/cli.rs) for the gateway process flag surface.

## Verifying the gateway
Expand Down Expand Up @@ -242,7 +246,7 @@ At startup, the driver reclaims inactive attempts in this directory. It preserve
Each sandbox gets its own sparse writable
`<state-dir>/sandboxes/<id>/overlay.ext4`. Guest init mounts overlayfs as `/`
with the prepared image rootfs as lowerdir when present, otherwise the bootstrap
rootfs is used directly. Writes to `/sandbox` and other mutable paths land in
rootfs is used directly. Writes to the workspace and other mutable paths land in
the overlay while cached image disks remain unchanged. The overlay disk must be
large enough to hold the compressed payload, unpacked rootfs, and sandbox writes
during the first prepare.
Expand Down
103 changes: 82 additions & 21 deletions crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,42 @@ root_path() {
printf '%s%s\n' "${ROOT_PREFIX:-}" "$path"
}

load_sandbox_workspace() {
# Read driver-owned overlay metadata directly. The workload image cannot
# supply workspace configuration through the merged root filesystem.
local config=/overlay/config/openshell-workspace extra
WORKSPACE_MODE=managed
WORKSPACE_ROOT=/sandbox
[ -e "$config" ] || return 0

if ! {
IFS= read -r WORKSPACE_MODE &&
IFS= read -r WORKSPACE_ROOT &&
! IFS= read -r extra &&
[ -z "$extra" ]
} < "$config"; then
ts "FATAL: invalid sandbox workspace configuration"
exit 1
fi
case "$WORKSPACE_MODE" in
managed|image) ;;
*)
ts "FATAL: invalid sandbox workspace mode"
exit 1
;;
esac
case "$WORKSPACE_ROOT/" in
/|*//*|*/./*|*/../*|[!/]*)
ts "FATAL: invalid sandbox workspace path"
exit 1
;;
esac
if [ "$WORKSPACE_MODE" = managed ] && [[ "$WORKSPACE_ROOT" == *:* ]]; then
ts "FATAL: managed workspace path contains a passwd field separator"
exit 1
fi
}

sandbox_owner() {
sandbox_owner_from_passwd "$(root_path /etc/passwd)"
}
Expand Down Expand Up @@ -377,7 +413,7 @@ setup_gpu() {
reconcile_sandbox_account() {
local sandbox_uid="${OPENSHELL_VM_SANDBOX_UID:-}"
local sandbox_gid="${OPENSHELL_VM_SANDBOX_GID:-}"
local etc
local etc home=""

[ -n "$sandbox_uid" ] && [ -n "$sandbox_gid" ] || return 0
[[ "$sandbox_uid" =~ ^[0-9]+$ ]] && [[ "$sandbox_gid" =~ ^[0-9]+$ ]] || {
Expand All @@ -399,19 +435,28 @@ reconcile_sandbox_account() {
else
printf 'sandbox:x:%s:\n' "$sandbox_gid" >> "$etc/group"
fi
if [ "$WORKSPACE_MODE" = managed ] && [ "$WORKSPACE_ROOT" != /sandbox ]; then
home="$WORKSPACE_ROOT"
fi
if grep -q '^sandbox:' "$etc/passwd"; then
# Preserve image-owned account metadata (home, shell, and description)
# while restoring the UID/GID contract recorded for this overlay.
if ! awk -F: -v OFS=: -v uid="$sandbox_uid" -v gid="$sandbox_gid" \
'$1 == "sandbox" { $3 = uid; $4 = gid } { print }' \
# Update the account home only for an explicitly managed custom path.
if ! OPENSHELL_VM_ACCOUNT_HOME="$home" \
awk -F: -v OFS=: -v uid="$sandbox_uid" -v gid="$sandbox_gid" \
'$1 == "sandbox" {
$3 = uid
$4 = gid
home = ENVIRON["OPENSHELL_VM_ACCOUNT_HOME"]
if (home != "") $6 = home
} { print }' \
"$etc/passwd" >"$etc/passwd.openshell"; then
rm -f "$etc/passwd.openshell"
ts "FATAL: failed to reconcile sandbox account"
exit 1
fi
mv "$etc/passwd.openshell" "$etc/passwd"
else
printf 'sandbox:x:%s:%s:OpenShell Sandbox:/sandbox:/bin/sh\n' "$sandbox_uid" "$sandbox_gid" >> "$etc/passwd"
printf 'sandbox:x:%s:%s:OpenShell Sandbox:%s:/bin/sh\n' \
"$sandbox_uid" "$sandbox_gid" "${home:-/sandbox}" >> "$etc/passwd"
fi
grep -q '^sandbox:' "$etc/gshadow" || printf 'sandbox:!::\n' >> "$etc/gshadow"
grep -q '^sandbox:' "$etc/shadow" || printf 'sandbox:!:20123:0:99999:7:::\n' >> "$etc/shadow"
Expand All @@ -422,8 +467,24 @@ setup_sandbox_workdir() {
local sandbox_dir
local owner
local current_owner
sandbox_dir="$(root_path /sandbox)"
if [ "$WORKSPACE_ROOT" != /sandbox ] && [ "$WORKSPACE_MODE" = image ]; then
# The boundary checks the image directory as the actual workload user.
return
fi
owner="$(sandbox_owner)"
if [ "$WORKSPACE_ROOT" != /sandbox ]; then
sandbox_dir="$(
umask 022
prepare_guest_directory "$WORKSPACE_ROOT" "$owner" "$WORKSPACE_ROOT"
)" || exit 1
if ! chown "$owner" "$sandbox_dir" || ! chmod u+rwx "$sandbox_dir"; then
ts "FATAL: failed to prepare workspace ${WORKSPACE_ROOT} (${owner})"
exit 1
fi
return
fi

sandbox_dir="$(root_path /sandbox)"
mkdir -p "$sandbox_dir"
current_owner="$(stat -c '%u:%g' "$sandbox_dir" 2>/dev/null || true)"
if [ "$owner" = "10001:10001" ]; then
Expand Down Expand Up @@ -561,43 +622,42 @@ run_openshell_init_dropins() {
done < <(LC_ALL=C sort -u "$manifest")
}

# Create a virtiofs mount point under ROOT_PREFIX without following symlinks,
# so an image cannot redirect a validated target onto another guest path.
# Directories created under /sandbox are handed to the sandbox user.
prepare_virtiofs_target() {
local target="$1" owner="$2"
# Walk a guest directory path without following image symlinks. New
# directories below the workspace belong to the sandbox account.
prepare_guest_directory() {
local target="$1" owner="$2" workspace_root="$3"
local path="${ROOT_PREFIX:-}" component
local -a components
IFS=/ read -r -a components <<<"${target#/}"
for component in "${components[@]}"; do
[ -n "$component" ] || continue
path="${path}/${component}"
if [ -L "$path" ]; then
ts >&2 "FATAL: virtiofs mount target ${target} traverses symlink ${path#"${ROOT_PREFIX:-}"}"
ts >&2 "FATAL: guest path ${target} traverses symlink ${path#"${ROOT_PREFIX:-}"}"
exit 1
fi
if [ ! -e "$path" ]; then
if ! mkdir "$path"; then
ts >&2 "FATAL: failed to create virtiofs mount point ${target}"
ts >&2 "FATAL: failed to create guest directory ${target}"
exit 1
fi
case "${path#"${ROOT_PREFIX:-}"}" in
/sandbox/*)
"$workspace_root"/*)
if ! chown "$owner" "$path"; then
ts >&2 "FATAL: failed to hand virtiofs mount point parent ${path#"${ROOT_PREFIX:-}"} to ${owner}"
ts >&2 "FATAL: failed to assign guest directory ${path#"${ROOT_PREFIX:-}"} to ${owner}"
exit 1
fi
;;
esac
elif [ ! -d "$path" ]; then
ts >&2 "FATAL: virtiofs mount target ${target} is not a directory"
ts >&2 "FATAL: guest path ${target} is not a directory"
exit 1
fi
done
printf '%s\n' "$path"
}

# Runs after every /sandbox ownership fixup and the root-run init drop-ins so
# Runs after workspace ownership setup and the root-run init drop-ins so
# nothing in init walks into host-backed shares.
mount_virtiofs_shares() {
local manifest
Expand All @@ -618,7 +678,7 @@ mount_virtiofs_shares() {
exit 1
;;
esac
guest_target="$(prepare_virtiofs_target "$target" "$owner")" || exit 1
guest_target="$(prepare_guest_directory "$target" "$owner" "$WORKSPACE_ROOT")" || exit 1
# shellcheck disable=SC2086
if mount -t virtiofs $mount_opts "$tag" "$guest_target"; then
ts " mounted virtiofs ${tag} -> ${target} (${mode})"
Expand Down Expand Up @@ -654,6 +714,7 @@ run_post_overlay_setup() {
mount -t cgroup2 cgroup2 "$(root_path /sys/fs/cgroup)" 2>/dev/null &
wait

load_sandbox_workspace
reconcile_sandbox_account
setup_sandbox_workdir

Expand All @@ -678,7 +739,7 @@ EOF
# The boundary transport mediates network and DNS requests. Only loopback is
# configured in the guest; no public resolver or guest NIC is needed.

export HOME=/sandbox
export HOME="$WORKSPACE_ROOT"
export USER=sandbox

# Fix /sandbox ownership. The host-side CLI extracts OCI layers as a non-root
Expand All @@ -688,7 +749,7 @@ export USER=sandbox
# chown succeeds. On Linux non-root hosts with virtiofs, guest root maps to
# the host user, so chown is denied — this is non-fatal because the
# supervisor's own filesystem preparation handles the paths that matter.
if [ -d /sandbox ]; then
if [ "$WORKSPACE_ROOT" = /sandbox ] && [ -d /sandbox ]; then
_sb_uid=$(id -u sandbox 2>/dev/null || true)
_sb_gid=$(id -g sandbox 2>/dev/null || true)
if [ -n "$_sb_uid" ] && [ -n "$_sb_gid" ]; then
Expand Down
Loading
Loading