Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 32 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,19 @@ URUNIT_SRC := ${SOURCE_DIR}/urunit.c ${SOURCE_DIR}/main.c ${SOURCE_DIR}/mode.c \
# Header files
URUNIT_HDR := $(wildcard ${SOURCE_DIR}/*.h)

#? TELEM_CAPTURE a prebuilt in-guest telemetry producer (vmi's telem-capture,
# static, for the target architecture) to embed in urunit. The early init then
# starts it from memory before the workload, when the kernel offers /dev/telem,
# so an introspecting initrd is one file. Unset, the init starts
# /vmi-telem-capture from the initrd when that exists, and nothing otherwise.
TELEM_CAPTURE ?=
ifneq ($(TELEM_CAPTURE),)
TELEM_CAPTURE_BLOB := $(BUILD_DIR)/telem_capture_blob.o
URUNIT_OBJ := $(TELEM_CAPTURE_BLOB)
else
URUNIT_OBJ :=
endif

# Unit tests: every source except urunit.c, plus src/tests. Linux only.
TEST_SRC := $(wildcard ${SOURCE_DIR}/tests/*.c)
TEST_HDR := $(wildcard ${SOURCE_DIR}/tests/*.h)
Expand Down Expand Up @@ -140,17 +153,28 @@ ifneq ($(TARGET_OS),Linux)
endif
$(CC) $(CFLAGS) -I${SOURCE_DIR} $(filter-out ${SOURCE_DIR}/urunit.c,$(URUNIT_SRC)) $(TEST_SRC) -o $@

$(URUNIT_BIN_DYNAMIC): $(URUNIT_SRC) $(URUNIT_HDR) | prepare
$(CC) $(CFLAGS) $(URUNIT_SRC) $(LDFLAGS) -o $@
# The embedded producer: the file's bytes in .rodata, between two symbols
# init.c declares weak, so a build without TELEM_CAPTURE leaves them null and
# the init falls back to the file in the initrd. .incbin keeps the bytes as they
# are; --noexecstack says what the assembler cannot know about them.
ifneq ($(TELEM_CAPTURE),)
$(TELEM_CAPTURE_BLOB): $(TELEM_CAPTURE) | prepare
printf '\t.section .rodata\n\t.balign 16\n\t.globl urunit_telem_capture_blob\nurunit_telem_capture_blob:\n\t.incbin "%s"\n\t.globl urunit_telem_capture_blob_end\nurunit_telem_capture_blob_end:\n' \
"$(TELEM_CAPTURE)" > $(BUILD_DIR)/telem_capture_blob.S
$(CC) -c -Wa,--noexecstack $(BUILD_DIR)/telem_capture_blob.S -o $@
endif

$(URUNIT_BIN_DYNAMIC): $(URUNIT_SRC) $(URUNIT_HDR) $(URUNIT_OBJ) | prepare
$(CC) $(CFLAGS) $(URUNIT_SRC) $(URUNIT_OBJ) $(LDFLAGS) -o $@

$(URUNIT_BIN_DYNAMIC_DEBUG): $(URUNIT_SRC) $(URUNIT_HDR) | prepare
$(CC) $(CFLAGS) -DDEBUG $(URUNIT_SRC) $(LDFLAGS) -o $@
$(URUNIT_BIN_DYNAMIC_DEBUG): $(URUNIT_SRC) $(URUNIT_HDR) $(URUNIT_OBJ) | prepare
$(CC) $(CFLAGS) -DDEBUG $(URUNIT_SRC) $(URUNIT_OBJ) $(LDFLAGS) -o $@

$(URUNIT_BIN_STATIC): $(URUNIT_SRC) $(URUNIT_HDR) | prepare
$(CC) $(CFLAGS) $(URUNIT_SRC) $(LDFLAGS) $(STATIC_LDFLAGS) -o $@
$(URUNIT_BIN_STATIC): $(URUNIT_SRC) $(URUNIT_HDR) $(URUNIT_OBJ) | prepare
$(CC) $(CFLAGS) $(URUNIT_SRC) $(URUNIT_OBJ) $(LDFLAGS) $(STATIC_LDFLAGS) -o $@

$(URUNIT_BIN_STATIC_DEBUG): $(URUNIT_SRC) $(URUNIT_HDR) | prepare
$(CC) $(CFLAGS) -DDEBUG $(URUNIT_SRC) $(LDFLAGS) $(STATIC_LDFLAGS) -o $@
$(URUNIT_BIN_STATIC_DEBUG): $(URUNIT_SRC) $(URUNIT_HDR) $(URUNIT_OBJ) | prepare
$(CC) $(CFLAGS) -DDEBUG $(URUNIT_SRC) $(URUNIT_OBJ) $(LDFLAGS) $(STATIC_LDFLAGS) -o $@

## install Install urunit in PREFIX
.PHONY: install
Expand Down
19 changes: 19 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,25 @@ virtio-blk serial, and every failure ends the boot.
A container boot initrd needs only `/urunit`, with `/init` and `/vz-init` as
symlinks to it. On FreeBSD `urunit` is always the supervisor.

### In-guest telemetry producer

An introspecting guest runs a telemetry producer (vmi's `telem-capture`) from
boot, which fills the export ring the kernel offers at `/dev/telem`. Both
early-init roles start it before the root is mounted, when that device exists,
from one of two places:

- the bytes embedded in `urunit` by `make static TELEM_CAPTURE=<path>`, a
prebuilt static producer for the target architecture, run from a `memfd`;
the initrd then stays one file;
- else `/vmi-telem-capture` in the initrd.

The producer is a child of PID 1, reaped like every other. It is given
`--ready-fd` on a pipe and the init waits up to three seconds for its readiness
byte, so the workload starts under capture; a producer that exits first or
stalls is logged and the boot goes on without the rich channel. A kernel
without `/dev/telem` boots as before, so the same initrd serves a stock and an
introspecting kernel.

## Building

Building `urunit` is as simple as running `make`, but make sure that `make` and
Expand Down
159 changes: 159 additions & 0 deletions src/init.c
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,15 @@
#define _GNU_SOURCE

#include <errno.h>
#include <fcntl.h>
#include <poll.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/mount.h>
#include <unistd.h>

#include "sysops.h"
#include "urunit.h"
Expand Down Expand Up @@ -1197,6 +1201,158 @@ static int init_layers(const struct init_ctx *c, int overlay, char **root,
return rc;
}

// The in-guest telemetry producer (vmi's telem-capture), started before the
// workload when the kernel offers the export ring at /dev/telem. It is the
// source of every rich event the sidecar sees; without it the host observes
// the boundary alone. It comes from one of two places: the bytes a build with
// TELEM_CAPTURE embedded in this binary (urunit_telem_capture_blob, weak, so a
// build without them leaves the symbols null), or /vmi-telem-capture in the
// initrd. Either way it runs from a memfd or the initrd file through fexecve,
// as a child of PID 1 that the reaping loop collects, while this initramfs is
// still the root: switch_root deletes the file, the running process survives.
//
// It reads the kernel's BTF from /sys and the tracepoint layouts from tracefs,
// so tracefs is mounted first. It reports readiness with one byte on the pipe
// it is given (--ready-fd); the init waits for that byte, bounded, so the
// workload starts under capture, and says so when the producer died or stalled
// instead. A kernel without /dev/telem boots on without any of this, so one
// initrd serves the stock and the introspecting kernel.
extern const unsigned char urunit_telem_capture_blob[] __attribute__((weak));
extern const unsigned char urunit_telem_capture_blob_end[] __attribute__((weak));

#define INIT_CAPTURE_FILE "/vmi-telem-capture"
#define INIT_CAPTURE_DEV "/dev/telem"
#define INIT_TRACEFS "/sys/kernel/tracing"
// How long the init waits for the producer's readiness. Attaching takes some
// tens of milliseconds; a second is already a sign of trouble, and a boot
// should not hang on a capture that never comes.
#define INIT_CAPTURE_READY_MS 3000

int init_capture_wait_ready(int fd, int timeout_ms)
{
struct pollfd pfd = { .fd = fd, .events = POLLIN };
unsigned char byte;
ssize_t n;
int rc;

do {
rc = poll(&pfd, 1, timeout_ms);
} while (rc < 0 && errno == EINTR);
if (rc <= 0)
return -1;
do {
n = read(fd, &byte, 1);
} while (n < 0 && errno == EINTR);
if (n < 0)
return -1;
return n == 1;
}

// Returns a descriptor to fexecve the producer from, or -1 with errno set
// when there is none: a memfd holding the embedded bytes, else the initrd
// file. *embedded says which.
static int init_capture_fd(const struct init_ctx *c, int *embedded)
{
const unsigned char *p = urunit_telem_capture_blob;
size_t left;
int fd;

*embedded = p != NULL && urunit_telem_capture_blob_end > p;
if (!*embedded) {
if (c->ops->access_x(INIT_CAPTURE_FILE) != 0)
return -1;
return open(INIT_CAPTURE_FILE, O_RDONLY | O_CLOEXEC);
}
left = (size_t)(urunit_telem_capture_blob_end - p);
fd = memfd_create("vmi-telem-capture", MFD_CLOEXEC);
if (fd < 0)
return -1;
while (left > 0) {
ssize_t n = write(fd, p, left);

if (n < 0 && errno == EINTR)
continue;
if (n <= 0) {
close(fd);
return -1;
}
p += n;
left -= (size_t)n;
}
return fd;
}

static void init_start_capture(const struct init_ctx *c)
{
char ready_arg[16];
char *argv[] = { "vmi-telem-capture", "--ready-fd", ready_arg, NULL };
int ready[2];
int embedded;
int fd;
int kmsg;
pid_t pid;

if (c->ops->exists(INIT_CAPTURE_DEV) != 0)
return;
fd = init_capture_fd(c, &embedded);
if (fd < 0) {
if (errno != ENOENT)
init_log(c, "could not stage the telemetry producer: %s",
strerror(errno));
return;
}
if (c->ops->mount("tracefs", INIT_TRACEFS, "tracefs",
MS_NOSUID | MS_NODEV | MS_NOEXEC, NULL) != 0 &&
errno != EBUSY)
init_log(c, "could not mount tracefs: %s", strerror(errno));
if (pipe2(ready, O_CLOEXEC) != 0) {
init_log(c, "could not make the readiness pipe: %s",
strerror(errno));
close(fd);
return;
}
pid = fork();
if (pid < 0) {
init_log(c, "could not fork for the telemetry producer: %s",
strerror(errno));
close(ready[0]);
close(ready[1]);
close(fd);
return;
}
if (pid == 0) {
// The write end crosses the exec; the agent closes it once ready.
fcntl(ready[1], F_SETFD, 0);
snprintf(ready_arg, sizeof(ready_arg), "%d", ready[1]);
kmsg = open("/dev/kmsg", O_WRONLY | O_CLOEXEC);
if (kmsg >= 0) {
dup2(kmsg, 1);
dup2(kmsg, 2);
}
fexecve(fd, argv, environ);
_exit(127);
}
close(ready[1]);
close(fd);
switch (init_capture_wait_ready(ready[0], INIT_CAPTURE_READY_MS)) {
case 1:
init_log(c, "telemetry producer ready (pid %d, %s)", (int)pid,
embedded ? "embedded" : INIT_CAPTURE_FILE);
break;
case 0:
init_log(c, "telemetry producer (pid %d) exited before it was "
"ready; the guest runs without the rich channel",
(int)pid);
break;
default:
init_log(c, "telemetry producer (pid %d) not ready after %d ms; "
"going on without waiting", (int)pid,
INIT_CAPTURE_READY_MS);
break;
}
close(ready[0]);
}

int init_run(const struct sysops *ops, enum urunit_mode mode, int argc,
char **argv, const char *cmdline, int *out_argc,
char ***out_argv)
Expand Down Expand Up @@ -1233,6 +1389,9 @@ int init_run(const struct sysops *ops, enum urunit_mode mode, int argc,

// The clock comes first, before anything checks a certificate.
init_set_clock(&c);
// Then the telemetry producer, so the root mount and the workload are
// already under capture.
init_start_capture(&c);
if (init_rosetta(&c))
return 1;

Expand Down
6 changes: 6 additions & 0 deletions src/sysops.h
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ struct sysops {

extern const struct sysops sysops_real;

// Waits up to timeout_ms for the telemetry producer's readiness byte on fd,
// the read end of the pipe whose write end the producer was given. Returns 1
// when the byte arrived, 0 when the producer closed the pipe without writing
// it (it exited first), and -1 when the wait timed out or failed.
int init_capture_wait_ready(int fd, int timeout_ms);

// Runs the early init sequence of mode through ops. cmdline is the text of
// /proc/cmdline. When it is NULL, init_run reads the file through
// ops->read_file. On success it returns 0 and sets *out_argv to a heap
Expand Down
20 changes: 20 additions & 0 deletions src/tests/test_init.c
Original file line number Diff line number Diff line change
Expand Up @@ -1340,10 +1340,30 @@ static void test_init_parsers(void)
init_free_mounts(m);
}

// The readiness wait: the byte, the producer exiting without it (end of
// file), and nothing at all within the timeout.
static void test_capture_wait_ready(void)
{
int fds[2];

CHECK(pipe(fds) == 0);
CHECK(write(fds[1], "1", 1) == 1);
CHECK(init_capture_wait_ready(fds[0], 1000) == 1);
close(fds[1]);
CHECK(init_capture_wait_ready(fds[0], 1000) == 0);
close(fds[0]);

CHECK(pipe(fds) == 0);
CHECK(init_capture_wait_ready(fds[0], 20) == -1);
close(fds[0]);
close(fds[1]);
}

int run_init_tests(void)
{
int start = harness_failures;

test_capture_wait_ready();
test_init_linux_virtiofs();
test_init_command_from_config();
test_init_hull_block();
Expand Down