From 4e97444041e92c45cf6bb6758e0afd9469e254e5 Mon Sep 17 00:00:00 2001 From: Anastassios Nanos Date: Mon, 16 Nov 2026 08:42:05 +0200 Subject: [PATCH] feat(init): Start the telemetry producer, embedded or from the initrd An introspecting guest runs vmi's telem-capture from boot: it fills the export ring the kernel offers at /dev/telem, and is the source of every rich event the host sidecar sees. So far only urunc's legacy BusyBox /init started it, which is why brig-standalone-linux-priv still builds that layout instead of the single-binary initrd hull-assets ships. Both early-init roles now start it, right after /dev, /proc and /sys are mounted and before the root, when /dev/telem exists. It comes from one of two places: the bytes a build with TELEM_CAPTURE= embeds in urunit (.incbin into .rodata between two symbols init.c declares weak, so a build without them leaves the symbols null and nothing else changes), run from a memfd; or /vmi-telem-capture in the initrd. Either way it is fexecve'd as a child of PID 1 while this initramfs is still the root, so switch_root deleting the file does not matter, and the reaping loop collects it like the agent. tracefs is mounted first, since the producer lays its probes out from the tracepoint format files. The producer is given --ready-fd on a pipe and the init waits up to three seconds for its readiness byte, so the workload starts under capture; a producer that exits first (end of file) or stalls (timeout) is logged and the boot goes on without the rich channel. A kernel without /dev/telem boots as before, so one initrd serves a stock and an introspecting kernel. Tested on an x86_64 KVM host with the hull-assets introspection kernel (6.18.60, ring built in) under cloud-hypervisor-vmi: the producer is ready 50 ms after fork, embedded, and the sidecar drains the workload's exec and syscall records with no drops. Unit tests cover the readiness wait: byte, end of file, timeout. Signed-off-by: Anastassios Nanos --- Makefile | 40 ++++++++--- README.md | 19 +++++ src/init.c | 159 ++++++++++++++++++++++++++++++++++++++++++ src/sysops.h | 6 ++ src/tests/test_init.c | 20 ++++++ 5 files changed, 236 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index e0b1749..d6abab8 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,19 @@ URUNIT_SRC := ${SOURCE_DIR}/urunit.c ${SOURCE_DIR}/main.c ${SOURCE_DIR}/mode.c \ # Header files URUNIT_HDR := $(wildcard ${SOURCE_DIR}/*.h) +#? TELEM_CAPTURE a prebuilt in-guest telemetry producer (vmi's telem-capture, +# static, for the target architecture) to embed in urunit. The early init then +# starts it from memory before the workload, when the kernel offers /dev/telem, +# so an introspecting initrd is one file. Unset, the init starts +# /vmi-telem-capture from the initrd when that exists, and nothing otherwise. +TELEM_CAPTURE ?= +ifneq ($(TELEM_CAPTURE),) + TELEM_CAPTURE_BLOB := $(BUILD_DIR)/telem_capture_blob.o + URUNIT_OBJ := $(TELEM_CAPTURE_BLOB) +else + URUNIT_OBJ := +endif + # Unit tests: every source except urunit.c, plus src/tests. Linux only. TEST_SRC := $(wildcard ${SOURCE_DIR}/tests/*.c) TEST_HDR := $(wildcard ${SOURCE_DIR}/tests/*.h) @@ -140,17 +153,28 @@ ifneq ($(TARGET_OS),Linux) endif $(CC) $(CFLAGS) -I${SOURCE_DIR} $(filter-out ${SOURCE_DIR}/urunit.c,$(URUNIT_SRC)) $(TEST_SRC) -o $@ -$(URUNIT_BIN_DYNAMIC): $(URUNIT_SRC) $(URUNIT_HDR) | prepare - $(CC) $(CFLAGS) $(URUNIT_SRC) $(LDFLAGS) -o $@ +# The embedded producer: the file's bytes in .rodata, between two symbols +# init.c declares weak, so a build without TELEM_CAPTURE leaves them null and +# the init falls back to the file in the initrd. .incbin keeps the bytes as they +# are; --noexecstack says what the assembler cannot know about them. +ifneq ($(TELEM_CAPTURE),) +$(TELEM_CAPTURE_BLOB): $(TELEM_CAPTURE) | prepare + printf '\t.section .rodata\n\t.balign 16\n\t.globl urunit_telem_capture_blob\nurunit_telem_capture_blob:\n\t.incbin "%s"\n\t.globl urunit_telem_capture_blob_end\nurunit_telem_capture_blob_end:\n' \ + "$(TELEM_CAPTURE)" > $(BUILD_DIR)/telem_capture_blob.S + $(CC) -c -Wa,--noexecstack $(BUILD_DIR)/telem_capture_blob.S -o $@ +endif + +$(URUNIT_BIN_DYNAMIC): $(URUNIT_SRC) $(URUNIT_HDR) $(URUNIT_OBJ) | prepare + $(CC) $(CFLAGS) $(URUNIT_SRC) $(URUNIT_OBJ) $(LDFLAGS) -o $@ -$(URUNIT_BIN_DYNAMIC_DEBUG): $(URUNIT_SRC) $(URUNIT_HDR) | prepare - $(CC) $(CFLAGS) -DDEBUG $(URUNIT_SRC) $(LDFLAGS) -o $@ +$(URUNIT_BIN_DYNAMIC_DEBUG): $(URUNIT_SRC) $(URUNIT_HDR) $(URUNIT_OBJ) | prepare + $(CC) $(CFLAGS) -DDEBUG $(URUNIT_SRC) $(URUNIT_OBJ) $(LDFLAGS) -o $@ -$(URUNIT_BIN_STATIC): $(URUNIT_SRC) $(URUNIT_HDR) | prepare - $(CC) $(CFLAGS) $(URUNIT_SRC) $(LDFLAGS) $(STATIC_LDFLAGS) -o $@ +$(URUNIT_BIN_STATIC): $(URUNIT_SRC) $(URUNIT_HDR) $(URUNIT_OBJ) | prepare + $(CC) $(CFLAGS) $(URUNIT_SRC) $(URUNIT_OBJ) $(LDFLAGS) $(STATIC_LDFLAGS) -o $@ -$(URUNIT_BIN_STATIC_DEBUG): $(URUNIT_SRC) $(URUNIT_HDR) | prepare - $(CC) $(CFLAGS) -DDEBUG $(URUNIT_SRC) $(LDFLAGS) $(STATIC_LDFLAGS) -o $@ +$(URUNIT_BIN_STATIC_DEBUG): $(URUNIT_SRC) $(URUNIT_HDR) $(URUNIT_OBJ) | prepare + $(CC) $(CFLAGS) -DDEBUG $(URUNIT_SRC) $(URUNIT_OBJ) $(LDFLAGS) $(STATIC_LDFLAGS) -o $@ ## install Install urunit in PREFIX .PHONY: install diff --git a/README.md b/README.md index 5708023..e7d5cdc 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,25 @@ virtio-blk serial, and every failure ends the boot. A container boot initrd needs only `/urunit`, with `/init` and `/vz-init` as symlinks to it. On FreeBSD `urunit` is always the supervisor. +### In-guest telemetry producer + +An introspecting guest runs a telemetry producer (vmi's `telem-capture`) from +boot, which fills the export ring the kernel offers at `/dev/telem`. Both +early-init roles start it before the root is mounted, when that device exists, +from one of two places: + +- the bytes embedded in `urunit` by `make static TELEM_CAPTURE=`, a + prebuilt static producer for the target architecture, run from a `memfd`; + the initrd then stays one file; +- else `/vmi-telem-capture` in the initrd. + +The producer is a child of PID 1, reaped like every other. It is given +`--ready-fd` on a pipe and the init waits up to three seconds for its readiness +byte, so the workload starts under capture; a producer that exits first or +stalls is logged and the boot goes on without the rich channel. A kernel +without `/dev/telem` boots as before, so the same initrd serves a stock and an +introspecting kernel. + ## Building Building `urunit` is as simple as running `make`, but make sure that `make` and diff --git a/src/init.c b/src/init.c index 7d61a72..86cfd37 100644 --- a/src/init.c +++ b/src/init.c @@ -71,11 +71,15 @@ #define _GNU_SOURCE #include +#include +#include #include #include #include #include +#include #include +#include #include "sysops.h" #include "urunit.h" @@ -1197,6 +1201,158 @@ static int init_layers(const struct init_ctx *c, int overlay, char **root, return rc; } +// The in-guest telemetry producer (vmi's telem-capture), started before the +// workload when the kernel offers the export ring at /dev/telem. It is the +// source of every rich event the sidecar sees; without it the host observes +// the boundary alone. It comes from one of two places: the bytes a build with +// TELEM_CAPTURE embedded in this binary (urunit_telem_capture_blob, weak, so a +// build without them leaves the symbols null), or /vmi-telem-capture in the +// initrd. Either way it runs from a memfd or the initrd file through fexecve, +// as a child of PID 1 that the reaping loop collects, while this initramfs is +// still the root: switch_root deletes the file, the running process survives. +// +// It reads the kernel's BTF from /sys and the tracepoint layouts from tracefs, +// so tracefs is mounted first. It reports readiness with one byte on the pipe +// it is given (--ready-fd); the init waits for that byte, bounded, so the +// workload starts under capture, and says so when the producer died or stalled +// instead. A kernel without /dev/telem boots on without any of this, so one +// initrd serves the stock and the introspecting kernel. +extern const unsigned char urunit_telem_capture_blob[] __attribute__((weak)); +extern const unsigned char urunit_telem_capture_blob_end[] __attribute__((weak)); + +#define INIT_CAPTURE_FILE "/vmi-telem-capture" +#define INIT_CAPTURE_DEV "/dev/telem" +#define INIT_TRACEFS "/sys/kernel/tracing" +// How long the init waits for the producer's readiness. Attaching takes some +// tens of milliseconds; a second is already a sign of trouble, and a boot +// should not hang on a capture that never comes. +#define INIT_CAPTURE_READY_MS 3000 + +int init_capture_wait_ready(int fd, int timeout_ms) +{ + struct pollfd pfd = { .fd = fd, .events = POLLIN }; + unsigned char byte; + ssize_t n; + int rc; + + do { + rc = poll(&pfd, 1, timeout_ms); + } while (rc < 0 && errno == EINTR); + if (rc <= 0) + return -1; + do { + n = read(fd, &byte, 1); + } while (n < 0 && errno == EINTR); + if (n < 0) + return -1; + return n == 1; +} + +// Returns a descriptor to fexecve the producer from, or -1 with errno set +// when there is none: a memfd holding the embedded bytes, else the initrd +// file. *embedded says which. +static int init_capture_fd(const struct init_ctx *c, int *embedded) +{ + const unsigned char *p = urunit_telem_capture_blob; + size_t left; + int fd; + + *embedded = p != NULL && urunit_telem_capture_blob_end > p; + if (!*embedded) { + if (c->ops->access_x(INIT_CAPTURE_FILE) != 0) + return -1; + return open(INIT_CAPTURE_FILE, O_RDONLY | O_CLOEXEC); + } + left = (size_t)(urunit_telem_capture_blob_end - p); + fd = memfd_create("vmi-telem-capture", MFD_CLOEXEC); + if (fd < 0) + return -1; + while (left > 0) { + ssize_t n = write(fd, p, left); + + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) { + close(fd); + return -1; + } + p += n; + left -= (size_t)n; + } + return fd; +} + +static void init_start_capture(const struct init_ctx *c) +{ + char ready_arg[16]; + char *argv[] = { "vmi-telem-capture", "--ready-fd", ready_arg, NULL }; + int ready[2]; + int embedded; + int fd; + int kmsg; + pid_t pid; + + if (c->ops->exists(INIT_CAPTURE_DEV) != 0) + return; + fd = init_capture_fd(c, &embedded); + if (fd < 0) { + if (errno != ENOENT) + init_log(c, "could not stage the telemetry producer: %s", + strerror(errno)); + return; + } + if (c->ops->mount("tracefs", INIT_TRACEFS, "tracefs", + MS_NOSUID | MS_NODEV | MS_NOEXEC, NULL) != 0 && + errno != EBUSY) + init_log(c, "could not mount tracefs: %s", strerror(errno)); + if (pipe2(ready, O_CLOEXEC) != 0) { + init_log(c, "could not make the readiness pipe: %s", + strerror(errno)); + close(fd); + return; + } + pid = fork(); + if (pid < 0) { + init_log(c, "could not fork for the telemetry producer: %s", + strerror(errno)); + close(ready[0]); + close(ready[1]); + close(fd); + return; + } + if (pid == 0) { + // The write end crosses the exec; the agent closes it once ready. + fcntl(ready[1], F_SETFD, 0); + snprintf(ready_arg, sizeof(ready_arg), "%d", ready[1]); + kmsg = open("/dev/kmsg", O_WRONLY | O_CLOEXEC); + if (kmsg >= 0) { + dup2(kmsg, 1); + dup2(kmsg, 2); + } + fexecve(fd, argv, environ); + _exit(127); + } + close(ready[1]); + close(fd); + switch (init_capture_wait_ready(ready[0], INIT_CAPTURE_READY_MS)) { + case 1: + init_log(c, "telemetry producer ready (pid %d, %s)", (int)pid, + embedded ? "embedded" : INIT_CAPTURE_FILE); + break; + case 0: + init_log(c, "telemetry producer (pid %d) exited before it was " + "ready; the guest runs without the rich channel", + (int)pid); + break; + default: + init_log(c, "telemetry producer (pid %d) not ready after %d ms; " + "going on without waiting", (int)pid, + INIT_CAPTURE_READY_MS); + break; + } + close(ready[0]); +} + int init_run(const struct sysops *ops, enum urunit_mode mode, int argc, char **argv, const char *cmdline, int *out_argc, char ***out_argv) @@ -1233,6 +1389,9 @@ int init_run(const struct sysops *ops, enum urunit_mode mode, int argc, // The clock comes first, before anything checks a certificate. init_set_clock(&c); + // Then the telemetry producer, so the root mount and the workload are + // already under capture. + init_start_capture(&c); if (init_rosetta(&c)) return 1; diff --git a/src/sysops.h b/src/sysops.h index 121c2ab..a7d48be 100644 --- a/src/sysops.h +++ b/src/sysops.h @@ -52,6 +52,12 @@ struct sysops { extern const struct sysops sysops_real; +// Waits up to timeout_ms for the telemetry producer's readiness byte on fd, +// the read end of the pipe whose write end the producer was given. Returns 1 +// when the byte arrived, 0 when the producer closed the pipe without writing +// it (it exited first), and -1 when the wait timed out or failed. +int init_capture_wait_ready(int fd, int timeout_ms); + // Runs the early init sequence of mode through ops. cmdline is the text of // /proc/cmdline. When it is NULL, init_run reads the file through // ops->read_file. On success it returns 0 and sets *out_argv to a heap diff --git a/src/tests/test_init.c b/src/tests/test_init.c index 74f7bfe..f81ed21 100644 --- a/src/tests/test_init.c +++ b/src/tests/test_init.c @@ -1340,10 +1340,30 @@ static void test_init_parsers(void) init_free_mounts(m); } +// The readiness wait: the byte, the producer exiting without it (end of +// file), and nothing at all within the timeout. +static void test_capture_wait_ready(void) +{ + int fds[2]; + + CHECK(pipe(fds) == 0); + CHECK(write(fds[1], "1", 1) == 1); + CHECK(init_capture_wait_ready(fds[0], 1000) == 1); + close(fds[1]); + CHECK(init_capture_wait_ready(fds[0], 1000) == 0); + close(fds[0]); + + CHECK(pipe(fds) == 0); + CHECK(init_capture_wait_ready(fds[0], 20) == -1); + close(fds[0]); + close(fds[1]); +} + int run_init_tests(void) { int start = harness_failures; + test_capture_wait_ready(); test_init_linux_virtiofs(); test_init_command_from_config(); test_init_hull_block();