From 50668276252eba1ed23db5221def8cfdfe174dd7 Mon Sep 17 00:00:00 2001 From: zaidoon Date: Sun, 8 Nov 2026 03:16:57 -0500 Subject: [PATCH 1/6] Keep logging from hanging on a stalled systemd journal systemd connects a service's stdout and stderr to a journald socket and names it in JOURNAL_STREAM. If journald stops reading that socket but keeps it open, as systemd/systemd#44521 describes after a journald restart, every blocking write to it waits for good: the logging thread, then TelemetryDriver::shutdown_logger, and any thread that panics, because the default panic hook writes its message to stderr. When stdout or stderr is the stream JOURNAL_STREAM names, log output to it now goes through a writer that never waits. The writer buffers each record until the formatter flushes it, then sends it with MSG_DONTWAIT, so it doesn't wait even if something sharing the descriptor makes it blocking again. A record that doesn't fit is dropped and counted in logging_records_dropped{reason="journal_full"}. A full stream isn't an I/O error, so it never stops the logging thread, whatever ignore_io_errors is. Other send errors are reported as before and counted as journal_error. When a record is cut short, the next one starts with a newline, so journald doesn't join the two. A JSON record that fails to format is discarded rather than sent with the next one. slog-term flushes a text record even when formatting fails, so that one goes out cut short, on its own line. Initialization also makes those streams nonblocking, whatever the log output is, so the default panic hook and other writers can't wait for journald either. The flag is shared by every process holding the stream. When the stream is full, println! and eprintln! panic and the panic hook's message can be lost. allow_blocking_journal_stdio leaves the streams as they are. --- foundations/Cargo.toml | 1 + foundations/src/telemetry/log/init.rs | 22 + foundations/src/telemetry/log/journal.rs | 436 ++++++++++++++++++ foundations/src/telemetry/log/mod.rs | 2 + foundations/src/telemetry/settings/logging.rs | 20 + foundations/tests/stdio_logging.rs | 356 ++++++++++++++ 6 files changed, 837 insertions(+) create mode 100644 foundations/src/telemetry/log/journal.rs create mode 100644 foundations/tests/stdio_logging.rs diff --git a/foundations/Cargo.toml b/foundations/Cargo.toml index 8a1e12c4..5712ad67 100644 --- a/foundations/Cargo.toml +++ b/foundations/Cargo.toml @@ -140,6 +140,7 @@ tokio-runtime-metrics = [ # Enables logging functionality. logging = [ "ratelimit", + "dep:libc", "dep:parking_lot", "dep:slog-async", "dep:slog-json", diff --git a/foundations/src/telemetry/log/init.rs b/foundations/src/telemetry/log/init.rs index 58989c3e..00db6446 100644 --- a/foundations/src/telemetry/log/init.rs +++ b/foundations/src/telemetry/log/init.rs @@ -2,6 +2,8 @@ use super::field_dedup::FieldDedupFilterFactory; use super::field_filtering::{FieldFilteringDrain, FilterFactory}; use super::field_redact::FieldRedactFilterFactory; use super::internal::{LoggerWithKvNestingTracking, SharedLog}; +#[cfg(target_os = "linux")] +use super::journal::{JournalStreams, JournalWriter, stdio_fd}; #[cfg(feature = "metrics")] use crate::telemetry::log::log_volume::LogVolumeMetricsDrain; @@ -90,7 +92,27 @@ pub(crate) fn init( // NOTE: OXY-178, default is 128 (https://docs.rs/slog-async/2.7.0/src/slog_async/lib.rs.html#251) const CHANNEL_SIZE: usize = 1024; + // Before building any drain, so that stderr is covered when logs go elsewhere too. + #[cfg(target_os = "linux")] + let journal = JournalStreams::detect(!settings.allow_blocking_journal_stdio); + let (async_drain, async_guard) = match (&settings.output, &settings.format) { + #[cfg(target_os = "linux")] + (output @ (LogOutput::Terminal | LogOutput::Stderr), LogFormat::Text) + if journal.includes(output) => + { + let drain = JournalWriter::drain(stdio_fd(output), |writer| { + TextDrain::new(PlainDecorator::new(writer)).build() + }); + build_async_drain(drain, settings, CHANNEL_SIZE) + } + #[cfg(target_os = "linux")] + (output @ (LogOutput::Terminal | LogOutput::Stderr), LogFormat::Json) + if journal.includes(output) => + { + let drain = JournalWriter::drain(stdio_fd(output), build_json_log_drain); + build_async_drain(drain, settings, CHANNEL_SIZE) + } (output @ (LogOutput::Terminal | LogOutput::Stderr), LogFormat::Text) => { let decorator = if matches!(output, LogOutput::Terminal) { TermDecorator::new().stdout().build() diff --git a/foundations/src/telemetry/log/journal.rs b/foundations/src/telemetry/log/journal.rs new file mode 100644 index 00000000..07a097d7 --- /dev/null +++ b/foundations/src/telemetry/log/journal.rs @@ -0,0 +1,436 @@ +//! Log output to the systemd journal that never waits for journald. +//! +//! systemd connects a service's stdout and stderr to a journald socket and names it in +//! `JOURNAL_STREAM`. If journald stops reading the socket while keeping it open, which a bug +//! can leave it doing for good, the socket's send buffer fills and every blocking write to it +//! waits for good: the logging thread, then [`TelemetryDriver::shutdown_logger`], and any thread +//! that panics, because the default panic hook writes the message to stderr. +//! +//! So log records go to the journal stream through [`JournalWriter`], which never waits, and +//! unless [`LoggingSettings::allow_blocking_journal_stdio`] is on, the stream is made nonblocking +//! for every other writer too. +//! +//! [`TelemetryDriver::shutdown_logger`]: crate::telemetry::TelemetryDriver::shutdown_logger +//! [`LoggingSettings::allow_blocking_journal_stdio`]: crate::telemetry::settings::LoggingSettings::allow_blocking_journal_stdio + +use crate::telemetry::settings::LogOutput; +use slog::{Drain, OwnedKVList, Record}; +use std::ffi::OsStr; +use std::io; +use std::os::fd::{AsFd, AsRawFd, BorrowedFd}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +// A record buffer larger than this is freed after use, rather than kept for the next record. +const MAX_RETAINED_RECORD_CAPACITY: usize = 64 * 1024; + +/// Which of stdout and stderr are the journal stream. +#[derive(Debug, Default, Clone, Copy)] +pub(super) struct JournalStreams { + stdout: bool, + stderr: bool, +} + +impl JournalStreams { + /// Finds which of stdout and stderr are the journal stream, and makes them nonblocking if + /// `nonblocking` is set. + /// + /// A check that fails counts as not the journal, and a descriptor whose flags can't be set + /// stays as it was, so this never stops logging from starting. + pub(super) fn detect(nonblocking: bool) -> Self { + let Some(journal_stream) = std::env::var_os("JOURNAL_STREAM") else { + return Self::default(); + }; + let detect = |fd: BorrowedFd<'_>| { + let is_journal = is_journal(fd, &journal_stream); + if is_journal && nonblocking { + set_nonblocking(fd); + } + is_journal + }; + + Self { + stdout: detect(io::stdout().as_fd()), + stderr: detect(io::stderr().as_fd()), + } + } + + /// Whether `output` writes to the journal stream. + pub(super) fn includes(&self, output: &LogOutput) -> bool { + match output { + LogOutput::Terminal => self.stdout, + LogOutput::Stderr => self.stderr, + _ => false, + } + } +} + +/// The stdout or stderr descriptor that `output` writes to. +pub(super) fn stdio_fd(output: &LogOutput) -> BorrowedFd<'static> { + let fd = if matches!(output, LogOutput::Terminal) { + libc::STDOUT_FILENO + } else { + libc::STDERR_FILENO + }; + // SAFETY: std treats stdout and stderr as open for the life of the process. Borrowing + // rather than owning them means the logging thread never closes them when it exits, which + // would let the next file or socket opened take their numbers. + unsafe { BorrowedFd::borrow_raw(fd) } +} + +/// Whether `fd` is the socket named by `journal_stream`, the way systemd names it. +fn is_journal(fd: BorrowedFd<'_>, journal_stream: &OsStr) -> bool { + let mut metadata = std::mem::MaybeUninit::::uninit(); + // SAFETY: `fd` is borrowed, and fstat initializes `metadata` on success. + if unsafe { libc::fstat(fd.as_raw_fd(), metadata.as_mut_ptr()) } == -1 { + return false; + } + // SAFETY: fstat succeeded. + let metadata = unsafe { metadata.assume_init() }; + + metadata.st_mode & libc::S_IFMT == libc::S_IFSOCK + && journal_stream.as_encoded_bytes() + == format!("{}:{}", metadata.st_dev, metadata.st_ino).as_bytes() +} + +/// Sets `O_NONBLOCK` on `fd`, leaving it as it was if that fails. +fn set_nonblocking(fd: BorrowedFd<'_>) { + // SAFETY: F_GETFL takes no argument and does not close `fd`. + let flags = unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFL) }; + if flags != -1 && flags & libc::O_NONBLOCK == 0 { + // SAFETY: F_SETFL takes an integer argument and does not close `fd`. + unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_SETFL, flags | libc::O_NONBLOCK) }; + } +} + +/// Writes each log record to the journal stream without ever waiting, and drops the record if +/// the stream has no room for it. +/// +/// Records are buffered until the formatter flushes at the end of each one, then sent at once. +/// Sends pass `MSG_DONTWAIT`, so they don't wait even if something sharing the descriptor makes +/// it blocking. A record that only partly fits is cut short, and the next record that gets +/// through starts with a newline, so journald doesn't join the two into one entry. +pub(super) struct JournalWriter { + out: W, + /// A newline, then the record being formatted. The newline is sent only to end a record + /// that was cut short. + buf: Vec, + cut_short: bool, + new_record: Arc, +} + +impl JournalWriter { + /// Builds the formatting drain for records to `out`, and tells its writer where each + /// record starts, including after a record that failed to format. + pub(super) fn drain(out: W, build: impl FnOnce(Self) -> D) -> JournalDrain { + metrics::register(); + let new_record = Arc::new(AtomicBool::new(true)); + let writer = Self { + out, + buf: vec![b'\n'], + cut_short: false, + new_record: Arc::clone(&new_record), + }; + + JournalDrain { + inner: build(writer), + new_record, + } + } + + fn clear(&mut self) { + self.buf.truncate(1); + self.buf.shrink_to(MAX_RETAINED_RECORD_CAPACITY); + } + + /// Sends the buffered record from `start`, in as many sends as the stream takes without + /// waiting, and counts it if it's dropped. + fn send(&mut self, start: usize) -> io::Result<()> { + let record = &self.buf[start..]; + let mut sent = 0; + let res = loop { + match send_nonblocking(self.out.as_fd(), &record[sent..]) { + Ok(0) => break Err(io::ErrorKind::WriteZero.into()), + Ok(n) => { + sent += n; + if sent == record.len() { + break Ok(()); + } + } + Err(err) if err.kind() == io::ErrorKind::Interrupted => {} + Err(err) => break Err(err), + } + }; + if let Some(&last) = record[..sent].last() { + self.cut_short = last != b'\n'; + } + + match res { + Err(err) if err.kind() == io::ErrorKind::WouldBlock => { + metrics::count(DropReason::JournalFull); + Ok(()) + } + Err(err) => { + metrics::count(DropReason::JournalError); + Err(err) + } + Ok(()) => Ok(()), + } + } +} + +/// Sends without waiting, whatever the descriptor's own blocking mode is. +fn send_nonblocking(fd: BorrowedFd<'_>, buf: &[u8]) -> io::Result { + // SAFETY: `buf` is valid for `buf.len()` bytes, and send does not take ownership of `fd`. + let sent = unsafe { + libc::send( + fd.as_raw_fd(), + buf.as_ptr().cast(), + buf.len(), + libc::MSG_DONTWAIT | libc::MSG_NOSIGNAL, + ) + }; + if sent < 0 { + return Err(io::Error::last_os_error()); + } + Ok(sent as usize) +} + +impl io::Write for JournalWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + if self.new_record.load(Ordering::Relaxed) { + self.new_record.store(false, Ordering::Relaxed); + // A formatter that fails part way through a record doesn't flush it. Discard what + // it wrote instead of sending it with the next record. + self.clear(); + } + self.buf.extend_from_slice(buf); + Ok(buf.len()) + } + + /// Sends the buffered record. The log formatters flush at the end of each record. + fn flush(&mut self) -> io::Result<()> { + if self.buf.len() == 1 { + return Ok(()); + } + let res = self.send(usize::from(!self.cut_short)); + self.clear(); + res + } +} + +/// Marks the start of each record for its [`JournalWriter`]. +pub(super) struct JournalDrain { + inner: D, + new_record: Arc, +} + +impl Drain for JournalDrain { + type Ok = D::Ok; + type Err = D::Err; + + fn log(&self, record: &Record, values: &OwnedKVList) -> Result { + // The formatter and writer run on the same logging thread. The shared flag only + // connects the drain to its writer; it does not synchronize threads. + self.new_record.store(true, Ordering::Relaxed); + self.inner.log(record, values) + } +} + +/// Why a log record was dropped instead of written. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum DropReason { + /// The journal stream had no room for the record. + JournalFull, + /// Sending to the journal stream failed, for example because journald closed it. + JournalError, +} + +#[cfg(feature = "metrics")] +mod metrics { + use super::DropReason; + use crate::telemetry::metrics::Counter; + use std::sync::LazyLock; + + #[crate::telemetry::metrics::metrics(crate_path = "crate", unprefixed)] + pub(crate) mod logging { + /// Number of log records dropped instead of written. + pub fn records_dropped(reason: DropReason) -> Counter; + } + + struct Counters { + journal_full: Counter, + journal_error: Counter, + } + + static COUNTERS: LazyLock = LazyLock::new(|| Counters { + journal_full: logging::records_dropped(DropReason::JournalFull), + journal_error: logging::records_dropped(DropReason::JournalError), + }); + + /// Creates the counters at zero, so they exist before the first drop. + pub(super) fn register() { + LazyLock::force(&COUNTERS); + } + + pub(super) fn count(reason: DropReason) { + match reason { + DropReason::JournalFull => &COUNTERS.journal_full, + DropReason::JournalError => &COUNTERS.journal_error, + } + .inc(); + } +} + +#[cfg(not(feature = "metrics"))] +mod metrics { + pub(super) fn register() {} + + pub(super) fn count(_reason: super::DropReason) {} +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read, Write}; + use std::os::unix::fs::MetadataExt; + use std::os::unix::net::UnixStream; + use std::sync::mpsc; + use std::time::Duration; + + fn stream_id(fd: &impl AsFd) -> String { + let metadata = std::fs::File::from(fd.as_fd().try_clone_to_owned().unwrap()) + .metadata() + .unwrap(); + format!("{}:{}", metadata.dev(), metadata.ino()) + } + + /// A writer to a blocking journal stream, and journald's end of it. + fn journal_pair() -> (JournalWriter, UnixStream) { + let (stream, journald) = UnixStream::pair().unwrap(); + let drain = JournalWriter::drain(stream, |writer| writer); + (drain.inner, journald) + } + + /// Fills `writer`'s stream so that nothing more fits, and leaves it blocking. + fn fill(writer: &JournalWriter) { + let stream = &writer.out; + stream.set_nonblocking(true).unwrap(); + loop { + match (&*stream).write(&[b'\n'; 8192]) { + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::WouldBlock => break, + Err(err) => panic!("filling the journal stream: {err}"), + } + } + stream.set_nonblocking(false).unwrap(); + } + + fn read_available(journald: &mut UnixStream) -> Vec { + journald.set_nonblocking(true).unwrap(); + let mut received = Vec::new(); + let mut buf = [0; 64 * 1024]; + loop { + match journald.read(&mut buf) { + Ok(0) => return received, + Ok(n) => received.extend_from_slice(&buf[..n]), + Err(err) if err.kind() == io::ErrorKind::WouldBlock => return received, + Err(err) => panic!("read failed: {err}"), + } + } + } + + /// Writes and flushes `record` on another thread, failing the test if that waits. + fn log_without_waiting( + mut writer: JournalWriter, + record: &[u8], + ) -> (JournalWriter, io::Result<()>) { + let record = record.to_vec(); + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + writer.new_record.store(true, Ordering::Relaxed); + writer.write_all(&record).unwrap(); + let res = writer.flush(); + done_tx.send((writer, res)).unwrap(); + }); + done_rx + .recv_timeout(Duration::from_secs(10)) + .expect("the write waited for journald") + } + + #[test] + fn only_the_named_socket_is_the_journal() { + let (stream, _journald) = UnixStream::pair().unwrap(); + + assert!(!is_journal(stream.as_fd(), OsStr::new("1:2"))); + assert!(is_journal(stream.as_fd(), OsStr::new(&stream_id(&stream)))); + } + + #[test] + fn a_file_is_not_the_journal() { + let file = tempfile::tempfile().unwrap(); + + assert!(!is_journal(file.as_fd(), OsStr::new(&stream_id(&file)))); + } + + #[test] + fn sends_each_record_when_flushed() { + let (mut writer, mut journald) = journal_pair(); + + writer.write_all(b"first ").unwrap(); + writer.write_all(b"record\n").unwrap(); + assert!(read_available(&mut journald).is_empty()); + writer.flush().unwrap(); + // The text formatter flushes a second time when it finishes a record. + writer.flush().unwrap(); + writer.write_all(b"second record\n").unwrap(); + writer.flush().unwrap(); + + assert_eq!( + read_available(&mut journald), + b"first record\nsecond record\n" + ); + } + + #[test] + fn drops_records_that_do_not_fit_without_waiting() { + let (writer, mut journald) = journal_pair(); + fill(&writer); + + let (writer, res) = log_without_waiting(writer, b"dropped record\n"); + res.unwrap(); + let filled = read_available(&mut journald); + assert!(filled.iter().all(|&b| b == b'\n')); + + let (_writer, res) = log_without_waiting(writer, b"next record\n"); + res.unwrap(); + assert_eq!(read_available(&mut journald), b"next record\n"); + } + + #[test] + fn ends_a_record_cut_short_before_the_next_one() { + let (writer, mut journald) = journal_pair(); + // Far larger than a socket send buffer normally is, so only part of it fits. + let big = format!("{}\n", "x".repeat(32 * 1024 * 1024)); + + let (writer, res) = log_without_waiting(writer, big.as_bytes()); + res.unwrap(); + let received = read_available(&mut journald); + assert!(!received.is_empty() && received.len() < big.len()); + assert!(!received.contains(&b'\n')); + + let (_writer, res) = log_without_waiting(writer, b"next record\n"); + res.unwrap(); + assert_eq!(read_available(&mut journald), b"\nnext record\n"); + } + + #[test] + fn reports_a_closed_journal_as_an_error() { + let (writer, journald) = journal_pair(); + drop(journald); + + let (_writer, res) = log_without_waiting(writer, b"record\n"); + assert_eq!(res.unwrap_err().kind(), io::ErrorKind::BrokenPipe); + } +} diff --git a/foundations/src/telemetry/log/mod.rs b/foundations/src/telemetry/log/mod.rs index 73717082..c4f05f24 100644 --- a/foundations/src/telemetry/log/mod.rs +++ b/foundations/src/telemetry/log/mod.rs @@ -3,6 +3,8 @@ mod field_dedup; mod field_filtering; mod field_redact; +#[cfg(target_os = "linux")] +mod journal; mod rate_limit; pub(crate) mod init; diff --git a/foundations/src/telemetry/settings/logging.rs b/foundations/src/telemetry/settings/logging.rs index 0e1a0ea4..9a3a74d0 100644 --- a/foundations/src/telemetry/settings/logging.rs +++ b/foundations/src/telemetry/settings/logging.rs @@ -46,6 +46,26 @@ pub struct LoggingSettings { /// running if a log sink returns an error such as [`std::io::ErrorKind::BrokenPipe`]. pub ignore_io_errors: bool, + /// Whether to let stdout and stderr block when they are the systemd journal stream. + /// + /// On Linux, logs written to the journal stream that systemd names in `JOURNAL_STREAM` never + /// wait for journald, whatever this setting is. Records that don't fit in the stream's buffer + /// are dropped and counted in the `logging_records_dropped` metric. Other writes to the + /// stream can wait, though, and if journald stops reading it for good, they wait for good. + /// That includes the default panic hook's message, so a thread that panics never finishes. + /// By default, foundations makes them nonblocking to prevent that, whatever the log output is. + /// + /// Nonblocking mode belongs to the stream, so it applies to every writer in the process and + /// in child processes that inherit the stream. When the stream's buffer is full, the default + /// panic hook's message can be lost, and `println!` and `eprintln!` panic instead of waiting. + /// Enabling this setting leaves the streams as they are; it doesn't make a nonblocking stream + /// blocking again. + /// + /// If seccomp sandboxing applies to the thread that initializes telemetry, the logging thread + /// needs `sendto` to write to the journal stream, and initialization needs `fcntl` unless this + /// setting is enabled. + pub allow_blocking_journal_stdio: bool, + /// Settings for rate limiting emission of log events pub rate_limit: RateLimitingSettings, diff --git a/foundations/tests/stdio_logging.rs b/foundations/tests/stdio_logging.rs new file mode 100644 index 00000000..a9292f11 --- /dev/null +++ b/foundations/tests/stdio_logging.rs @@ -0,0 +1,356 @@ +#![cfg(all(target_os = "linux", feature = "telemetry"))] +//! These tests replace stdout and stderr and initialize telemetry, so they assume a separate +//! process is used for each test. Make sure you run with `cargo nextest run`. + +mod common; + +use common::collect_text; +use foundations::service_info; +use foundations::telemetry::settings::{LogFormat, LogOutput, LoggingSettings, TelemetrySettings}; +use foundations::telemetry::{self, TelemetryConfig, TelemetryDriver}; +use std::io; +use std::io::{Read, Write}; +use std::os::fd::{AsFd, AsRawFd, BorrowedFd, OwnedFd}; +use std::os::unix::fs::MetadataExt; +use std::os::unix::net::UnixStream; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +fn init_logging(output: LogOutput, format: LogFormat) -> TelemetryDriver { + init_logging_with_settings(LoggingSettings { + output, + format, + ..Default::default() + }) +} + +fn init_logging_with_settings(logging: LoggingSettings) -> TelemetryDriver { + let settings = TelemetrySettings { + logging, + ..Default::default() + }; + + telemetry::init(TelemetryConfig { + service_info: &service_info!(), + settings: &settings, + custom_server_routes: Default::default(), + }) + .unwrap() +} + +/// Puts `stream` on stderr and names it as the journal stream, as systemd does. +/// +/// Returns the original stderr. +fn pose_as_journal(stream: &UnixStream) -> OwnedFd { + let original = io::stderr().as_fd().try_clone_to_owned().unwrap(); + nix::unistd::dup2_stderr(stream).unwrap(); + // SAFETY: nothing else in this test process reads or writes the environment meanwhile. + unsafe { std::env::set_var("JOURNAL_STREAM", stream_id(stream.as_fd())) }; + + original +} + +fn stream_id(fd: BorrowedFd<'_>) -> String { + let metadata = std::fs::File::from(fd.try_clone_to_owned().unwrap()) + .metadata() + .unwrap(); + format!("{}:{}", metadata.dev(), metadata.ino()) +} + +fn is_nonblocking(fd: BorrowedFd<'_>) -> bool { + // SAFETY: F_GETFL only reads the borrowed descriptor's flags. + let flags = unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFL) }; + assert_ne!(flags, -1); + flags & libc::O_NONBLOCK != 0 +} + +/// Fills `stream` so that nothing more fits, leaving its blocking mode as it was. +fn fill_stream(stream: &UnixStream) { + let nonblocking = is_nonblocking(stream.as_fd()); + stream.set_nonblocking(true).unwrap(); + loop { + match (&*stream).write(&[b'\n'; 8192]) { + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::WouldBlock => break, + Err(err) => panic!("filling the journal stream: {err}"), + } + } + stream.set_nonblocking(nonblocking).unwrap(); +} + +fn read_available(journald: &mut UnixStream) -> String { + journald.set_nonblocking(true).unwrap(); + let mut received = Vec::new(); + let mut buf = [0; 64 * 1024]; + loop { + match journald.read(&mut buf) { + Ok(0) => break, + Ok(n) => received.extend_from_slice(&buf[..n]), + Err(err) if err.kind() == io::ErrorKind::WouldBlock => break, + Err(err) => panic!("read failed: {err}"), + } + } + String::from_utf8(received).unwrap() +} + +fn dropped(reason: &str) -> Option { + let series = format!("logging_records_dropped{{reason=\"{reason}\"}} "); + collect_text(&Default::default()) + .lines() + .find_map(|line| line.strip_prefix(&series)?.parse().ok()) +} + +/// Waits until at least `count` records were dropped for `reason`. +fn wait_for_drops(reason: &str, count: u64, original_stderr: &OwnedFd) { + let deadline = Instant::now() + Duration::from_secs(10); + while dropped(reason) < Some(count) { + if Instant::now() >= deadline { + nix::unistd::dup2_stderr(original_stderr).unwrap(); + panic!("the logging thread did not drop records for {reason}"); + } + std::thread::sleep(Duration::from_millis(5)); + } +} + +fn finishes_in_time(f: impl FnOnce() + Send + 'static) -> bool { + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + f(); + let _ = done_tx.send(()); + }); + done_rx.recv_timeout(Duration::from_secs(10)).is_ok() +} + +/// Fails the test without going through the panic hook. +/// +/// A panic stuck writing to the journal holds the default hook's backtrace lock, so the +/// panic of a failed assertion would wait for that lock forever instead of failing the test. +fn fail_without_panicking(message: &str) -> ! { + let _ = nix::unistd::write(io::stderr(), format!("{message}\n").as_bytes()); + std::process::exit(101); +} + +#[tokio::test] +async fn stalled_journal_does_not_hang_logging_panics_or_shutdown() { + let (stream, mut journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging(LogOutput::Stderr, LogFormat::Json); + let nonblocking = is_nonblocking(io::stderr().as_fd()); + + // journald isn't reading, and the stream is full. + fill_stream(&stream); + for _ in 0..3 { + telemetry::log::warn!("dropped record"); + } + wait_for_drops("journal_full", 3, &original_stderr); + let panic_finished = finishes_in_time(|| { + let _ = std::thread::spawn(|| panic!("panic while the journal is stalled")).join(); + }); + // Once journald reads again, the logging thread writes the next record. + read_available(&mut journald); + telemetry::log::warn!("after the stall"); + let shutdown_finished = finishes_in_time(move || driver.shutdown_logger()); + + // Restore stderr, so a failure can report itself. + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + if !nonblocking { + fail_without_panicking("the journal stream is still blocking"); + } + if !panic_finished { + fail_without_panicking("the panic waited for the journal"); + } + if !shutdown_finished { + fail_without_panicking("logger shutdown waited for the journal"); + } + + let received = read_available(&mut journald); + let last = received.lines().last().unwrap_or_default(); + let record: serde_json::Value = serde_json::from_str(last).unwrap(); + assert_eq!(record["msg"], "after the stall", "{received}"); +} + +#[tokio::test] +async fn logging_does_not_wait_for_a_blocking_journal_stream() { + let (stream, _journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging_with_settings(LoggingSettings { + output: LogOutput::Stderr, + format: LogFormat::Json, + allow_blocking_journal_stdio: true, + ..Default::default() + }); + let nonblocking = is_nonblocking(io::stderr().as_fd()); + fill_stream(&stream); + + telemetry::log::warn!("written while the stream blocks"); + wait_for_drops("journal_full", 1, &original_stderr); + let shutdown_finished = finishes_in_time(move || driver.shutdown_logger()); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + + assert!(!nonblocking, "the setting didn't leave the stream alone"); + assert!(shutdown_finished, "logger shutdown waited for the journal"); +} + +#[tokio::test] +async fn records_dropped_after_journald_closes_the_stream_are_counted() { + let (stream, journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging_with_settings(LoggingSettings { + output: LogOutput::Stderr, + format: LogFormat::Json, + ignore_io_errors: true, + ..Default::default() + }); + assert_eq!(dropped("journal_error"), Some(0)); + drop(journald); + + telemetry::log::warn!("first record after journald closed the stream"); + telemetry::log::warn!("second record after journald closed the stream"); + wait_for_drops("journal_error", 2, &original_stderr); + driver.shutdown_logger(); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); +} + +#[tokio::test] +async fn failed_json_records_are_discarded_before_the_next_record() { + struct FailingContext { + fail: Arc, + failures: Arc, + } + + impl slog::Value for FailingContext { + fn serialize( + &self, + _record: &slog::Record, + key: slog::Key, + serializer: &mut dyn slog::Serializer, + ) -> slog::Result { + serializer.emit_str(key, &"x".repeat(256))?; + if self.fail.load(Ordering::Relaxed) { + self.failures.fetch_add(1, Ordering::Relaxed); + return Err(io::Error::other("context serialization failed").into()); + } + Ok(()) + } + } + + let (stream, mut journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging_with_settings(LoggingSettings { + output: LogOutput::Stderr, + format: LogFormat::Json, + ignore_io_errors: true, + ..Default::default() + }); + let fail = Arc::new(AtomicBool::new(true)); + let failures = Arc::new(AtomicUsize::new(0)); + telemetry::log::add_fields!("context" => FailingContext { + fail: Arc::clone(&fail), + failures: Arc::clone(&failures), + }); + for _ in 0..128 { + telemetry::log::warn!("failed record"); + } + let deadline = Instant::now() + Duration::from_secs(10); + while failures.load(Ordering::Relaxed) < 128 { + if Instant::now() >= deadline { + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + panic!("the logging thread did not process the failed records"); + } + std::thread::yield_now(); + } + fail.store(false, Ordering::Relaxed); + telemetry::log::warn!("successful record"); + driver.shutdown_logger(); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + + let received = read_available(&mut journald); + let record: serde_json::Value = serde_json::from_str(&received).unwrap(); + assert_eq!(record["msg"], "successful record"); + assert_eq!(received.lines().count(), 1, "{received}"); +} + +#[tokio::test] +async fn text_records_reach_the_journal_one_per_line() { + let (stream, mut journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging(LogOutput::Stderr, LogFormat::Text); + + telemetry::log::warn!("first record"); + telemetry::log::warn!("second record"); + driver.shutdown_logger(); + + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + let received = read_available(&mut journald); + let lines: Vec<_> = received.lines().collect(); + assert_eq!(lines.len(), 2, "{received}"); + assert!(lines[0].contains("first record"), "{received}"); + assert!(lines[1].contains("second record"), "{received}"); + // The drop counters exist from the start, so alerts can use them. + assert_eq!(dropped("journal_full"), Some(0)); + assert_eq!(dropped("journal_error"), Some(0)); +} + +#[tokio::test] +async fn terminal_output_to_the_journal_does_not_block() { + let (stream, mut journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let original_stdout = io::stdout().as_fd().try_clone_to_owned().unwrap(); + nix::unistd::dup2_stdout(&stream).unwrap(); + let mut driver = init_logging(LogOutput::Terminal, LogFormat::Json); + + telemetry::log::warn!("to stdout"); + driver.shutdown_logger(); + + nix::unistd::dup2_stdout(&original_stdout).unwrap(); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + assert!(is_nonblocking(stream.as_fd())); + let received = read_available(&mut journald); + assert!(received.contains("\"msg\":\"to stdout\""), "{received}"); +} + +#[tokio::test] +async fn stderr_does_not_block_when_logs_go_to_a_file() { + let (stream, _journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let dir = tempfile::tempdir().unwrap(); + let _driver = init_logging(LogOutput::File(dir.path().join("log")), LogFormat::Json); + + let nonblocking = is_nonblocking(io::stderr().as_fd()); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + assert!(nonblocking); +} + +#[tokio::test] +async fn stderr_that_is_not_the_journal_is_left_alone() { + let (stream, _journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let (other, _other_reader) = UnixStream::pair().unwrap(); + nix::unistd::dup2_stderr(&other).unwrap(); + let _driver = init_logging(LogOutput::Stderr, LogFormat::Json); + + let nonblocking = is_nonblocking(io::stderr().as_fd()); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + assert!(!nonblocking); +} + +#[tokio::test] +async fn journal_output_leaves_stderr_open_after_shutdown() { + let (stream, _journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging(LogOutput::Stderr, LogFormat::Json); + + telemetry::log::warn!("logged before shutdown"); + driver.shutdown_logger(); + let stderr = io::stderr().as_fd().try_clone_to_owned(); + + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + let stderr = stderr.expect("shutting down the logger closed stderr"); + assert_eq!( + stream_id(stderr.as_fd()), + stream_id(stream.as_fd()), + "stderr no longer refers to the journal stream" + ); +} From 4c538560f7dd3cb6c45d24bb1d050f69ec5a174e Mon Sep 17 00:00:00 2001 From: zaidoon Date: Tue, 10 Nov 2026 23:44:15 -0500 Subject: [PATCH 2/6] Keep stdout and stderr open after the logger shuts down When JSON logs go to a stdout or stderr that isn't the journal stream, the drain writes through a File made from fd 1 or 2, and dropping it at shutdown closed the descriptor. The next file or socket opened took that number, so anything written to stderr afterwards, like a panic message, went into it. The writer wraps the File in ManuallyDrop, so dropping the drain leaves the descriptor open, the same as the journal writer does. --- foundations/src/telemetry/log/init.rs | 38 +++++++++++++++++++++------ foundations/tests/stdio_logging.rs | 32 ++++++++++++++++++++++ 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/foundations/src/telemetry/log/init.rs b/foundations/src/telemetry/log/init.rs index 00db6446..dc121454 100644 --- a/foundations/src/telemetry/log/init.rs +++ b/foundations/src/telemetry/log/init.rs @@ -25,6 +25,7 @@ use std::fmt::Debug; use std::fs::File; use std::io; use std::io::BufWriter; +use std::mem::ManuallyDrop; use std::sync::{Arc, LazyLock, OnceLock}; type BoxedDebug = Box; @@ -176,48 +177,69 @@ pub(crate) fn init( Ok(Some(async_guard)) } +/// Writes to stdout or stderr through a [`File`] that never closes them. +/// +/// The log drain is dropped when the logger shuts down. Closing the descriptor then would +/// let the next file or socket opened take its number, and later writes to stdout or stderr, +/// such as a panic message, would go there. +struct StdioWriter(ManuallyDrop); + +impl io::Write for StdioWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.write(buf) + } + + fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result { + self.0.write_vectored(bufs) + } + + fn flush(&mut self) -> io::Result<()> { + self.0.flush() + } +} + /// Opens fd 1 directly and wraps with a [`BufWriter`] with [`BUF_SIZE`] capacity. /// /// [`io::Stdout`] uses a [`io::LineWriter`] which may cause unnecessary flushing. #[cfg(not(target_family = "windows"))] -fn stdout_writer_without_line_buffering() -> BufWriter { +fn stdout_writer_without_line_buffering() -> BufWriter { use std::os::fd::{AsRawFd, FromRawFd}; let stdout = std::io::stdout(); let stdout = unsafe { File::from_raw_fd(stdout.as_raw_fd()) }; - BufWriter::with_capacity(BUF_SIZE, stdout) + BufWriter::with_capacity(BUF_SIZE, StdioWriter(ManuallyDrop::new(stdout))) } /// Opens fd 1 directly and wraps with a [`BufWriter`] with [`BUF_SIZE`] capacity. /// /// [`io::Stdout`] uses a [`io::LineWriter`] which may cause unnecessary flushing. #[cfg(target_family = "windows")] -fn stdout_writer_without_line_buffering() -> BufWriter { +fn stdout_writer_without_line_buffering() -> BufWriter { use std::os::windows::io::{AsRawHandle, FromRawHandle}; let stdout = std::io::stdout(); let stdout = unsafe { File::from_raw_handle(stdout.as_raw_handle()) }; - BufWriter::with_capacity(BUF_SIZE, stdout) + BufWriter::with_capacity(BUF_SIZE, StdioWriter(ManuallyDrop::new(stdout))) } /// Opens fd 2 directly and wraps with a [`BufWriter`] with [`BUF_SIZE`] capacity. #[cfg(not(target_family = "windows"))] -fn stderr_writer_without_line_buffering() -> BufWriter { +fn stderr_writer_without_line_buffering() -> BufWriter { use std::os::fd::{AsRawFd, FromRawFd}; let stderr = std::io::stderr(); let stderr = unsafe { File::from_raw_fd(stderr.as_raw_fd()) }; - BufWriter::with_capacity(BUF_SIZE, stderr) + BufWriter::with_capacity(BUF_SIZE, StdioWriter(ManuallyDrop::new(stderr))) } /// Opens fd 2 directly and wraps with a [`BufWriter`] with [`BUF_SIZE`] capacity. #[cfg(target_family = "windows")] -fn stderr_writer_without_line_buffering() -> BufWriter { +fn stderr_writer_without_line_buffering() -> BufWriter { use std::os::windows::io::{AsRawHandle, FromRawHandle}; let stderr = std::io::stderr(); let stderr = unsafe { File::from_raw_handle(stderr.as_raw_handle()) }; - BufWriter::with_capacity(BUF_SIZE, stderr) + BufWriter::with_capacity(BUF_SIZE, StdioWriter(ManuallyDrop::new(stderr))) } pub(crate) fn wrap_root_drain(settings: &LoggingSettings, drain: D) -> SharedDrain diff --git a/foundations/tests/stdio_logging.rs b/foundations/tests/stdio_logging.rs index a9292f11..2f2cf11b 100644 --- a/foundations/tests/stdio_logging.rs +++ b/foundations/tests/stdio_logging.rs @@ -336,6 +336,38 @@ async fn stderr_that_is_not_the_journal_is_left_alone() { assert!(!nonblocking); } +#[tokio::test] +async fn stderr_output_leaves_stderr_open_after_shutdown() { + let (stream, _reader) = UnixStream::pair().unwrap(); + let original_stderr = io::stderr().as_fd().try_clone_to_owned().unwrap(); + nix::unistd::dup2_stderr(&stream).unwrap(); + let mut driver = init_logging(LogOutput::Stderr, LogFormat::Json); + + telemetry::log::warn!("logged before shutdown"); + driver.shutdown_logger(); + let stderr = io::stderr().as_fd().try_clone_to_owned(); + + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + let stderr = stderr.expect("shutting down the logger closed stderr"); + assert_eq!(stream_id(stderr.as_fd()), stream_id(stream.as_fd())); +} + +#[tokio::test] +async fn terminal_output_leaves_stdout_open_after_shutdown() { + let (stream, _reader) = UnixStream::pair().unwrap(); + let original_stdout = io::stdout().as_fd().try_clone_to_owned().unwrap(); + nix::unistd::dup2_stdout(&stream).unwrap(); + let mut driver = init_logging(LogOutput::Terminal, LogFormat::Json); + + telemetry::log::warn!("logged before shutdown"); + driver.shutdown_logger(); + let stdout = io::stdout().as_fd().try_clone_to_owned(); + + nix::unistd::dup2_stdout(&original_stdout).unwrap(); + let stdout = stdout.expect("shutting down the logger closed stdout"); + assert_eq!(stream_id(stdout.as_fd()), stream_id(stream.as_fd())); +} + #[tokio::test] async fn journal_output_leaves_stderr_open_after_shutdown() { let (stream, _journald) = UnixStream::pair().unwrap(); From c8b7e50a559191db423435c1073c67b20e1699a2 Mon Sep 17 00:00:00 2001 From: zaidoon Date: Tue, 10 Nov 2026 23:45:01 -0500 Subject: [PATCH 3/6] Write to stdout or stderr when it stops being a socket The journal writer sends with MSG_DONTWAIT, and send fails with ENOTSOCK if something replaces stdout or stderr with a regular file or a pipe after logging started. With ignore_io_errors off, that stopped the logging thread on the next record, so every log after the swap was lost. On ENOTSOCK the writer writes to the descriptor instead, the same way the other stdout and stderr writers do, so logs follow the redirect. --- foundations/src/telemetry/log/journal.rs | 17 +++++++++++++++-- foundations/tests/stdio_logging.rs | 24 +++++++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/foundations/src/telemetry/log/journal.rs b/foundations/src/telemetry/log/journal.rs index 07a097d7..d8b136ed 100644 --- a/foundations/src/telemetry/log/journal.rs +++ b/foundations/src/telemetry/log/journal.rs @@ -180,6 +180,9 @@ impl JournalWriter { } /// Sends without waiting, whatever the descriptor's own blocking mode is. +/// +/// If something replaced stdout or stderr with a file that isn't a socket after logging +/// started, this writes to it the way any other writer would, which waits if that file does. fn send_nonblocking(fd: BorrowedFd<'_>, buf: &[u8]) -> io::Result { // SAFETY: `buf` is valid for `buf.len()` bytes, and send does not take ownership of `fd`. let sent = unsafe { @@ -190,10 +193,20 @@ fn send_nonblocking(fd: BorrowedFd<'_>, buf: &[u8]) -> io::Result { libc::MSG_DONTWAIT | libc::MSG_NOSIGNAL, ) }; - if sent < 0 { + if sent >= 0 { + return Ok(sent as usize); + } + let err = io::Error::last_os_error(); + if err.raw_os_error() != Some(libc::ENOTSOCK) { + return Err(err); + } + + // SAFETY: `buf` is valid for `buf.len()` bytes, and write does not take ownership of `fd`. + let written = unsafe { libc::write(fd.as_raw_fd(), buf.as_ptr().cast(), buf.len()) }; + if written < 0 { return Err(io::Error::last_os_error()); } - Ok(sent as usize) + Ok(written as usize) } impl io::Write for JournalWriter { diff --git a/foundations/tests/stdio_logging.rs b/foundations/tests/stdio_logging.rs index 2f2cf11b..a54d7c53 100644 --- a/foundations/tests/stdio_logging.rs +++ b/foundations/tests/stdio_logging.rs @@ -9,7 +9,7 @@ use foundations::service_info; use foundations::telemetry::settings::{LogFormat, LogOutput, LoggingSettings, TelemetrySettings}; use foundations::telemetry::{self, TelemetryConfig, TelemetryDriver}; use std::io; -use std::io::{Read, Write}; +use std::io::{Read, Seek, Write}; use std::os::fd::{AsFd, AsRawFd, BorrowedFd, OwnedFd}; use std::os::unix::fs::MetadataExt; use std::os::unix::net::UnixStream; @@ -368,6 +368,28 @@ async fn terminal_output_leaves_stdout_open_after_shutdown() { assert_eq!(stream_id(stdout.as_fd()), stream_id(stream.as_fd())); } +#[tokio::test] +async fn logs_follow_stderr_when_something_replaces_the_journal() { + let (stream, _journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging(LogOutput::Stderr, LogFormat::Json); + let mut file = tempfile::tempfile().unwrap(); + nix::unistd::dup2_stderr(&file).unwrap(); + + telemetry::log::warn!("logged after stderr was replaced"); + driver.shutdown_logger(); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + + let mut written = String::new(); + file.seek(io::SeekFrom::Start(0)).unwrap(); + file.read_to_string(&mut written).unwrap(); + assert!( + written.contains("\"msg\":\"logged after stderr was replaced\""), + "{written}" + ); + assert_eq!(dropped("journal_error"), Some(0)); +} + #[tokio::test] async fn journal_output_leaves_stderr_open_after_shutdown() { let (stream, _journald) = UnixStream::pair().unwrap(); From c19403ab7adc146876d4643cb11ea9ff0ecbaf23 Mon Sep 17 00:00:00 2001 From: zaidoon Date: Tue, 10 Nov 2026 23:47:53 -0500 Subject: [PATCH 4/6] Make the journal tests fail instead of hanging When the journal stream blocks, a panic on the logging thread can get stuck writing its message in the default panic hook, which holds the hook's lock. A failing assert! in the test then waits for that lock forever, and nextest has no timeout here, so CI would hang instead of reporting the failure. The checks that can run in that state go through fail_without_panicking, like the stalled journal test already does. Also test that the journal writer keeps the buffer of a normal record for the next one and frees the buffer of a huge one. --- foundations/src/telemetry/log/journal.rs | 15 +++++++++++++++ foundations/tests/stdio_logging.rs | 13 ++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/foundations/src/telemetry/log/journal.rs b/foundations/src/telemetry/log/journal.rs index d8b136ed..7ece8638 100644 --- a/foundations/src/telemetry/log/journal.rs +++ b/foundations/src/telemetry/log/journal.rs @@ -438,6 +438,21 @@ mod tests { assert_eq!(read_available(&mut journald), b"\nnext record\n"); } + #[test] + fn keeps_the_buffer_of_a_normal_record_and_frees_a_huge_one() { + let (writer, _journald) = journal_pair(); + + let (writer, res) = log_without_waiting(writer, &[b'x'; 8 * 1024]); + res.unwrap(); + let capacity = writer.buf.capacity(); + assert!(capacity > 8 * 1024, "{capacity}"); + + let (writer, res) = log_without_waiting(writer, &vec![b'x'; 1024 * 1024]); + res.unwrap(); + let capacity = writer.buf.capacity(); + assert!(capacity <= 64 * 1024, "{capacity}"); + } + #[test] fn reports_a_closed_journal_as_an_error() { let (writer, journald) = journal_pair(); diff --git a/foundations/tests/stdio_logging.rs b/foundations/tests/stdio_logging.rs index a54d7c53..5d45f2e8 100644 --- a/foundations/tests/stdio_logging.rs +++ b/foundations/tests/stdio_logging.rs @@ -108,7 +108,9 @@ fn wait_for_drops(reason: &str, count: u64, original_stderr: &OwnedFd) { while dropped(reason) < Some(count) { if Instant::now() >= deadline { nix::unistd::dup2_stderr(original_stderr).unwrap(); - panic!("the logging thread did not drop records for {reason}"); + fail_without_panicking(&format!( + "the logging thread did not drop records for {reason}" + )); } std::thread::sleep(Duration::from_millis(5)); } @@ -189,8 +191,13 @@ async fn logging_does_not_wait_for_a_blocking_journal_stream() { let shutdown_finished = finishes_in_time(move || driver.shutdown_logger()); nix::unistd::dup2_stderr(&original_stderr).unwrap(); - assert!(!nonblocking, "the setting didn't leave the stream alone"); - assert!(shutdown_finished, "logger shutdown waited for the journal"); + // The stream blocks here, so a panic on the logging thread can be stuck in the panic hook. + if nonblocking { + fail_without_panicking("the setting didn't leave the stream alone"); + } + if !shutdown_finished { + fail_without_panicking("logger shutdown waited for the journal"); + } } #[tokio::test] From e71c7cafa00e220e5097d2c167af8e0a5455a4fd Mon Sep 17 00:00:00 2001 From: zaidoon Date: Tue, 10 Nov 2026 23:49:22 -0500 Subject: [PATCH 5/6] Say what journal_error means and what seccomp needs to allow With ignore_io_errors off, the logging thread stops after the first journal_error, so the counter stays at 1 while every later record is lost. The metric's help text and the DropReason docs say so, so an alert can fire on any increase. ignore_io_errors also says that a full journal stream isn't an I/O error. The allow_blocking_journal_stdio docs say that a stream stays nonblocking until the service restarts, and which of foundations' seccomp allow lists has sendto and fcntl. The comment on the record reset says what each formatter does with a record it fails to finish: JSON never flushes it, so it's discarded, and text flushes it cut short, so it goes out on its own line. --- foundations/src/telemetry/log/journal.rs | 11 +++++++---- foundations/src/telemetry/settings/logging.rs | 11 ++++++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/foundations/src/telemetry/log/journal.rs b/foundations/src/telemetry/log/journal.rs index 7ece8638..9b4609fc 100644 --- a/foundations/src/telemetry/log/journal.rs +++ b/foundations/src/telemetry/log/journal.rs @@ -213,8 +213,9 @@ impl io::Write for JournalWriter { fn write(&mut self, buf: &[u8]) -> io::Result { if self.new_record.load(Ordering::Relaxed) { self.new_record.store(false, Ordering::Relaxed); - // A formatter that fails part way through a record doesn't flush it. Discard what - // it wrote instead of sending it with the next record. + // The JSON formatter doesn't flush a record it fails to finish, so discard what it + // wrote instead of sending it with the next record. The text formatter flushes such + // a record anyway; it goes out cut short, and the next record starts a new line. self.clear(); } self.buf.extend_from_slice(buf); @@ -256,7 +257,8 @@ impl Drain for JournalDrain { pub(crate) enum DropReason { /// The journal stream had no room for the record. JournalFull, - /// Sending to the journal stream failed, for example because journald closed it. + /// Sending to the journal stream failed, for example because journald closed it. Unless + /// `ignore_io_errors` is set, the logging thread stops after the first one. JournalError, } @@ -268,7 +270,8 @@ mod metrics { #[crate::telemetry::metrics::metrics(crate_path = "crate", unprefixed)] pub(crate) mod logging { - /// Number of log records dropped instead of written. + /// Number of log records dropped instead of written to the systemd journal stream. + /// Unless `ignore_io_errors` is set, logging stops after the first `journal_error`. pub fn records_dropped(reason: DropReason) -> Counter; } diff --git a/foundations/src/telemetry/settings/logging.rs b/foundations/src/telemetry/settings/logging.rs index 9a3a74d0..7a45d43e 100644 --- a/foundations/src/telemetry/settings/logging.rs +++ b/foundations/src/telemetry/settings/logging.rs @@ -44,6 +44,9 @@ pub struct LoggingSettings { /// By default, I/O errors from terminal, stderr, and file outputs are treated /// as fatal logging failures. Enabling this setting allows services to keep /// running if a log sink returns an error such as [`std::io::ErrorKind::BrokenPipe`]. + /// + /// A systemd journal stream with no room for a record isn't an I/O error: the record is + /// dropped and counted in the `logging_records_dropped` metric, and logging carries on. pub ignore_io_errors: bool, /// Whether to let stdout and stderr block when they are the systemd journal stream. @@ -59,11 +62,13 @@ pub struct LoggingSettings { /// in child processes that inherit the stream. When the stream's buffer is full, the default /// panic hook's message can be lost, and `println!` and `eprintln!` panic instead of waiting. /// Enabling this setting leaves the streams as they are; it doesn't make a nonblocking stream - /// blocking again. + /// blocking again. A stream stays nonblocking until the service restarts and systemd gives it + /// a new one, including when a process that inherits the stream enables this setting. /// /// If seccomp sandboxing applies to the thread that initializes telemetry, the logging thread - /// needs `sendto` to write to the journal stream, and initialization needs `fcntl` unless this - /// setting is enabled. + /// needs `sendto` to write to the journal stream. `SERVICE_BASICS` doesn't allow it, and + /// `NET_SOCKET_API` does. Initialization also needs `fcntl` unless this setting is enabled, + /// and `RUST_BASICS` only allows `fcntl` in debug builds. pub allow_blocking_journal_stdio: bool, /// Settings for rate limiting emission of log events From 380fa721bcc5b399a275d3abe48ce8c55d501db0 Mon Sep 17 00:00:00 2001 From: zaidoon Date: Tue, 10 Nov 2026 23:49:54 -0500 Subject: [PATCH 6/6] Address review: compare JOURNAL_STREAM as a str, explain the flag check OsStr compares with str directly, so is_journal doesn't need to go through encoded bytes. The record-start flag check in JournalWriter::write is a load followed by a store, which reads like a missing compare-and-swap. A comment explains that only the logging thread touches the flag, and that the load keeps every write after a record's first to a plain read. --- foundations/src/telemetry/log/journal.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/foundations/src/telemetry/log/journal.rs b/foundations/src/telemetry/log/journal.rs index 9b4609fc..37ff8c77 100644 --- a/foundations/src/telemetry/log/journal.rs +++ b/foundations/src/telemetry/log/journal.rs @@ -89,8 +89,7 @@ fn is_journal(fd: BorrowedFd<'_>, journal_stream: &OsStr) -> bool { let metadata = unsafe { metadata.assume_init() }; metadata.st_mode & libc::S_IFMT == libc::S_IFSOCK - && journal_stream.as_encoded_bytes() - == format!("{}:{}", metadata.st_dev, metadata.st_ino).as_bytes() + && journal_stream == format!("{}:{}", metadata.st_dev, metadata.st_ino).as_str() } /// Sets `O_NONBLOCK` on `fd`, leaving it as it was if that fails. @@ -211,6 +210,10 @@ fn send_nonblocking(fd: BorrowedFd<'_>, buf: &[u8]) -> io::Result { impl io::Write for JournalWriter { fn write(&mut self, buf: &[u8]) -> io::Result { + // Only the logging thread touches this flag: the drain sets it before formatting a + // record, and the first write of the record clears it. So nothing can change it between + // the load and the store, and the load keeps every later write to a plain read, where a + // swap would be a read-modify-write each time. if self.new_record.load(Ordering::Relaxed) { self.new_record.store(false, Ordering::Relaxed); // The JSON formatter doesn't flush a record it fails to finish, so discard what it