diff --git a/foundations/Cargo.toml b/foundations/Cargo.toml index 8a1e12c4..5712ad67 100644 --- a/foundations/Cargo.toml +++ b/foundations/Cargo.toml @@ -140,6 +140,7 @@ tokio-runtime-metrics = [ # Enables logging functionality. logging = [ "ratelimit", + "dep:libc", "dep:parking_lot", "dep:slog-async", "dep:slog-json", diff --git a/foundations/src/telemetry/log/init.rs b/foundations/src/telemetry/log/init.rs index 58989c3e..dc121454 100644 --- a/foundations/src/telemetry/log/init.rs +++ b/foundations/src/telemetry/log/init.rs @@ -2,6 +2,8 @@ use super::field_dedup::FieldDedupFilterFactory; use super::field_filtering::{FieldFilteringDrain, FilterFactory}; use super::field_redact::FieldRedactFilterFactory; use super::internal::{LoggerWithKvNestingTracking, SharedLog}; +#[cfg(target_os = "linux")] +use super::journal::{JournalStreams, JournalWriter, stdio_fd}; #[cfg(feature = "metrics")] use crate::telemetry::log::log_volume::LogVolumeMetricsDrain; @@ -23,6 +25,7 @@ use std::fmt::Debug; use std::fs::File; use std::io; use std::io::BufWriter; +use std::mem::ManuallyDrop; use std::sync::{Arc, LazyLock, OnceLock}; type BoxedDebug = Box; @@ -90,7 +93,27 @@ pub(crate) fn init( // NOTE: OXY-178, default is 128 (https://docs.rs/slog-async/2.7.0/src/slog_async/lib.rs.html#251) const CHANNEL_SIZE: usize = 1024; + // Before building any drain, so that stderr is covered when logs go elsewhere too. + #[cfg(target_os = "linux")] + let journal = JournalStreams::detect(!settings.allow_blocking_journal_stdio); + let (async_drain, async_guard) = match (&settings.output, &settings.format) { + #[cfg(target_os = "linux")] + (output @ (LogOutput::Terminal | LogOutput::Stderr), LogFormat::Text) + if journal.includes(output) => + { + let drain = JournalWriter::drain(stdio_fd(output), |writer| { + TextDrain::new(PlainDecorator::new(writer)).build() + }); + build_async_drain(drain, settings, CHANNEL_SIZE) + } + #[cfg(target_os = "linux")] + (output @ (LogOutput::Terminal | LogOutput::Stderr), LogFormat::Json) + if journal.includes(output) => + { + let drain = JournalWriter::drain(stdio_fd(output), build_json_log_drain); + build_async_drain(drain, settings, CHANNEL_SIZE) + } (output @ (LogOutput::Terminal | LogOutput::Stderr), LogFormat::Text) => { let decorator = if matches!(output, LogOutput::Terminal) { TermDecorator::new().stdout().build() @@ -154,48 +177,69 @@ pub(crate) fn init( Ok(Some(async_guard)) } +/// Writes to stdout or stderr through a [`File`] that never closes them. +/// +/// The log drain is dropped when the logger shuts down. Closing the descriptor then would +/// let the next file or socket opened take its number, and later writes to stdout or stderr, +/// such as a panic message, would go there. +struct StdioWriter(ManuallyDrop); + +impl io::Write for StdioWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.write(buf) + } + + fn write_vectored(&mut self, bufs: &[io::IoSlice<'_>]) -> io::Result { + self.0.write_vectored(bufs) + } + + fn flush(&mut self) -> io::Result<()> { + self.0.flush() + } +} + /// Opens fd 1 directly and wraps with a [`BufWriter`] with [`BUF_SIZE`] capacity. /// /// [`io::Stdout`] uses a [`io::LineWriter`] which may cause unnecessary flushing. #[cfg(not(target_family = "windows"))] -fn stdout_writer_without_line_buffering() -> BufWriter { +fn stdout_writer_without_line_buffering() -> BufWriter { use std::os::fd::{AsRawFd, FromRawFd}; let stdout = std::io::stdout(); let stdout = unsafe { File::from_raw_fd(stdout.as_raw_fd()) }; - BufWriter::with_capacity(BUF_SIZE, stdout) + BufWriter::with_capacity(BUF_SIZE, StdioWriter(ManuallyDrop::new(stdout))) } /// Opens fd 1 directly and wraps with a [`BufWriter`] with [`BUF_SIZE`] capacity. /// /// [`io::Stdout`] uses a [`io::LineWriter`] which may cause unnecessary flushing. #[cfg(target_family = "windows")] -fn stdout_writer_without_line_buffering() -> BufWriter { +fn stdout_writer_without_line_buffering() -> BufWriter { use std::os::windows::io::{AsRawHandle, FromRawHandle}; let stdout = std::io::stdout(); let stdout = unsafe { File::from_raw_handle(stdout.as_raw_handle()) }; - BufWriter::with_capacity(BUF_SIZE, stdout) + BufWriter::with_capacity(BUF_SIZE, StdioWriter(ManuallyDrop::new(stdout))) } /// Opens fd 2 directly and wraps with a [`BufWriter`] with [`BUF_SIZE`] capacity. #[cfg(not(target_family = "windows"))] -fn stderr_writer_without_line_buffering() -> BufWriter { +fn stderr_writer_without_line_buffering() -> BufWriter { use std::os::fd::{AsRawFd, FromRawFd}; let stderr = std::io::stderr(); let stderr = unsafe { File::from_raw_fd(stderr.as_raw_fd()) }; - BufWriter::with_capacity(BUF_SIZE, stderr) + BufWriter::with_capacity(BUF_SIZE, StdioWriter(ManuallyDrop::new(stderr))) } /// Opens fd 2 directly and wraps with a [`BufWriter`] with [`BUF_SIZE`] capacity. #[cfg(target_family = "windows")] -fn stderr_writer_without_line_buffering() -> BufWriter { +fn stderr_writer_without_line_buffering() -> BufWriter { use std::os::windows::io::{AsRawHandle, FromRawHandle}; let stderr = std::io::stderr(); let stderr = unsafe { File::from_raw_handle(stderr.as_raw_handle()) }; - BufWriter::with_capacity(BUF_SIZE, stderr) + BufWriter::with_capacity(BUF_SIZE, StdioWriter(ManuallyDrop::new(stderr))) } pub(crate) fn wrap_root_drain(settings: &LoggingSettings, drain: D) -> SharedDrain diff --git a/foundations/src/telemetry/log/journal.rs b/foundations/src/telemetry/log/journal.rs new file mode 100644 index 00000000..37ff8c77 --- /dev/null +++ b/foundations/src/telemetry/log/journal.rs @@ -0,0 +1,470 @@ +//! Log output to the systemd journal that never waits for journald. +//! +//! systemd connects a service's stdout and stderr to a journald socket and names it in +//! `JOURNAL_STREAM`. If journald stops reading the socket while keeping it open, which a bug +//! can leave it doing for good, the socket's send buffer fills and every blocking write to it +//! waits for good: the logging thread, then [`TelemetryDriver::shutdown_logger`], and any thread +//! that panics, because the default panic hook writes the message to stderr. +//! +//! So log records go to the journal stream through [`JournalWriter`], which never waits, and +//! unless [`LoggingSettings::allow_blocking_journal_stdio`] is on, the stream is made nonblocking +//! for every other writer too. +//! +//! [`TelemetryDriver::shutdown_logger`]: crate::telemetry::TelemetryDriver::shutdown_logger +//! [`LoggingSettings::allow_blocking_journal_stdio`]: crate::telemetry::settings::LoggingSettings::allow_blocking_journal_stdio + +use crate::telemetry::settings::LogOutput; +use slog::{Drain, OwnedKVList, Record}; +use std::ffi::OsStr; +use std::io; +use std::os::fd::{AsFd, AsRawFd, BorrowedFd}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +// A record buffer larger than this is freed after use, rather than kept for the next record. +const MAX_RETAINED_RECORD_CAPACITY: usize = 64 * 1024; + +/// Which of stdout and stderr are the journal stream. +#[derive(Debug, Default, Clone, Copy)] +pub(super) struct JournalStreams { + stdout: bool, + stderr: bool, +} + +impl JournalStreams { + /// Finds which of stdout and stderr are the journal stream, and makes them nonblocking if + /// `nonblocking` is set. + /// + /// A check that fails counts as not the journal, and a descriptor whose flags can't be set + /// stays as it was, so this never stops logging from starting. + pub(super) fn detect(nonblocking: bool) -> Self { + let Some(journal_stream) = std::env::var_os("JOURNAL_STREAM") else { + return Self::default(); + }; + let detect = |fd: BorrowedFd<'_>| { + let is_journal = is_journal(fd, &journal_stream); + if is_journal && nonblocking { + set_nonblocking(fd); + } + is_journal + }; + + Self { + stdout: detect(io::stdout().as_fd()), + stderr: detect(io::stderr().as_fd()), + } + } + + /// Whether `output` writes to the journal stream. + pub(super) fn includes(&self, output: &LogOutput) -> bool { + match output { + LogOutput::Terminal => self.stdout, + LogOutput::Stderr => self.stderr, + _ => false, + } + } +} + +/// The stdout or stderr descriptor that `output` writes to. +pub(super) fn stdio_fd(output: &LogOutput) -> BorrowedFd<'static> { + let fd = if matches!(output, LogOutput::Terminal) { + libc::STDOUT_FILENO + } else { + libc::STDERR_FILENO + }; + // SAFETY: std treats stdout and stderr as open for the life of the process. Borrowing + // rather than owning them means the logging thread never closes them when it exits, which + // would let the next file or socket opened take their numbers. + unsafe { BorrowedFd::borrow_raw(fd) } +} + +/// Whether `fd` is the socket named by `journal_stream`, the way systemd names it. +fn is_journal(fd: BorrowedFd<'_>, journal_stream: &OsStr) -> bool { + let mut metadata = std::mem::MaybeUninit::::uninit(); + // SAFETY: `fd` is borrowed, and fstat initializes `metadata` on success. + if unsafe { libc::fstat(fd.as_raw_fd(), metadata.as_mut_ptr()) } == -1 { + return false; + } + // SAFETY: fstat succeeded. + let metadata = unsafe { metadata.assume_init() }; + + metadata.st_mode & libc::S_IFMT == libc::S_IFSOCK + && journal_stream == format!("{}:{}", metadata.st_dev, metadata.st_ino).as_str() +} + +/// Sets `O_NONBLOCK` on `fd`, leaving it as it was if that fails. +fn set_nonblocking(fd: BorrowedFd<'_>) { + // SAFETY: F_GETFL takes no argument and does not close `fd`. + let flags = unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFL) }; + if flags != -1 && flags & libc::O_NONBLOCK == 0 { + // SAFETY: F_SETFL takes an integer argument and does not close `fd`. + unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_SETFL, flags | libc::O_NONBLOCK) }; + } +} + +/// Writes each log record to the journal stream without ever waiting, and drops the record if +/// the stream has no room for it. +/// +/// Records are buffered until the formatter flushes at the end of each one, then sent at once. +/// Sends pass `MSG_DONTWAIT`, so they don't wait even if something sharing the descriptor makes +/// it blocking. A record that only partly fits is cut short, and the next record that gets +/// through starts with a newline, so journald doesn't join the two into one entry. +pub(super) struct JournalWriter { + out: W, + /// A newline, then the record being formatted. The newline is sent only to end a record + /// that was cut short. + buf: Vec, + cut_short: bool, + new_record: Arc, +} + +impl JournalWriter { + /// Builds the formatting drain for records to `out`, and tells its writer where each + /// record starts, including after a record that failed to format. + pub(super) fn drain(out: W, build: impl FnOnce(Self) -> D) -> JournalDrain { + metrics::register(); + let new_record = Arc::new(AtomicBool::new(true)); + let writer = Self { + out, + buf: vec![b'\n'], + cut_short: false, + new_record: Arc::clone(&new_record), + }; + + JournalDrain { + inner: build(writer), + new_record, + } + } + + fn clear(&mut self) { + self.buf.truncate(1); + self.buf.shrink_to(MAX_RETAINED_RECORD_CAPACITY); + } + + /// Sends the buffered record from `start`, in as many sends as the stream takes without + /// waiting, and counts it if it's dropped. + fn send(&mut self, start: usize) -> io::Result<()> { + let record = &self.buf[start..]; + let mut sent = 0; + let res = loop { + match send_nonblocking(self.out.as_fd(), &record[sent..]) { + Ok(0) => break Err(io::ErrorKind::WriteZero.into()), + Ok(n) => { + sent += n; + if sent == record.len() { + break Ok(()); + } + } + Err(err) if err.kind() == io::ErrorKind::Interrupted => {} + Err(err) => break Err(err), + } + }; + if let Some(&last) = record[..sent].last() { + self.cut_short = last != b'\n'; + } + + match res { + Err(err) if err.kind() == io::ErrorKind::WouldBlock => { + metrics::count(DropReason::JournalFull); + Ok(()) + } + Err(err) => { + metrics::count(DropReason::JournalError); + Err(err) + } + Ok(()) => Ok(()), + } + } +} + +/// Sends without waiting, whatever the descriptor's own blocking mode is. +/// +/// If something replaced stdout or stderr with a file that isn't a socket after logging +/// started, this writes to it the way any other writer would, which waits if that file does. +fn send_nonblocking(fd: BorrowedFd<'_>, buf: &[u8]) -> io::Result { + // SAFETY: `buf` is valid for `buf.len()` bytes, and send does not take ownership of `fd`. + let sent = unsafe { + libc::send( + fd.as_raw_fd(), + buf.as_ptr().cast(), + buf.len(), + libc::MSG_DONTWAIT | libc::MSG_NOSIGNAL, + ) + }; + if sent >= 0 { + return Ok(sent as usize); + } + let err = io::Error::last_os_error(); + if err.raw_os_error() != Some(libc::ENOTSOCK) { + return Err(err); + } + + // SAFETY: `buf` is valid for `buf.len()` bytes, and write does not take ownership of `fd`. + let written = unsafe { libc::write(fd.as_raw_fd(), buf.as_ptr().cast(), buf.len()) }; + if written < 0 { + return Err(io::Error::last_os_error()); + } + Ok(written as usize) +} + +impl io::Write for JournalWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + // Only the logging thread touches this flag: the drain sets it before formatting a + // record, and the first write of the record clears it. So nothing can change it between + // the load and the store, and the load keeps every later write to a plain read, where a + // swap would be a read-modify-write each time. + if self.new_record.load(Ordering::Relaxed) { + self.new_record.store(false, Ordering::Relaxed); + // The JSON formatter doesn't flush a record it fails to finish, so discard what it + // wrote instead of sending it with the next record. The text formatter flushes such + // a record anyway; it goes out cut short, and the next record starts a new line. + self.clear(); + } + self.buf.extend_from_slice(buf); + Ok(buf.len()) + } + + /// Sends the buffered record. The log formatters flush at the end of each record. + fn flush(&mut self) -> io::Result<()> { + if self.buf.len() == 1 { + return Ok(()); + } + let res = self.send(usize::from(!self.cut_short)); + self.clear(); + res + } +} + +/// Marks the start of each record for its [`JournalWriter`]. +pub(super) struct JournalDrain { + inner: D, + new_record: Arc, +} + +impl Drain for JournalDrain { + type Ok = D::Ok; + type Err = D::Err; + + fn log(&self, record: &Record, values: &OwnedKVList) -> Result { + // The formatter and writer run on the same logging thread. The shared flag only + // connects the drain to its writer; it does not synchronize threads. + self.new_record.store(true, Ordering::Relaxed); + self.inner.log(record, values) + } +} + +/// Why a log record was dropped instead of written. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum DropReason { + /// The journal stream had no room for the record. + JournalFull, + /// Sending to the journal stream failed, for example because journald closed it. Unless + /// `ignore_io_errors` is set, the logging thread stops after the first one. + JournalError, +} + +#[cfg(feature = "metrics")] +mod metrics { + use super::DropReason; + use crate::telemetry::metrics::Counter; + use std::sync::LazyLock; + + #[crate::telemetry::metrics::metrics(crate_path = "crate", unprefixed)] + pub(crate) mod logging { + /// Number of log records dropped instead of written to the systemd journal stream. + /// Unless `ignore_io_errors` is set, logging stops after the first `journal_error`. + pub fn records_dropped(reason: DropReason) -> Counter; + } + + struct Counters { + journal_full: Counter, + journal_error: Counter, + } + + static COUNTERS: LazyLock = LazyLock::new(|| Counters { + journal_full: logging::records_dropped(DropReason::JournalFull), + journal_error: logging::records_dropped(DropReason::JournalError), + }); + + /// Creates the counters at zero, so they exist before the first drop. + pub(super) fn register() { + LazyLock::force(&COUNTERS); + } + + pub(super) fn count(reason: DropReason) { + match reason { + DropReason::JournalFull => &COUNTERS.journal_full, + DropReason::JournalError => &COUNTERS.journal_error, + } + .inc(); + } +} + +#[cfg(not(feature = "metrics"))] +mod metrics { + pub(super) fn register() {} + + pub(super) fn count(_reason: super::DropReason) {} +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read, Write}; + use std::os::unix::fs::MetadataExt; + use std::os::unix::net::UnixStream; + use std::sync::mpsc; + use std::time::Duration; + + fn stream_id(fd: &impl AsFd) -> String { + let metadata = std::fs::File::from(fd.as_fd().try_clone_to_owned().unwrap()) + .metadata() + .unwrap(); + format!("{}:{}", metadata.dev(), metadata.ino()) + } + + /// A writer to a blocking journal stream, and journald's end of it. + fn journal_pair() -> (JournalWriter, UnixStream) { + let (stream, journald) = UnixStream::pair().unwrap(); + let drain = JournalWriter::drain(stream, |writer| writer); + (drain.inner, journald) + } + + /// Fills `writer`'s stream so that nothing more fits, and leaves it blocking. + fn fill(writer: &JournalWriter) { + let stream = &writer.out; + stream.set_nonblocking(true).unwrap(); + loop { + match (&*stream).write(&[b'\n'; 8192]) { + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::WouldBlock => break, + Err(err) => panic!("filling the journal stream: {err}"), + } + } + stream.set_nonblocking(false).unwrap(); + } + + fn read_available(journald: &mut UnixStream) -> Vec { + journald.set_nonblocking(true).unwrap(); + let mut received = Vec::new(); + let mut buf = [0; 64 * 1024]; + loop { + match journald.read(&mut buf) { + Ok(0) => return received, + Ok(n) => received.extend_from_slice(&buf[..n]), + Err(err) if err.kind() == io::ErrorKind::WouldBlock => return received, + Err(err) => panic!("read failed: {err}"), + } + } + } + + /// Writes and flushes `record` on another thread, failing the test if that waits. + fn log_without_waiting( + mut writer: JournalWriter, + record: &[u8], + ) -> (JournalWriter, io::Result<()>) { + let record = record.to_vec(); + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + writer.new_record.store(true, Ordering::Relaxed); + writer.write_all(&record).unwrap(); + let res = writer.flush(); + done_tx.send((writer, res)).unwrap(); + }); + done_rx + .recv_timeout(Duration::from_secs(10)) + .expect("the write waited for journald") + } + + #[test] + fn only_the_named_socket_is_the_journal() { + let (stream, _journald) = UnixStream::pair().unwrap(); + + assert!(!is_journal(stream.as_fd(), OsStr::new("1:2"))); + assert!(is_journal(stream.as_fd(), OsStr::new(&stream_id(&stream)))); + } + + #[test] + fn a_file_is_not_the_journal() { + let file = tempfile::tempfile().unwrap(); + + assert!(!is_journal(file.as_fd(), OsStr::new(&stream_id(&file)))); + } + + #[test] + fn sends_each_record_when_flushed() { + let (mut writer, mut journald) = journal_pair(); + + writer.write_all(b"first ").unwrap(); + writer.write_all(b"record\n").unwrap(); + assert!(read_available(&mut journald).is_empty()); + writer.flush().unwrap(); + // The text formatter flushes a second time when it finishes a record. + writer.flush().unwrap(); + writer.write_all(b"second record\n").unwrap(); + writer.flush().unwrap(); + + assert_eq!( + read_available(&mut journald), + b"first record\nsecond record\n" + ); + } + + #[test] + fn drops_records_that_do_not_fit_without_waiting() { + let (writer, mut journald) = journal_pair(); + fill(&writer); + + let (writer, res) = log_without_waiting(writer, b"dropped record\n"); + res.unwrap(); + let filled = read_available(&mut journald); + assert!(filled.iter().all(|&b| b == b'\n')); + + let (_writer, res) = log_without_waiting(writer, b"next record\n"); + res.unwrap(); + assert_eq!(read_available(&mut journald), b"next record\n"); + } + + #[test] + fn ends_a_record_cut_short_before_the_next_one() { + let (writer, mut journald) = journal_pair(); + // Far larger than a socket send buffer normally is, so only part of it fits. + let big = format!("{}\n", "x".repeat(32 * 1024 * 1024)); + + let (writer, res) = log_without_waiting(writer, big.as_bytes()); + res.unwrap(); + let received = read_available(&mut journald); + assert!(!received.is_empty() && received.len() < big.len()); + assert!(!received.contains(&b'\n')); + + let (_writer, res) = log_without_waiting(writer, b"next record\n"); + res.unwrap(); + assert_eq!(read_available(&mut journald), b"\nnext record\n"); + } + + #[test] + fn keeps_the_buffer_of_a_normal_record_and_frees_a_huge_one() { + let (writer, _journald) = journal_pair(); + + let (writer, res) = log_without_waiting(writer, &[b'x'; 8 * 1024]); + res.unwrap(); + let capacity = writer.buf.capacity(); + assert!(capacity > 8 * 1024, "{capacity}"); + + let (writer, res) = log_without_waiting(writer, &vec![b'x'; 1024 * 1024]); + res.unwrap(); + let capacity = writer.buf.capacity(); + assert!(capacity <= 64 * 1024, "{capacity}"); + } + + #[test] + fn reports_a_closed_journal_as_an_error() { + let (writer, journald) = journal_pair(); + drop(journald); + + let (_writer, res) = log_without_waiting(writer, b"record\n"); + assert_eq!(res.unwrap_err().kind(), io::ErrorKind::BrokenPipe); + } +} diff --git a/foundations/src/telemetry/log/mod.rs b/foundations/src/telemetry/log/mod.rs index 73717082..c4f05f24 100644 --- a/foundations/src/telemetry/log/mod.rs +++ b/foundations/src/telemetry/log/mod.rs @@ -3,6 +3,8 @@ mod field_dedup; mod field_filtering; mod field_redact; +#[cfg(target_os = "linux")] +mod journal; mod rate_limit; pub(crate) mod init; diff --git a/foundations/src/telemetry/settings/logging.rs b/foundations/src/telemetry/settings/logging.rs index 0e1a0ea4..7a45d43e 100644 --- a/foundations/src/telemetry/settings/logging.rs +++ b/foundations/src/telemetry/settings/logging.rs @@ -44,8 +44,33 @@ pub struct LoggingSettings { /// By default, I/O errors from terminal, stderr, and file outputs are treated /// as fatal logging failures. Enabling this setting allows services to keep /// running if a log sink returns an error such as [`std::io::ErrorKind::BrokenPipe`]. + /// + /// A systemd journal stream with no room for a record isn't an I/O error: the record is + /// dropped and counted in the `logging_records_dropped` metric, and logging carries on. pub ignore_io_errors: bool, + /// Whether to let stdout and stderr block when they are the systemd journal stream. + /// + /// On Linux, logs written to the journal stream that systemd names in `JOURNAL_STREAM` never + /// wait for journald, whatever this setting is. Records that don't fit in the stream's buffer + /// are dropped and counted in the `logging_records_dropped` metric. Other writes to the + /// stream can wait, though, and if journald stops reading it for good, they wait for good. + /// That includes the default panic hook's message, so a thread that panics never finishes. + /// By default, foundations makes them nonblocking to prevent that, whatever the log output is. + /// + /// Nonblocking mode belongs to the stream, so it applies to every writer in the process and + /// in child processes that inherit the stream. When the stream's buffer is full, the default + /// panic hook's message can be lost, and `println!` and `eprintln!` panic instead of waiting. + /// Enabling this setting leaves the streams as they are; it doesn't make a nonblocking stream + /// blocking again. A stream stays nonblocking until the service restarts and systemd gives it + /// a new one, including when a process that inherits the stream enables this setting. + /// + /// If seccomp sandboxing applies to the thread that initializes telemetry, the logging thread + /// needs `sendto` to write to the journal stream. `SERVICE_BASICS` doesn't allow it, and + /// `NET_SOCKET_API` does. Initialization also needs `fcntl` unless this setting is enabled, + /// and `RUST_BASICS` only allows `fcntl` in debug builds. + pub allow_blocking_journal_stdio: bool, + /// Settings for rate limiting emission of log events pub rate_limit: RateLimitingSettings, diff --git a/foundations/tests/stdio_logging.rs b/foundations/tests/stdio_logging.rs new file mode 100644 index 00000000..5d45f2e8 --- /dev/null +++ b/foundations/tests/stdio_logging.rs @@ -0,0 +1,417 @@ +#![cfg(all(target_os = "linux", feature = "telemetry"))] +//! These tests replace stdout and stderr and initialize telemetry, so they assume a separate +//! process is used for each test. Make sure you run with `cargo nextest run`. + +mod common; + +use common::collect_text; +use foundations::service_info; +use foundations::telemetry::settings::{LogFormat, LogOutput, LoggingSettings, TelemetrySettings}; +use foundations::telemetry::{self, TelemetryConfig, TelemetryDriver}; +use std::io; +use std::io::{Read, Seek, Write}; +use std::os::fd::{AsFd, AsRawFd, BorrowedFd, OwnedFd}; +use std::os::unix::fs::MetadataExt; +use std::os::unix::net::UnixStream; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +fn init_logging(output: LogOutput, format: LogFormat) -> TelemetryDriver { + init_logging_with_settings(LoggingSettings { + output, + format, + ..Default::default() + }) +} + +fn init_logging_with_settings(logging: LoggingSettings) -> TelemetryDriver { + let settings = TelemetrySettings { + logging, + ..Default::default() + }; + + telemetry::init(TelemetryConfig { + service_info: &service_info!(), + settings: &settings, + custom_server_routes: Default::default(), + }) + .unwrap() +} + +/// Puts `stream` on stderr and names it as the journal stream, as systemd does. +/// +/// Returns the original stderr. +fn pose_as_journal(stream: &UnixStream) -> OwnedFd { + let original = io::stderr().as_fd().try_clone_to_owned().unwrap(); + nix::unistd::dup2_stderr(stream).unwrap(); + // SAFETY: nothing else in this test process reads or writes the environment meanwhile. + unsafe { std::env::set_var("JOURNAL_STREAM", stream_id(stream.as_fd())) }; + + original +} + +fn stream_id(fd: BorrowedFd<'_>) -> String { + let metadata = std::fs::File::from(fd.try_clone_to_owned().unwrap()) + .metadata() + .unwrap(); + format!("{}:{}", metadata.dev(), metadata.ino()) +} + +fn is_nonblocking(fd: BorrowedFd<'_>) -> bool { + // SAFETY: F_GETFL only reads the borrowed descriptor's flags. + let flags = unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFL) }; + assert_ne!(flags, -1); + flags & libc::O_NONBLOCK != 0 +} + +/// Fills `stream` so that nothing more fits, leaving its blocking mode as it was. +fn fill_stream(stream: &UnixStream) { + let nonblocking = is_nonblocking(stream.as_fd()); + stream.set_nonblocking(true).unwrap(); + loop { + match (&*stream).write(&[b'\n'; 8192]) { + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::WouldBlock => break, + Err(err) => panic!("filling the journal stream: {err}"), + } + } + stream.set_nonblocking(nonblocking).unwrap(); +} + +fn read_available(journald: &mut UnixStream) -> String { + journald.set_nonblocking(true).unwrap(); + let mut received = Vec::new(); + let mut buf = [0; 64 * 1024]; + loop { + match journald.read(&mut buf) { + Ok(0) => break, + Ok(n) => received.extend_from_slice(&buf[..n]), + Err(err) if err.kind() == io::ErrorKind::WouldBlock => break, + Err(err) => panic!("read failed: {err}"), + } + } + String::from_utf8(received).unwrap() +} + +fn dropped(reason: &str) -> Option { + let series = format!("logging_records_dropped{{reason=\"{reason}\"}} "); + collect_text(&Default::default()) + .lines() + .find_map(|line| line.strip_prefix(&series)?.parse().ok()) +} + +/// Waits until at least `count` records were dropped for `reason`. +fn wait_for_drops(reason: &str, count: u64, original_stderr: &OwnedFd) { + let deadline = Instant::now() + Duration::from_secs(10); + while dropped(reason) < Some(count) { + if Instant::now() >= deadline { + nix::unistd::dup2_stderr(original_stderr).unwrap(); + fail_without_panicking(&format!( + "the logging thread did not drop records for {reason}" + )); + } + std::thread::sleep(Duration::from_millis(5)); + } +} + +fn finishes_in_time(f: impl FnOnce() + Send + 'static) -> bool { + let (done_tx, done_rx) = mpsc::channel(); + std::thread::spawn(move || { + f(); + let _ = done_tx.send(()); + }); + done_rx.recv_timeout(Duration::from_secs(10)).is_ok() +} + +/// Fails the test without going through the panic hook. +/// +/// A panic stuck writing to the journal holds the default hook's backtrace lock, so the +/// panic of a failed assertion would wait for that lock forever instead of failing the test. +fn fail_without_panicking(message: &str) -> ! { + let _ = nix::unistd::write(io::stderr(), format!("{message}\n").as_bytes()); + std::process::exit(101); +} + +#[tokio::test] +async fn stalled_journal_does_not_hang_logging_panics_or_shutdown() { + let (stream, mut journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging(LogOutput::Stderr, LogFormat::Json); + let nonblocking = is_nonblocking(io::stderr().as_fd()); + + // journald isn't reading, and the stream is full. + fill_stream(&stream); + for _ in 0..3 { + telemetry::log::warn!("dropped record"); + } + wait_for_drops("journal_full", 3, &original_stderr); + let panic_finished = finishes_in_time(|| { + let _ = std::thread::spawn(|| panic!("panic while the journal is stalled")).join(); + }); + // Once journald reads again, the logging thread writes the next record. + read_available(&mut journald); + telemetry::log::warn!("after the stall"); + let shutdown_finished = finishes_in_time(move || driver.shutdown_logger()); + + // Restore stderr, so a failure can report itself. + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + if !nonblocking { + fail_without_panicking("the journal stream is still blocking"); + } + if !panic_finished { + fail_without_panicking("the panic waited for the journal"); + } + if !shutdown_finished { + fail_without_panicking("logger shutdown waited for the journal"); + } + + let received = read_available(&mut journald); + let last = received.lines().last().unwrap_or_default(); + let record: serde_json::Value = serde_json::from_str(last).unwrap(); + assert_eq!(record["msg"], "after the stall", "{received}"); +} + +#[tokio::test] +async fn logging_does_not_wait_for_a_blocking_journal_stream() { + let (stream, _journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging_with_settings(LoggingSettings { + output: LogOutput::Stderr, + format: LogFormat::Json, + allow_blocking_journal_stdio: true, + ..Default::default() + }); + let nonblocking = is_nonblocking(io::stderr().as_fd()); + fill_stream(&stream); + + telemetry::log::warn!("written while the stream blocks"); + wait_for_drops("journal_full", 1, &original_stderr); + let shutdown_finished = finishes_in_time(move || driver.shutdown_logger()); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + + // The stream blocks here, so a panic on the logging thread can be stuck in the panic hook. + if nonblocking { + fail_without_panicking("the setting didn't leave the stream alone"); + } + if !shutdown_finished { + fail_without_panicking("logger shutdown waited for the journal"); + } +} + +#[tokio::test] +async fn records_dropped_after_journald_closes_the_stream_are_counted() { + let (stream, journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging_with_settings(LoggingSettings { + output: LogOutput::Stderr, + format: LogFormat::Json, + ignore_io_errors: true, + ..Default::default() + }); + assert_eq!(dropped("journal_error"), Some(0)); + drop(journald); + + telemetry::log::warn!("first record after journald closed the stream"); + telemetry::log::warn!("second record after journald closed the stream"); + wait_for_drops("journal_error", 2, &original_stderr); + driver.shutdown_logger(); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); +} + +#[tokio::test] +async fn failed_json_records_are_discarded_before_the_next_record() { + struct FailingContext { + fail: Arc, + failures: Arc, + } + + impl slog::Value for FailingContext { + fn serialize( + &self, + _record: &slog::Record, + key: slog::Key, + serializer: &mut dyn slog::Serializer, + ) -> slog::Result { + serializer.emit_str(key, &"x".repeat(256))?; + if self.fail.load(Ordering::Relaxed) { + self.failures.fetch_add(1, Ordering::Relaxed); + return Err(io::Error::other("context serialization failed").into()); + } + Ok(()) + } + } + + let (stream, mut journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging_with_settings(LoggingSettings { + output: LogOutput::Stderr, + format: LogFormat::Json, + ignore_io_errors: true, + ..Default::default() + }); + let fail = Arc::new(AtomicBool::new(true)); + let failures = Arc::new(AtomicUsize::new(0)); + telemetry::log::add_fields!("context" => FailingContext { + fail: Arc::clone(&fail), + failures: Arc::clone(&failures), + }); + for _ in 0..128 { + telemetry::log::warn!("failed record"); + } + let deadline = Instant::now() + Duration::from_secs(10); + while failures.load(Ordering::Relaxed) < 128 { + if Instant::now() >= deadline { + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + panic!("the logging thread did not process the failed records"); + } + std::thread::yield_now(); + } + fail.store(false, Ordering::Relaxed); + telemetry::log::warn!("successful record"); + driver.shutdown_logger(); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + + let received = read_available(&mut journald); + let record: serde_json::Value = serde_json::from_str(&received).unwrap(); + assert_eq!(record["msg"], "successful record"); + assert_eq!(received.lines().count(), 1, "{received}"); +} + +#[tokio::test] +async fn text_records_reach_the_journal_one_per_line() { + let (stream, mut journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging(LogOutput::Stderr, LogFormat::Text); + + telemetry::log::warn!("first record"); + telemetry::log::warn!("second record"); + driver.shutdown_logger(); + + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + let received = read_available(&mut journald); + let lines: Vec<_> = received.lines().collect(); + assert_eq!(lines.len(), 2, "{received}"); + assert!(lines[0].contains("first record"), "{received}"); + assert!(lines[1].contains("second record"), "{received}"); + // The drop counters exist from the start, so alerts can use them. + assert_eq!(dropped("journal_full"), Some(0)); + assert_eq!(dropped("journal_error"), Some(0)); +} + +#[tokio::test] +async fn terminal_output_to_the_journal_does_not_block() { + let (stream, mut journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let original_stdout = io::stdout().as_fd().try_clone_to_owned().unwrap(); + nix::unistd::dup2_stdout(&stream).unwrap(); + let mut driver = init_logging(LogOutput::Terminal, LogFormat::Json); + + telemetry::log::warn!("to stdout"); + driver.shutdown_logger(); + + nix::unistd::dup2_stdout(&original_stdout).unwrap(); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + assert!(is_nonblocking(stream.as_fd())); + let received = read_available(&mut journald); + assert!(received.contains("\"msg\":\"to stdout\""), "{received}"); +} + +#[tokio::test] +async fn stderr_does_not_block_when_logs_go_to_a_file() { + let (stream, _journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let dir = tempfile::tempdir().unwrap(); + let _driver = init_logging(LogOutput::File(dir.path().join("log")), LogFormat::Json); + + let nonblocking = is_nonblocking(io::stderr().as_fd()); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + assert!(nonblocking); +} + +#[tokio::test] +async fn stderr_that_is_not_the_journal_is_left_alone() { + let (stream, _journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let (other, _other_reader) = UnixStream::pair().unwrap(); + nix::unistd::dup2_stderr(&other).unwrap(); + let _driver = init_logging(LogOutput::Stderr, LogFormat::Json); + + let nonblocking = is_nonblocking(io::stderr().as_fd()); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + assert!(!nonblocking); +} + +#[tokio::test] +async fn stderr_output_leaves_stderr_open_after_shutdown() { + let (stream, _reader) = UnixStream::pair().unwrap(); + let original_stderr = io::stderr().as_fd().try_clone_to_owned().unwrap(); + nix::unistd::dup2_stderr(&stream).unwrap(); + let mut driver = init_logging(LogOutput::Stderr, LogFormat::Json); + + telemetry::log::warn!("logged before shutdown"); + driver.shutdown_logger(); + let stderr = io::stderr().as_fd().try_clone_to_owned(); + + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + let stderr = stderr.expect("shutting down the logger closed stderr"); + assert_eq!(stream_id(stderr.as_fd()), stream_id(stream.as_fd())); +} + +#[tokio::test] +async fn terminal_output_leaves_stdout_open_after_shutdown() { + let (stream, _reader) = UnixStream::pair().unwrap(); + let original_stdout = io::stdout().as_fd().try_clone_to_owned().unwrap(); + nix::unistd::dup2_stdout(&stream).unwrap(); + let mut driver = init_logging(LogOutput::Terminal, LogFormat::Json); + + telemetry::log::warn!("logged before shutdown"); + driver.shutdown_logger(); + let stdout = io::stdout().as_fd().try_clone_to_owned(); + + nix::unistd::dup2_stdout(&original_stdout).unwrap(); + let stdout = stdout.expect("shutting down the logger closed stdout"); + assert_eq!(stream_id(stdout.as_fd()), stream_id(stream.as_fd())); +} + +#[tokio::test] +async fn logs_follow_stderr_when_something_replaces_the_journal() { + let (stream, _journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging(LogOutput::Stderr, LogFormat::Json); + let mut file = tempfile::tempfile().unwrap(); + nix::unistd::dup2_stderr(&file).unwrap(); + + telemetry::log::warn!("logged after stderr was replaced"); + driver.shutdown_logger(); + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + + let mut written = String::new(); + file.seek(io::SeekFrom::Start(0)).unwrap(); + file.read_to_string(&mut written).unwrap(); + assert!( + written.contains("\"msg\":\"logged after stderr was replaced\""), + "{written}" + ); + assert_eq!(dropped("journal_error"), Some(0)); +} + +#[tokio::test] +async fn journal_output_leaves_stderr_open_after_shutdown() { + let (stream, _journald) = UnixStream::pair().unwrap(); + let original_stderr = pose_as_journal(&stream); + let mut driver = init_logging(LogOutput::Stderr, LogFormat::Json); + + telemetry::log::warn!("logged before shutdown"); + driver.shutdown_logger(); + let stderr = io::stderr().as_fd().try_clone_to_owned(); + + nix::unistd::dup2_stderr(&original_stderr).unwrap(); + let stderr = stderr.expect("shutting down the logger closed stderr"); + assert_eq!( + stream_id(stderr.as_fd()), + stream_id(stream.as_fd()), + "stderr no longer refers to the journal stream" + ); +}