diff --git a/foundations-sentry/src/hook.rs b/foundations-sentry/src/hook.rs index d3af9fa4..295dcb22 100644 --- a/foundations-sentry/src/hook.rs +++ b/foundations-sentry/src/hook.rs @@ -1,106 +1,53 @@ //! Sentry hook implementation for tracking sentry events and rate-limiting them. use crate::SentrySettings; -use foundations::ratelimit::StaticQuantaClock; -use governor::{Quota, RateLimiter}; -use std::borrow::Cow; -use std::num::NonZeroU32; +use crate::metrics; +use crate::rate_limit::{EventRateLimiter, RateLimitKind}; use std::sync::Arc; -type Fingerprint = Cow<'static, str>; - -/// Clean up keys in the sentry rate limiter once every 20 minutes -/// (3 times per hour), with no burst allowed. -const SENTRY_LIMITER_CLEANUP_QUOTA: Quota = - Quota::per_hour(NonZeroU32::new(3).unwrap()).allow_burst(NonZeroU32::new(1).unwrap()); - /// Install the sentry hook on the provided client options. /// /// This installs a `before_send` hook that increments `sentry_events_total` /// and performs rate limiting, if configured. If a previous `before_send` -/// hook exists, it will be called after rate limiting has been applied. -/// Only unfiltered events are counted. +/// hook exists, it will be called after the per-fingerprint rate limit and +/// before the global and fatal rate limits, so that events it drops don't use +/// up their allowance. Only unfiltered events are counted. Events dropped by +/// the rate limits are counted by `sentry_events_rate_limited_total`. /// /// See the [`foundations-sentry`](crate) crate-level docs for more information. pub fn install_hook_with_settings( options: &mut sentry_core::ClientOptions, settings: &SentrySettings, ) { - let rate_limiter = settings.max_events_per_second.map(|rl| { - RateLimiter::dashmap_with_clock(Quota::per_second(rl), StaticQuantaClock::default()) - }); + let rate_limiter = EventRateLimiter::from_settings(settings); let previous = options.before_send.take(); options.before_send = Some(Arc::new(move |mut event| { - if let Some(limiter) = &rate_limiter { - foundations::ratelimit!(SENTRY_LIMITER_CLEANUP_QUOTA; limiter.retain_recent()); - - let fp = extract_fingerprint(&event); - if limiter.check_key(&fp).is_err() { - return None; - } + if let Some(limiter) = &rate_limiter + && let Err(limit) = limiter.check_fingerprint(&event) + { + record_rate_limited(&event, limit); + return None; } if let Some(prev) = &previous { event = prev(event)?; } - super::metrics::sentry::events_total(event.level).inc(); - - Some(event) - })); -} - -/// Derive a fingerprint for a sentry event to perform rate limiting. -/// -/// We check for the following event attributes, in order: -/// 1. Explicit fingerprint (if set and not defaulted) -/// 2. Event message -/// 3. First exception value/type -/// 4. Fallback: event level name -fn extract_fingerprint(event: &sentry_core::protocol::Event<'static>) -> Fingerprint { - use sentry_core::Level; - - // Try the explicitly-specified fingerprint first, but only if its not defaulted - let explicit_fp = &event.fingerprint; - if !explicit_fp.is_empty() && !is_sentry_default_fingerprint(explicit_fp) { - if let [fp] = explicit_fp.as_ref() { - // Just clone if the explicit fingerprint is a single element - return fp.clone(); + if let Some(limiter) = &rate_limiter + && let Err(limit) = limiter.check_shared(&event) + { + record_rate_limited(&event, limit); + return None; } - return explicit_fp.join("::").into(); - } - // Try the event message, if there is one - if let Some(msg) = &event.message { - return msg.clone().into(); - } + metrics::sentry::events_total(event.level).inc(); - // Try the first attached exception, if there is one - if let Some(exc) = event.exception.first() { - if let Some(val) = &exc.value { - return val.clone().into(); - } - if !exc.ty.is_empty() { - return exc.ty.clone().into(); - } - } - - // Finally, fall back to the event level - Cow::Borrowed(match event.level { - Level::Debug => "level::debug", - Level::Info => "level::info", - Level::Warning => "level::warning", - Level::Error => "level::error", - Level::Fatal => "level::fatal", - }) + Some(event) + })); } -// Adapted from https://github.com/getsentry/sentry-rust/blob/0.47.0/sentry-types/src/protocol/v7.rs#L1619 -fn is_sentry_default_fingerprint(fp: &[Cow<'_, str>]) -> bool { - if let [fp] = fp { - return matches!(fp.as_ref(), "{{ default }}" | "{{default}}"); - } - false +fn record_rate_limited(event: &sentry_core::protocol::Event<'_>, limit: RateLimitKind) { + metrics::sentry::events_rate_limited_total(event.level, limit.as_str()).inc(); } diff --git a/foundations-sentry/src/lib.rs b/foundations-sentry/src/lib.rs index 136177ec..375492aa 100644 --- a/foundations-sentry/src/lib.rs +++ b/foundations-sentry/src/lib.rs @@ -4,17 +4,42 @@ //! //! This crate provides a sentry hook that increments the //! `sentry_events_total{level=<...>}` metric for each sentry event. If a -//! previous `before_send` hook exists, it will be executed after rate limiting -//! and before the metric is incremented. Only unfiltered events are counted. +//! previous `before_send` hook exists, it will be executed after the +//! per-fingerprint rate limit and before the global and fatal rate limits (see +//! below) and the metric. Only unfiltered events are counted. //! -//! For rate-limiting, we group events by fingerprint. Each group has a separate -//! rate limiter. The fingerprint of a sentry event is the first out of the following -//! attributes that is present and not defaulted: +//! # Rate limiting +//! +//! All rate limits are disabled by default, and can be enabled in [`SentrySettings`]: +//! +//! - **Per fingerprint**: each group of events with the same fingerprint has a separate rate +//! limit (`per_fingerprint_rate_limit`, or the `max_events_per_second` shorthand). +//! - **Global**: a single rate limit shared by all events (`global_rate_limit`). +//! - **Fatal**: a single rate limit shared by fatal events, which then don't count against the +//! global one (`fatal_rate_limit`). This keeps a flood of other events from crowding out +//! panics. +//! +//! An event is checked against the per-fingerprint rate limit first, so that the events of a +//! fingerprint that is already rate limited don't use up the global allowance. The global and +//! fatal rate limits are checked after the previous `before_send` hook, so that events it drops +//! don't use up their allowance either. Rate-limited events +//! are dropped, and counted by the `sentry_events_rate_limited_total{level=<...>,limit=<...>}` +//! metric. +//! +//! The fingerprint of a sentry event is the first out of the following attributes that is +//! present and not defaulted: //! //! 1. Explicit `event.fingerprint` //! 2. Event message //! 3. First exception value, or exception type if no value is set -//! 4. Fallback: event level name (e.g., `error`) +//! 4. Fallback: event level name (e.g., `level::error`) +//! +//! Up to `max_tracked_fingerprints` fingerprints are tracked at once. Beyond that, events with +//! new fingerprints are only subject to the global and fatal rate limits, until idle fingerprints +//! are forgotten. +//! +//! The hook runs after Sentry's integrations have processed the event, so the rate limits don't +//! save the cost of e.g. capturing a stacktrace for dropped events. //! //! **note**: a clone of a client's [`sentry_core::ClientOptions`] will have the //! hook installed. This means "child" sentry clients will inherit the hook. A @@ -34,6 +59,21 @@ //! } //! ``` //! +//! For instance, these settings allow 10 events per minute for each fingerprint, and 60 events +//! per minute in total, plus 60 fatal events per minute: +//! +//! ```rust +//! let sentry_settings: foundations_sentry::SentrySettings = foundations::settings::from_yaml_str( +//! r#" +//! per_fingerprint_rate_limit: { max_events: 10, period_secs: 60 } +//! global_rate_limit: { max_events: 60, period_secs: 60 } +//! fatal_rate_limit: { max_events: 60, period_secs: 60 } +//! "#, +//! ) +//! .unwrap(); +//! # assert_eq!(sentry_settings.global_rate_limit.max_events, 60); +//! ``` +//! //! # Server-side symbolication //! //! Stacktraces are symbolicated locally by default. To capture instruction addresses @@ -73,7 +113,8 @@ pub mod metrics; pub mod panic; mod hook; +mod rate_limit; mod settings; pub use self::hook::install_hook_with_settings; -pub use self::settings::SentrySettings; +pub use self::settings::{EventRateLimitSettings, SentrySettings}; diff --git a/foundations-sentry/src/metrics.rs b/foundations-sentry/src/metrics.rs index bffda47e..16c7899c 100644 --- a/foundations-sentry/src/metrics.rs +++ b/foundations-sentry/src/metrics.rs @@ -8,4 +8,8 @@ use sentry_core::Level; pub mod sentry { /// Total number of sentry events observed. pub fn events_total(level: Level) -> Counter; + + /// Total number of sentry events dropped by the rate limits, by the rate limit that dropped + /// them: `fingerprint`, `global`, or `fatal`. + pub fn events_rate_limited_total(level: Level, limit: &'static str) -> Counter; } diff --git a/foundations-sentry/src/rate_limit.rs b/foundations-sentry/src/rate_limit.rs new file mode 100644 index 00000000..935c4a3e --- /dev/null +++ b/foundations-sentry/src/rate_limit.rs @@ -0,0 +1,543 @@ +//! Rate limits for Sentry events. + +use crate::settings::{EventQuota, SentrySettings}; +use foundations::ratelimit::StaticQuantaClock; +use governor::clock::Clock; +use governor::middleware::NoOpMiddleware; +use governor::nanos::Nanos; +use governor::state::keyed::{DashMapStateStore, ShrinkableKeyedStateStore}; +use governor::state::{InMemoryState, NotKeyed, StateStore}; +use governor::{Quota, RateLimiter}; +use sentry_core::Level; +use sentry_core::protocol::Event; +use std::borrow::Cow; +use std::hash::{BuildHasher, RandomState}; +use std::num::{NonZeroU32, NonZeroUsize}; +use std::time::Duration; + +/// Clean up idle fingerprints once every 20 minutes (3 times per hour), with no burst allowed. +const PERIODIC_CLEANUP_QUOTA: Quota = + Quota::per_hour(NonZeroU32::new(3).unwrap()).allow_burst(NonZeroU32::new(1).unwrap()); + +/// Clean up idle fingerprints at most once per second while the fingerprint cap is reached. +const FULL_CLEANUP_QUOTA: Quota = Quota::per_second(NonZeroU32::new(1).unwrap()); + +type DirectLimiter = + RateLimiter::Instant>>; + +type FingerprintLimiter = + RateLimiter::Instant>>; + +/// The rate limit that rejected an event. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RateLimitKind { + Fingerprint, + Global, + Fatal, +} + +impl RateLimitKind { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Fingerprint => "fingerprint", + Self::Global => "global", + Self::Fatal => "fatal", + } + } +} + +/// Enforces the rate limits configured in [`SentrySettings`]. +pub(crate) struct EventRateLimiter { + /// Fingerprints are tracked by their hash, so that tracking a fingerprint takes a small, + /// fixed amount of memory. A collision only makes two fingerprints share a rate limit. + fingerprint_hasher: RandomState, + per_fingerprint: Option>, + max_tracked_fingerprints: Option, + global: Option>, + fatal: Option>, + periodic_cleanup: DirectLimiter, + full_cleanup: DirectLimiter, +} + +impl EventRateLimiter { + /// Returns `None` if no rate limit is enabled. + pub(crate) fn from_settings(settings: &SentrySettings) -> Option { + Self::with_clock(settings, StaticQuantaClock::default()) + } +} + +impl EventRateLimiter { + fn with_clock(settings: &SentrySettings, clock: C) -> Option { + let max_tracked_fingerprints = settings.max_tracked_fingerprints(); + let per_fingerprint = settings + .effective_per_fingerprint_rate_limit() + .map(|quota| { + RateLimiter::new( + to_governor_quota(quota), + BoundedStateStore::new(max_tracked_fingerprints), + clock.clone(), + ) + }); + let direct = |quota: EventQuota| { + RateLimiter::direct_with_clock(to_governor_quota(quota), clock.clone()) + }; + let global = settings.global_rate_limit.quota().map(direct); + let fatal = settings.fatal_rate_limit.quota().map(direct); + + if per_fingerprint.is_none() && global.is_none() && fatal.is_none() { + return None; + } + + Some(Self { + fingerprint_hasher: RandomState::new(), + per_fingerprint, + max_tracked_fingerprints, + global, + fatal, + periodic_cleanup: RateLimiter::direct_with_clock(PERIODIC_CLEANUP_QUOTA, clock.clone()), + full_cleanup: RateLimiter::direct_with_clock(FULL_CLEANUP_QUOTA, clock), + }) + } + + /// Takes the allowance of `event` from the per-fingerprint rate limit, if enabled. + /// + /// This should be checked before [`Self::check_shared`], so that events of a fingerprint + /// that is already rate limited don't use up the allowance of the shared rate limits. + pub(crate) fn check_fingerprint(&self, event: &Event<'_>) -> Result<(), RateLimitKind> { + if let Some(limiter) = &self.per_fingerprint { + self.clean_up(limiter); + + let fingerprint = extract_fingerprint(event); + let key = self.fingerprint_hasher.hash_one(fingerprint.as_ref()); + if limiter.check_key(&key).is_err() { + return Err(RateLimitKind::Fingerprint); + } + } + + Ok(()) + } + + /// Takes the allowance of `event` from the shared (global or fatal) rate limit that applies + /// to it, if enabled. + pub(crate) fn check_shared(&self, event: &Event<'_>) -> Result<(), RateLimitKind> { + let (shared, kind) = match (&self.fatal, event.level) { + (Some(fatal), Level::Fatal) => (Some(fatal), RateLimitKind::Fatal), + _ => (self.global.as_ref(), RateLimitKind::Global), + }; + if let Some(limiter) = shared + && limiter.check().is_err() + { + return Err(kind); + } + + Ok(()) + } + + /// Checks `event` against every rate limit, in the order used by the hook. + #[cfg(test)] + fn check(&self, event: &Event<'_>) -> Result<(), RateLimitKind> { + self.check_fingerprint(event)?; + self.check_shared(event) + } + + /// Forgets fingerprints whose rate limit is back to its initial state. + fn clean_up(&self, limiter: &FingerprintLimiter) { + let is_full = self + .max_tracked_fingerprints + .is_some_and(|max| limiter.len() >= max.get()); + let is_due = + (is_full && self.full_cleanup.check().is_ok()) || self.periodic_cleanup.check().is_ok(); + + if is_due { + limiter.retain_recent(); + } + } + + #[cfg(test)] + fn tracked_fingerprints(&self) -> usize { + self.per_fingerprint.as_ref().map_or(0, RateLimiter::len) + } +} + +/// Allows `max_events` at once, replenished evenly over `period`. +fn to_governor_quota(quota: EventQuota) -> Quota { + // The period can only be zero for absurdly high limits: replenish as fast as possible then. + let replenish_one_per = (quota.period / quota.max_events.get()).max(Duration::from_nanos(1)); + + Quota::with_period(replenish_one_per) + .expect("the period is non-zero") + .allow_burst(quota.max_events) +} + +/// A state store that tracks up to a maximum number of keys. +/// +/// Once full, keys that aren't tracked yet are treated as if they were seen for the first time, +/// and are not tracked. The number of keys can slightly exceed the maximum when new keys are +/// inserted concurrently. +struct BoundedStateStore { + entries: DashMapStateStore, + max_entries: Option, +} + +impl BoundedStateStore { + fn new(max_entries: Option) -> Self { + Self { + entries: Default::default(), + max_entries, + } + } +} + +impl StateStore for BoundedStateStore { + type Key = u64; + + fn measure_and_replace(&self, key: &u64, f: F) -> Result + where + F: Fn(Option) -> Result<(T, Nanos), E>, + { + // Only untracked keys need the size of the map, which locks every shard. + let is_full = || { + self.max_entries + .is_some_and(|max| self.entries.len() >= max.get()) + }; + + if !self.entries.contains_key(key) && is_full() { + return f(None).map(|(outcome, _)| outcome); + } + + StateStore::measure_and_replace(&self.entries, key, f) + } +} + +impl ShrinkableKeyedStateStore for BoundedStateStore { + fn retain_recent(&self, drop_below: Nanos) { + ShrinkableKeyedStateStore::retain_recent(&self.entries, drop_below); + } + + fn shrink_to_fit(&self) { + ShrinkableKeyedStateStore::shrink_to_fit(&self.entries); + } + + fn len(&self) -> usize { + self.entries.len() + } + + fn is_empty(&self) -> bool { + self.entries.is_empty() + } +} + +/// Derive a fingerprint for a sentry event to perform rate limiting. +/// +/// We check for the following event attributes, in order: +/// 1. Explicit fingerprint (if set and not defaulted) +/// 2. Event message +/// 3. First exception value/type +/// 4. Fallback: event level name +fn extract_fingerprint<'a>(event: &'a Event<'_>) -> Cow<'a, str> { + // Try the explicitly-specified fingerprint first, but only if its not defaulted + let explicit_fp = &event.fingerprint; + if !explicit_fp.is_empty() && !is_sentry_default_fingerprint(explicit_fp) { + if let [fp] = explicit_fp.as_ref() { + return Cow::Borrowed(fp); + } + return explicit_fp.join("::").into(); + } + + // Try the event message, if there is one + if let Some(msg) = &event.message { + return Cow::Borrowed(msg); + } + + // Try the first attached exception, if there is one + if let Some(exc) = event.exception.first() { + if let Some(val) = &exc.value { + return Cow::Borrowed(val); + } + if !exc.ty.is_empty() { + return Cow::Borrowed(&exc.ty); + } + } + + // Finally, fall back to the event level + Cow::Borrowed(match event.level { + Level::Debug => "level::debug", + Level::Info => "level::info", + Level::Warning => "level::warning", + Level::Error => "level::error", + Level::Fatal => "level::fatal", + }) +} + +// Adapted from https://github.com/getsentry/sentry-rust/blob/0.47.0/sentry-types/src/protocol/v7.rs#L1619 +fn is_sentry_default_fingerprint(fp: &[Cow<'_, str>]) -> bool { + if let [fp] = fp { + return matches!(fp.as_ref(), "{{ default }}" | "{{default}}"); + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::settings::EventRateLimitSettings; + use governor::clock::FakeRelativeClock; + use sentry_core::protocol::Exception; + + fn rate_limit(max_events: u32, period_secs: u64) -> EventRateLimitSettings { + EventRateLimitSettings { + max_events, + period_secs, + } + } + + fn limiter( + settings: SentrySettings, + ) -> (EventRateLimiter, FakeRelativeClock) { + let clock = FakeRelativeClock::default(); + let limiter = EventRateLimiter::with_clock(&settings, clock.clone()) + .expect("a rate limit is enabled"); + (limiter, clock) + } + + fn event(fingerprint: &str) -> Event<'static> { + Event { + fingerprint: Cow::Owned(vec![Cow::Owned(fingerprint.to_owned())]), + ..Event::default() + } + } + + fn fatal_event(fingerprint: &str) -> Event<'static> { + Event { + level: Level::Fatal, + ..event(fingerprint) + } + } + + #[test] + fn no_limiter_when_no_rate_limit_is_enabled() { + assert!(EventRateLimiter::from_settings(&SentrySettings::default()).is_none()); + + let disabled = SentrySettings { + per_fingerprint_rate_limit: rate_limit(0, 60), + global_rate_limit: rate_limit(10, 0), + ..SentrySettings::default() + }; + assert!(EventRateLimiter::from_settings(&disabled).is_none()); + } + + #[test] + fn per_fingerprint_rate_limit_allows_a_burst_then_replenishes_evenly() { + let (limiter, clock) = limiter(SentrySettings { + per_fingerprint_rate_limit: rate_limit(10, 60), + ..SentrySettings::default() + }); + + for _ in 0..10 { + assert_eq!(limiter.check(&event("a")), Ok(())); + } + assert_eq!(limiter.check(&event("a")), Err(RateLimitKind::Fingerprint)); + assert_eq!(limiter.check(&event("b")), Ok(())); + + // One event is replenished every `60s / 10`. + clock.advance(Duration::from_secs(5)); + assert_eq!(limiter.check(&event("a")), Err(RateLimitKind::Fingerprint)); + clock.advance(Duration::from_secs(1)); + assert_eq!(limiter.check(&event("a")), Ok(())); + assert_eq!(limiter.check(&event("a")), Err(RateLimitKind::Fingerprint)); + } + + #[test] + fn max_events_per_second_is_a_per_fingerprint_rate_limit() { + let (limiter, clock) = limiter(SentrySettings { + max_events_per_second: NonZeroU32::new(2), + ..SentrySettings::default() + }); + + for _ in 0..2 { + assert_eq!(limiter.check(&event("a")), Ok(())); + } + assert_eq!(limiter.check(&event("a")), Err(RateLimitKind::Fingerprint)); + assert_eq!(limiter.check(&event("b")), Ok(())); + + clock.advance(Duration::from_millis(500)); + assert_eq!(limiter.check(&event("a")), Ok(())); + } + + #[test] + fn per_fingerprint_rate_limit_overrides_max_events_per_second() { + let (limiter, _clock) = limiter(SentrySettings { + max_events_per_second: NonZeroU32::new(100), + per_fingerprint_rate_limit: rate_limit(1, 60), + ..SentrySettings::default() + }); + + assert_eq!(limiter.check(&event("a")), Ok(())); + assert_eq!(limiter.check(&event("a")), Err(RateLimitKind::Fingerprint)); + } + + #[test] + fn global_rate_limit_is_shared_by_all_fingerprints() { + let (limiter, clock) = limiter(SentrySettings { + global_rate_limit: rate_limit(3, 60), + ..SentrySettings::default() + }); + + for fingerprint in ["a", "b", "c"] { + assert_eq!(limiter.check(&event(fingerprint)), Ok(())); + } + assert_eq!(limiter.check(&event("d")), Err(RateLimitKind::Global)); + assert_eq!(limiter.check(&fatal_event("e")), Err(RateLimitKind::Global)); + + clock.advance(Duration::from_secs(20)); + assert_eq!(limiter.check(&event("d")), Ok(())); + } + + #[test] + fn fatal_events_have_their_own_rate_limit_when_enabled() { + let (limiter, _clock) = limiter(SentrySettings { + global_rate_limit: rate_limit(2, 60), + fatal_rate_limit: rate_limit(1, 60), + ..SentrySettings::default() + }); + + for fingerprint in ["a", "b"] { + assert_eq!(limiter.check(&event(fingerprint)), Ok(())); + } + assert_eq!(limiter.check(&event("c")), Err(RateLimitKind::Global)); + + assert_eq!(limiter.check(&fatal_event("d")), Ok(())); + assert_eq!(limiter.check(&fatal_event("e")), Err(RateLimitKind::Fatal)); + } + + #[test] + fn fatal_rate_limit_does_not_limit_other_events() { + let (limiter, _clock) = limiter(SentrySettings { + fatal_rate_limit: rate_limit(1, 60), + ..SentrySettings::default() + }); + + assert_eq!(limiter.check(&fatal_event("a")), Ok(())); + assert_eq!(limiter.check(&fatal_event("b")), Err(RateLimitKind::Fatal)); + for fingerprint in ["c", "d", "e"] { + assert_eq!(limiter.check(&event(fingerprint)), Ok(())); + } + } + + #[test] + fn events_rejected_per_fingerprint_do_not_use_the_global_allowance() { + let (limiter, _clock) = limiter(SentrySettings { + per_fingerprint_rate_limit: rate_limit(1, 60), + global_rate_limit: rate_limit(2, 60), + ..SentrySettings::default() + }); + + assert_eq!(limiter.check(&event("a")), Ok(())); + for _ in 0..10 { + assert_eq!(limiter.check(&event("a")), Err(RateLimitKind::Fingerprint)); + } + assert_eq!(limiter.check(&event("b")), Ok(())); + assert_eq!(limiter.check(&event("c")), Err(RateLimitKind::Global)); + } + + #[test] + fn untracked_fingerprints_are_not_limited_per_fingerprint_once_the_cap_is_reached() { + let (limiter, _clock) = limiter(SentrySettings { + per_fingerprint_rate_limit: rate_limit(1, 60), + global_rate_limit: rate_limit(5, 60), + max_tracked_fingerprints: 2, + ..SentrySettings::default() + }); + + assert_eq!(limiter.check(&event("a")), Ok(())); + assert_eq!(limiter.check(&event("b")), Ok(())); + assert_eq!(limiter.tracked_fingerprints(), 2); + + // Tracked fingerprints are still limited. + assert_eq!(limiter.check(&event("a")), Err(RateLimitKind::Fingerprint)); + + // Untracked ones are only subject to the global rate limit. + for _ in 0..3 { + assert_eq!(limiter.check(&event("c")), Ok(())); + } + assert_eq!(limiter.check(&event("c")), Err(RateLimitKind::Global)); + assert_eq!(limiter.tracked_fingerprints(), 2); + } + + #[test] + fn idle_fingerprints_are_forgotten_once_the_cap_is_reached() { + let (limiter, clock) = limiter(SentrySettings { + per_fingerprint_rate_limit: rate_limit(1, 60), + max_tracked_fingerprints: 2, + ..SentrySettings::default() + }); + + assert_eq!(limiter.check(&event("a")), Ok(())); + assert_eq!(limiter.check(&event("b")), Ok(())); + assert_eq!(limiter.tracked_fingerprints(), 2); + + // Once "a" and "b" are idle, they make room for new fingerprints. `governor` only forgets + // a key one replenish interval after its rate limit is back to its initial state. + clock.advance(Duration::from_secs(120)); + assert_eq!(limiter.check(&event("c")), Ok(())); + assert_eq!(limiter.tracked_fingerprints(), 1); + assert_eq!(limiter.check(&event("c")), Err(RateLimitKind::Fingerprint)); + } + + #[test] + fn fingerprints_are_not_capped_when_the_cap_is_zero() { + let (limiter, _clock) = limiter(SentrySettings { + per_fingerprint_rate_limit: rate_limit(1, 60), + max_tracked_fingerprints: 0, + ..SentrySettings::default() + }); + + for i in 0..100 { + assert_eq!(limiter.check(&event(&i.to_string())), Ok(())); + } + assert_eq!(limiter.tracked_fingerprints(), 100); + assert_eq!(limiter.check(&event("0")), Err(RateLimitKind::Fingerprint)); + } + + #[test] + fn quota_replenishes_as_fast_as_possible_for_absurdly_high_limits() { + let quota = to_governor_quota(EventQuota { + max_events: NonZeroU32::MAX, + period: Duration::from_secs(1), + }); + + assert_eq!(quota.replenish_interval(), Duration::from_nanos(1)); + assert_eq!(quota.burst_size(), NonZeroU32::MAX); + } + + #[test] + fn fingerprint_uses_the_first_available_attribute() { + let mut event = Event { + level: Level::Warning, + ..Event::default() + }; + assert_eq!(extract_fingerprint(&event), "level::warning"); + + event.exception = vec![Exception { + ty: "panic".to_owned(), + ..Exception::default() + }] + .into(); + assert_eq!(extract_fingerprint(&event), "panic"); + + event.exception.values[0].value = Some("exception value".to_owned()); + assert_eq!(extract_fingerprint(&event), "exception value"); + + event.message = Some("message".to_owned()); + assert_eq!(extract_fingerprint(&event), "message"); + + event.fingerprint = Cow::Owned(vec!["{{ default }}".into()]); + assert_eq!(extract_fingerprint(&event), "message"); + + event.fingerprint = Cow::Owned(vec!["msg".into(), "file.rs".into(), "42".into()]); + assert_eq!(extract_fingerprint(&event), "msg::file.rs::42"); + + event.fingerprint = Cow::Owned(vec!["explicit".into()]); + assert_eq!(extract_fingerprint(&event), "explicit"); + } +} diff --git a/foundations-sentry/src/settings.rs b/foundations-sentry/src/settings.rs index 54eec57d..66fe2f29 100644 --- a/foundations-sentry/src/settings.rs +++ b/foundations-sentry/src/settings.rs @@ -1,12 +1,96 @@ use foundations::settings::settings; -use std::num::NonZeroU32; +use std::num::{NonZeroU32, NonZeroUsize}; +use std::time::Duration; /// Sentry hook settings. +/// +/// All rate limits are disabled by default. #[settings] pub struct SentrySettings { /// Maximum number of events that can be emitted per second, per fingerprint. + /// + /// Shorthand for a `per_fingerprint_rate_limit` of `max_events_per_second` events every + /// second. Ignored if `per_fingerprint_rate_limit` is enabled. pub max_events_per_second: Option, - // In the future, we may offer different fingerprinting modes here. - // For example, we could offer a "dummy" mode that assigns the same fingerprint - // to all events so they all share a single rate limit. + + /// Rate limit applied to each event fingerprint separately. + pub per_fingerprint_rate_limit: EventRateLimitSettings, + + /// Rate limit shared by all events. + /// + /// If `fatal_rate_limit` is enabled, fatal events are limited by it instead. + pub global_rate_limit: EventRateLimitSettings, + + /// Rate limit shared by all fatal events (e.g. panics). + /// + /// This keeps a flood of non-fatal events from crowding out fatal ones. If disabled, fatal + /// events are limited by `global_rate_limit`. + pub fatal_rate_limit: EventRateLimitSettings, + + /// Maximum number of fingerprints tracked by the per-fingerprint rate limit. + /// + /// Once reached, events with fingerprints that aren't tracked yet are not limited per + /// fingerprint, but they are still subject to the global and fatal rate limits. `0` removes + /// the cap. + #[serde(default = "SentrySettings::default_max_tracked_fingerprints")] + pub max_tracked_fingerprints: usize, +} + +impl SentrySettings { + fn default_max_tracked_fingerprints() -> usize { + 10_000 + } + + pub(crate) fn effective_per_fingerprint_rate_limit(&self) -> Option { + self.per_fingerprint_rate_limit.quota().or_else(|| { + self.max_events_per_second.map(|max_events| EventQuota { + max_events, + period: Duration::from_secs(1), + }) + }) + } + + pub(crate) fn max_tracked_fingerprints(&self) -> Option { + NonZeroUsize::new(self.max_tracked_fingerprints) + } +} + +/// Settings of a Sentry event rate limit. +/// +/// Up to `max_events` events are allowed at once, and the allowance is replenished evenly over +/// `period_secs`: e.g. `max_events: 10` and `period_secs: 60` allow a burst of 10 events, followed +/// by one event every 6 seconds. +#[settings] +pub struct EventRateLimitSettings { + /// Maximum number of events allowed per `period_secs`. `0` disables the rate limit. + pub max_events: u32, + + /// The period of the rate limit, in seconds. `0` disables the rate limit. + #[serde(default = "EventRateLimitSettings::default_period_secs")] + pub period_secs: u64, +} + +impl EventRateLimitSettings { + fn default_period_secs() -> u64 { + 1 + } + + pub(crate) fn quota(&self) -> Option { + let max_events = NonZeroU32::new(self.max_events)?; + if self.period_secs == 0 { + return None; + } + + Some(EventQuota { + max_events, + period: Duration::from_secs(self.period_secs), + }) + } +} + +/// A validated, enabled event rate limit. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct EventQuota { + pub(crate) max_events: NonZeroU32, + pub(crate) period: Duration, } diff --git a/foundations-sentry/tests/sentry_hook.rs b/foundations-sentry/tests/sentry_hook.rs index dbb309ea..4caf9978 100644 --- a/foundations-sentry/tests/sentry_hook.rs +++ b/foundations-sentry/tests/sentry_hook.rs @@ -6,7 +6,7 @@ use std::num::NonZeroU32; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use foundations_sentry::{SentrySettings, metrics}; +use foundations_sentry::{EventRateLimitSettings, SentrySettings, metrics}; const TEST_DSN: &str = "https://example@sentry.io/123"; @@ -61,6 +61,7 @@ fn sentry_hook_increments_metric_on_multiple_events() { fn sentry_hook_rate_limits_events() { let settings = SentrySettings { max_events_per_second: Some(NonZeroU32::new(1).unwrap()), + ..Default::default() }; let hub = hub_with_settings(&settings); @@ -140,3 +141,184 @@ fn cloned_client_options_have_hook_installed() { // The hook should have been called via the cloned options assert_eq!(metrics::sentry::events_total(Level::Error).get(), 1); } + +fn rate_limit(max_events: u32, period_secs: u64) -> EventRateLimitSettings { + EventRateLimitSettings { + max_events, + period_secs, + } +} + +#[test] +fn sentry_hook_drops_events_over_the_global_rate_limit_and_counts_them() { + let settings = SentrySettings { + global_rate_limit: rate_limit(2, 3600), + ..Default::default() + }; + let hub = hub_with_settings(&settings); + + for message in ["a", "b", "c"] { + hub.capture_message(message, Level::Error); + } + + assert_eq!(metrics::sentry::events_total(Level::Error).get(), 2); + assert_eq!( + metrics::sentry::events_rate_limited_total(Level::Error, "global").get(), + 1 + ); + + const SERIES: &str = "sentry_events_rate_limited_total{level=\"error\",limit=\"global\"} "; + + #[allow(deprecated)] + let metrics = foundations::telemetry::metrics::collect(&Default::default()).unwrap(); + let has_metric = metrics.lines().any(|line| { + line.strip_prefix(SERIES) + .and_then(|value| value.parse::().ok()) + == Some(1.0) + }); + assert!(has_metric, "{metrics}"); +} + +#[test] +fn sentry_hook_limits_fatal_events_separately() { + let settings = SentrySettings { + global_rate_limit: rate_limit(1, 3600), + fatal_rate_limit: rate_limit(1, 3600), + ..Default::default() + }; + let hub = hub_with_settings(&settings); + + hub.capture_message("error a", Level::Error); + hub.capture_message("error b", Level::Error); + hub.capture_message("panic a", Level::Fatal); + hub.capture_message("panic b", Level::Fatal); + + assert_eq!(metrics::sentry::events_total(Level::Error).get(), 1); + assert_eq!(metrics::sentry::events_total(Level::Fatal).get(), 1); + assert_eq!( + metrics::sentry::events_rate_limited_total(Level::Error, "global").get(), + 1 + ); + assert_eq!( + metrics::sentry::events_rate_limited_total(Level::Fatal, "fatal").get(), + 1 + ); +} + +#[test] +fn sentry_hook_does_not_run_previous_hook_for_rate_limited_events() { + let previous_hook_count = Arc::new(AtomicU64::new(0)); + let counter = Arc::clone(&previous_hook_count); + + let mut options = ClientOptions { + before_send: Some(Arc::new(move |event| { + counter.fetch_add(1, Ordering::Relaxed); + Some(event) + })), + ..Default::default() + }; + let settings = SentrySettings { + per_fingerprint_rate_limit: rate_limit(1, 3600), + ..Default::default() + }; + foundations_sentry::install_hook_with_settings(&mut options, &settings); + let hub = hub_with_options(options); + + simulate_sentry_event(&hub); + simulate_sentry_event(&hub); + + assert_eq!(previous_hook_count.load(Ordering::Relaxed), 1); + assert_eq!(metrics::sentry::events_total(Level::Error).get(), 1); + assert_eq!( + metrics::sentry::events_rate_limited_total(Level::Error, "fingerprint").get(), + 1 + ); +} + +#[test] +fn events_dropped_by_previous_hook_do_not_use_the_shared_allowance() { + let mut options = ClientOptions { + before_send: Some(Arc::new(|event| { + (event.message.as_deref() != Some("noise")).then_some(event) + })), + ..Default::default() + }; + let settings = SentrySettings { + global_rate_limit: rate_limit(1, 3600), + fatal_rate_limit: rate_limit(1, 3600), + ..Default::default() + }; + foundations_sentry::install_hook_with_settings(&mut options, &settings); + let hub = hub_with_options(options); + + for _ in 0..10 { + hub.capture_message("noise", Level::Error); + hub.capture_message("noise", Level::Fatal); + } + hub.capture_message("error", Level::Error); + hub.capture_message("panic", Level::Fatal); + + assert_eq!(metrics::sentry::events_total(Level::Error).get(), 1); + assert_eq!(metrics::sentry::events_total(Level::Fatal).get(), 1); + assert_eq!( + metrics::sentry::events_rate_limited_total(Level::Error, "global").get(), + 0 + ); + assert_eq!( + metrics::sentry::events_rate_limited_total(Level::Fatal, "fatal").get(), + 0 + ); +} + +#[test] +fn cloned_client_options_share_the_rate_limits() { + let settings = SentrySettings { + global_rate_limit: rate_limit(1, 3600), + ..Default::default() + }; + let hub1 = hub_with_settings(&settings); + let cloned_options = hub1.client().unwrap().options().clone(); + let hub2 = Hub::new( + Some(Arc::new(sentry_core::Client::with_options(cloned_options))), + Default::default(), + ); + + hub1.capture_message("a", Level::Error); + hub2.capture_message("b", Level::Error); + + assert_eq!(metrics::sentry::events_total(Level::Error).get(), 1); + assert_eq!( + metrics::sentry::events_rate_limited_total(Level::Error, "global").get(), + 1 + ); +} + +#[test] +fn settings_are_parsed_from_yaml() { + let settings: SentrySettings = foundations::settings::from_yaml_str( + r#" + per_fingerprint_rate_limit: { max_events: 10, period_secs: 60 } + global_rate_limit: { max_events: 60 } + "#, + ) + .unwrap(); + + assert_eq!(settings.per_fingerprint_rate_limit.max_events, 10); + assert_eq!(settings.per_fingerprint_rate_limit.period_secs, 60); + assert_eq!(settings.global_rate_limit.max_events, 60); + assert_eq!(settings.global_rate_limit.period_secs, 1); + assert_eq!(settings.fatal_rate_limit.max_events, 0); + assert_eq!(settings.max_events_per_second, None); + assert_eq!(settings.max_tracked_fingerprints, 10_000); +} + +#[test] +fn default_settings_keep_rate_limits_disabled() { + let hub = hub_with_settings(&SentrySettings::default()); + + for _ in 0..100 { + simulate_sentry_event(&hub); + } + + assert_eq!(metrics::sentry::events_total(Level::Error).get(), 100); +}