Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 22 additions & 75 deletions foundations-sentry/src/hook.rs
Original file line number Diff line number Diff line change
@@ -1,106 +1,53 @@
//! Sentry hook implementation for tracking sentry events and rate-limiting them.

use crate::SentrySettings;
use foundations::ratelimit::StaticQuantaClock;
use governor::{Quota, RateLimiter};
use std::borrow::Cow;
use std::num::NonZeroU32;
use crate::metrics;
use crate::rate_limit::{EventRateLimiter, RateLimitKind};
use std::sync::Arc;

type Fingerprint = Cow<'static, str>;

/// Clean up keys in the sentry rate limiter once every 20 minutes
/// (3 times per hour), with no burst allowed.
const SENTRY_LIMITER_CLEANUP_QUOTA: Quota =
Quota::per_hour(NonZeroU32::new(3).unwrap()).allow_burst(NonZeroU32::new(1).unwrap());

/// Install the sentry hook on the provided client options.
///
/// This installs a `before_send` hook that increments `sentry_events_total`
/// and performs rate limiting, if configured. If a previous `before_send`
/// hook exists, it will be called after rate limiting has been applied.
/// Only unfiltered events are counted.
/// hook exists, it will be called after the per-fingerprint rate limit and
/// before the global and fatal rate limits, so that events it drops don't use
/// up their allowance. Only unfiltered events are counted. Events dropped by
/// the rate limits are counted by `sentry_events_rate_limited_total`.
///
/// See the [`foundations-sentry`](crate) crate-level docs for more information.
pub fn install_hook_with_settings(
options: &mut sentry_core::ClientOptions,
settings: &SentrySettings,
) {
let rate_limiter = settings.max_events_per_second.map(|rl| {
RateLimiter::dashmap_with_clock(Quota::per_second(rl), StaticQuantaClock::default())
});
let rate_limiter = EventRateLimiter::from_settings(settings);

let previous = options.before_send.take();

options.before_send = Some(Arc::new(move |mut event| {
if let Some(limiter) = &rate_limiter {
foundations::ratelimit!(SENTRY_LIMITER_CLEANUP_QUOTA; limiter.retain_recent());

let fp = extract_fingerprint(&event);
if limiter.check_key(&fp).is_err() {
return None;
}
if let Some(limiter) = &rate_limiter
&& let Err(limit) = limiter.check_fingerprint(&event)
{
record_rate_limited(&event, limit);
return None;
}

if let Some(prev) = &previous {
event = prev(event)?;
}

super::metrics::sentry::events_total(event.level).inc();

Some(event)
}));
}

/// Derive a fingerprint for a sentry event to perform rate limiting.
///
/// We check for the following event attributes, in order:
/// 1. Explicit fingerprint (if set and not defaulted)
/// 2. Event message
/// 3. First exception value/type
/// 4. Fallback: event level name
fn extract_fingerprint(event: &sentry_core::protocol::Event<'static>) -> Fingerprint {
use sentry_core::Level;

// Try the explicitly-specified fingerprint first, but only if its not defaulted
let explicit_fp = &event.fingerprint;
if !explicit_fp.is_empty() && !is_sentry_default_fingerprint(explicit_fp) {
if let [fp] = explicit_fp.as_ref() {
// Just clone if the explicit fingerprint is a single element
return fp.clone();
if let Some(limiter) = &rate_limiter
&& let Err(limit) = limiter.check_shared(&event)
{
record_rate_limited(&event, limit);
return None;
}
return explicit_fp.join("::").into();
}

// Try the event message, if there is one
if let Some(msg) = &event.message {
return msg.clone().into();
}
metrics::sentry::events_total(event.level).inc();

// Try the first attached exception, if there is one
if let Some(exc) = event.exception.first() {
if let Some(val) = &exc.value {
return val.clone().into();
}
if !exc.ty.is_empty() {
return exc.ty.clone().into();
}
}

// Finally, fall back to the event level
Cow::Borrowed(match event.level {
Level::Debug => "level::debug",
Level::Info => "level::info",
Level::Warning => "level::warning",
Level::Error => "level::error",
Level::Fatal => "level::fatal",
})
Some(event)
}));
}

// Adapted from https://github.com/getsentry/sentry-rust/blob/0.47.0/sentry-types/src/protocol/v7.rs#L1619
fn is_sentry_default_fingerprint(fp: &[Cow<'_, str>]) -> bool {
if let [fp] = fp {
return matches!(fp.as_ref(), "{{ default }}" | "{{default}}");
}
false
fn record_rate_limited(event: &sentry_core::protocol::Event<'_>, limit: RateLimitKind) {
metrics::sentry::events_rate_limited_total(event.level, limit.as_str()).inc();
}
55 changes: 48 additions & 7 deletions foundations-sentry/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,42 @@
//!
//! This crate provides a sentry hook that increments the
//! `sentry_events_total{level=<...>}` metric for each sentry event. If a
//! previous `before_send` hook exists, it will be executed after rate limiting
//! and before the metric is incremented. Only unfiltered events are counted.
//! previous `before_send` hook exists, it will be executed after the
//! per-fingerprint rate limit and before the global and fatal rate limits (see
//! below) and the metric. Only unfiltered events are counted.
//!
//! For rate-limiting, we group events by fingerprint. Each group has a separate
//! rate limiter. The fingerprint of a sentry event is the first out of the following
//! attributes that is present and not defaulted:
//! # Rate limiting
//!
//! All rate limits are disabled by default, and can be enabled in [`SentrySettings`]:
//!
//! - **Per fingerprint**: each group of events with the same fingerprint has a separate rate
//! limit (`per_fingerprint_rate_limit`, or the `max_events_per_second` shorthand).
//! - **Global**: a single rate limit shared by all events (`global_rate_limit`).
//! - **Fatal**: a single rate limit shared by fatal events, which then don't count against the
//! global one (`fatal_rate_limit`). This keeps a flood of other events from crowding out
//! panics.
//!
//! An event is checked against the per-fingerprint rate limit first, so that the events of a
//! fingerprint that is already rate limited don't use up the global allowance. The global and
//! fatal rate limits are checked after the previous `before_send` hook, so that events it drops
//! don't use up their allowance either. Rate-limited events
//! are dropped, and counted by the `sentry_events_rate_limited_total{level=<...>,limit=<...>}`
//! metric.
//!
//! The fingerprint of a sentry event is the first out of the following attributes that is
//! present and not defaulted:
//!
//! 1. Explicit `event.fingerprint`
//! 2. Event message
//! 3. First exception value, or exception type if no value is set
//! 4. Fallback: event level name (e.g., `error`)
//! 4. Fallback: event level name (e.g., `level::error`)
//!
//! Up to `max_tracked_fingerprints` fingerprints are tracked at once. Beyond that, events with
//! new fingerprints are only subject to the global and fatal rate limits, until idle fingerprints
//! are forgotten.
//!
//! The hook runs after Sentry's integrations have processed the event, so the rate limits don't
//! save the cost of e.g. capturing a stacktrace for dropped events.
//!
//! **note**: a clone of a client's [`sentry_core::ClientOptions`] will have the
//! hook installed. This means "child" sentry clients will inherit the hook. A
Expand All @@ -34,6 +59,21 @@
//! }
//! ```
//!
//! For instance, these settings allow 10 events per minute for each fingerprint, and 60 events
//! per minute in total, plus 60 fatal events per minute:
//!
//! ```rust
//! let sentry_settings: foundations_sentry::SentrySettings = foundations::settings::from_yaml_str(
//! r#"
//! per_fingerprint_rate_limit: { max_events: 10, period_secs: 60 }
//! global_rate_limit: { max_events: 60, period_secs: 60 }
//! fatal_rate_limit: { max_events: 60, period_secs: 60 }
//! "#,
//! )
//! .unwrap();
//! # assert_eq!(sentry_settings.global_rate_limit.max_events, 60);
//! ```
//!
//! # Server-side symbolication
//!
//! Stacktraces are symbolicated locally by default. To capture instruction addresses
Expand Down Expand Up @@ -73,7 +113,8 @@ pub mod metrics;
pub mod panic;

mod hook;
mod rate_limit;
mod settings;

pub use self::hook::install_hook_with_settings;
pub use self::settings::SentrySettings;
pub use self::settings::{EventRateLimitSettings, SentrySettings};
4 changes: 4 additions & 0 deletions foundations-sentry/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,8 @@ use sentry_core::Level;
pub mod sentry {
/// Total number of sentry events observed.
pub fn events_total(level: Level) -> Counter;

/// Total number of sentry events dropped by the rate limits, by the rate limit that dropped
/// them: `fingerprint`, `global`, or `fatal`.
pub fn events_rate_limited_total(level: Level, limit: &'static str) -> Counter;
}
Loading