From 5d65d9eea6a3ee1bfca4f54ca66bfe7ad15d8a93 Mon Sep 17 00:00:00 2001 From: David Yaffe Date: Thu, 10 Sep 2026 17:55:35 -0400 Subject: [PATCH] Make Datadog metric submission explicit --- README.md | 25 +++++++---- src/commands/push_metrics.rs | 82 ++++++++++++++++++++++++++++++++++++ src/main.rs | 21 ++++++--- 3 files changed, 115 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 855c34b..e70406f 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Fetch: fetch-labels Fetch all labels for a repository Pipeline: - sync-metrics Fetch GitHub data locally, rebuild DB, and submit Datadog metrics + sync-metrics Fetch GitHub data locally, rebuild DB, and prepare or submit metrics push-metrics Submit a Datadog snapshot from an existing local DB generate-all Build DB + summaries + charts for all repos (workflow) build-db Load issues JSON into SQLite database @@ -160,17 +160,20 @@ Re-run `generate-all` after collecting stats to update the dashboard with the AI op run --env-file secrets.env -- github-tools sync-metrics \ --repo vectordotdev/vector \ --lookback 7d \ - --activity-window 30d + --activity-window 30d \ + --submit ``` The command: 1. Fetches a complete GitHub snapshot into the runner-local `data/{owner}_{repo}/` directory. 2. Rebuilds `out/db/{owner}_{repo}.db` from that local snapshot. -3. Reconstructs one snapshot at each UTC midnight in `--lookback`, adds the current snapshot, and sends them to Datadog. `--activity-window` independently controls the rolling period summarized by the closed-item metrics. +3. Reconstructs one snapshot at each UTC midnight in `--lookback`, adds the current snapshot, and sends them to Datadog when `--submit` is present. `--activity-window` independently controls the rolling period summarized by the closed-item metrics. The fetched JSON and SQLite database are temporary automation inputs. The command never stages, commits, or pushes them. +`sync-metrics` requires exactly one output mode: `--submit`, `--dry-run`, or `--output-json`. Submission is explicit so an incomplete automation command cannot write metrics accidentally. `--submit` reads `DD_API_KEY` from the environment; do not pass secrets as command-line arguments. + Use `--dry-run` to perform the fetch and database rebuild while printing, but not submitting, the resulting metrics. To preview metrics entirely offline after a database has been built: ```shell @@ -181,7 +184,7 @@ github-tools push-metrics \ --dry-run ``` -Use `--output-json` when a Datadog workflow or agent owns the Datadog connection. It performs the same calculation without submitting and emits one final JSON envelope. Each object in `batches` is a size-safe request body that can be sent directly to `POST /api/v2/series`: +Use `--output-json` when another process owns the Datadog connection and can transfer the generated batches without routing them through an LLM context. It performs the same calculation without submitting and emits one final JSON envelope. Each object in `batches` is a size-safe Datadog API request body that can be sent directly to `POST /api/v2/series`: ```shell github-tools sync-metrics \ @@ -195,6 +198,12 @@ github-tools sync-metrics \ {"format":"datadog-series-batches-v1","series_count":1,"point_count":1,"batches":[{"series":[{"metric":"github.health.v2.issues","type":3,"points":[{"timestamp":1788278400,"value":42}],"tags":["repo:quickwit-oss/quickwit"]}]}]} ``` +After `--submit` succeeds, the final output line is a compact receipt suitable for an automation agent to validate without reading metric payloads: + +```json +{"format":"datadog-submission-result-v1","success":true,"repository":"quickwit-oss/quickwit","metric_prefix":"github.health.v2","lookback":"7d","activity_window":"30d","series_count":91,"point_count":705,"batch_count":1,"batch_http_statuses":[202],"earliest_timestamp_utc":"2026-09-04T00:00:00Z","latest_timestamp_utc":"2026-09-10T20:00:00Z"} +``` + The default prefix is `github.health.v2`; override it with `--prefix`. Increment the prefix version before changing the metric dimensions so a clean backfill cannot mix incompatible tag schemas. The metrics are: | Metric | Meaning | Important tags | @@ -212,10 +221,10 @@ All are gauges. Rolling-window totals remain gauges because each point is a comp Historical Metrics Ingestion must be enabled before submitting the backfill. For a new prefix: -1. Submit a one-day seed using `--lookback 1d --activity-window 30d` so the metric names exist. +1. Submit a one-day seed using `--lookback 1d --activity-window 30d --submit` so the metric names exist. 2. In Datadog **Metrics Summary**, choose **Configure Metrics → Enable historical metrics** and select the `github.health.v2` namespace. -3. Submit the one-time backfill with `--lookback 450d --activity-window 30d`. -4. Schedule subsequent runs at a fixed UTC time with `--lookback 7d --activity-window 30d`. +3. Submit the one-time backfill with `--lookback 450d --activity-window 30d --submit`. +4. Schedule subsequent runs at a fixed UTC time with `--lookback 7d --activity-window 30d --submit`. Closed metrics are sparse: a metric with no matching closures is not emitted. If a closed metric is absent after the seed, use a wider activity window to create its name before enabling Historical Metrics Ingestion. @@ -245,6 +254,6 @@ sum:github.health.v2.discussions.closed{repo:quickwit-oss/quickwit,window:30d} Do not use `avg:` for repository totals: each metric is split into multiple tag combinations for its breakdown dimensions. -An external scheduler invokes `sync-metrics` once per repository. Its GitHub token needs read access to the source repository. Direct submission needs `DD_API_KEY`; alternatively, `--output-json` lets a Datadog workflow submit each generated batch through a managed HTTP connection. Non-US1 accounts should also set `DD_SITE` (for example, `datadoghq.eu`) for direct submission. +An external scheduler invokes `sync-metrics --submit` once per repository. Its GitHub token needs read access to the source repository, and `DD_API_KEY` must be injected as a secret environment variable. Non-US1 accounts should also set `DD_SITE` (for example, `datadoghq.eu`). Use `--output-json` only when the caller can transfer the generated batches directly; sandbox files are not shared automatically with managed HTTP actions. Tags and series are emitted in deterministic order, and an identical metric name, timestamp, and tag combination is safe to resubmit because Datadog retains the most recently submitted value. Current GitHub snapshots do not contain the full timelines for labels, issue types, PR draft transitions, discussion categories/answers, or close/reopen cycles. Historical breakdowns using those mutable fields are therefore current-state approximations. Use a new prefix version for a clean corrective backfill if the schema changes; exact lifecycle reconstruction would require fetching GitHub timeline events. diff --git a/src/commands/push_metrics.rs b/src/commands/push_metrics.rs index 7ce80c9..7e44812 100644 --- a/src/commands/push_metrics.rs +++ b/src/commands/push_metrics.rs @@ -149,6 +149,7 @@ pub fn run(config: &Config, options: MetricsOptions<'_>) -> Result<()> { "Pushing {} metric series with {point_count} historical/current points to Datadog...", series.len() ); + let mut batch_http_statuses = Vec::with_capacity(batches.len()); for (index, chunk) in batches.iter().enumerate() { let response = client .post(&api_url) @@ -172,10 +173,22 @@ pub fn run(config: &Config, options: MetricsOptions<'_>) -> Result<()> { ); } + batch_http_statuses.push(status.as_u16()); println!(" Batch {}: {} series accepted", index + 1, chunk.len()); } println!("Done."); + println!( + "{}", + serde_json::to_string(&submission_result( + config, + history, + activity_window, + prefix, + &series, + &batch_http_statuses, + )?)? + ); Ok(()) } @@ -646,6 +659,47 @@ fn json_batch_output(series: &[MetricSeries]) -> Result { })) } +fn submission_result( + config: &Config, + history: &str, + activity_window: &str, + prefix: &str, + series: &[MetricSeries], + batch_http_statuses: &[u16], +) -> Result { + let point_count = series + .iter() + .map(|metric| metric.points.len()) + .sum::(); + let earliest_timestamp = series + .iter() + .flat_map(|metric| metric.points.iter()) + .map(|point| point.timestamp) + .min() + .context("submitted metrics contain no points")?; + let latest_timestamp = series + .iter() + .flat_map(|metric| metric.points.iter()) + .map(|point| point.timestamp) + .max() + .context("submitted metrics contain no points")?; + + Ok(serde_json::json!({ + "format": "datadog-submission-result-v1", + "success": true, + "repository": format!("{}/{}", config.org, config.repo), + "metric_prefix": prefix, + "lookback": history, + "activity_window": activity_window, + "series_count": series.len(), + "point_count": point_count, + "batch_count": batch_http_statuses.len(), + "batch_http_statuses": batch_http_statuses, + "earliest_timestamp_utc": format_timestamp(earliest_timestamp)?, + "latest_timestamp_utc": format_timestamp(latest_timestamp)?, + })) +} + fn print_dry_run(series: &[MetricSeries]) { let point_count = series .iter() @@ -875,4 +929,32 @@ mod tests { assert_eq!(output["batches"][0]["series"][0]["metric"], "test.metric"); assert_eq!(output["batches"][0]["series"][0]["points"][0]["value"], 7); } + + #[test] + fn emits_compact_submission_result() { + let config = Config { + github_token: String::new(), + org: "example".to_string(), + repo: "repo".to_string(), + }; + let series = vec![MetricSeries::gauge( + "github.health.v2.issues".to_string(), + 7, + 1_788_134_400, + vec!["repo:example/repo".to_string()], + )]; + + let output = + submission_result(&config, "7d", "30d", "github.health.v2", &series, &[202]).unwrap(); + + assert_eq!(output["format"], "datadog-submission-result-v1"); + assert_eq!(output["success"], true); + assert_eq!(output["repository"], "example/repo"); + assert_eq!(output["series_count"], 1); + assert_eq!(output["point_count"], 1); + assert_eq!(output["batch_count"], 1); + assert_eq!(output["batch_http_statuses"][0], 202); + assert_eq!(output["earliest_timestamp_utc"], "2026-08-31T00:00:00Z"); + assert_eq!(output["latest_timestamp_utc"], "2026-08-31T00:00:00Z"); + } } diff --git a/src/main.rs b/src/main.rs index e01bf9d..ea03e48 100644 --- a/src/main.rs +++ b/src/main.rs @@ -124,14 +124,12 @@ enum Command { )] output_json: bool, }, - /// Fetch GitHub data and submit Datadog metrics + /// Fetch GitHub data and prepare or submit Datadog metrics SyncMetrics { #[arg(long, help = "Repository, e.g. vectordotdev/vector")] repo: String, #[arg(long, help = "Path to .env file (may contain GITHUB_TOKEN, DD_API_KEY, DD_SITE)")] env_file: Option, - #[arg(long, help = "Datadog API key (prefer DD_API_KEY in automation)")] - dd_api_key: Option, #[arg(long, help = "Datadog site hostname, e.g. datadoghq.eu")] dd_site: Option, #[arg( @@ -150,6 +148,13 @@ enum Command { prefix: String, #[arg( long, + conflicts_with_all = ["dry_run", "output_json"], + help = "Submit metrics to Datadog using DD_API_KEY from the environment" + )] + submit: bool, + #[arg( + long, + conflicts_with = "output_json", help = "Fetch data and build metrics, but do not send to Datadog" )] dry_run: bool, @@ -361,16 +366,22 @@ fn main() -> Result<()> { Command::SyncMetrics { repo, env_file, - dd_api_key, dd_site, lookback, activity_window, prefix, + submit, dry_run, output_json, } => { + if !submit && !dry_run && !output_json { + anyhow::bail!("sync-metrics requires one of --submit, --dry-run, or --output-json"); + } let config = Config::load(&Repo::parse(&repo)?, env_file.as_deref())?; - let api_key = dd_api_key.or_else(|| std::env::var("DD_API_KEY").ok()); + let api_key = std::env::var("DD_API_KEY").ok(); + if submit && api_key.as_deref().is_none_or(|key| key.trim().is_empty()) { + anyhow::bail!("DD_API_KEY not set"); + } let site = dd_site.or_else(|| std::env::var("DD_SITE").ok()); workflows::sync_metrics( &config,