From 21dedc2acae534c31fa40ddf81ba6bf64bbc822f Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Fri, 4 Sep 2026 22:44:28 +0500 Subject: [PATCH 1/2] fix(observability): ignore reboot OOM resets and 30s inventory blips host_oom_detected used unsigned max-min span of a boot-scoped counter, so gha-runner-2's 09:43Z kernel reboot replayed as observed 3 with zero CONSTRAINT_NONE lines this boot. lifecycle_inventory_gap used an aggregated subquery that dropped zero steps and paged two 30-second missing_instances=1 listing blips. Both replacements replay as 0 against the live store at those timestamps. --- CHANGELOG.md | 11 ++++ config/observability-rules.yaml | 31 +++++++-- docs/runbooks/fleet-alerts.md | 15 ++++- internal/observabilityrules/rules_test.go | 77 ++++++++++++++++++++++- 4 files changed, 124 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4054adf..a17f1666 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +- Stop paging a kernel reboot as a host-global OOM, and stop paging a + 30-second inventory listing blip as `lifecycle_inventory_gap`. + `host_oom_detected` used max-min span of the boot-scoped counter, so + gha-runner-2's 09:43Z reboot (6.8.0-138 → 6.8.0-139, zero CONSTRAINT_NONE + lines this boot) replayed as observed 3 — the previous boot's three real + kills from 2026-09-03 18:09–18:22Z. last-min of the same window is 0. + `lifecycle_inventory_gap` used `min_over_time((max(a)+max(b)+max(c))[2m:30s])`, + which dropped the zero steps and returned 1 at 07:30:16Z and 14:41:31Z while + `missing_instances` was 1 for one sample. Raw `min_over_time(metric[2m])` + is 0 at both timestamps. Replayed against the live store before shipping. + - Write a world-readable `/run/nddev/image-fingerprint` from the root assignment so an unprivileged job can attest the live image without the Incus guest API, which answers HTTP 401 to uid runner. The parent diff --git a/config/observability-rules.yaml b/config/observability-rules.yaml index 9e83d4d3..99a5779e 100644 --- a/config/observability-rules.yaml +++ b/config/observability-rules.yaml @@ -311,10 +311,19 @@ rules: # republishes the same total after a restart, OpenObserve starts a new # series at that value, and increase() treats 0→N as N new kills. Measured # 2026-09-04 01:36Z: observed 4 and 3 with no dmesg after the image-cutover - # observer restart. The in-window span is zero across a republish and one - # when the boot total actually moves. `round` still strips PromQL's - # boundary extrapolation so a single integer event does not page as 1.05. - expression: round(max by (host_name) (max_over_time(gha_fleet_host_oom_kills_total[5m]) - min_over_time(gha_fleet_host_oom_kills_total[5m]))) + # observer restart. `round` still strips PromQL's boundary extrapolation + # so a single integer event does not page as 1.05. + # + # Do not use max_over_time - min_over_time either. That span is unsigned: + # a kernel reboot that resets the boot total 3→0 has the same span as + # three new kills. Measured 2026-09-04 09:43Z on gha-runner-2: the host + # rebooted 6.8.0-138 → 6.8.0-139, this boot's journal has zero + # CONSTRAINT_NONE lines, and the page still said observed 3 because the + # previous boot's three real kills (2026-09-03 18:09–18:22Z) were still + # in the five-minute window. last_over_time - min_over_time is zero across + # that reset and one when the boot total actually moves up. Replayed + # against the live store at 09:43:02Z and 09:46:08Z. + expression: round(max by (host_name) (last_over_time(gha_fleet_host_oom_kills_total[5m]) - min_over_time(gha_fleet_host_oom_kills_total[5m]))) operator: ">" threshold: 0 evaluation_seconds: 30 @@ -539,11 +548,21 @@ rules: # minutes of normal traffic the raw counter was non-zero in two samples of # twenty-four. Paging on that buries the case worth waking for, which is an # intent still uncovered after nothing is coming for it. - expression: max(gha_fleet_incus_orphan_instances) + max(gha_fleet_journal_missing_instances) + max(gha_fleet_queue_uncovered_running_beyond_grace) + # + # Hold cannot be the renderer's aggregated subquery. Measured 2026-09-04 + # against the live engine: min_over_time((max(missing)+max(orphan)+max( + # uncovered_bg))[2m:30s]) returned 1 at 07:30:16Z and 14:41:31Z while + # missing_instances was 1 for one 30-second sample and 0 for the rest of + # the window. The subquery drops the zero steps of an aggregated gauge, + # so min of the remaining 1 is 1 and the page fires on a listing blip. + # Raw min_over_time(metric[2m]) keeps those zeros (0 at both timestamps) + # and is the hold. evaluation_seconds equals hold_seconds so the renderer + # does not wrap this again. + expression: max(min_over_time(gha_fleet_incus_orphan_instances[2m]) + min_over_time(gha_fleet_journal_missing_instances[2m]) + min_over_time(gha_fleet_queue_uncovered_running_beyond_grace[2m])) operator: ">" threshold: 0 evaluation_seconds: 30 - hold_seconds: 120 + hold_seconds: 30 destination_ref: fleet_oncall enabled: true owner: fleet-operations diff --git a/docs/runbooks/fleet-alerts.md b/docs/runbooks/fleet-alerts.md index 968feb61..79d3e934 100644 --- a/docs/runbooks/fleet-alerts.md +++ b/docs/runbooks/fleet-alerts.md @@ -71,9 +71,18 @@ gha-fleet reconcile-openobserve-alerts \ - OOM or pressure pages: close admission; never stop an already running worker merely to make utilization look healthy. `host_oom_detected` is host-global `CONSTRAINT_NONE` only. Memory-cgroup kills are the envelope working; they - must not close the member. The detector is the in-window span of - `gha_fleet_host_oom_kills_total`, not `increase()`, because an observer - restart republishes the boot total and `increase()` treats that as a burst. + must not close the member. The detector is last minus min of + `gha_fleet_host_oom_kills_total` over five minutes, not `increase()` and + not max minus min: an observer restart republishes the boot total and + `increase()` treats that as a burst, while a kernel reboot that resets the + counter N→0 has the same unsigned span as N new kills. Confirm a new + `constraint=CONSTRAINT_NONE` journal line on the current boot before + treating the page as a live kill. +- Inventory-gap pages: a 30-second `missing_instances=1` listing blip is not + the page. The rule requires each of orphan, missing and uncovered-beyond-grace + to stay non-zero for two minutes of raw samples. Preserve all three states + before recovering; a member reboot without a drain marker is still a gap if + it lasts that long. - Host-signal tickets: use `gha_fleet_host_signal_events` cumulative deltas. LVM activation and overlay `xino=off` are workload-volume context; audit suppression and workqueue-hog alerts act only on their bounded burst budget. diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index b138b7ce..3e93286f 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -34,7 +34,8 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) { "audit_suppression_burst": `signal_class="audit_suppressed"`, "kernel_workqueue_hog": `signal_class="kernel_workqueue_hog"`, "host_compliance_observer_missing": "gha_fleet_host_compliance_observer_up", - "host_oom_detected": "round(max by (host_name) (max_over_time(gha_fleet_host_oom_kills_total[5m]) - min_over_time(gha_fleet_host_oom_kills_total[5m])))", + "host_oom_detected": "round(max by (host_name) (last_over_time(gha_fleet_host_oom_kills_total[5m]) - min_over_time(gha_fleet_host_oom_kills_total[5m])))", + "lifecycle_inventory_gap": "min_over_time(gha_fleet_journal_missing_instances[2m])", "host_package_inventory_stale": "gha_fleet_host_package_inventory_age_seconds", "host_reboot_required": "gha_fleet_host_reboot_required", "host_standard_updates_available": "gha_fleet_host_standard_updates_available", @@ -361,6 +362,80 @@ func TestQueueWaitRulesPartitionTheSameSeries(t *testing.T) { // Non-negative counters compared against zero say the same thing with `+`, and // a window with no events says it with an empty result, which is honest and // does not fire. +// host_oom_detected and lifecycle_inventory_gap both paged on 2026-09-04 +// for facts the live store still holds, and both expressions replayed as +// firing at those timestamps. The replacements replay as 0. +// +// host_oom_detected: gha-runner-2 rebooted 09:43:43Z (6.8.0-138 → 6.8.0-139). +// The previous boot's three CONSTRAINT_NONE kills were 2026-09-03 18:09–18:22Z. +// max_over_time - min_over_time of the boot-scoped counter was 3 at 09:43:02Z +// and 09:46:08Z; last_over_time - min_over_time was 0 at both. +// +// lifecycle_inventory_gap: missing_instances was 1 for one 30-second sample +// at 07:30:16Z and 14:41:31Z. min_over_time((max(a)+max(b)+max(c))[2m:30s]) +// returned 1 — the subquery dropped the zero steps — while +// min_over_time(metric[2m]) returned 0. Hold lives in that raw range, so +// evaluation_seconds equals hold_seconds and the renderer must not wrap it. +func TestAlertHoldsThatTheBackendActuallyEvaluates(t *testing.T) { + bundle, err := Load("../../config/observability-rules.yaml") + if err != nil { + t.Fatal(err) + } + rendered, err := RenderOpenObserve(bundle, "fleet_oncall", true) + if err != nil { + t.Fatal(err) + } + byID := map[string]Rule{} + for _, rule := range bundle.Rules { + byID[rule.ID] = rule + } + oom, ok := byID["host_oom_detected"] + if !ok { + t.Fatal("host_oom_detected rule is missing") + } + if strings.Contains(oom.Expression, "max_over_time(gha_fleet_host_oom_kills_total[5m]) - min_over_time") { + t.Fatalf("host_oom_detected still pages a counter reset: %s", oom.Expression) + } + if !strings.Contains(oom.Expression, "last_over_time(gha_fleet_host_oom_kills_total[5m]) - min_over_time(gha_fleet_host_oom_kills_total[5m])") { + t.Fatalf("host_oom_detected does not count only upward movement: %s", oom.Expression) + } + gap, ok := byID["lifecycle_inventory_gap"] + if !ok { + t.Fatal("lifecycle_inventory_gap rule is missing") + } + if gap.HoldSecs > gap.EvaluationSecs { + t.Fatalf("lifecycle_inventory_gap hold %d > eval %d would wrap the raw range in the aggregated subquery that dropped zeros", gap.HoldSecs, gap.EvaluationSecs) + } + if strings.Contains(gap.Expression, "max(gha_fleet_incus_orphan_instances) + max(gha_fleet_journal_missing_instances)") { + t.Fatalf("lifecycle_inventory_gap still sums instant max() gauges: %s", gap.Expression) + } + for _, metric := range []string{ + "gha_fleet_incus_orphan_instances", + "gha_fleet_journal_missing_instances", + "gha_fleet_queue_uncovered_running_beyond_grace", + } { + want := "min_over_time(" + metric + "[2m])" + if !strings.Contains(gap.Expression, want) { + t.Fatalf("lifecycle_inventory_gap missing %s in %s", want, gap.Expression) + } + } + for _, alert := range rendered.Alerts { + switch alert.Name { + case "host_oom_detected": + if !strings.Contains(alert.QueryCondition.PromQL, "last_over_time(gha_fleet_host_oom_kills_total[5m])") { + t.Fatalf("rendered host_oom_detected lost last_over_time: %s", alert.QueryCondition.PromQL) + } + case "lifecycle_inventory_gap": + if strings.Contains(alert.QueryCondition.PromQL, "[2m:30s]") || strings.Contains(alert.QueryCondition.PromQL, "[2m:15s]") { + t.Fatalf("rendered lifecycle_inventory_gap was wrapped in the aggregated subquery: %s", alert.QueryCondition.PromQL) + } + if !strings.Contains(alert.QueryCondition.PromQL, "min_over_time(gha_fleet_journal_missing_instances[2m])") { + t.Fatalf("rendered lifecycle_inventory_gap lost the raw missing range: %s", alert.QueryCondition.PromQL) + } + } + } +} + func TestNoRuleDependsOnASetOperatorTheBackendDiscards(t *testing.T) { bundle, err := Load("../../config/observability-rules.yaml") if err != nil { From 86c7d6aa096f611fff3638c269b5813fcf3347e1 Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Fri, 4 Sep 2026 23:39:31 +0500 Subject: [PATCH 2/2] fix(observability): ignore OpenObserve ingest gaps and scalar max()+max() holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instant count(up==1) paged compute_pressure_observer_missing and host_compliance_observer_missing at 09:54–09:58Z while all members were up: OpenObserve had not replayed every series after the services reboot. count(last_over_time([10m])==1) was 4 and 5 at those timestamps. github_correlation_persistent used the same scalar max(a)+max(b) subquery that dropped zeros for lifecycle_inventory_gap. Bake min_over_time on the raw series and keep hold equal to evaluation so the renderer does not wrap it again. --- CHANGELOG.md | 7 +++ config/observability-rules.yaml | 28 +++++++-- docs/runbooks/fleet-alerts.md | 5 ++ internal/observabilityrules/rules_test.go | 74 ++++++++++++++++++++++- 4 files changed, 105 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a17f1666..e6711b50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,13 @@ `missing_instances` was 1 for one sample. Raw `min_over_time(metric[2m])` is 0 at both timestamps. Replayed against the live store before shipping. +- Stop paging OpenObserve ingest after a services reboot as missing + observers, and apply the inventory-gap hold to `github_correlation_persistent`. + Instant `count(up==1)` was 2 at 09:54Z and 3 at 09:58:43Z while all four + members were up; `count(last_over_time(up[10m])==1)` was 4 at both. + Compliance was 3 vs 5 on the same window. The sibling `max(a)+max(b)` + correlation page used the same scalar subquery that dropped zeros. + - Write a world-readable `/run/nddev/image-fingerprint` from the root assignment so an unprivileged job can attest the live image without the Incus guest API, which answers HTTP 401 to uid runner. The parent diff --git a/config/observability-rules.yaml b/config/observability-rules.yaml index 99a5779e..c397844e 100644 --- a/config/observability-rules.yaml +++ b/config/observability-rules.yaml @@ -114,11 +114,18 @@ rules: severity: page query_language: promql stream_name: up - expression: count(up{service_name="pressure-state"} == 1) + # Instant count(up==1) is the current ingest set, not the live observers. + # Measured 2026-09-04 after gha-services rebooted at 09:49Z: count was 2 + # at 09:54Z and 3 at 09:58:43Z while all four members were up, because + # OpenObserve had not yet replayed every series. last_over_time[10m] was + # 4 at both timestamps — the pre-reboot samples still identify the hosts. + # A member whose collector has been silent for ten minutes falls out. + # hold equals evaluation so the renderer does not wrap this in a subquery. + expression: count(last_over_time(up{service_name="pressure-state"}[10m]) == 1) operator: "<" threshold: 4 evaluation_seconds: 30 - hold_seconds: 120 + hold_seconds: 30 destination_ref: fleet_oncall enabled: true owner: fleet-operations @@ -275,11 +282,16 @@ rules: # are non-negative counts compared against zero, so their sum is positive # exactly when either is -- the same alert, in an operator the backend # evaluates. - expression: max(gha_fleet_queue_missing_workflow_run_id_beyond_grace) + max(gha_fleet_queue_unbound_repository_beyond_grace) + # + # Same hold as lifecycle_inventory_gap: min_over_time((max(a)+max(b))[2m:30s]) + # drops the zero steps of a scalar aggregation, so a 30-second blip pages. + # Raw min_over_time(metric[2m]) keeps those zeros. hold equals evaluation + # so the renderer does not wrap this again. + expression: max(min_over_time(gha_fleet_queue_missing_workflow_run_id_beyond_grace[2m]) + min_over_time(gha_fleet_queue_unbound_repository_beyond_grace[2m])) operator: ">" threshold: 0 evaluation_seconds: 30 - hold_seconds: 120 + hold_seconds: 30 destination_ref: fleet_oncall enabled: true owner: fleet-operations @@ -291,11 +303,15 @@ rules: severity: page query_language: promql stream_name: gha_fleet_host_compliance_observer_up - expression: count(gha_fleet_host_compliance_observer_up == 1) + # Same ingest-set vs live-host distinction as + # compute_pressure_observer_missing. Measured 2026-09-04 09:54Z: instant + # count was 3 of 5 while last_over_time[10m] was 5. A host that has not + # exported a complete sample for ten minutes is the page. + expression: count(last_over_time(gha_fleet_host_compliance_observer_up[10m]) == 1) operator: "<" threshold: 5 evaluation_seconds: 60 - hold_seconds: 180 + hold_seconds: 60 destination_ref: fleet_oncall enabled: true owner: fleet-operations diff --git a/docs/runbooks/fleet-alerts.md b/docs/runbooks/fleet-alerts.md index 79d3e934..5eaff05f 100644 --- a/docs/runbooks/fleet-alerts.md +++ b/docs/runbooks/fleet-alerts.md @@ -87,6 +87,11 @@ gha-fleet reconcile-openobserve-alerts \ LVM activation and overlay `xino=off` are workload-volume context; audit suppression and workqueue-hog alerts act only on their bounded burst budget. Raw host-signal logs are intentionally not duplicated into the fleet stream. +- Compliance and pressure-observer pages: `count(up==1)` is the current + ingest set, not the live hosts. After OpenObserve restarts it fills in over + minutes while the members are already up. The page counts hosts whose last + sample in ten minutes is 1; a collector silent for ten minutes is the + page, a backend that has not yet replayed a live series is not. - Compliance alerts: require complete coverage from every declared host before trusting package, reboot, kernel or SRSO state. A kernel-reported hardware or microcode boundary is not a software rollout failure. diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index 3e93286f..da11438f 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -22,18 +22,18 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) { t.Fatal(err) } wanted := map[string]string{ - "github_correlation_persistent": "gha_fleet_queue_missing_workflow_run_id_beyond_grace", + "github_correlation_persistent": "min_over_time(gha_fleet_queue_missing_workflow_run_id_beyond_grace[2m])", "lifecycle_queued_delivery_stall": "gha_fleet_queue_oldest_queued_wait_seconds_by_scale_set", "memory_psi_slow_burn": `window_seconds="10"`, "queue_wait_slow_burn": "gha_fleet_queue_oldest_queued_wait_seconds_by_scale_set", "provider_retry_error_persistent": `gha_fleet_provider_retry_deferred_records_by_error_class{error_class=~"identity|intent|provider|timeout|unknown"}`, - "compute_pressure_observer_missing": `count(up{service_name="pressure-state"} == 1)`, + "compute_pressure_observer_missing": `count(last_over_time(up{service_name="pressure-state"}[10m]) == 1)`, "compute_pressure_state_stale": "gha_fleet_pressure_observer_up", "compute_root_disk_low": "gha_fleet_host_root_free_percent", "kernel_slab_unreclaimable": "gha_fleet_host_slab_unreclaimable_attributed_bytes", "audit_suppression_burst": `signal_class="audit_suppressed"`, "kernel_workqueue_hog": `signal_class="kernel_workqueue_hog"`, - "host_compliance_observer_missing": "gha_fleet_host_compliance_observer_up", + "host_compliance_observer_missing": "last_over_time(gha_fleet_host_compliance_observer_up[10m])", "host_oom_detected": "round(max by (host_name) (last_over_time(gha_fleet_host_oom_kills_total[5m]) - min_over_time(gha_fleet_host_oom_kills_total[5m])))", "lifecycle_inventory_gap": "min_over_time(gha_fleet_journal_missing_instances[2m])", "host_package_inventory_stale": "gha_fleet_host_package_inventory_age_seconds", @@ -419,6 +419,36 @@ func TestAlertHoldsThatTheBackendActuallyEvaluates(t *testing.T) { t.Fatalf("lifecycle_inventory_gap missing %s in %s", want, gap.Expression) } } + pressure, ok := byID["compute_pressure_observer_missing"] + if !ok { + t.Fatal("compute_pressure_observer_missing rule is missing") + } + if !strings.Contains(pressure.Expression, "last_over_time(up{service_name=\"pressure-state\"}[10m])") { + t.Fatalf("compute_pressure_observer_missing still counts the ingest set: %s", pressure.Expression) + } + if pressure.HoldSecs > pressure.EvaluationSecs { + t.Fatalf("compute_pressure_observer_missing hold %d > eval %d would wrap last_over_time in a subquery", pressure.HoldSecs, pressure.EvaluationSecs) + } + compliance, ok := byID["host_compliance_observer_missing"] + if !ok { + t.Fatal("host_compliance_observer_missing rule is missing") + } + if !strings.Contains(compliance.Expression, "last_over_time(gha_fleet_host_compliance_observer_up[10m])") { + t.Fatalf("host_compliance_observer_missing still counts the ingest set: %s", compliance.Expression) + } + if compliance.HoldSecs > compliance.EvaluationSecs { + t.Fatalf("host_compliance_observer_missing hold %d > eval %d would wrap last_over_time in a subquery", compliance.HoldSecs, compliance.EvaluationSecs) + } + corr, ok := byID["github_correlation_persistent"] + if !ok { + t.Fatal("github_correlation_persistent rule is missing") + } + if corr.HoldSecs > corr.EvaluationSecs { + t.Fatalf("github_correlation_persistent hold %d > eval %d would wrap the scalar max()+max() subquery that drops zeros", corr.HoldSecs, corr.EvaluationSecs) + } + if strings.Contains(corr.Expression, "max(gha_fleet_queue_missing_workflow_run_id_beyond_grace) + max(gha_fleet_queue_unbound_repository_beyond_grace)") { + t.Fatalf("github_correlation_persistent still sums instant max() gauges: %s", corr.Expression) + } for _, alert := range rendered.Alerts { switch alert.Name { case "host_oom_detected": @@ -432,6 +462,44 @@ func TestAlertHoldsThatTheBackendActuallyEvaluates(t *testing.T) { if !strings.Contains(alert.QueryCondition.PromQL, "min_over_time(gha_fleet_journal_missing_instances[2m])") { t.Fatalf("rendered lifecycle_inventory_gap lost the raw missing range: %s", alert.QueryCondition.PromQL) } + case "compute_pressure_observer_missing": + if strings.Contains(alert.QueryCondition.PromQL, "count(up{service_name=\"pressure-state\"} == 1)") && + !strings.Contains(alert.QueryCondition.PromQL, "last_over_time") { + t.Fatalf("rendered compute_pressure_observer_missing still counts the ingest set: %s", alert.QueryCondition.PromQL) + } + case "github_correlation_persistent": + if strings.Contains(alert.QueryCondition.PromQL, "[2m:30s]") { + t.Fatalf("rendered github_correlation_persistent was wrapped in the aggregated subquery: %s", alert.QueryCondition.PromQL) + } + } + } +} + +// Instant max(a)+max(b) of gauges that are 0 when healthy cannot take the +// renderer's min_over_time((expr)[hold:eval]) wrap: the subquery drops zero +// steps and a 30-second blip pages. Those rules bake min_over_time on the +// raw series and set hold equal to evaluation. +func TestScalarZeroGaugesDoNotUseAggregatedSubqueryHold(t *testing.T) { + bundle, err := Load("../../config/observability-rules.yaml") + if err != nil { + t.Fatal(err) + } + for _, rule := range bundle.Rules { + if rule.QueryLanguage != "" && rule.QueryLanguage != "promql" { + continue + } + if rule.Operator != ">" || rule.Threshold != 0 { + continue + } + if !strings.Contains(rule.Expression, "max(") || !strings.Contains(rule.Expression, ") + ") { + continue + } + if rule.HoldSecs > rule.EvaluationSecs { + t.Errorf("%s sums scalar max() gauges with a renderer wrap that drops zeros: hold %d eval %d expr %s", + rule.ID, rule.HoldSecs, rule.EvaluationSecs, rule.Expression) + } + if !strings.Contains(rule.Expression, "min_over_time(") { + t.Errorf("%s sums scalar max() gauges without a raw min_over_time hold: %s", rule.ID, rule.Expression) } } }