From 92d1a0fa50dcd9739f36068416e56a3689faae3a Mon Sep 17 00:00:00 2001 From: Dustin Smith Date: Sun, 22 Nov 2026 23:04:29 +0700 Subject: [PATCH] fix: compare inclusion dates in the schedule's time zone The missed-inclusion check compared inclusion dates with the UTC date. West of UTC an inclusion date became due at UTC midnight, the local evening before, where the schedule.time gate did not apply, so IRREGULAR_SCHEDULE_MISSED fired early and its once-per-date marker then suppressed the alert for a real miss. East of UTC it waited for UTC midnight. PastInclusionDates now takes the location and compares with today's date in schedule.timezone, the same zone the time gate uses. --- CHANGELOG.md | 1 + docs/content/docs/architecture/watchdog.md | 2 +- docs/content/docs/configuration/schedules.md | 4 +- docs/content/docs/reference/alerting.md | 2 +- internal/lambda/date.go | 10 ++--- internal/lambda/inclusion_test.go | 39 +++++++++++++++++--- internal/lambda/watchdog/missed.go | 5 +-- internal/lambda/watchdog/missed_test.go | 31 ++++++++++++++++ 8 files changed, 77 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d481fa..3c2c67f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`IRREGULAR_SCHEDULE_MISSED` compared inclusion dates with the UTC date, so west of UTC it fired the evening before the date and was then never sent for a real miss; east of UTC it waited for UTC midnight.** Dates are now compared in `schedule.timezone`. - **When an alert could not be published, the watchdog still closed the stale run as `FAILED_FINAL`, or still wrote the once-per-date marker for a missed inclusion date or a missing post-run sensor, so the alert was lost for good.** The run now stays `RUNNING` and the marker is not written, and the next pass tries again. - **A run whose execution died could be deleted by DynamoDB before the watchdog reported it: no `SFN_TIMEOUT` was sent and the watchdog then started the date a second time.** A running trigger row now carries its stale deadline as `staleAt` and no `ttl`, so DynamoDB never deletes it and a dead run is closed only by the watchdog (a live run's row is closed by its own execution). A pipeline switched to dry-run has its leftover running row closed the same way. - **A rerun whose execution could not start deleted the date's trigger row and still spent the rerun budget, so the retried request was rejected and the date lost its record.** The date now keeps its status, no budget is spent, and a retry of the request can start the rerun. diff --git a/docs/content/docs/architecture/watchdog.md b/docs/content/docs/architecture/watchdog.md index 28e04e7..43c8547 100644 --- a/docs/content/docs/architecture/watchdog.md +++ b/docs/content/docs/architecture/watchdog.md @@ -8,7 +8,7 @@ The watchdog is one of six Lambda functions in the Interlock framework. It runs 1. **Stale triggers** -- a Step Function execution started but never completed (timeout, infrastructure failure) 2. **Missed schedules** -- a cron pipeline's job has not started by the cron start-by time for a `RUN#` date -3. **Missed inclusion schedules** -- a pipeline with an inclusion calendar has no trigger on a scheduled date. A date whose evaluation was released before its cutoff has no trigger row either, so `IRREGULAR_SCHEDULE_MISSED` fires once for it, as for a date that never started. The once-per-date marker (`SENSOR#irregular-missed-check#`) is written only after the publish succeeds, so a failed publish is retried on the next pass +3. **Missed inclusion schedules** -- a pipeline with an inclusion calendar has no trigger on a scheduled date. The check starts when the date begins in `schedule.timezone` (UTC when unset), or at `schedule.time` on that date when it is set; without `schedule.time` it fires at local midnight of the date, so set `time` to avoid an alert before the run is due. A date whose evaluation was released before its cutoff has no trigger row either, so `IRREGULAR_SCHEDULE_MISSED` fires once for it, as for a date that never started. The once-per-date marker (`SENSOR#irregular-missed-check#`) is written only after the publish succeeds, so a failed publish is retried on the next pass 4. **Sensor-trigger reconciliation** -- a sensor-triggered pipeline's conditions (with or without `schedule.cron`) are met but no trigger exists (self-heals missed triggers); past the date's `schedule.trigger.cutoff` it closes the date with `FAILED_FINAL` and one `SENSOR_DEADLINE_EXPIRED` instead of starting it 5. **SLA scheduling** -- proactively ensures EventBridge Scheduler entries exist for each expected run date (`RUN#` row) of a pipeline with an SLA config 6. **Cutoffs** -- an expected run date (`RUN#` row) passed its `schedule.trigger.cutoff` without starting; it is closed with `FAILED_FINAL` and one `SENSOR_DEADLINE_EXPIRED` diff --git a/docs/content/docs/configuration/schedules.md b/docs/content/docs/configuration/schedules.md index 55cd0f2..b92ab19 100644 --- a/docs/content/docs/configuration/schedules.md +++ b/docs/content/docs/configuration/schedules.md @@ -31,8 +31,8 @@ schedule: | `trigger` | object | no | Condition that activates the evaluation loop (see below) | | `evaluation` | object | no | Window and interval settings for the evaluation loop | | `exclude` | object | no | Dates the pipeline does not run (see [Calendar Exclusions](#calendar-exclusions)) | -| `include` | object | no | `dates` (`YYYY-MM-DD` list) the pipeline is expected to run on; the watchdog publishes `IRREGULAR_SCHEDULE_MISSED` for a past date with no trigger row. It does not stop runs on other dates. Not allowed with `cron` | -| `time` | string | no | `HH:MM` in `timezone`: on an `include` date, the missed-date check waits until this time. Not allowed with `cron` | +| `include` | object | no | `dates` (`YYYY-MM-DD` list) the pipeline is expected to run on; the watchdog publishes `IRREGULAR_SCHEDULE_MISSED` for a date with no trigger row once that date has begun in `timezone` (UTC when unset). It does not stop runs on other dates. Not allowed with `cron` | +| `time` | string | no | `HH:MM` in `timezone`: on an `include` date, the missed-date check waits until this time. Without it the check fires at local midnight of the date, so set `time` to avoid an alert before the run is due. Not allowed with `cron` | ### Trigger Condition diff --git a/docs/content/docs/reference/alerting.md b/docs/content/docs/reference/alerting.md index e8deaba..ff0ff70 100644 --- a/docs/content/docs/reference/alerting.md +++ b/docs/content/docs/reference/alerting.md @@ -102,7 +102,7 @@ Published by the **watchdog** Lambda, invoked on an EventBridge schedule (defaul |---|---|---| | `RELATIVE_SLA_WARNING` | A relative SLA (`sla.maxDuration`) is at risk | Once per date whose run has not finished, on the first watchdog pass at or after first sensor arrival + `maxDuration` - `expectedDuration` (75% of `maxDuration` without `expectedDuration`) and before the breach; detail carries `maxDuration`, `expectedDuration`, `sensorArrivalAt`, `warningAt` and `breachAt` | | `RELATIVE_SLA_BREACH` | A relative SLA (`sla.maxDuration`) was exceeded | Once per date whose run has not finished, on the first watchdog pass at or after first sensor arrival + `maxDuration`; detail carries `maxDuration`, `sensorArrivalAt` and `breachAt` | -| `IRREGULAR_SCHEDULE_MISSED` | An inclusion-calendar date has no run | Once per past `schedule.include` date with no trigger row (after `schedule.time` on that date); retried on the next pass if the publish fails | +| `IRREGULAR_SCHEDULE_MISSED` | An inclusion-calendar date has no run | Once per `schedule.include` date with no trigger row, once the date has begun in `schedule.timezone` (UTC when unset) and, when set, `schedule.time` on that date has passed; without `schedule.time` it fires at local midnight of the date. Retried on the next pass if the publish fails | | `SCHEDULE_MISSED` | The job for a run date has not started by the cron's start-by time, so the SLA is at risk | Once per `RUN#` date of a cron pipeline, on the first watchdog pass after the start-by time; detail carries `cron` and `startBy` | | `TRIGGER_RECOVERED` | Sensor trigger condition met but no trigger existed | Watchdog re-evaluated sensor data and self-healed a trigger the stream-router missed. A restart of a date whose evaluation was released publishes `JOB_TRIGGERED` instead | | `WATCHDOG_DEGRADED` | One or more watchdog checks failed | Detail lists the failed checks; `pipelineId` is empty. Not an alert | diff --git a/internal/lambda/date.go b/internal/lambda/date.go index 4d09ff3..e082837 100644 --- a/internal/lambda/date.go +++ b/internal/lambda/date.go @@ -89,13 +89,13 @@ func ResolveTimezone(tz string) *time.Location { // Caps DynamoDB reads when the watchdog has been down for an extended period. const maxInclusionLookback = 3 -// PastInclusionDates returns dates from the list that are on or before now, -// sorted most recent first and capped at maxInclusionLookback (3) entries. -// The cap bounds DynamoDB reads when the watchdog has been down for an +// PastInclusionDates returns dates from the list that are on or before now's +// date in loc, sorted most recent first and capped at maxInclusionLookback (3) +// entries. The cap bounds DynamoDB reads when the watchdog has been down for an // extended period. Dates must be YYYY-MM-DD strings; unparseable entries // are silently skipped. Returns nil if no dates qualify. -func PastInclusionDates(dates []string, now time.Time) []string { - nowDate := now.Format("2006-01-02") +func PastInclusionDates(dates []string, now time.Time, loc *time.Location) []string { + nowDate := now.In(loc).Format("2006-01-02") var past []string for _, d := range dates { if _, err := time.Parse("2006-01-02", d); err != nil { diff --git a/internal/lambda/inclusion_test.go b/internal/lambda/inclusion_test.go index 9826a41..cf9adf2 100644 --- a/internal/lambda/inclusion_test.go +++ b/internal/lambda/inclusion_test.go @@ -13,7 +13,7 @@ func TestPastInclusionDates_AllFuture(t *testing.T) { now := time.Date(2026, 3, 10, 14, 0, 0, 0, time.UTC) dates := []string{"2026-03-11", "2026-04-01", "2026-06-30"} - got := lambda.PastInclusionDates(dates, now) + got := lambda.PastInclusionDates(dates, now, time.UTC) assert.Empty(t, got, "all future dates should return empty slice") } @@ -21,7 +21,7 @@ func TestPastInclusionDates_MultiplePast(t *testing.T) { now := time.Date(2026, 3, 10, 14, 0, 0, 0, time.UTC) dates := []string{"2026-01-15", "2026-02-28", "2026-03-05"} - got := lambda.PastInclusionDates(dates, now) + got := lambda.PastInclusionDates(dates, now, time.UTC) assert.Equal(t, []string{"2026-03-05", "2026-02-28", "2026-01-15"}, got, "should return all past dates, most recent first") } @@ -30,7 +30,7 @@ func TestPastInclusionDates_MixedPastFuture(t *testing.T) { now := time.Date(2026, 3, 10, 14, 0, 0, 0, time.UTC) dates := []string{"2026-01-01", "2026-03-08", "2026-03-12", "2026-04-01"} - got := lambda.PastInclusionDates(dates, now) + got := lambda.PastInclusionDates(dates, now, time.UTC) assert.Equal(t, []string{"2026-03-08", "2026-01-01"}, got, "should only return past dates, most recent first") } @@ -39,7 +39,7 @@ func TestPastInclusionDates_TodayIncluded(t *testing.T) { now := time.Date(2026, 3, 10, 14, 0, 0, 0, time.UTC) dates := []string{"2026-03-05", "2026-03-10", "2026-03-15"} - got := lambda.PastInclusionDates(dates, now) + got := lambda.PastInclusionDates(dates, now, time.UTC) assert.Equal(t, []string{"2026-03-10", "2026-03-05"}, got, "today's date should be included in results") } @@ -50,8 +50,37 @@ func TestPastInclusionDates_CappedAt3(t *testing.T) { "2026-01-01", "2026-01-15", "2026-02-01", "2026-02-15", "2026-03-01", } - got := lambda.PastInclusionDates(dates, now) + got := lambda.PastInclusionDates(dates, now, time.UTC) assert.Len(t, got, 3, "result should be capped at 3 entries") assert.Equal(t, []string{"2026-03-01", "2026-02-15", "2026-02-01"}, got, "should return the 3 most recent past dates, most recent first") } + +// TestPastInclusionDates_UsesLocation: a date is past once it has begun in +// loc, whatever the UTC date is. +func TestPastInclusionDates_UsesLocation(t *testing.T) { + load := func(name string) *time.Location { + loc, err := time.LoadLocation(name) + if err != nil { + t.Fatalf("load %s: %v", name, err) + } + return loc + } + dates := []string{"2026-03-10"} + tests := []struct { + name string + now time.Time + loc *time.Location + want []string + }{ + {"New York, the evening before", time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC), load("America/New_York"), nil}, + {"UTC midnight", time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC), time.UTC, dates}, + {"Tokyo, date begun", time.Date(2026, 3, 9, 15, 30, 0, 0, time.UTC), load("Asia/Tokyo"), dates}, + {"UTC, the day before", time.Date(2026, 3, 9, 15, 30, 0, 0, time.UTC), time.UTC, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, lambda.PastInclusionDates(dates, tt.now, tt.loc)) + }) + } +} diff --git a/internal/lambda/watchdog/missed.go b/internal/lambda/watchdog/missed.go index 67e8f46..e1ac32e 100644 --- a/internal/lambda/watchdog/missed.go +++ b/internal/lambda/watchdog/missed.go @@ -117,14 +117,13 @@ func detectMissedInclusionSchedules(ctx context.Context, d *lambda.Deps) error { continue } - pastDates := lambda.PastInclusionDates(cfg.Schedule.Include.Dates, now) + tzLoc := lambda.ResolveTimezone(cfg.Schedule.Timezone) + pastDates := lambda.PastInclusionDates(cfg.Schedule.Include.Dates, now, tzLoc) if len(pastDates) == 0 { continue } scheduleID := lambda.ResolveScheduleID(cfg) - - tzLoc := lambda.ResolveTimezone(cfg.Schedule.Timezone) today := now.In(tzLoc).Format("2006-01-02") for _, date := range pastDates { diff --git a/internal/lambda/watchdog/missed_test.go b/internal/lambda/watchdog/missed_test.go index 61db832..1317e11 100644 --- a/internal/lambda/watchdog/missed_test.go +++ b/internal/lambda/watchdog/missed_test.go @@ -147,6 +147,32 @@ func TestDetectMissedInclusionSchedules_PublishFailureRetries(t *testing.T) { } } +// TestDetectMissedInclusionSchedules_EveningBeforeThenMissed: a pass on the +// evening before the date in New York neither alerts nor marks the date, so +// the pass once the date's time has passed still alerts. +func TestDetectMissedInclusionSchedules_EveningBeforeThenMissed(t *testing.T) { + d, _, rec := slaDeps(t, "", "", func(c *types.PipelineConfig) { + c.SLA, c.Schedule.Time, c.Schedule.Timezone = nil, "09:00", "America/New_York" + c.Schedule.Include = &types.InclusionConfig{Dates: []string{"2026-03-31"}} + }) + ctx := context.Background() + now := time.Date(2026, 3, 31, 0, 0, 0, 0, time.UTC) + d.NowFunc = func() time.Time { return now } + for pass, at := range []time.Time{now, time.Date(2026, 3, 31, 13, 30, 0, 0, time.UTC)} { + now = at + if err := detectMissedInclusionSchedules(ctx, d); err != nil { + t.Fatalf("pass %d: %v", pass, err) + } + marker, err := d.Store.GetSensorData(ctx, "p", "irregular-missed-check#2026-03-31") + if err != nil { + t.Fatal(err) + } + if marked := marker != nil; len(rec.events) != pass || marked != (pass == 1) { + t.Fatalf("pass %d at %s: events %v, marker %v; want %d events and marker %v", pass, at.Format(time.RFC3339), rec.events, marked, pass, pass == 1) + } + } +} + // TestDetectMissedInclusionSchedules: one IRREGULAR_SCHEDULE_MISSED per past // inclusion date (the newest three) with no trigger row and no marker, newest // first, each followed by its marker. The clock is 09:00 on slaTestDate @@ -241,6 +267,11 @@ func TestDetectMissedInclusionSchedules(t *testing.T) { // 22:00 on the date in New York is already the next day in UTC. {name: "New York evening, before the time", tm: "23:30", tz: "America/New_York", now: time.Date(2026, 4, 1, 2, 0, 0, 0, time.UTC)}, {name: "no time", now: at(0, 5), want: end}, + // New York is UTC-4 here, so 00:00Z is 20:00 the evening before. + {name: "New York, the evening before", tm: "09:00", tz: "America/New_York", now: at(0, 0)}, + {name: "New York, the evening before, no time", tz: "America/New_York", now: at(0, 0)}, + {name: "New York, on the date past the time", tm: "09:00", tz: "America/New_York", now: at(13, 30), want: end}, + {name: "Tokyo, date begun before UTC midnight", tz: "Asia/Tokyo", now: time.Date(2026, 3, 30, 15, 30, 0, 0, time.UTC), want: end}, } for _, tt := range tests { tt.dates = end