From 5d9460877de5bfabae37503f64a3221db00229e5 Mon Sep 17 00:00:00 2001 From: Alex Guerrieri Date: Mon, 19 Oct 2026 12:56:26 +0200 Subject: [PATCH] feat: log every failed attempt that will be retried - only a parked task was logged, so retries were invisible until the budget ran out Claude-Session: https://claude.ai/code/session_01JeE7nT7xaWppRm9ngjdD6B --- executor.go | 4 ++++ executor_test.go | 20 ++++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index 8d33867..a07e1f3 100644 --- a/executor.go +++ b/executor.go @@ -205,6 +205,10 @@ func (m *Manager) executePlan(ctx context.Context, claimed *store.ClaimedRun) St return StatusStuck } + taskLog.WarnContext(ctx, "workflow: task attempt failed", + slog.Int("attempt", attempt), slog.Int("budget", budget), + slog.Duration("backoff", backoff), slog.Any("error", err)) + if backoff <= m.cfg.MaxInProcessBackoff { persisted, perr := m.db.PersistTaskFailure(ctx, runID, seq, failure, backoff, false, claimID) if perr != nil { diff --git a/executor_test.go b/executor_test.go index d297af7..04cb4c0 100644 --- a/executor_test.go +++ b/executor_test.go @@ -177,6 +177,16 @@ func (h *recordingHandler) messages() []string { return slices.Clone(h.msgs) } +func countMessage(h *recordingHandler, msg string) int { + n := 0 + for _, m := range h.messages() { + if m == msg { + n++ + } + } + return n +} + func TestNextBackoff(t *testing.T) { cfg := Config{BackoffBase: time.Second, BackoffMax: time.Minute} schedule := []time.Duration{time.Second, 5 * time.Second, 30 * time.Second} @@ -673,11 +683,14 @@ func TestExecutor(t *testing.T) { return map[string]string{"v": "ok"}, nil }, Backoff(10*time.Millisecond, 50*time.Millisecond), MaxAttempts(5)), ))) - mgr := newManager(t, db, reg, func(c *Config) { c.MaxInProcessBackoff = time.Second }) + rec := new(recordingHandler) + mgr, err := NewManager(t.Context(), db.DB, reg, Config{MaxInProcessBackoff: time.Second}, slog.New(rec)) + require.NoError(t, err) plan := seedPlan(t, db, ctx, mgr, "test.retry", testInput{Seed: "x"}) require.Equal(t, StatusSucceeded, mgr.executePlan(ctx, claim(t, db, ctx, plan.ID))) require.EqualValues(t, 3, calls.Load()) + require.Equal(t, 2, countMessage(rec, "workflow: task attempt failed")) got, err := db.GetRun(ctx, plan.ID) require.NoError(t, err) @@ -695,11 +708,14 @@ func TestExecutor(t *testing.T) { return nil, errors.New("transient") }, Backoff(20*time.Second, time.Minute), MaxAttempts(5)), ))) - mgr := newManager(t, db, reg) + rec := new(recordingHandler) + mgr, err := NewManager(t.Context(), db.DB, reg, Config{}, slog.New(rec)) + require.NoError(t, err) plan := seedPlan(t, db, ctx, mgr, "test.longbackoff", testInput{Seed: "x"}) before := time.Now() require.Equal(t, StatusRunning, mgr.executePlan(ctx, claim(t, db, ctx, plan.ID))) + require.Equal(t, 1, countMessage(rec, "workflow: task attempt failed")) got, err := db.GetRun(ctx, plan.ID) require.NoError(t, err)