fix: dashboard failures and retry summary timing out on big databases

SW.Bitween.Api/Resources/Dashboard/FailuresToActOn.cs

@@ -0,0 +1,47 @@
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.EntityFrameworkCore;
+using SW.Bitween.Domain;
+using SW.Bitween.Resources.Xchanges;
+using SW.PrimitiveTypes;
+
+namespace SW.Bitween.Resources.Dashboard;
+
+/// <summary>
+/// How many failures nothing has been retried from yet: the ends of chains, the problems still to
+/// deal with. The number the exchange list's failed + newest-attempt view counts, counted here from
+/// the failures rather than from the exchanges.
+/// </summary>
+/// <remarks>
+/// The dashboard used to ask the list for it, and the list fetches its rows newest first: on a
+/// database where nearly every failure gets retried, that walked back through millions of exchanges
+/// before reaching one, past the 30s limit - for a row the dashboard never showed. Here the failures
+/// come from their own index (ix_xchange_result_failed on Postgres) and the retry check from
+/// RetryFor's, so no exchange row is read. Capped like the list's count, so the client shows
+/// "10,000+" for either.
+/// </remarks>
+[HandlerName("failurestoacton")]
+public class FailuresToActOn : IQueryHandler<object>
+{
+    private readonly BitweenDbContext _dbContext;
+    private readonly RequestContext _requestContext;
+
+    public FailuresToActOn(BitweenDbContext dbContext, RequestContext requestContext)
+    {
+        _dbContext = dbContext;
+        _requestContext = requestContext;
+    }
+
+    public async Task<object> Handle()
+    {
+        await _requestContext.EnsurePermission(_dbContext, Model.Permissions.Dashboard.View);
+
+        var total = await _dbContext.Set<XchangeResult>().AsNoTracking()
+            .Where(result => !result.Success
+                             && !_dbContext.Set<Xchange>().Any(child => child.RetryFor == result.Id))
+            .Take(Search.CountCap + 1)
+            .CountAsync();
+
+        return new { Total = total };
+    }
+}

SW.Bitween.Api/Resources/Dashboard/RetrySummary.cs

@@ -67,32 +67,42 @@ group result by result.Success && !result.ResponseBad into worked
 
         // Chains that keep failing: a failure nothing has been retried from - so it is where that
         // chain has got to - which is itself a retry, so the chain is at least two attempts long.
+        //
+        // Over the same seven days as the number above, and asked so that indexes answer all of it:
+        // the retries from their own (ix_xchange_retries_started_on on Postgres), whether each
+        // failed from the failures' and whether it was retried from RetryFor's. With no time bound
+        // it walked back through every retry ever made until it found five hundred failures - on a
+        // database where every failure gets retried, millions of exchanges, past the 30s limit.
         var candidates = await (
             from xchange in _dbContext.Set<Xchange>()
-            join result in _dbContext.Set<XchangeResult>() on xchange.Id equals result.Id
             where xchange.RetryFor != null
-                  && !result.Success
+                  && xchange.StartedOn >= since
+                  && _dbContext.Set<XchangeResult>().Any(result => result.Id == xchange.Id && !result.Success)
                   && !_dbContext.Set<Xchange>().Any(child => child.RetryFor == xchange.Id)
             orderby xchange.StartedOn descending
-            select new
-            {
-                xchange.Id,
-                xchange.RetryFor,
-                xchange.SubscriptionId,
-                xchange.DocumentId,
-                xchange.StartedOn,
-                result.Exception
-            }).AsNoTracking().Take(Candidates).ToListAsync();
+            select new { xchange.Id, xchange.RetryFor, xchange.StartedOn })
+            .AsNoTracking().Take(Candidates).ToListAsync();
 
         var attempts = await CountAttempts(candidates.ToDictionary(c => c.Id, c => c.RetryFor));
 
-        var worst = candidates
+        var worstIds = candidates
             .OrderByDescending(c => attempts[c.Id])
             .ThenByDescending(c => c.StartedOn)
             .Take(Listed)
+            .Select(c => c.Id)
+            .ToList();
+
+        // What is shown of them, and the names below, for the few that are actually listed rather
+        // than for all five hundred.
+        var worst = (await (
+                from xchange in _dbContext.Set<Xchange>()
+                join result in _dbContext.Set<XchangeResult>() on xchange.Id equals result.Id
+                where worstIds.Contains(xchange.Id)
+                select new { xchange.Id, xchange.SubscriptionId, xchange.DocumentId, xchange.StartedOn, result.Exception })
+            .AsNoTracking().ToListAsync())
+            .OrderBy(w => worstIds.IndexOf(w.Id))
             .ToList();
 
-        // Names for the few that are actually listed, rather than for all five hundred.
         var subscriptionIds = worst.Where(w => w.SubscriptionId != null).Select(w => w.SubscriptionId!.Value).ToList();
         var subscriptionNames = await _dbContext.Set<Subscription>().AsNoTracking()
             .Where(s => subscriptionIds.Contains(s.Id))

SW.Bitween.IntegrationTests/Tests/RetryChainTests.cs

@@ -530,6 +530,73 @@ public async Task RetrySummary_ranks_the_chains_that_keep_failing_and_says_wheth
         Assert.True(retries.GetProperty("Finished").GetInt32() >= retries.GetProperty("Succeeded").GetInt32());
     }
 
+    /// <summary>
+    /// The panel looks at the same seven days as its retry numbers: a chain nothing has retried for
+    /// longer is still a failure to act on, but no longer one that keeps failing.
+    /// </summary>
+    [Fact]
+    public async Task RetrySummary_leaves_out_chains_nothing_has_retried_for_a_week()
+    {
+        await using var scope = _fixture.CreateScope();
+        var db = scope.ServiceProvider.GetRequiredService<BitweenDbContext>();
+        var xs = scope.ServiceProvider.GetRequiredService<XchangeService>();
+        var ctx = scope.Superuser();
+
+        var (_, first) = await FailedXchange(db, xs, "Summary Old Doc");
+        var second = await RetryOnce(db, xs, ctx, first.Id);
+        var third = await RetryOnce(db, xs, ctx, second.Id);
+        var chain = new[] { first.Id, second.Id, third.Id };
+        await db.Set<Xchange>().Where(x => chain.Contains(x.Id))
+            .ExecuteUpdateAsync(s => s.SetProperty(x => x.StartedOn, x => x.StartedOn.AddDays(-8)));
+
+        var summary = await new Resources.Dashboard.RetrySummary(db, ctx).Handle();
+        var json = System.Text.Json.JsonSerializer.SerializeToElement(summary);
+
+        Assert.DoesNotContain(json.GetProperty("FailingChains").EnumerateArray(),
+            c => c.GetProperty("Id").GetString() == third.Id);
+    }
+
+    /// <summary>
+    /// The dashboard's failures-to-act-on tile counts from the failures rather than through the
+    /// exchange list, and has to come to the number the list shows for the view the tile opens.
+    /// </summary>
+    [Fact]
+    public async Task FailuresToActOn_counts_what_the_list_of_failures_to_act_on_counts()
+    {
+        await using var scope = _fixture.CreateScope();
+        var db = scope.ServiceProvider.GetRequiredService<BitweenDbContext>();
+        var xs = scope.ServiceProvider.GetRequiredService<XchangeService>();
+        var ctx = scope.Superuser();
+
+        var tile = new Resources.Dashboard.FailuresToActOn(db, ctx);
+        async Task<int> Total() =>
+            System.Text.Json.JsonSerializer.SerializeToElement(await tile.Handle()).GetProperty("Total").GetInt32();
+        var before = await Total();
+
+        // A chain of three failed attempts is one failure to act on: the attempt at its end.
+        var (sub, first) = await FailedXchange(db, xs, "To Act On Doc");
+        var second = await RetryOnce(db, xs, ctx, first.Id);
+        await RetryOnce(db, xs, ctx, second.Id);
+
+        // A failure whose retry worked leaves nothing to act on.
+        var healed = await xs.CreateXchange(sub, new XchangeFile("{}"));
+        await db.SaveChangesAsync();
+        db.Set<XchangeResult>().Add(new XchangeResult(healed.Id, null, null, exception: "boom"));
+        await db.SaveChangesAsync();
+        await new Resources.Xchanges.Retry(db, ctx, xs).Handle(healed.Id, new XchangeRetry());
+        await db.SaveChangesAsync();
+        var healedRetry = await db.Set<Xchange>().FirstAsync(x => x.RetryFor == healed.Id);
+        db.Set<XchangeResult>().Add(new XchangeResult(healedRetry.Id, null, null));
+        await db.SaveChangesAsync();
+
+        var after = await Total();
+        Assert.Equal(before + 1, after);
+
+        var listed = (SearchyResponse<XchangeRow>)await new Resources.Xchanges.Search(db, xs, ctx)
+            .Handle(new SearchyRequest("filter=StatusFilter:1:3&filter=LatestOnly:1:true") { PageSize = 1 });
+        Assert.Equal(listed.TotalCount, after);
+    }
+
     // ─── Reading the chain back ───────────────────────────────────────────────
 
     [Fact]

SW.Bitween.PgSql/BitweenDbContext.cs

@@ -407,6 +407,13 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 b.HasIndex(i => i.SubscriptionId);
                 b.HasIndex(i => i.StartedOn);
                 b.HasIndex(i => i.RetryFor);
+                // Retries newest first, for the dashboard's retry panel: a few percent of the
+                // exchanges, read without touching the rows.
+                b.HasIndex(i => i.StartedOn, "ix_xchange_retries_started_on")
+                    .HasDatabaseName("ix_xchange_retries_started_on")
+                    .HasFilter("retry_for IS NOT NULL")
+                    .IncludeProperties(i => new { i.Id, i.RetryFor })
+                    .IsCreatedConcurrently();
             });
 
             modelBuilder.Entity<OnHoldXchange>(b =>
@@ -438,6 +445,12 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
                 b.Property(p => p.HandlerVersion).HasMaxLength(50);
                 // A subscription's latest result: what auto-pause counts from.
                 b.HasIndex(p => new { p.SubscriptionId, p.FinishedOn }).IsCreatedConcurrently();
+                // The failures alone, a few percent of the results: what the dashboard counts the
+                // failures still to act on from, and how it asks whether a retry failed.
+                b.HasIndex(p => p.Id, "ix_xchange_result_failed")
+                    .HasDatabaseName("ix_xchange_result_failed")
+                    .HasFilter("NOT success")
+                    .IsCreatedConcurrently();
 
                 b.HasOne<Xchange>().WithOne().HasForeignKey<XchangeResult>(p => p.Id).OnDelete(DeleteBehavior.Cascade);
             });

SW.Bitween.PgSql/ConcurrentIndex.cs

@@ -10,14 +10,20 @@ namespace SW.Bitween.PgSql;
 /// table of tens of millions of rows, inside the startup migration. CONCURRENTLY doesn't, but it
 /// cannot run in a transaction, and a build that fails part-way leaves the index behind marked
 /// invalid: the IF NOT EXISTS would then skip the rebuild and the migration would pass with an
-/// index Postgres never uses. So an invalid leftover is dropped first, as
-/// PromotedPropertiesTrigramIndex does.
+/// index Postgres never uses. So an invalid leftover goes first.
+///
+/// Not with a plain DROP INDEX, which locks the whole table: queued behind whatever is reading it,
+/// it holds up every read and insert that comes after. The leftover is renamed out of the way
+/// instead - a lock reads and writes don't wait for - and dropped concurrently, which a DO block
+/// can't do. A valid index of the name is left as it is.
 /// </remarks>
 internal static class ConcurrentIndex
 {
     /// <param name="definition">What follows <c>ON infolink.table</c>: the columns, and any WHERE.</param>
     public static void Create(MigrationBuilder migrationBuilder, string table, string name, string definition)
     {
+        var leftover = name + "_invalid";
+
         migrationBuilder.Sql(
             $@"DO $$
                BEGIN
@@ -28,10 +34,12 @@ FROM pg_index i
                        JOIN pg_namespace n ON n.oid = c.relnamespace
                        WHERE n.nspname = '{BitweenDbContext.Schema}' AND c.relname = '{name}' AND NOT i.indisvalid
                    ) THEN
-                       DROP INDEX {BitweenDbContext.Schema}.{name};
+                       ALTER INDEX {BitweenDbContext.Schema}.{name} RENAME TO {leftover};
                    END IF;
                END $$;");
 
+        Drop(migrationBuilder, leftover);
+
         migrationBuilder.Sql(
             $"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} ON {BitweenDbContext.Schema}.{table} {definition};",
             suppressTransaction: true);