Exchanges page shows its rows at once, and slow counts finish on the server

SW.Bitween.Api/Resources/Xchanges/Search.cs

@@ -156,8 +156,12 @@ from delayedRetry in drGroup.DefaultIfEmpty()
                 // as "10,000+". 36ms drops to 2.3ms unfiltered, 264ms to 25ms on Success. Anything
                 // filtered narrowly enough to act on still gets an exact number; only views far too
                 // broad to page through lose it, and 10,000 rows is 400 pages of Next.
-                TotalCount = await query.AsNoTracking().Search(searchyRequest.Conditions)
-                    .Take(CountCap + 1).CountAsync()
+                //
+                // Not at all with count=false, as Searchy's own search does: the Exchanges page asks
+                // for its rows that way so they show at once, and has the total from the summary.
+                TotalCount = searchyRequest.CountRows
+                    ? await query.AsNoTracking().Search(searchyRequest.Conditions).Take(CountCap + 1).CountAsync()
+                    : 0
             };
 
             return searchyResponse;

SW.Bitween.Api/Resources/Xchanges/Summary.cs

@@ -1,7 +1,9 @@
 using System;
 using System.Collections.Generic;
 using System.Data;
+using System.Globalization;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.EntityFrameworkCore;
 using SW.Bitween.Model;
@@ -27,11 +29,16 @@ namespace SW.Bitween.Resources.Xchanges;
 /// Every special filter the search understands is applied the same way, so a count is never of a
 /// different set than the list it sits over.
 /// </para>
+/// <para>
+/// A count or a histogram that takes longer than <see cref="Budget"/> is finished in the background
+/// by <see cref="SlowCounts"/>, which keeps it for the next ask: until then it is missing from the
+/// answer, and <see cref="XchangeSummary.Counting"/> says so.
+/// </para>
 /// </remarks>
 [HandlerName("summary")]
-public class Summary(BitweenDbContext dbContext, RequestContext requestContext) : ISearchyHandler
+public class Summary(BitweenDbContext dbContext, RequestContext requestContext, SlowCounts slowCounts) : ISearchyHandler
 {
-    /// <summary>The status numbers <see cref="XchangeFilters.WhereStatus"/> reads, in the order they are counted.</summary>
+    /// <summary>The status numbers <see cref="XchangeFilters.WhereStatus"/> reads.</summary>
     private const string Processing = "0", Success = "1", BadResponse = "2", Failed = "3";
 
     public async Task<object> Handle(SearchyRequest searchyRequest, bool lookup = false, string searchPhrase = null)
@@ -47,6 +54,11 @@ public async Task<object> Handle(SearchyRequest searchyRequest, bool lookup = fa
             foreach (var statusFilter in condition.Filters.Where(f => f.Field == "StatusFilter").ToList())
                 condition.Filters.Remove(statusFilter);
 
+        // Taken before FilteredRowsAsync takes the special filters out of the request: a count finished
+        // in the background filters again from this, with a database of its own.
+        var asked = (SearchyRequest)searchyRequest.Clone();
+        var view = ViewKey(searchyRequest, DateTime.UtcNow);
+
         var rows = await XchangeFilters.FilteredRowsAsync(dbContext, searchyRequest);
 
         // Read after FilteredRowsAsync, which is what moves the request's dates to UTC. The
@@ -58,15 +70,24 @@ public async Task<object> Handle(SearchyRequest searchyRequest, bool lookup = fa
         // One capped count per status rather than a single grouped count: each can stop at the
         // cap, as the search's total does, where a grouping has to visit every matching row. Four
         // short queries over a broad filter cost far less than one exhaustive one.
-        var counts = new XchangeStatusCounts
+        //
+        // All four share one budget, so the answer never waits longer than it, however many are slow.
+        // The ones with an index of their own go first; still running, which can't have one, last.
+        var counts = new XchangeStatusCounts();
+        using (var deadline = new CancellationTokenSource(Budget))
         {
-            Processing = await CountCapped(rows, Processing),
-            Success = await CountCapped(rows, Success),
-            BadResponse = await CountCapped(rows, BadResponse),
-            Failed = await CountCapped(rows, Failed),
-        };
+            counts.BadResponse = await Count(BadResponse, deadline.Token);
+            counts.Failed = await Count(Failed, deadline.Token);
+            counts.Success = await Count(Success, deadline.Token);
+            counts.Processing = await Count(Processing, deadline.Token);
+        }
 
-        var summary = new XchangeSummary { Counts = counts };
+        var summary = new XchangeSummary
+        {
+            Counts = counts,
+            Counting = counts.Processing == null || counts.Success == null || counts.BadResponse == null ||
+                       counts.Failed == null
+        };
         if (rangeStart is not { } start)
             return summary;
 
@@ -75,14 +96,117 @@ public async Task<object> Handle(SearchyRequest searchyRequest, bool lookup = fa
             return summary;
 
         var bucketMinutes = XchangeHistogram.ChooseBucketMinutes(start, end);
+        using var histogramDeadline = new CancellationTokenSource(Budget);
+        var started = await Answer($"histogram:{bucketMinutes}|{view}", histogramDeadline.Token,
+            ct => StartedCounts(rows, bucketMinutes, ct),
+            async (db, ct) => await StartedCounts(await XchangeFilters.FilteredRowsAsync(db, Copy(asked)), bucketMinutes, ct));
+        if (started == null)
+        {
+            summary.Counting = true;
+            return summary;
+        }
+
         summary.BucketMinutes = bucketMinutes;
-        summary.Buckets = XchangeHistogram.Fold(start, end, bucketMinutes,
-            await StartedCounts(rows, bucketMinutes));
+        summary.Buckets = XchangeHistogram.Fold(start, end, bucketMinutes, started);
         return summary;
+
+        Task<int?> Count(string status, CancellationToken deadline) =>
+            Answer($"count:{status}|{view}", deadline,
+                ct => CountCapped(rows, status, dbContext, ct),
+                async (db, ct) => await CountCapped(await XchangeFilters.FilteredRowsAsync(db, Copy(asked)), status, db, ct));
+    }
+
+    /// <summary>
+    /// The answer <see cref="SlowCounts"/> keeps for <paramref name="key"/>, when it is one that was too
+    /// slow before. Otherwise worked out <paramref name="here"/> within the budget, and when that runs
+    /// out, left to <see cref="SlowCounts"/> to work out <paramref name="later"/>: null until then.
+    /// </summary>
+    private async Task<T> Answer<T>(string key, CancellationToken deadline, Func<CancellationToken, Task<T>> here,
+        Func<BitweenDbContext, CancellationToken, Task<T>> later)
+    {
+        if (slowCounts.TryGet(key, later, out var kept))
+            return kept;
+
+        var answer = await WithinBudget(dbContext, deadline, here);
+        if (answer == null)
+            slowCounts.Start(key, later);
+        return answer;
+    }
+
+    private static SearchyRequest Copy(SearchyRequest request) => (SearchyRequest)request.Clone();
+
+    /// <summary>
+    /// The filters as text, naming what was counted for <see cref="SlowCounts"/>. A start with no end -
+    /// what "last 24 hours" sends, a little later on every ask - is named by how far back it reaches,
+    /// to five minutes, so the asks of one page find the one count rather than each starting another.
+    /// </summary>
+    internal static string ViewKey(SearchyRequest request, DateTime now) =>
+        string.Join("|", request.Conditions.Select(condition =>
+        {
+            var openEnded = !condition.Filters.Any(f =>
+                f.Field == nameof(XchangeRow.StartedOn) && f.Rule == SearchyRule.LessThanOrEquals);
+            return string.Join("&", condition.Filters
+                .Select(f =>
+                    openEnded && f.Field == nameof(XchangeRow.StartedOn) && f.Rule == SearchyRule.GreaterThanOrEquals &&
+                    DateTime.TryParse(f.Value?.ToString(), CultureInfo.InvariantCulture,
+                        DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out var since)
+                        ? $"StartedOn:since:{Math.Round((now - since).TotalMinutes / 5)}"
+                        : f.ToString())
+                .OrderBy(f => f, StringComparer.Ordinal));
+        }));
+
+    /// <summary>How long the counts together, and then the histogram, may take before the rest is left to the background.</summary>
+    /// <remarks>
+    /// A status that is rare across a large table can only be counted by visiting every exchange:
+    /// with no result row to look up, "processing" can't be found any other way. On millions of
+    /// exchanges that ran past the database's own time limit and failed the whole request, taking
+    /// every other tab's count with it.
+    /// </remarks>
+    internal static readonly TimeSpan Budget = TimeSpan.FromSeconds(5);
+
+    /// <summary>
+    /// What <paramref name="query"/> returns, or the default - null - when <paramref name="deadline"/>
+    /// passes first. Runs inside the summary's transaction, under a savepoint: a statement cancelled
+    /// part-way leaves Postgres refusing every later one in the transaction until it is rolled back to
+    /// before it.
+    /// </summary>
+    internal static async Task<T> WithinBudget<T>(BitweenDbContext dbContext, CancellationToken deadline,
+        Func<CancellationToken, Task<T>> query)
+    {
+        const string savepoint = "within_budget";
+        var transaction = dbContext.Database.CurrentTransaction!;
+        await transaction.CreateSavepointAsync(savepoint);
+
+        try
+        {
+            return await query(deadline);
+        }
+        // Whatever the provider throws for a cancelled command: Npgsql an OperationCanceledException,
+        // SqlClient a SqlException. Only once the deadline has passed: an error before it surfaces, and
+        // one after it is taken for the deadline, then met again by the count in the background, which
+        // logs it.
+        catch (Exception) when (deadline.IsCancellationRequested)
+        {
+            await transaction.RollbackToSavepointAsync(savepoint);
+            return default;
+        }
     }
 
-    private Task<int> CountCapped(IQueryable<XchangeRow> rows, string status) =>
-        rows.WhereStatus(status, dbContext).Take(Search.CountCap + 1).CountAsync();
+    private static async Task<int?> CountCapped(IQueryable<XchangeRow> rows, string status, BitweenDbContext dbContext,
+        CancellationToken cancellationToken)
+    {
+        var matching = rows.WhereStatus(status, dbContext);
+
+        // Nearly every exchange succeeds, so the newest 10,001 successes are among about the newest
+        // 10,001 exchanges. Unordered, Postgres matched the two tables by id instead, and ids aren't in
+        // the order exchanges were written, so nearly every row was read from a different place on
+        // disk: 12s on 5.8M exchanges, against under a second this way. The rarer statuses stay
+        // unordered, so they keep the plan that reads their own small index.
+        if (status == Success)
+            matching = matching.OrderByDescending(r => r.StartedOn);
+
+        return await matching.Take(Search.CountCap + 1).CountAsync(cancellationToken);
+    }
 
     /// <summary>
     /// The matching exchanges grouped by when they started, to the precision the bucket size
@@ -105,7 +229,7 @@ private Task<int> CountCapped(IQueryable<XchangeRow> rows, string status) =>
     /// </para>
     /// </remarks>
     private static async Task<List<XchangeStartedCount>> StartedCounts(IQueryable<XchangeRow> rows,
-        int bucketMinutes)
+        int bucketMinutes, CancellationToken cancellationToken)
     {
         if (bucketMinutes >= 1440)
             return await rows
@@ -119,7 +243,7 @@ private static async Task<List<XchangeStartedCount>> StartedCounts(IQueryable<Xc
                     ResponseBad = g.Key.ResponseBad,
                     Count = g.Count()
                 })
-                .ToListAsync();
+                .ToListAsync(cancellationToken);
 
         if (bucketMinutes >= 60)
             return await rows
@@ -138,7 +262,7 @@ private static async Task<List<XchangeStartedCount>> StartedCounts(IQueryable<Xc
                     ResponseBad = g.Key.ResponseBad,
                     Count = g.Count()
                 })
-                .ToListAsync();
+                .ToListAsync(cancellationToken);
 
         return await rows
             .GroupBy(r => new
@@ -157,7 +281,7 @@ private static async Task<List<XchangeStartedCount>> StartedCounts(IQueryable<Xc
                 ResponseBad = g.Key.ResponseBad,
                 Count = g.Count()
             })
-            .ToListAsync();
+            .ToListAsync(cancellationToken);
     }
 
     /// <summary>

SW.Bitween.Api/Services/Retention/RetentionPlanner.cs

@@ -13,7 +13,7 @@ namespace SW.Bitween;
 /// <summary>
 /// Works out what the retention settings do - as they stand, or as someone is about to change them - so
 /// the settings page can say it before anything is saved: how long files and exchanges are kept, what
-/// the next run removes, and which exchanges are left without their files.
+/// the next run removes, and whether exchanges are kept longer than their files.
 /// </summary>
 public class RetentionPlanner(BitweenDbContext dbContext, BitweenOptions options, StorageRetention storageRetention,
     StorageAccess storageAccess)
@@ -35,8 +35,8 @@ public async Task<RetentionStatus> Plan(RetentionProposal proposal = null, bool
         var now = DateTime.UtcNow;
 
         var files = Retention(prefix, rules);
-        // Exchanges already written keep the prefix they were written under, so how many are left
-        // without files depends on the prefix in force now, not on a proposed one.
+        // Exchanges already written keep the prefix they were written under, so whether they outlive
+        // their files depends on the prefix in force now, not on a proposed one.
         var currentFiles = prefix == options.DocumentPrefix ? files : Retention(options.DocumentPrefix, rules);
         var legacyPrefix = options.LegacyDocumentPrefix;
         var legacyFiles = !string.IsNullOrEmpty(legacyPrefix) && legacyPrefix != prefix ? Retention(legacyPrefix, rules) : null;
@@ -51,29 +51,10 @@ public async Task<RetentionStatus> Plan(RetentionProposal proposal = null, bool
             ? await ExchangeRetention.DueBefore(dbContext, now.AddDays(-days)).Take(CountCap).CountAsync()
             : null;
 
-        // As things stand now, whatever the proposal: those the next run removes are still listed until then.
-        // Each exchange's files follow the rule of the prefix they were written under, which a later change
-        // of prefix doesn't move; only those older than the shortest rule can have lost them.
-        int? withoutFiles = currentFiles.Days == null ? null : 0;
-        if (rules.Problem == null &&
-            rules.Lifecycle?.Rules.Where(r => r.Enabled).Select(r => (int?)r.Days).Min() is { } shortest)
-        {
-            var unrecordedPrefix = options.LegacyDocumentPrefix ?? options.DocumentPrefix;
-            var writtenUnder = await dbContext.Set<Xchange>()
-                .Where(x => x.StartedOn < now.AddDays(-shortest))
-                .Select(x => x.FilesPrefix).Distinct().ToListAsync();
-            foreach (var recorded in writtenUnder)
-            {
-                if (Retention(recorded ?? unrecordedPrefix, rules).Days is not { } filesDays) continue;
-                var filesGone = now.AddDays(-filesDays);
-                var counted = withoutFiles ?? 0;
-                withoutFiles = counted + await dbContext.Set<Xchange>()
-                    .Where(x => x.FilesPrefix == recorded && x.StartedOn < filesGone)
-                    .Take(CountCap - counted).CountAsync();
-                if (withoutFiles >= CountCap) break;
-            }
-        }
-
+        // How many exchanges have lost their files isn't counted: which rule applies depends on the
+        // prefix each was written under, and finding those meant reading every exchange older than the
+        // bucket's shortest rule - a day, with the standard temp1/ rule - which took over 30s on 5.8M
+        // exchanges and failed the page. When the oldest started is given instead.
         var oldest = await dbContext.Set<Xchange>().OrderBy(x => x.StartedOn)
             .Select(x => (DateTime?)x.StartedOn).FirstOrDefaultAsync();
 
@@ -106,7 +87,6 @@ public async Task<RetentionStatus> Plan(RetentionProposal proposal = null, bool
                 Cron = cron,
                 NextRun = nextRun,
                 DueNow = dueNow,
-                WithoutFiles = withoutFiles,
                 CountCap = CountCap,
                 Oldest = oldest
             }
@@ -145,7 +125,7 @@ private List<RetentionNotice> Notices(RetentionStatus status, PrefixRetention cu
         if (currentFiles.Days is { } filesDays && (days == 0 || days > filesDays))
             warnings.Add(Warning("OUTLIVE_FILES",
                 $"Exchanges older than {filesDays} days are listed without their files: they can't be opened " +
-                $"or retried. {(exchanges.WithoutFiles == 1 ? "There is 1" : $"There are {Count(exchanges.WithoutFiles, exchanges.CountCap)}")} right now." +
+                "or retried." +
                 (newOutliveFiles ? $" New exchanges lose theirs after {newFilesDays} days." : "")));
         else if (newOutliveFiles)
             warnings.Add(Warning("OUTLIVE_FILES",