diff --git a/src/NoteBookmark.Api.Tests/Endpoints/DeltaApiTests.cs b/src/NoteBookmark.Api.Tests/Endpoints/DeltaApiTests.cs index ca597ec..7c794a8 100644 --- a/src/NoteBookmark.Api.Tests/Endpoints/DeltaApiTests.cs +++ b/src/NoteBookmark.Api.Tests/Endpoints/DeltaApiTests.cs @@ -290,6 +290,45 @@ private async Task CreateAndSaveTestPost(string rowKey) return post; } + // ── Post ids ───────────────────────────────────────────────────────────── + + [Fact] + public async Task GetPostIds_ReturnsIdsOfReadAndUnreadPosts() + { + // Arrange + var unreadPost = CreateTestPost("ids-unread-post-1"); + var readPost = CreateTestPost("ids-read-post-1"); + readPost.is_read = true; + await _client.PostAsJsonAsync("/api/posts/", unreadPost); + await _client.PostAsJsonAsync("/api/posts/", readPost); + + // Act + var response = await _client.GetAsync("/api/posts/ids"); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + var ids = await response.Content.ReadFromJsonAsync>(); + ids.Should().NotBeNull(); + ids.Should().Contain(unreadPost.RowKey); + ids.Should().Contain(readPost.RowKey); + } + + [Fact] + public async Task GetPostIds_DoesNotReturnDeletedPosts() + { + // Arrange + var post = CreateTestPost("ids-deleted-post-1"); + await _client.PostAsJsonAsync("/api/posts/", post); + await _client.DeleteAsync($"/api/posts/{post.RowKey}"); + + // Act + var ids = await _client.GetFromJsonAsync>("/api/posts/ids"); + + // Assert + ids.Should().NotBeNull(); + ids.Should().NotContain(post.RowKey); + } + private static Post CreateTestPost(string rowKey) { return new Post diff --git a/src/NoteBookmark.Api/DataStorageService.cs b/src/NoteBookmark.Api/DataStorageService.cs index 45abbb4..6bfc50f 100644 --- a/src/NoteBookmark.Api/DataStorageService.cs +++ b/src/NoteBookmark.Api/DataStorageService.cs @@ -123,6 +123,16 @@ orderby post.Timestamp return lstPosts; } + public List GetPostIds() + { + var tblPosts = GetPostTable(); + // Only project the key columns so the query stays small even with thousands of posts. + // Clients identify posts by Id, falling back to RowKey, so return the same value. + return tblPosts.Query(select: new[] { "RowKey", "id", "Id" }) + .Select(p => p.Id ?? p.RowKey) + .ToList(); + } + public Post? GetPost(string rowKey) { var tblPost = GetPostTable(); diff --git a/src/NoteBookmark.Api/IDataStorageService.cs b/src/NoteBookmark.Api/IDataStorageService.cs index 85f9473..fbc8704 100644 --- a/src/NoteBookmark.Api/IDataStorageService.cs +++ b/src/NoteBookmark.Api/IDataStorageService.cs @@ -7,6 +7,8 @@ namespace NoteBookmark.Api; public interface IDataStorageService { public List GetFilteredPosts(string filter); + + public List GetPostIds(); public Post? GetPost(string rowKey); diff --git a/src/NoteBookmark.Api/PostEndpoints.cs b/src/NoteBookmark.Api/PostEndpoints.cs index 66a8a7e..ec85fd2 100644 --- a/src/NoteBookmark.Api/PostEndpoints.cs +++ b/src/NoteBookmark.Api/PostEndpoints.cs @@ -19,6 +19,8 @@ public static void MapPostEndpoints(this IEndpointRouteBuilder app) .WithDescription("Get all unread posts"); endpoints.MapGet("/read", GetReadPosts) .WithDescription("Get all read posts"); + endpoints.MapGet("/ids", GetPostIds) + .WithDescription("Get the ids of all existing posts (read and unread)"); endpoints.MapGet("/{id}", Get) .WithDescription("Get a post by id"); endpoints.MapGet("/{postId}/html", GetPostHtml) @@ -57,6 +59,12 @@ static List GetReadPosts(TableServiceClient tblClient, BlobServiceClient return posts; } + static List GetPostIds(TableServiceClient tblClient, BlobServiceClient blobClient) + { + var dataStorageService = new DataStorageService(tblClient, blobClient); + return dataStorageService.GetPostIds(); + } + static async Task>> GetPostHtml( string postId, BlobServiceClient blobClient) { diff --git a/src/NoteBookmark.MauiApp.Tests/LocalDataServiceTests.cs b/src/NoteBookmark.MauiApp.Tests/LocalDataServiceTests.cs index c3311d0..92b3830 100644 --- a/src/NoteBookmark.MauiApp.Tests/LocalDataServiceTests.cs +++ b/src/NoteBookmark.MauiApp.Tests/LocalDataServiceTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading.Tasks; using FluentAssertions; using NoteBookmark.Domain; @@ -161,4 +162,43 @@ public async Task SaveSettings_ShouldStoreAndRetrieveFontSize() retrieved.Should().NotBeNull(); retrieved!.FontSize.Should().Be("large"); } + + [Fact] + public async Task RemovePostsAsync_ShouldPermanentlyRemoveOnlyGivenPosts() + { + await _sut.SavePostsAsync(new[] + { + new Post { Id = "p1", PartitionKey = "pk", RowKey = "p1", Title = "One", DateModified = DateTime.UtcNow }, + new Post { Id = "p2", PartitionKey = "pk", RowKey = "p2", Title = "Two", DateModified = DateTime.UtcNow }, + new Post { Id = "p3", PartitionKey = "pk", RowKey = "p3", Title = "Three", DateModified = DateTime.UtcNow } + }); + await _sut.SavePostAsync(new Post { Id = "p2", PartitionKey = "pk", RowKey = "p2", Title = "Two edited" }, isPendingSync: true); + + await _sut.RemovePostsAsync(new[] { "p1", "p2", "missing" }); + + var remaining = await _sut.GetPostsAsync(); + remaining.Select(p => p.Id).Should().BeEquivalentTo(new[] { "p3" }); + (await _sut.GetPendingSyncPostsAsync()).Should().BeEmpty(); + } + + [Fact] + public async Task RemovePostsAsync_WithNoIds_ShouldDoNothing() + { + await _sut.SavePostAsync(new Post { Id = "p1", PartitionKey = "pk", RowKey = "p1", Title = "One" }); + + await _sut.RemovePostsAsync(Array.Empty()); + + (await _sut.GetPostsAsync()).Should().HaveCount(1); + } + + [Fact] + public async Task SavePostsAsync_ShouldPreservePendingSyncFlag_ForLocallyEditedPosts() + { + await _sut.SavePostAsync(new Post { Id = "p1", PartitionKey = "pk", RowKey = "p1", Title = "Local edit" }, isPendingSync: true); + + await _sut.SavePostsAsync(new[] { new Post { Id = "p1", PartitionKey = "pk", RowKey = "p1", Title = "From server" } }); + + var pending = await _sut.GetPendingSyncPostsAsync(); + pending.Should().ContainSingle(p => p.Id == "p1"); + } } diff --git a/src/NoteBookmark.MauiApp.Tests/SyncServiceTests.cs b/src/NoteBookmark.MauiApp.Tests/SyncServiceTests.cs index 4204ab5..b1fe4f2 100644 --- a/src/NoteBookmark.MauiApp.Tests/SyncServiceTests.cs +++ b/src/NoteBookmark.MauiApp.Tests/SyncServiceTests.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Linq; +using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Moq; @@ -123,11 +125,11 @@ public async Task PullPhase_RemoteNewer_ShouldOverwriteLocal() .ReturnsAsync(new List { remotePostL }); _apiClientMock.Setup(c => c.GetPost("post1")).ReturnsAsync(remotePost); _apiClientMock.Setup(c => c.GetNotesModifiedAfter(It.IsAny())).ReturnsAsync(new List()); - _localDataServiceMock.Setup(c => c.GetPostAsync("post1")).ReturnsAsync(localPost); + _localDataServiceMock.Setup(c => c.GetPostsAsync()).ReturnsAsync(new List { localPost }); await _sut.SyncAsync(); - _localDataServiceMock.Verify(c => c.SavePostAsync(It.Is(p => p.Title == "Remote"), false), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Single().Title == "Remote")), Times.Once); } [Fact] @@ -156,11 +158,12 @@ public async Task PullPhase_LocalNewer_ShouldKeepLocal() _apiClientMock.Setup(c => c.GetPostsModifiedAfter(It.IsAny())) .ReturnsAsync(new List { remotePostL }); _apiClientMock.Setup(c => c.GetNotesModifiedAfter(It.IsAny())).ReturnsAsync(new List()); - _localDataServiceMock.Setup(c => c.GetPostAsync("post1")).ReturnsAsync(localPost); + _localDataServiceMock.Setup(c => c.GetPostsAsync()).ReturnsAsync(new List { localPost }); await _sut.SyncAsync(); - _localDataServiceMock.Verify(c => c.SavePostAsync(It.IsAny(), false), Times.Never); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.IsAny>()), Times.Never); + _localDataServiceMock.Verify(c => c.SavePostAsync(It.IsAny(), It.IsAny()), Times.Never); } [Fact] @@ -190,11 +193,10 @@ public async Task PullPhase_NoLocal_ShouldSaveRemote() .ReturnsAsync(new List { remotePostL }); _apiClientMock.Setup(c => c.GetPost("post1")).ReturnsAsync(remotePost); _apiClientMock.Setup(c => c.GetNotesModifiedAfter(It.IsAny())).ReturnsAsync(new List()); - _localDataServiceMock.Setup(c => c.GetPostAsync("post1")).ReturnsAsync((Post?)null); await _sut.SyncAsync(); - _localDataServiceMock.Verify(c => c.SavePostAsync(It.Is(p => p.Title == "Remote"), false), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Single().Title == "Remote")), Times.Once); } [Fact] @@ -266,8 +268,7 @@ public async Task PullPhase_ShouldDeleteLocalPosts_WhenDeletedOnServer() await _sut.SyncAsync(); - _localDataServiceMock.Verify(c => c.DeletePostAsync("post1", false), Times.Once); - _localDataServiceMock.Verify(c => c.MarkSyncedAsync("post1", true), Times.Once); + _localDataServiceMock.Verify(c => c.RemovePostsAsync(It.Is>(ids => ids.SequenceEqual(new[] { "post1" }))), Times.Once); } [Fact] @@ -296,14 +297,13 @@ public async Task PullPhase_ShouldAddNewPosts_WhenAddedOnServer() _localDataServiceMock.Setup(c => c.GetPostsAsync()).ReturnsAsync(new List()); _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)) .ReturnsAsync(new List { remotePostL }); - _localDataServiceMock.Setup(c => c.GetPostAsync("post1")).ReturnsAsync((Post?)null); _apiClientMock.Setup(c => c.GetPost("post1")).ReturnsAsync(remotePost); _apiClientMock.Setup(c => c.GetNotesModifiedAfter(It.IsAny())).ReturnsAsync(new List()); await _sut.SyncAsync(); _apiClientMock.Verify(c => c.GetPost("post1"), Times.Once); - _localDataServiceMock.Verify(c => c.SavePostAsync(remotePost, false), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Single() == remotePost)), Times.Once); } [Fact] @@ -467,7 +467,7 @@ public async Task SyncAsync_ShouldRaiseSyncProgressChanged_WhenDownloadingPostHt progressEvents.Should().NotBeEmpty(); progressEvents.Should().Contain(e => e.Status == "Cleaning..."); - progressEvents.Should().Contain(e => e.Status == "Downloading 1 of 2 posts..." && e.Current == 1 && e.Total == 2); + progressEvents.Should().Contain(e => e.Status == "Downloading 0 of 2 posts..." && e.Current == 0 && e.Total == 2); progressEvents.Should().Contain(e => e.Status == "Downloading 2 of 2 posts..." && e.Current == 2 && e.Total == 2); progressEvents.Last().Status.Should().Be("Synchronization complete!"); progressEvents.Last().IsComplete.Should().BeTrue(); @@ -495,7 +495,7 @@ public async Task PullPhase_ReadPosts_ShouldNotCallGetPost_AndShouldSaveDirectly // GetPost should NOT be called for read posts _apiClientMock.Verify(c => c.GetPost("read1"), Times.Never); - _localDataServiceMock.Verify(c => c.SavePostAsync(It.Is(p => p.Id == "read1" && p.is_read == true), false), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Single().Id == "read1" && ps.Single().is_read == true)), Times.Once); } [Fact] @@ -520,7 +520,7 @@ public async Task PullPhase_UnreadPost_WhenGetPostFails_ShouldFallbackToBasicPos await _sut.SyncAsync(); // Should fall back and save basic post without throwing - _localDataServiceMock.Verify(c => c.SavePostAsync(It.Is(p => p.Id == "unread1" && p.Title == "Unread Post"), false), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Single().Id == "unread1" && ps.Single().Title == "Unread Post")), Times.Once); } [Fact] @@ -540,7 +540,6 @@ public async Task PullPhase_ShouldReportProgress_WhenPullingPosts() await _sut.SyncAsync(); progressEvents.Should().Contain(e => e.Status == "Pulling 0 of 2 posts..." && e.Current == 0 && e.Total == 2); - progressEvents.Should().Contain(e => e.Status == "Pulling 1 of 2 posts..." && e.Current == 1 && e.Total == 2); progressEvents.Should().Contain(e => e.Status == "Pulling 2 of 2 posts..." && e.Current == 2 && e.Total == 2); } @@ -581,5 +580,170 @@ public async Task IsSyncing_ShouldReflectActiveSyncTask() _sut.IsSyncing.Should().BeFalse(); } -} + // ── Issue #208: delta pull, deletion detection, batched save ───────────── + + private static PostL RemotePostL(string id, bool isRead = true, DateTime? modified = null) => new() + { + Id = id, + RowKey = id, + PartitionKey = "pk", + Title = $"Title {id}", + is_read = isRead, + DateModified = modified ?? DateTime.UtcNow + }; + + private static Post LocalPost(string id, DateTime? modified = null) => new() + { + Id = id, + RowKey = id, + PartitionKey = "pk", + Title = $"Title {id}", + is_read = true, + DateModified = modified ?? DateTime.UtcNow.AddDays(-1) + }; + + private void SetupEmptyPushAndNotes() + { + _localDataServiceMock.Setup(c => c.GetPendingSyncNotesAsync()).ReturnsAsync(new List()); + _apiClientMock.Setup(c => c.GetNotesModifiedAfter(It.IsAny())).ReturnsAsync(new List()); + } + + [Fact] + public async Task PullPhase_WithLastSync_ShouldOnlyPullDelta_AndUsePostIdsForDeletions() + { + var lastSync = DateTime.UtcNow.AddHours(-1); + SyncService.SetInMemoryPreference("LastSyncTimestamp", lastSync.ToString("O")); + SetupEmptyPushAndNotes(); + + _localDataServiceMock.Setup(c => c.GetPostsAsync()) + .ReturnsAsync(new List { LocalPost("p1"), LocalPost("p2"), LocalPost("p3") }); + _apiClientMock.Setup(c => c.GetPostIds()).ReturnsAsync(new List { "p1", "p3", "new" }); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(It.Is(d => d > DateTime.MinValue))) + .ReturnsAsync(new List { RemotePostL("new") }); + + await _sut.SyncAsync(); + + // The full post list is never requested once we have a last sync time. + _apiClientMock.Verify(c => c.GetPostsModifiedAfter(DateTime.MinValue), Times.Never); + _apiClientMock.Verify(c => c.GetPostsModifiedAfter(It.Is(d => Math.Abs((d - lastSync).TotalSeconds) < 1)), Times.Once); + _localDataServiceMock.Verify(c => c.RemovePostsAsync(It.Is>(ids => ids.SequenceEqual(new[] { "p2" }))), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Select(p => p.Id).SequenceEqual(new[] { "new" }))), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task PullPhase_WithLastSync_WhenPostIdsEndpointUnavailable_ShouldFallBackToFullListForDeletions() + { + SyncService.SetInMemoryPreference("LastSyncTimestamp", DateTime.UtcNow.AddHours(-1).ToString("O")); + SetupEmptyPushAndNotes(); + + _localDataServiceMock.Setup(c => c.GetPostsAsync()) + .ReturnsAsync(new List { LocalPost("p1"), LocalPost("p2") }); + _apiClientMock.Setup(c => c.GetPostIds()).ReturnsAsync((List?)null); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(It.Is(d => d > DateTime.MinValue))) + .ReturnsAsync(new List()); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)) + .ReturnsAsync(new List { RemotePostL("p1", modified: DateTime.UtcNow.AddDays(-2)) }); + + await _sut.SyncAsync(); + + _localDataServiceMock.Verify(c => c.RemovePostsAsync(It.Is>(ids => ids.SequenceEqual(new[] { "p2" }))), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.IsAny>()), Times.Never); + } + + [Fact] + public async Task PullPhase_FirstSync_ShouldUseFullListForDeletions_WithoutCallingPostIds() + { + SetupEmptyPushAndNotes(); + _localDataServiceMock.Setup(c => c.GetPostsAsync()).ReturnsAsync(new List { LocalPost("gone") }); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)) + .ReturnsAsync(new List { RemotePostL("p1") }); + + await _sut.SyncAsync(); + + _apiClientMock.Verify(c => c.GetPostIds(), Times.Never); + _localDataServiceMock.Verify(c => c.RemovePostsAsync(It.Is>(ids => ids.SequenceEqual(new[] { "gone" }))), Times.Once); + } + + [Fact] + public async Task PullPhase_NoDeletedPosts_ShouldNotCallRemovePosts() + { + SetupEmptyPushAndNotes(); + _localDataServiceMock.Setup(c => c.GetPostsAsync()).ReturnsAsync(new List { LocalPost("p1") }); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)) + .ReturnsAsync(new List { RemotePostL("p1", modified: DateTime.UtcNow.AddDays(-2)) }); + + await _sut.SyncAsync(); + + _localDataServiceMock.Verify(c => c.RemovePostsAsync(It.IsAny>()), Times.Never); + } + + [Fact] + public async Task PullPhase_ManyPosts_ShouldSaveInOneBatch() + { + SetupEmptyPushAndNotes(); + var remotePosts = Enumerable.Range(0, 1500).Select(i => RemotePostL($"p{i}")).ToList(); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)).ReturnsAsync(remotePosts); + + await _sut.SyncAsync(); + + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Count() == 1500)), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task PullPhase_DuplicateRowsForSamePost_ShouldSaveItOnce() + { + // The API joins posts with notes, so a post with two notes comes back twice. + SetupEmptyPushAndNotes(); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)) + .ReturnsAsync(new List { RemotePostL("p1"), RemotePostL("p1") }); + + await _sut.SyncAsync(); + + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Count() == 1)), Times.Once); + } + + [Fact] + public async Task PullPhase_ManyPosts_ShouldThrottleProgressEvents() + { + SetupEmptyPushAndNotes(); + var remotePosts = Enumerable.Range(0, 500).Select(i => RemotePostL($"p{i}")).ToList(); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)).ReturnsAsync(remotePosts); + + var progressEvents = new List(); + _sut.SyncProgressChanged += (sender, args) => { lock (progressEvents) progressEvents.Add(args); }; + + await _sut.SyncAsync(); + + var pullingEvents = progressEvents.Where(e => e.Status.StartsWith("Pulling ") && e.Total == 500).ToList(); + pullingEvents.Should().HaveCountLessThan(50); + pullingEvents.Should().Contain(e => e.Current == 0); + pullingEvents.Should().Contain(e => e.Current == 500 && e.Status == "Pulling 500 of 500 posts..."); + } + + [Fact] + public async Task PullPhase_UnreadPosts_ShouldFetchFullPostsWithLimitedConcurrency() + { + SetupEmptyPushAndNotes(); + var remotePosts = Enumerable.Range(0, 20).Select(i => RemotePostL($"u{i}", isRead: false)).ToList(); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)).ReturnsAsync(remotePosts); + + int inFlight = 0, maxInFlight = 0; + _apiClientMock.Setup(c => c.GetPost(It.IsAny())).Returns(async (string id) => + { + var now = Interlocked.Increment(ref inFlight); + lock (_apiClientMock) maxInFlight = Math.Max(maxInFlight, now); + await Task.Delay(20); + Interlocked.Decrement(ref inFlight); + return new Post { Id = id, RowKey = id, PartitionKey = "pk", Title = $"Full {id}", is_read = false }; + }); + + await _sut.SyncAsync(); + + maxInFlight.Should().BeGreaterThan(1).And.BeLessThanOrEqualTo(5); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => + ps.Count() == 20 && ps.All(p => p.Title!.StartsWith("Full ")))), Times.Once); + } +} diff --git a/src/NoteBookmark.MauiApp/Data/ILocalDataService.cs b/src/NoteBookmark.MauiApp/Data/ILocalDataService.cs index a20da8b..9689add 100644 --- a/src/NoteBookmark.MauiApp/Data/ILocalDataService.cs +++ b/src/NoteBookmark.MauiApp/Data/ILocalDataService.cs @@ -15,6 +15,8 @@ public interface ILocalDataService Task SaveNoteAsync(Note note, bool isPendingSync = false); Task DeleteNoteAsync(string rowKey, bool isPendingSync = false); Task DeletePostAsync(string rowKey, bool isPendingSync = false); + /// Permanently removes posts that no longer exist on the server, in a single transaction. + Task RemovePostsAsync(IEnumerable ids); Task> GetSummariesAsync(); Task SaveSummariesAsync(IEnumerable summaries); Task GetSettingsAsync(); diff --git a/src/NoteBookmark.MauiApp/Data/ISyncApiClient.cs b/src/NoteBookmark.MauiApp/Data/ISyncApiClient.cs index c71807a..52da789 100644 --- a/src/NoteBookmark.MauiApp/Data/ISyncApiClient.cs +++ b/src/NoteBookmark.MauiApp/Data/ISyncApiClient.cs @@ -9,6 +9,8 @@ namespace NoteBookmark.MauiApp.Data; public interface ISyncApiClient { Task> GetPostsModifiedAfter(DateTime modifiedAfter); + /// Ids of all posts on the server, or null if the server does not support it. + Task?> GetPostIds(); Task> GetNotesModifiedAfter(DateTime modifiedAfter); Task GetPost(string id); Task GetNote(string rowKey); diff --git a/src/NoteBookmark.MauiApp/Data/LocalDataService.cs b/src/NoteBookmark.MauiApp/Data/LocalDataService.cs index a9da479..4c3bd46 100644 --- a/src/NoteBookmark.MauiApp/Data/LocalDataService.cs +++ b/src/NoteBookmark.MauiApp/Data/LocalDataService.cs @@ -170,6 +170,22 @@ public async Task DeletePostAsync(string id, bool isPendingSync = false) } } + public async Task RemovePostsAsync(IEnumerable ids) + { + await InitAsync(); + var idList = ids.ToList(); + if (idList.Count == 0) + return; + + await _database.RunInTransactionAsync(conn => + { + foreach (var id in idList) + { + conn.Delete(id); + } + }); + } + public async Task> GetSummariesAsync() { await InitAsync(); diff --git a/src/NoteBookmark.MauiApp/Data/SyncApiClient.cs b/src/NoteBookmark.MauiApp/Data/SyncApiClient.cs index 61a76ef..da32823 100644 --- a/src/NoteBookmark.MauiApp/Data/SyncApiClient.cs +++ b/src/NoteBookmark.MauiApp/Data/SyncApiClient.cs @@ -10,6 +10,7 @@ namespace NoteBookmark.MauiApp.Data; public class SyncApiClient(PostNoteClient client) : ISyncApiClient { public Task> GetPostsModifiedAfter(DateTime modifiedAfter) => client.GetPostsModifiedAfter(modifiedAfter); + public Task?> GetPostIds() => client.GetPostIds(); public Task> GetNotesModifiedAfter(DateTime modifiedAfter) => client.GetNotesModifiedAfter(modifiedAfter); public Task GetPost(string id) => client.GetPost(id); public Task GetNote(string rowKey) => client.GetNote(rowKey); diff --git a/src/NoteBookmark.MauiApp/Data/SyncService.cs b/src/NoteBookmark.MauiApp/Data/SyncService.cs index a82698b..ba0a877 100644 --- a/src/NoteBookmark.MauiApp/Data/SyncService.cs +++ b/src/NoteBookmark.MauiApp/Data/SyncService.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NoteBookmark.Domain; @@ -27,6 +28,8 @@ public class SyncService( ILocalHtmlStorageService localHtmlStorageService) : ISyncService { private const string LastSyncTimestampKey = "LastSyncTimestamp"; + private const int MaxConcurrentRequests = 5; + private static readonly TimeSpan ProgressReportInterval = TimeSpan.FromMilliseconds(250); private readonly object _syncLock = new(); private Task? _currentSyncTask; @@ -53,7 +56,8 @@ public Task SyncAsync() return _currentSyncTask; } - _currentSyncTask = DoSyncAsync(); + // Run on the thread pool so callers on the UI thread (e.g. Blazor pages) never block on sync work. + _currentSyncTask = Task.Run(DoSyncAsync); return _currentSyncTask; } } @@ -209,108 +213,48 @@ private async Task PushAsync(DateTime? lastSync) private async Task PullAsync(DateTime? lastSync) { - // 1. Get all remote posts - var allRemotePosts = await apiClient.GetPostsModifiedAfter(DateTime.MinValue) ?? new List(); - var remotePostIds = allRemotePosts.Select(p => p.Id ?? p.RowKey).ToHashSet(); + // 1. Only fetch posts modified since the last sync. On the first sync this is every post. + var changedRemotePosts = await apiClient.GetPostsModifiedAfter(lastSync ?? DateTime.MinValue) ?? new List(); // 2. Any post that was deleted on the online database while offline should be deleted locally. + var remotePostIds = await GetRemotePostIdsAsync(lastSync, changedRemotePosts); var localPosts = await localDataService.GetPostsAsync() ?? new List(); var localPostMap = localPosts.ToDictionary(p => p.Id ?? p.RowKey); - foreach (var localPost in localPosts) + var deletedIds = localPostMap.Keys.Where(id => !remotePostIds.Contains(id)).ToList(); + if (deletedIds.Count > 0) { - var id = localPost.Id ?? localPost.RowKey; - if (!remotePostIds.Contains(id)) + await localDataService.RemovePostsAsync(deletedIds); + foreach (var id in deletedIds) { - await localDataService.DeletePostAsync(id, isPendingSync: false); - await localDataService.MarkSyncedAsync(id, isPost: true); localPostMap.Remove(id); } } - // 3. Pull new/modified posts - var postsToPull = new List(); - foreach (var remotePostL in allRemotePosts) - { - var id = remotePostL.Id ?? remotePostL.RowKey; - if (!localPostMap.TryGetValue(id, out var lp)) - { - lp = await localDataService.GetPostAsync(id); - } - - if (lp is null || remotePostL.DateModified > lp.DateModified) - { - postsToPull.Add(remotePostL); - } - } + // 3. Pull new/modified posts. The API returns one row per post/note pair, so de-duplicate by id. + var postsToPull = changedRemotePosts + .DistinctBy(p => p.Id ?? p.RowKey) + .Where(remote => !localPostMap.TryGetValue(remote.Id ?? remote.RowKey, out var lp) || remote.DateModified > lp.DateModified) + .ToList(); if (postsToPull.Count > 0) { - SyncProgressChanged?.Invoke(this, new SyncProgressEventArgs(0, postsToPull.Count, $"Pulling 0 of {postsToPull.Count} posts...")); - - for (int i = 0; i < postsToPull.Count; i++) - { - var remotePostL = postsToPull[i]; - var id = remotePostL.Id ?? remotePostL.RowKey; - - Post postToSave; - if (remotePostL.is_read == true) - { - postToSave = new Post - { - Id = id, - RowKey = remotePostL.RowKey, - PartitionKey = remotePostL.PartitionKey, - Title = remotePostL.Title, - Url = remotePostL.Url, - Date_published = remotePostL.Date_published, - Excerpt = remotePostL.Excerpt, - is_read = remotePostL.is_read, - DateModified = remotePostL.DateModified - }; - } - else + int total = postsToPull.Count; + var progress = new ProgressThrottle(this, total, current => $"Pulling {current} of {total} posts..."); + progress.Report(0, force: true); + + var postsToSave = new Post[total]; + await Parallel.ForEachAsync( + Enumerable.Range(0, total), + new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRequests }, + async (i, _) => { - try - { - var fullPost = await apiClient.GetPost(id); - postToSave = fullPost ?? new Post - { - Id = id, - RowKey = remotePostL.RowKey, - PartitionKey = remotePostL.PartitionKey, - Title = remotePostL.Title, - Url = remotePostL.Url, - Date_published = remotePostL.Date_published, - Excerpt = remotePostL.Excerpt, - is_read = remotePostL.is_read, - DateModified = remotePostL.DateModified - }; - } - catch (Exception ex) - { - logger.LogWarning(ex, "Failed to retrieve full post for {PostId}, saving summary metadata", id); - postToSave = new Post - { - Id = id, - RowKey = remotePostL.RowKey, - PartitionKey = remotePostL.PartitionKey, - Title = remotePostL.Title, - Url = remotePostL.Url, - Date_published = remotePostL.Date_published, - Excerpt = remotePostL.Excerpt, - is_read = remotePostL.is_read, - DateModified = remotePostL.DateModified - }; - } - } + postsToSave[i] = await BuildPostToSaveAsync(postsToPull[i]); + progress.Increment(); + }); - await localDataService.SavePostAsync(postToSave, isPendingSync: false); - localPostMap[id] = postToSave; - - int current = i + 1; - SyncProgressChanged?.Invoke(this, new SyncProgressEventArgs(current, postsToPull.Count, $"Pulling {current} of {postsToPull.Count} posts...")); - } + await localDataService.SavePostsAsync(postsToSave); + progress.Report(total, force: true); } // 4. Pull notes modified since lastSync @@ -334,6 +278,90 @@ private async Task PullAsync(DateTime? lastSync) } } + private async Task> GetRemotePostIdsAsync(DateTime? lastSync, List changedRemotePosts) + { + if (lastSync is null) + { + // First sync: the delta already holds every post. + return changedRemotePosts.Select(p => p.Id ?? p.RowKey).ToHashSet(); + } + + var ids = await apiClient.GetPostIds(); + if (ids is not null) + { + return ids.ToHashSet(); + } + + // Older servers don't expose the ids endpoint; fall back to the full post list. + logger.LogInformation("Post ids endpoint unavailable, falling back to the full post list to detect deletions."); + var allRemotePosts = await apiClient.GetPostsModifiedAfter(DateTime.MinValue) ?? new List(); + return allRemotePosts.Select(p => p.Id ?? p.RowKey).ToHashSet(); + } + + private async Task BuildPostToSaveAsync(PostL remotePostL) + { + var id = remotePostL.Id ?? remotePostL.RowKey; + + // Read posts only need the summary metadata; unread posts get the full details for offline reading. + if (remotePostL.is_read != true) + { + try + { + var fullPost = await apiClient.GetPost(id); + if (fullPost is not null) + { + return fullPost; + } + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to retrieve full post for {PostId}, saving summary metadata", id); + } + } + + return new Post + { + Id = id, + RowKey = remotePostL.RowKey, + PartitionKey = remotePostL.PartitionKey, + Title = remotePostL.Title, + Url = remotePostL.Url, + Date_published = remotePostL.Date_published, + Excerpt = remotePostL.Excerpt, + is_read = remotePostL.is_read, + DateModified = remotePostL.DateModified + }; + } + + /// + /// Limits how often progress events fire so the UI isn't re-rendered for every single post. + /// + private sealed class ProgressThrottle(SyncService owner, int total, Func formatStatus) + { + private readonly object _lock = new(); + private readonly System.Diagnostics.Stopwatch _sinceLastReport = System.Diagnostics.Stopwatch.StartNew(); + private int _current; + + public void Increment() + { + var current = Interlocked.Increment(ref _current); + Report(current, force: current == total); + } + + public void Report(int current, bool force = false) + { + lock (_lock) + { + if (!force && _sinceLastReport.Elapsed < ProgressReportInterval) + { + return; + } + _sinceLastReport.Restart(); + } + owner.SyncProgressChanged?.Invoke(owner, new SyncProgressEventArgs(current, total, formatStatus(current))); + } + } + private async Task SyncHtmlAsync() { var posts = await localDataService.GetPostsAsync(); @@ -357,7 +385,8 @@ private async Task SyncHtmlAsync() if (total > 0) { - SyncProgressChanged?.Invoke(this, new SyncProgressEventArgs(0, total, $"Downloading 0 of {total} posts...")); + var progress = new ProgressThrottle(this, total, current => $"Downloading {current} of {total} posts..."); + progress.Report(0, force: true); for (int i = 0; i < unreadToDownload.Count; i++) { @@ -377,8 +406,7 @@ private async Task SyncHtmlAsync() logger.LogWarning(ex, "Failed to download HTML for post {PostId}", id); } - int current = i + 1; - SyncProgressChanged?.Invoke(this, new SyncProgressEventArgs(current, total, $"Downloading {current} of {total} posts...")); + progress.Increment(); } } } diff --git a/src/NoteBookmark.SharedUI/PostNoteClient.cs b/src/NoteBookmark.SharedUI/PostNoteClient.cs index 8b8acd2..1debee1 100644 --- a/src/NoteBookmark.SharedUI/PostNoteClient.cs +++ b/src/NoteBookmark.SharedUI/PostNoteClient.cs @@ -198,6 +198,17 @@ public async Task> GetPostsModifiedAfter(DateTime modifiedAfter) return unread.Concat(read).ToList(); } + /// + /// Returns the ids of every post on the server, or null when the server does not expose the endpoint. + /// + public async Task?> GetPostIds() + { + var response = await httpClient.GetAsync("api/posts/ids"); + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) return null; + response.EnsureSuccessStatusCode(); + return await response.Content.ReadFromJsonAsync>(); + } + public async Task> GetNotesModifiedAfter(DateTime modifiedAfter) { try