From 458b264255fd50f1b7bc314c1259477672ab04bb Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 12:57:40 +0000 Subject: [PATCH 1/2] Speed up MAUI sync: delta pull, batched saves, off-UI-thread (#208) - Pull only posts modified since the last sync; detect server deletions with a new lightweight GET api/posts/ids endpoint (falls back to the full list on first sync or on servers without the endpoint). - Save pulled posts in one SQLite transaction and remove deleted posts in one batch. - Fetch full details for unread posts with at most 5 concurrent requests. - Run sync on the thread pool and throttle progress events to ~250 ms. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Kq68qA7Jv7puAF3Kc6GSFY --- .../Endpoints/DeltaApiTests.cs | 39 ++ src/NoteBookmark.Api/DataStorageService.cs | 10 + src/NoteBookmark.Api/IDataStorageService.cs | 2 + src/NoteBookmark.Api/PostEndpoints.cs | 8 + .../LocalDataServiceTests.cs | 40 ++ .../SyncServiceTests.cs | 194 +++++- .../Data/ILocalDataService.cs | 2 + .../Data/ISyncApiClient.cs | 2 + .../Data/LocalDataService.cs | 582 +++++++++--------- .../Data/SyncApiClient.cs | 1 + src/NoteBookmark.MauiApp/Data/SyncService.cs | 208 ++++--- src/NoteBookmark.SharedUI/PostNoteClient.cs | 451 +++++++------- 12 files changed, 931 insertions(+), 608 deletions(-) diff --git a/src/NoteBookmark.Api.Tests/Endpoints/DeltaApiTests.cs b/src/NoteBookmark.Api.Tests/Endpoints/DeltaApiTests.cs index ca597ec..7c794a8 100644 --- a/src/NoteBookmark.Api.Tests/Endpoints/DeltaApiTests.cs +++ b/src/NoteBookmark.Api.Tests/Endpoints/DeltaApiTests.cs @@ -290,6 +290,45 @@ private async Task CreateAndSaveTestPost(string rowKey) return post; } + // ── Post ids ───────────────────────────────────────────────────────────── + + [Fact] + public async Task GetPostIds_ReturnsIdsOfReadAndUnreadPosts() + { + // Arrange + var unreadPost = CreateTestPost("ids-unread-post-1"); + var readPost = CreateTestPost("ids-read-post-1"); + readPost.is_read = true; + await _client.PostAsJsonAsync("/api/posts/", unreadPost); + await _client.PostAsJsonAsync("/api/posts/", readPost); + + // Act + var response = await _client.GetAsync("/api/posts/ids"); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + var ids = await response.Content.ReadFromJsonAsync>(); + ids.Should().NotBeNull(); + ids.Should().Contain(unreadPost.RowKey); + ids.Should().Contain(readPost.RowKey); + } + + [Fact] + public async Task GetPostIds_DoesNotReturnDeletedPosts() + { + // Arrange + var post = CreateTestPost("ids-deleted-post-1"); + await _client.PostAsJsonAsync("/api/posts/", post); + await _client.DeleteAsync($"/api/posts/{post.RowKey}"); + + // Act + var ids = await _client.GetFromJsonAsync>("/api/posts/ids"); + + // Assert + ids.Should().NotBeNull(); + ids.Should().NotContain(post.RowKey); + } + private static Post CreateTestPost(string rowKey) { return new Post diff --git a/src/NoteBookmark.Api/DataStorageService.cs b/src/NoteBookmark.Api/DataStorageService.cs index 45abbb4..6bfc50f 100644 --- a/src/NoteBookmark.Api/DataStorageService.cs +++ b/src/NoteBookmark.Api/DataStorageService.cs @@ -123,6 +123,16 @@ orderby post.Timestamp return lstPosts; } + public List GetPostIds() + { + var tblPosts = GetPostTable(); + // Only project the key columns so the query stays small even with thousands of posts. + // Clients identify posts by Id, falling back to RowKey, so return the same value. + return tblPosts.Query(select: new[] { "RowKey", "id", "Id" }) + .Select(p => p.Id ?? p.RowKey) + .ToList(); + } + public Post? GetPost(string rowKey) { var tblPost = GetPostTable(); diff --git a/src/NoteBookmark.Api/IDataStorageService.cs b/src/NoteBookmark.Api/IDataStorageService.cs index 85f9473..fbc8704 100644 --- a/src/NoteBookmark.Api/IDataStorageService.cs +++ b/src/NoteBookmark.Api/IDataStorageService.cs @@ -7,6 +7,8 @@ namespace NoteBookmark.Api; public interface IDataStorageService { public List GetFilteredPosts(string filter); + + public List GetPostIds(); public Post? GetPost(string rowKey); diff --git a/src/NoteBookmark.Api/PostEndpoints.cs b/src/NoteBookmark.Api/PostEndpoints.cs index 66a8a7e..ec85fd2 100644 --- a/src/NoteBookmark.Api/PostEndpoints.cs +++ b/src/NoteBookmark.Api/PostEndpoints.cs @@ -19,6 +19,8 @@ public static void MapPostEndpoints(this IEndpointRouteBuilder app) .WithDescription("Get all unread posts"); endpoints.MapGet("/read", GetReadPosts) .WithDescription("Get all read posts"); + endpoints.MapGet("/ids", GetPostIds) + .WithDescription("Get the ids of all existing posts (read and unread)"); endpoints.MapGet("/{id}", Get) .WithDescription("Get a post by id"); endpoints.MapGet("/{postId}/html", GetPostHtml) @@ -57,6 +59,12 @@ static List GetReadPosts(TableServiceClient tblClient, BlobServiceClient return posts; } + static List GetPostIds(TableServiceClient tblClient, BlobServiceClient blobClient) + { + var dataStorageService = new DataStorageService(tblClient, blobClient); + return dataStorageService.GetPostIds(); + } + static async Task>> GetPostHtml( string postId, BlobServiceClient blobClient) { diff --git a/src/NoteBookmark.MauiApp.Tests/LocalDataServiceTests.cs b/src/NoteBookmark.MauiApp.Tests/LocalDataServiceTests.cs index c3311d0..92b3830 100644 --- a/src/NoteBookmark.MauiApp.Tests/LocalDataServiceTests.cs +++ b/src/NoteBookmark.MauiApp.Tests/LocalDataServiceTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading.Tasks; using FluentAssertions; using NoteBookmark.Domain; @@ -161,4 +162,43 @@ public async Task SaveSettings_ShouldStoreAndRetrieveFontSize() retrieved.Should().NotBeNull(); retrieved!.FontSize.Should().Be("large"); } + + [Fact] + public async Task RemovePostsAsync_ShouldPermanentlyRemoveOnlyGivenPosts() + { + await _sut.SavePostsAsync(new[] + { + new Post { Id = "p1", PartitionKey = "pk", RowKey = "p1", Title = "One", DateModified = DateTime.UtcNow }, + new Post { Id = "p2", PartitionKey = "pk", RowKey = "p2", Title = "Two", DateModified = DateTime.UtcNow }, + new Post { Id = "p3", PartitionKey = "pk", RowKey = "p3", Title = "Three", DateModified = DateTime.UtcNow } + }); + await _sut.SavePostAsync(new Post { Id = "p2", PartitionKey = "pk", RowKey = "p2", Title = "Two edited" }, isPendingSync: true); + + await _sut.RemovePostsAsync(new[] { "p1", "p2", "missing" }); + + var remaining = await _sut.GetPostsAsync(); + remaining.Select(p => p.Id).Should().BeEquivalentTo(new[] { "p3" }); + (await _sut.GetPendingSyncPostsAsync()).Should().BeEmpty(); + } + + [Fact] + public async Task RemovePostsAsync_WithNoIds_ShouldDoNothing() + { + await _sut.SavePostAsync(new Post { Id = "p1", PartitionKey = "pk", RowKey = "p1", Title = "One" }); + + await _sut.RemovePostsAsync(Array.Empty()); + + (await _sut.GetPostsAsync()).Should().HaveCount(1); + } + + [Fact] + public async Task SavePostsAsync_ShouldPreservePendingSyncFlag_ForLocallyEditedPosts() + { + await _sut.SavePostAsync(new Post { Id = "p1", PartitionKey = "pk", RowKey = "p1", Title = "Local edit" }, isPendingSync: true); + + await _sut.SavePostsAsync(new[] { new Post { Id = "p1", PartitionKey = "pk", RowKey = "p1", Title = "From server" } }); + + var pending = await _sut.GetPendingSyncPostsAsync(); + pending.Should().ContainSingle(p => p.Id == "p1"); + } } diff --git a/src/NoteBookmark.MauiApp.Tests/SyncServiceTests.cs b/src/NoteBookmark.MauiApp.Tests/SyncServiceTests.cs index 4204ab5..b1fe4f2 100644 --- a/src/NoteBookmark.MauiApp.Tests/SyncServiceTests.cs +++ b/src/NoteBookmark.MauiApp.Tests/SyncServiceTests.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Linq; +using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Moq; @@ -123,11 +125,11 @@ public async Task PullPhase_RemoteNewer_ShouldOverwriteLocal() .ReturnsAsync(new List { remotePostL }); _apiClientMock.Setup(c => c.GetPost("post1")).ReturnsAsync(remotePost); _apiClientMock.Setup(c => c.GetNotesModifiedAfter(It.IsAny())).ReturnsAsync(new List()); - _localDataServiceMock.Setup(c => c.GetPostAsync("post1")).ReturnsAsync(localPost); + _localDataServiceMock.Setup(c => c.GetPostsAsync()).ReturnsAsync(new List { localPost }); await _sut.SyncAsync(); - _localDataServiceMock.Verify(c => c.SavePostAsync(It.Is(p => p.Title == "Remote"), false), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Single().Title == "Remote")), Times.Once); } [Fact] @@ -156,11 +158,12 @@ public async Task PullPhase_LocalNewer_ShouldKeepLocal() _apiClientMock.Setup(c => c.GetPostsModifiedAfter(It.IsAny())) .ReturnsAsync(new List { remotePostL }); _apiClientMock.Setup(c => c.GetNotesModifiedAfter(It.IsAny())).ReturnsAsync(new List()); - _localDataServiceMock.Setup(c => c.GetPostAsync("post1")).ReturnsAsync(localPost); + _localDataServiceMock.Setup(c => c.GetPostsAsync()).ReturnsAsync(new List { localPost }); await _sut.SyncAsync(); - _localDataServiceMock.Verify(c => c.SavePostAsync(It.IsAny(), false), Times.Never); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.IsAny>()), Times.Never); + _localDataServiceMock.Verify(c => c.SavePostAsync(It.IsAny(), It.IsAny()), Times.Never); } [Fact] @@ -190,11 +193,10 @@ public async Task PullPhase_NoLocal_ShouldSaveRemote() .ReturnsAsync(new List { remotePostL }); _apiClientMock.Setup(c => c.GetPost("post1")).ReturnsAsync(remotePost); _apiClientMock.Setup(c => c.GetNotesModifiedAfter(It.IsAny())).ReturnsAsync(new List()); - _localDataServiceMock.Setup(c => c.GetPostAsync("post1")).ReturnsAsync((Post?)null); await _sut.SyncAsync(); - _localDataServiceMock.Verify(c => c.SavePostAsync(It.Is(p => p.Title == "Remote"), false), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Single().Title == "Remote")), Times.Once); } [Fact] @@ -266,8 +268,7 @@ public async Task PullPhase_ShouldDeleteLocalPosts_WhenDeletedOnServer() await _sut.SyncAsync(); - _localDataServiceMock.Verify(c => c.DeletePostAsync("post1", false), Times.Once); - _localDataServiceMock.Verify(c => c.MarkSyncedAsync("post1", true), Times.Once); + _localDataServiceMock.Verify(c => c.RemovePostsAsync(It.Is>(ids => ids.SequenceEqual(new[] { "post1" }))), Times.Once); } [Fact] @@ -296,14 +297,13 @@ public async Task PullPhase_ShouldAddNewPosts_WhenAddedOnServer() _localDataServiceMock.Setup(c => c.GetPostsAsync()).ReturnsAsync(new List()); _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)) .ReturnsAsync(new List { remotePostL }); - _localDataServiceMock.Setup(c => c.GetPostAsync("post1")).ReturnsAsync((Post?)null); _apiClientMock.Setup(c => c.GetPost("post1")).ReturnsAsync(remotePost); _apiClientMock.Setup(c => c.GetNotesModifiedAfter(It.IsAny())).ReturnsAsync(new List()); await _sut.SyncAsync(); _apiClientMock.Verify(c => c.GetPost("post1"), Times.Once); - _localDataServiceMock.Verify(c => c.SavePostAsync(remotePost, false), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Single() == remotePost)), Times.Once); } [Fact] @@ -467,7 +467,7 @@ public async Task SyncAsync_ShouldRaiseSyncProgressChanged_WhenDownloadingPostHt progressEvents.Should().NotBeEmpty(); progressEvents.Should().Contain(e => e.Status == "Cleaning..."); - progressEvents.Should().Contain(e => e.Status == "Downloading 1 of 2 posts..." && e.Current == 1 && e.Total == 2); + progressEvents.Should().Contain(e => e.Status == "Downloading 0 of 2 posts..." && e.Current == 0 && e.Total == 2); progressEvents.Should().Contain(e => e.Status == "Downloading 2 of 2 posts..." && e.Current == 2 && e.Total == 2); progressEvents.Last().Status.Should().Be("Synchronization complete!"); progressEvents.Last().IsComplete.Should().BeTrue(); @@ -495,7 +495,7 @@ public async Task PullPhase_ReadPosts_ShouldNotCallGetPost_AndShouldSaveDirectly // GetPost should NOT be called for read posts _apiClientMock.Verify(c => c.GetPost("read1"), Times.Never); - _localDataServiceMock.Verify(c => c.SavePostAsync(It.Is(p => p.Id == "read1" && p.is_read == true), false), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Single().Id == "read1" && ps.Single().is_read == true)), Times.Once); } [Fact] @@ -520,7 +520,7 @@ public async Task PullPhase_UnreadPost_WhenGetPostFails_ShouldFallbackToBasicPos await _sut.SyncAsync(); // Should fall back and save basic post without throwing - _localDataServiceMock.Verify(c => c.SavePostAsync(It.Is(p => p.Id == "unread1" && p.Title == "Unread Post"), false), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Single().Id == "unread1" && ps.Single().Title == "Unread Post")), Times.Once); } [Fact] @@ -540,7 +540,6 @@ public async Task PullPhase_ShouldReportProgress_WhenPullingPosts() await _sut.SyncAsync(); progressEvents.Should().Contain(e => e.Status == "Pulling 0 of 2 posts..." && e.Current == 0 && e.Total == 2); - progressEvents.Should().Contain(e => e.Status == "Pulling 1 of 2 posts..." && e.Current == 1 && e.Total == 2); progressEvents.Should().Contain(e => e.Status == "Pulling 2 of 2 posts..." && e.Current == 2 && e.Total == 2); } @@ -581,5 +580,170 @@ public async Task IsSyncing_ShouldReflectActiveSyncTask() _sut.IsSyncing.Should().BeFalse(); } -} + // ── Issue #208: delta pull, deletion detection, batched save ───────────── + + private static PostL RemotePostL(string id, bool isRead = true, DateTime? modified = null) => new() + { + Id = id, + RowKey = id, + PartitionKey = "pk", + Title = $"Title {id}", + is_read = isRead, + DateModified = modified ?? DateTime.UtcNow + }; + + private static Post LocalPost(string id, DateTime? modified = null) => new() + { + Id = id, + RowKey = id, + PartitionKey = "pk", + Title = $"Title {id}", + is_read = true, + DateModified = modified ?? DateTime.UtcNow.AddDays(-1) + }; + + private void SetupEmptyPushAndNotes() + { + _localDataServiceMock.Setup(c => c.GetPendingSyncNotesAsync()).ReturnsAsync(new List()); + _apiClientMock.Setup(c => c.GetNotesModifiedAfter(It.IsAny())).ReturnsAsync(new List()); + } + + [Fact] + public async Task PullPhase_WithLastSync_ShouldOnlyPullDelta_AndUsePostIdsForDeletions() + { + var lastSync = DateTime.UtcNow.AddHours(-1); + SyncService.SetInMemoryPreference("LastSyncTimestamp", lastSync.ToString("O")); + SetupEmptyPushAndNotes(); + + _localDataServiceMock.Setup(c => c.GetPostsAsync()) + .ReturnsAsync(new List { LocalPost("p1"), LocalPost("p2"), LocalPost("p3") }); + _apiClientMock.Setup(c => c.GetPostIds()).ReturnsAsync(new List { "p1", "p3", "new" }); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(It.Is(d => d > DateTime.MinValue))) + .ReturnsAsync(new List { RemotePostL("new") }); + + await _sut.SyncAsync(); + + // The full post list is never requested once we have a last sync time. + _apiClientMock.Verify(c => c.GetPostsModifiedAfter(DateTime.MinValue), Times.Never); + _apiClientMock.Verify(c => c.GetPostsModifiedAfter(It.Is(d => Math.Abs((d - lastSync).TotalSeconds) < 1)), Times.Once); + _localDataServiceMock.Verify(c => c.RemovePostsAsync(It.Is>(ids => ids.SequenceEqual(new[] { "p2" }))), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Select(p => p.Id).SequenceEqual(new[] { "new" }))), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task PullPhase_WithLastSync_WhenPostIdsEndpointUnavailable_ShouldFallBackToFullListForDeletions() + { + SyncService.SetInMemoryPreference("LastSyncTimestamp", DateTime.UtcNow.AddHours(-1).ToString("O")); + SetupEmptyPushAndNotes(); + + _localDataServiceMock.Setup(c => c.GetPostsAsync()) + .ReturnsAsync(new List { LocalPost("p1"), LocalPost("p2") }); + _apiClientMock.Setup(c => c.GetPostIds()).ReturnsAsync((List?)null); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(It.Is(d => d > DateTime.MinValue))) + .ReturnsAsync(new List()); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)) + .ReturnsAsync(new List { RemotePostL("p1", modified: DateTime.UtcNow.AddDays(-2)) }); + + await _sut.SyncAsync(); + + _localDataServiceMock.Verify(c => c.RemovePostsAsync(It.Is>(ids => ids.SequenceEqual(new[] { "p2" }))), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.IsAny>()), Times.Never); + } + + [Fact] + public async Task PullPhase_FirstSync_ShouldUseFullListForDeletions_WithoutCallingPostIds() + { + SetupEmptyPushAndNotes(); + _localDataServiceMock.Setup(c => c.GetPostsAsync()).ReturnsAsync(new List { LocalPost("gone") }); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)) + .ReturnsAsync(new List { RemotePostL("p1") }); + + await _sut.SyncAsync(); + + _apiClientMock.Verify(c => c.GetPostIds(), Times.Never); + _localDataServiceMock.Verify(c => c.RemovePostsAsync(It.Is>(ids => ids.SequenceEqual(new[] { "gone" }))), Times.Once); + } + + [Fact] + public async Task PullPhase_NoDeletedPosts_ShouldNotCallRemovePosts() + { + SetupEmptyPushAndNotes(); + _localDataServiceMock.Setup(c => c.GetPostsAsync()).ReturnsAsync(new List { LocalPost("p1") }); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)) + .ReturnsAsync(new List { RemotePostL("p1", modified: DateTime.UtcNow.AddDays(-2)) }); + + await _sut.SyncAsync(); + + _localDataServiceMock.Verify(c => c.RemovePostsAsync(It.IsAny>()), Times.Never); + } + + [Fact] + public async Task PullPhase_ManyPosts_ShouldSaveInOneBatch() + { + SetupEmptyPushAndNotes(); + var remotePosts = Enumerable.Range(0, 1500).Select(i => RemotePostL($"p{i}")).ToList(); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)).ReturnsAsync(remotePosts); + + await _sut.SyncAsync(); + + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Count() == 1500)), Times.Once); + _localDataServiceMock.Verify(c => c.SavePostAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task PullPhase_DuplicateRowsForSamePost_ShouldSaveItOnce() + { + // The API joins posts with notes, so a post with two notes comes back twice. + SetupEmptyPushAndNotes(); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)) + .ReturnsAsync(new List { RemotePostL("p1"), RemotePostL("p1") }); + + await _sut.SyncAsync(); + + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => ps.Count() == 1)), Times.Once); + } + + [Fact] + public async Task PullPhase_ManyPosts_ShouldThrottleProgressEvents() + { + SetupEmptyPushAndNotes(); + var remotePosts = Enumerable.Range(0, 500).Select(i => RemotePostL($"p{i}")).ToList(); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)).ReturnsAsync(remotePosts); + + var progressEvents = new List(); + _sut.SyncProgressChanged += (sender, args) => { lock (progressEvents) progressEvents.Add(args); }; + + await _sut.SyncAsync(); + + var pullingEvents = progressEvents.Where(e => e.Status.StartsWith("Pulling ") && e.Total == 500).ToList(); + pullingEvents.Should().HaveCountLessThan(50); + pullingEvents.Should().Contain(e => e.Current == 0); + pullingEvents.Should().Contain(e => e.Current == 500 && e.Status == "Pulling 500 of 500 posts..."); + } + + [Fact] + public async Task PullPhase_UnreadPosts_ShouldFetchFullPostsWithLimitedConcurrency() + { + SetupEmptyPushAndNotes(); + var remotePosts = Enumerable.Range(0, 20).Select(i => RemotePostL($"u{i}", isRead: false)).ToList(); + _apiClientMock.Setup(c => c.GetPostsModifiedAfter(DateTime.MinValue)).ReturnsAsync(remotePosts); + + int inFlight = 0, maxInFlight = 0; + _apiClientMock.Setup(c => c.GetPost(It.IsAny())).Returns(async (string id) => + { + var now = Interlocked.Increment(ref inFlight); + lock (_apiClientMock) maxInFlight = Math.Max(maxInFlight, now); + await Task.Delay(20); + Interlocked.Decrement(ref inFlight); + return new Post { Id = id, RowKey = id, PartitionKey = "pk", Title = $"Full {id}", is_read = false }; + }); + + await _sut.SyncAsync(); + + maxInFlight.Should().BeGreaterThan(1).And.BeLessThanOrEqualTo(5); + _localDataServiceMock.Verify(c => c.SavePostsAsync(It.Is>(ps => + ps.Count() == 20 && ps.All(p => p.Title!.StartsWith("Full ")))), Times.Once); + } +} diff --git a/src/NoteBookmark.MauiApp/Data/ILocalDataService.cs b/src/NoteBookmark.MauiApp/Data/ILocalDataService.cs index a20da8b..9689add 100644 --- a/src/NoteBookmark.MauiApp/Data/ILocalDataService.cs +++ b/src/NoteBookmark.MauiApp/Data/ILocalDataService.cs @@ -15,6 +15,8 @@ public interface ILocalDataService Task SaveNoteAsync(Note note, bool isPendingSync = false); Task DeleteNoteAsync(string rowKey, bool isPendingSync = false); Task DeletePostAsync(string rowKey, bool isPendingSync = false); + /// Permanently removes posts that no longer exist on the server, in a single transaction. + Task RemovePostsAsync(IEnumerable ids); Task> GetSummariesAsync(); Task SaveSummariesAsync(IEnumerable summaries); Task GetSettingsAsync(); diff --git a/src/NoteBookmark.MauiApp/Data/ISyncApiClient.cs b/src/NoteBookmark.MauiApp/Data/ISyncApiClient.cs index c71807a..52da789 100644 --- a/src/NoteBookmark.MauiApp/Data/ISyncApiClient.cs +++ b/src/NoteBookmark.MauiApp/Data/ISyncApiClient.cs @@ -9,6 +9,8 @@ namespace NoteBookmark.MauiApp.Data; public interface ISyncApiClient { Task> GetPostsModifiedAfter(DateTime modifiedAfter); + /// Ids of all posts on the server, or null if the server does not support it. + Task?> GetPostIds(); Task> GetNotesModifiedAfter(DateTime modifiedAfter); Task GetPost(string id); Task GetNote(string rowKey); diff --git a/src/NoteBookmark.MauiApp/Data/LocalDataService.cs b/src/NoteBookmark.MauiApp/Data/LocalDataService.cs index a9da479..bf6dfbe 100644 --- a/src/NoteBookmark.MauiApp/Data/LocalDataService.cs +++ b/src/NoteBookmark.MauiApp/Data/LocalDataService.cs @@ -1,283 +1,299 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -#if !NOT_MAUI -using Microsoft.Maui.Storage; -#endif -using NoteBookmark.Domain; -using SQLite; - -namespace NoteBookmark.MauiApp.Data; - -public class LocalDataService : ILocalDataService -{ - private const string DatabaseFilename = "NoteBookmark.db3"; - - private const SQLiteOpenFlags Flags = - SQLiteOpenFlags.ReadWrite | - SQLiteOpenFlags.Create | - SQLiteOpenFlags.SharedCache; - - private readonly string _databasePath; - private SQLiteAsyncConnection? _database; - - public LocalDataService() - { -#if !NOT_MAUI - _databasePath = Path.Combine(FileSystem.AppDataDirectory, DatabaseFilename); -#else - _databasePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), DatabaseFilename); -#endif - } - - public LocalDataService(string dbPath) - { - _databasePath = dbPath; - } - - [System.Diagnostics.CodeAnalysis.MemberNotNull(nameof(_database))] - private async Task InitAsync() - { - if (_database is not null) - return; - - _database = new SQLiteAsyncConnection(_databasePath, Flags); - await _database.CreateTableAsync(); - await _database.CreateTableAsync(); - await _database.CreateTableAsync(); - await _database.CreateTableAsync(); - } - - public async Task> GetPostsAsync() - { - await InitAsync(); - var localPosts = await _database.Table() - .Where(p => !p.IsDeleted) - .ToListAsync(); - return localPosts.Select(p => p.ToDomain()).ToList(); - } - - public async Task GetPostAsync(string id) - { - await InitAsync(); - var localPost = await _database.Table() - .Where(p => p.Id == id && !p.IsDeleted) - .FirstOrDefaultAsync(); - return localPost?.ToDomain(); - } - - public async Task SavePostAsync(Post post, bool isPendingSync = false) - { - await InitAsync(); - var localPost = LocalPost.FromDomain(post, isPendingSync); - var existing = await _database.Table().Where(p => p.Id == localPost.Id).FirstOrDefaultAsync(); - - if (existing is not null) - { - localPost.IsPendingSync = isPendingSync || existing.IsPendingSync; // Preserve flag unless we are intentionally overwriting it - await _database.UpdateAsync(localPost); - } - else - { - await _database.InsertAsync(localPost); - } - } - - public async Task SavePostsAsync(IEnumerable posts) - { - await InitAsync(); - var localPosts = posts.Select(p => LocalPost.FromDomain(p)).ToList(); - - // Use RunInTransactionAsync for bulk operations - await _database.RunInTransactionAsync(conn => - { - foreach (var localPost in localPosts) - { - var existing = conn.Table().Where(p => p.Id == localPost.Id).FirstOrDefault(); - if (existing is not null) - { - localPost.IsPendingSync = existing.IsPendingSync; - conn.Update(localPost); - } - else - { - conn.Insert(localPost); - } - } - }); - } - - public async Task> GetNotesAsync() - { - await InitAsync(); - var localNotes = await _database.Table() - .Where(n => !n.IsDeleted) - .ToListAsync(); - return localNotes.Select(n => n.ToDomain()).ToList(); - } - - public async Task GetNoteAsync(string rowKey) - { - await InitAsync(); - var localNote = await _database.Table() - .Where(n => n.RowKey == rowKey && !n.IsDeleted) - .FirstOrDefaultAsync(); - return localNote?.ToDomain(); - } - - public async Task SaveNoteAsync(Note note, bool isPendingSync = false) - { - await InitAsync(); - var localNote = LocalNote.FromDomain(note, isPendingSync); - var existing = await _database.Table().Where(n => n.RowKey == localNote.RowKey).FirstOrDefaultAsync(); - - if (existing is not null) - { - localNote.IsPendingSync = isPendingSync || existing.IsPendingSync; - await _database.UpdateAsync(localNote); - } - else - { - await _database.InsertAsync(localNote); - } - } - - public async Task DeleteNoteAsync(string rowKey, bool isPendingSync = false) - { - await InitAsync(); - var existing = await _database.Table().Where(n => n.RowKey == rowKey).FirstOrDefaultAsync(); - if (existing is not null) - { - existing.IsDeleted = true; - existing.IsPendingSync = isPendingSync || existing.IsPendingSync; - existing.DateModified = DateTime.UtcNow; - await _database.UpdateAsync(existing); - } - } - - public async Task DeletePostAsync(string id, bool isPendingSync = false) - { - await InitAsync(); - var existing = await _database.Table().Where(p => p.Id == id).FirstOrDefaultAsync(); - if (existing is not null) - { - existing.IsDeleted = true; - existing.IsPendingSync = isPendingSync || existing.IsPendingSync; - existing.DateModified = DateTime.UtcNow; - await _database.UpdateAsync(existing); - } - } - - public async Task> GetSummariesAsync() - { - await InitAsync(); - var localSummaries = await _database.Table() - .Where(s => !s.IsDeleted) - .ToListAsync(); - return localSummaries.Select(s => s.ToDomain()).ToList(); - } - - public async Task SaveSummariesAsync(IEnumerable summaries) - { - await InitAsync(); - var localSummaries = summaries.Select(s => LocalSummary.FromDomain(s)).ToList(); - - await _database.RunInTransactionAsync(conn => - { - foreach (var localSummary in localSummaries) - { - var existing = conn.Table().Where(s => s.RowKey == localSummary.RowKey).FirstOrDefault(); - if (existing is not null) - { - localSummary.IsPendingSync = existing.IsPendingSync; - conn.Update(localSummary); - } - else - { - conn.Insert(localSummary); - } - } - }); - } - - public async Task GetSettingsAsync() - { - await InitAsync(); - var localSettings = await _database.Table().FirstOrDefaultAsync(); - return localSettings?.ToDomain(); - } - - public async Task SaveSettingsAsync(Settings settings) - { - await InitAsync(); - var localSettings = LocalSettings.FromDomain(settings); - var existing = await _database.Table().Where(s => s.RowKey == localSettings.RowKey).FirstOrDefaultAsync(); - - if (existing is not null) - { - localSettings.IsPendingSync = existing.IsPendingSync; - await _database.UpdateAsync(localSettings); - } - else - { - await _database.InsertAsync(localSettings); - } - } - - public async Task> GetPendingSyncPostsAsync() - { - await InitAsync(); - var pending = await _database.Table() - .Where(p => p.IsPendingSync) - .ToListAsync(); - return pending.Select(p => p.ToDomain()).ToList(); - } - - public async Task> GetPendingSyncNotesAsync() - { - await InitAsync(); - var pending = await _database.Table() - .Where(n => n.IsPendingSync) - .ToListAsync(); - return pending.Select(n => n.ToDomain()).ToList(); - } - - public async Task MarkSyncedAsync(string id, bool isPost) - { - await InitAsync(); - if (isPost) - { - var existing = await _database.Table().Where(p => p.Id == id).FirstOrDefaultAsync(); - if (existing is not null) - { - if (existing.IsDeleted) - { - await _database.DeleteAsync(existing); - } - else - { - existing.IsPendingSync = false; - await _database.UpdateAsync(existing); - } - } - } - else - { - var existing = await _database.Table().Where(n => n.RowKey == id).FirstOrDefaultAsync(); - if (existing is not null) - { - if (existing.IsDeleted) - { - await _database.DeleteAsync(existing); - } - else - { - existing.IsPendingSync = false; - await _database.UpdateAsync(existing); - } - } - } - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +#if !NOT_MAUI +using Microsoft.Maui.Storage; +#endif +using NoteBookmark.Domain; +using SQLite; + +namespace NoteBookmark.MauiApp.Data; + +public class LocalDataService : ILocalDataService +{ + private const string DatabaseFilename = "NoteBookmark.db3"; + + private const SQLiteOpenFlags Flags = + SQLiteOpenFlags.ReadWrite | + SQLiteOpenFlags.Create | + SQLiteOpenFlags.SharedCache; + + private readonly string _databasePath; + private SQLiteAsyncConnection? _database; + + public LocalDataService() + { +#if !NOT_MAUI + _databasePath = Path.Combine(FileSystem.AppDataDirectory, DatabaseFilename); +#else + _databasePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), DatabaseFilename); +#endif + } + + public LocalDataService(string dbPath) + { + _databasePath = dbPath; + } + + [System.Diagnostics.CodeAnalysis.MemberNotNull(nameof(_database))] + private async Task InitAsync() + { + if (_database is not null) + return; + + _database = new SQLiteAsyncConnection(_databasePath, Flags); + await _database.CreateTableAsync(); + await _database.CreateTableAsync(); + await _database.CreateTableAsync(); + await _database.CreateTableAsync(); + } + + public async Task> GetPostsAsync() + { + await InitAsync(); + var localPosts = await _database.Table() + .Where(p => !p.IsDeleted) + .ToListAsync(); + return localPosts.Select(p => p.ToDomain()).ToList(); + } + + public async Task GetPostAsync(string id) + { + await InitAsync(); + var localPost = await _database.Table() + .Where(p => p.Id == id && !p.IsDeleted) + .FirstOrDefaultAsync(); + return localPost?.ToDomain(); + } + + public async Task SavePostAsync(Post post, bool isPendingSync = false) + { + await InitAsync(); + var localPost = LocalPost.FromDomain(post, isPendingSync); + var existing = await _database.Table().Where(p => p.Id == localPost.Id).FirstOrDefaultAsync(); + + if (existing is not null) + { + localPost.IsPendingSync = isPendingSync || existing.IsPendingSync; // Preserve flag unless we are intentionally overwriting it + await _database.UpdateAsync(localPost); + } + else + { + await _database.InsertAsync(localPost); + } + } + + public async Task SavePostsAsync(IEnumerable posts) + { + await InitAsync(); + var localPosts = posts.Select(p => LocalPost.FromDomain(p)).ToList(); + + // Use RunInTransactionAsync for bulk operations + await _database.RunInTransactionAsync(conn => + { + foreach (var localPost in localPosts) + { + var existing = conn.Table().Where(p => p.Id == localPost.Id).FirstOrDefault(); + if (existing is not null) + { + localPost.IsPendingSync = existing.IsPendingSync; + conn.Update(localPost); + } + else + { + conn.Insert(localPost); + } + } + }); + } + + public async Task> GetNotesAsync() + { + await InitAsync(); + var localNotes = await _database.Table() + .Where(n => !n.IsDeleted) + .ToListAsync(); + return localNotes.Select(n => n.ToDomain()).ToList(); + } + + public async Task GetNoteAsync(string rowKey) + { + await InitAsync(); + var localNote = await _database.Table() + .Where(n => n.RowKey == rowKey && !n.IsDeleted) + .FirstOrDefaultAsync(); + return localNote?.ToDomain(); + } + + public async Task SaveNoteAsync(Note note, bool isPendingSync = false) + { + await InitAsync(); + var localNote = LocalNote.FromDomain(note, isPendingSync); + var existing = await _database.Table().Where(n => n.RowKey == localNote.RowKey).FirstOrDefaultAsync(); + + if (existing is not null) + { + localNote.IsPendingSync = isPendingSync || existing.IsPendingSync; + await _database.UpdateAsync(localNote); + } + else + { + await _database.InsertAsync(localNote); + } + } + + public async Task DeleteNoteAsync(string rowKey, bool isPendingSync = false) + { + await InitAsync(); + var existing = await _database.Table().Where(n => n.RowKey == rowKey).FirstOrDefaultAsync(); + if (existing is not null) + { + existing.IsDeleted = true; + existing.IsPendingSync = isPendingSync || existing.IsPendingSync; + existing.DateModified = DateTime.UtcNow; + await _database.UpdateAsync(existing); + } + } + + public async Task DeletePostAsync(string id, bool isPendingSync = false) + { + await InitAsync(); + var existing = await _database.Table().Where(p => p.Id == id).FirstOrDefaultAsync(); + if (existing is not null) + { + existing.IsDeleted = true; + existing.IsPendingSync = isPendingSync || existing.IsPendingSync; + existing.DateModified = DateTime.UtcNow; + await _database.UpdateAsync(existing); + } + } + + public async Task RemovePostsAsync(IEnumerable ids) + { + await InitAsync(); + var idList = ids.ToList(); + if (idList.Count == 0) + return; + + await _database.RunInTransactionAsync(conn => + { + foreach (var id in idList) + { + conn.Delete(id); + } + }); + } + + public async Task> GetSummariesAsync() + { + await InitAsync(); + var localSummaries = await _database.Table() + .Where(s => !s.IsDeleted) + .ToListAsync(); + return localSummaries.Select(s => s.ToDomain()).ToList(); + } + + public async Task SaveSummariesAsync(IEnumerable summaries) + { + await InitAsync(); + var localSummaries = summaries.Select(s => LocalSummary.FromDomain(s)).ToList(); + + await _database.RunInTransactionAsync(conn => + { + foreach (var localSummary in localSummaries) + { + var existing = conn.Table().Where(s => s.RowKey == localSummary.RowKey).FirstOrDefault(); + if (existing is not null) + { + localSummary.IsPendingSync = existing.IsPendingSync; + conn.Update(localSummary); + } + else + { + conn.Insert(localSummary); + } + } + }); + } + + public async Task GetSettingsAsync() + { + await InitAsync(); + var localSettings = await _database.Table().FirstOrDefaultAsync(); + return localSettings?.ToDomain(); + } + + public async Task SaveSettingsAsync(Settings settings) + { + await InitAsync(); + var localSettings = LocalSettings.FromDomain(settings); + var existing = await _database.Table().Where(s => s.RowKey == localSettings.RowKey).FirstOrDefaultAsync(); + + if (existing is not null) + { + localSettings.IsPendingSync = existing.IsPendingSync; + await _database.UpdateAsync(localSettings); + } + else + { + await _database.InsertAsync(localSettings); + } + } + + public async Task> GetPendingSyncPostsAsync() + { + await InitAsync(); + var pending = await _database.Table() + .Where(p => p.IsPendingSync) + .ToListAsync(); + return pending.Select(p => p.ToDomain()).ToList(); + } + + public async Task> GetPendingSyncNotesAsync() + { + await InitAsync(); + var pending = await _database.Table() + .Where(n => n.IsPendingSync) + .ToListAsync(); + return pending.Select(n => n.ToDomain()).ToList(); + } + + public async Task MarkSyncedAsync(string id, bool isPost) + { + await InitAsync(); + if (isPost) + { + var existing = await _database.Table().Where(p => p.Id == id).FirstOrDefaultAsync(); + if (existing is not null) + { + if (existing.IsDeleted) + { + await _database.DeleteAsync(existing); + } + else + { + existing.IsPendingSync = false; + await _database.UpdateAsync(existing); + } + } + } + else + { + var existing = await _database.Table().Where(n => n.RowKey == id).FirstOrDefaultAsync(); + if (existing is not null) + { + if (existing.IsDeleted) + { + await _database.DeleteAsync(existing); + } + else + { + existing.IsPendingSync = false; + await _database.UpdateAsync(existing); + } + } + } + } +} diff --git a/src/NoteBookmark.MauiApp/Data/SyncApiClient.cs b/src/NoteBookmark.MauiApp/Data/SyncApiClient.cs index 61a76ef..da32823 100644 --- a/src/NoteBookmark.MauiApp/Data/SyncApiClient.cs +++ b/src/NoteBookmark.MauiApp/Data/SyncApiClient.cs @@ -10,6 +10,7 @@ namespace NoteBookmark.MauiApp.Data; public class SyncApiClient(PostNoteClient client) : ISyncApiClient { public Task> GetPostsModifiedAfter(DateTime modifiedAfter) => client.GetPostsModifiedAfter(modifiedAfter); + public Task?> GetPostIds() => client.GetPostIds(); public Task> GetNotesModifiedAfter(DateTime modifiedAfter) => client.GetNotesModifiedAfter(modifiedAfter); public Task GetPost(string id) => client.GetPost(id); public Task GetNote(string rowKey) => client.GetNote(rowKey); diff --git a/src/NoteBookmark.MauiApp/Data/SyncService.cs b/src/NoteBookmark.MauiApp/Data/SyncService.cs index a82698b..ba0a877 100644 --- a/src/NoteBookmark.MauiApp/Data/SyncService.cs +++ b/src/NoteBookmark.MauiApp/Data/SyncService.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NoteBookmark.Domain; @@ -27,6 +28,8 @@ public class SyncService( ILocalHtmlStorageService localHtmlStorageService) : ISyncService { private const string LastSyncTimestampKey = "LastSyncTimestamp"; + private const int MaxConcurrentRequests = 5; + private static readonly TimeSpan ProgressReportInterval = TimeSpan.FromMilliseconds(250); private readonly object _syncLock = new(); private Task? _currentSyncTask; @@ -53,7 +56,8 @@ public Task SyncAsync() return _currentSyncTask; } - _currentSyncTask = DoSyncAsync(); + // Run on the thread pool so callers on the UI thread (e.g. Blazor pages) never block on sync work. + _currentSyncTask = Task.Run(DoSyncAsync); return _currentSyncTask; } } @@ -209,108 +213,48 @@ private async Task PushAsync(DateTime? lastSync) private async Task PullAsync(DateTime? lastSync) { - // 1. Get all remote posts - var allRemotePosts = await apiClient.GetPostsModifiedAfter(DateTime.MinValue) ?? new List(); - var remotePostIds = allRemotePosts.Select(p => p.Id ?? p.RowKey).ToHashSet(); + // 1. Only fetch posts modified since the last sync. On the first sync this is every post. + var changedRemotePosts = await apiClient.GetPostsModifiedAfter(lastSync ?? DateTime.MinValue) ?? new List(); // 2. Any post that was deleted on the online database while offline should be deleted locally. + var remotePostIds = await GetRemotePostIdsAsync(lastSync, changedRemotePosts); var localPosts = await localDataService.GetPostsAsync() ?? new List(); var localPostMap = localPosts.ToDictionary(p => p.Id ?? p.RowKey); - foreach (var localPost in localPosts) + var deletedIds = localPostMap.Keys.Where(id => !remotePostIds.Contains(id)).ToList(); + if (deletedIds.Count > 0) { - var id = localPost.Id ?? localPost.RowKey; - if (!remotePostIds.Contains(id)) + await localDataService.RemovePostsAsync(deletedIds); + foreach (var id in deletedIds) { - await localDataService.DeletePostAsync(id, isPendingSync: false); - await localDataService.MarkSyncedAsync(id, isPost: true); localPostMap.Remove(id); } } - // 3. Pull new/modified posts - var postsToPull = new List(); - foreach (var remotePostL in allRemotePosts) - { - var id = remotePostL.Id ?? remotePostL.RowKey; - if (!localPostMap.TryGetValue(id, out var lp)) - { - lp = await localDataService.GetPostAsync(id); - } - - if (lp is null || remotePostL.DateModified > lp.DateModified) - { - postsToPull.Add(remotePostL); - } - } + // 3. Pull new/modified posts. The API returns one row per post/note pair, so de-duplicate by id. + var postsToPull = changedRemotePosts + .DistinctBy(p => p.Id ?? p.RowKey) + .Where(remote => !localPostMap.TryGetValue(remote.Id ?? remote.RowKey, out var lp) || remote.DateModified > lp.DateModified) + .ToList(); if (postsToPull.Count > 0) { - SyncProgressChanged?.Invoke(this, new SyncProgressEventArgs(0, postsToPull.Count, $"Pulling 0 of {postsToPull.Count} posts...")); - - for (int i = 0; i < postsToPull.Count; i++) - { - var remotePostL = postsToPull[i]; - var id = remotePostL.Id ?? remotePostL.RowKey; - - Post postToSave; - if (remotePostL.is_read == true) - { - postToSave = new Post - { - Id = id, - RowKey = remotePostL.RowKey, - PartitionKey = remotePostL.PartitionKey, - Title = remotePostL.Title, - Url = remotePostL.Url, - Date_published = remotePostL.Date_published, - Excerpt = remotePostL.Excerpt, - is_read = remotePostL.is_read, - DateModified = remotePostL.DateModified - }; - } - else + int total = postsToPull.Count; + var progress = new ProgressThrottle(this, total, current => $"Pulling {current} of {total} posts..."); + progress.Report(0, force: true); + + var postsToSave = new Post[total]; + await Parallel.ForEachAsync( + Enumerable.Range(0, total), + new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRequests }, + async (i, _) => { - try - { - var fullPost = await apiClient.GetPost(id); - postToSave = fullPost ?? new Post - { - Id = id, - RowKey = remotePostL.RowKey, - PartitionKey = remotePostL.PartitionKey, - Title = remotePostL.Title, - Url = remotePostL.Url, - Date_published = remotePostL.Date_published, - Excerpt = remotePostL.Excerpt, - is_read = remotePostL.is_read, - DateModified = remotePostL.DateModified - }; - } - catch (Exception ex) - { - logger.LogWarning(ex, "Failed to retrieve full post for {PostId}, saving summary metadata", id); - postToSave = new Post - { - Id = id, - RowKey = remotePostL.RowKey, - PartitionKey = remotePostL.PartitionKey, - Title = remotePostL.Title, - Url = remotePostL.Url, - Date_published = remotePostL.Date_published, - Excerpt = remotePostL.Excerpt, - is_read = remotePostL.is_read, - DateModified = remotePostL.DateModified - }; - } - } + postsToSave[i] = await BuildPostToSaveAsync(postsToPull[i]); + progress.Increment(); + }); - await localDataService.SavePostAsync(postToSave, isPendingSync: false); - localPostMap[id] = postToSave; - - int current = i + 1; - SyncProgressChanged?.Invoke(this, new SyncProgressEventArgs(current, postsToPull.Count, $"Pulling {current} of {postsToPull.Count} posts...")); - } + await localDataService.SavePostsAsync(postsToSave); + progress.Report(total, force: true); } // 4. Pull notes modified since lastSync @@ -334,6 +278,90 @@ private async Task PullAsync(DateTime? lastSync) } } + private async Task> GetRemotePostIdsAsync(DateTime? lastSync, List changedRemotePosts) + { + if (lastSync is null) + { + // First sync: the delta already holds every post. + return changedRemotePosts.Select(p => p.Id ?? p.RowKey).ToHashSet(); + } + + var ids = await apiClient.GetPostIds(); + if (ids is not null) + { + return ids.ToHashSet(); + } + + // Older servers don't expose the ids endpoint; fall back to the full post list. + logger.LogInformation("Post ids endpoint unavailable, falling back to the full post list to detect deletions."); + var allRemotePosts = await apiClient.GetPostsModifiedAfter(DateTime.MinValue) ?? new List(); + return allRemotePosts.Select(p => p.Id ?? p.RowKey).ToHashSet(); + } + + private async Task BuildPostToSaveAsync(PostL remotePostL) + { + var id = remotePostL.Id ?? remotePostL.RowKey; + + // Read posts only need the summary metadata; unread posts get the full details for offline reading. + if (remotePostL.is_read != true) + { + try + { + var fullPost = await apiClient.GetPost(id); + if (fullPost is not null) + { + return fullPost; + } + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to retrieve full post for {PostId}, saving summary metadata", id); + } + } + + return new Post + { + Id = id, + RowKey = remotePostL.RowKey, + PartitionKey = remotePostL.PartitionKey, + Title = remotePostL.Title, + Url = remotePostL.Url, + Date_published = remotePostL.Date_published, + Excerpt = remotePostL.Excerpt, + is_read = remotePostL.is_read, + DateModified = remotePostL.DateModified + }; + } + + /// + /// Limits how often progress events fire so the UI isn't re-rendered for every single post. + /// + private sealed class ProgressThrottle(SyncService owner, int total, Func formatStatus) + { + private readonly object _lock = new(); + private readonly System.Diagnostics.Stopwatch _sinceLastReport = System.Diagnostics.Stopwatch.StartNew(); + private int _current; + + public void Increment() + { + var current = Interlocked.Increment(ref _current); + Report(current, force: current == total); + } + + public void Report(int current, bool force = false) + { + lock (_lock) + { + if (!force && _sinceLastReport.Elapsed < ProgressReportInterval) + { + return; + } + _sinceLastReport.Restart(); + } + owner.SyncProgressChanged?.Invoke(owner, new SyncProgressEventArgs(current, total, formatStatus(current))); + } + } + private async Task SyncHtmlAsync() { var posts = await localDataService.GetPostsAsync(); @@ -357,7 +385,8 @@ private async Task SyncHtmlAsync() if (total > 0) { - SyncProgressChanged?.Invoke(this, new SyncProgressEventArgs(0, total, $"Downloading 0 of {total} posts...")); + var progress = new ProgressThrottle(this, total, current => $"Downloading {current} of {total} posts..."); + progress.Report(0, force: true); for (int i = 0; i < unreadToDownload.Count; i++) { @@ -377,8 +406,7 @@ private async Task SyncHtmlAsync() logger.LogWarning(ex, "Failed to download HTML for post {PostId}", id); } - int current = i + 1; - SyncProgressChanged?.Invoke(this, new SyncProgressEventArgs(current, total, $"Downloading {current} of {total} posts...")); + progress.Increment(); } } } diff --git a/src/NoteBookmark.SharedUI/PostNoteClient.cs b/src/NoteBookmark.SharedUI/PostNoteClient.cs index 8b8acd2..8eb6494 100644 --- a/src/NoteBookmark.SharedUI/PostNoteClient.cs +++ b/src/NoteBookmark.SharedUI/PostNoteClient.cs @@ -1,220 +1,231 @@ -using System; -using System.Net.Http.Json; -using NoteBookmark.Domain; - -namespace NoteBookmark.SharedUI; - -public class PostNoteClient(HttpClient httpClient) : IDataService -{ - public async Task> GetUnreadPosts() - { - var posts = await httpClient.GetFromJsonAsync>("api/posts"); - return posts ?? new List(); - } - - public async Task> GetReadPosts() - { - var posts = await httpClient.GetFromJsonAsync>("api/posts/read"); - return posts ?? new List(); - } - - public async Task> GetSummaries() - { - var summaries = await httpClient.GetFromJsonAsync>("api/summary"); - return summaries ?? new List(); - } - - public async Task CreateNote(Note note) - { - var rnCounter = await httpClient.GetStringAsync("api/settings/GetNextReadingNotesCounter"); - note.PartitionKey = rnCounter; - var response = await httpClient.PostAsJsonAsync("api/notes/note", note); - if (!response.IsSuccessStatusCode) - { - var errorContent = await response.Content.ReadAsStringAsync(); - throw new HttpRequestException($"Server returned Bad Request: {errorContent}", null, response.StatusCode); - } - } - - public async Task GetNote(string noteId) - { - var note = await httpClient.GetFromJsonAsync($"api/notes/note/{noteId}"); - return note; - } - - public async Task UpdateNote(Note note) - { - var response = await httpClient.PutAsJsonAsync("api/notes/note", note); - return response.IsSuccessStatusCode; - } - - public async Task DeleteNote(string noteId) - { - var response = await httpClient.DeleteAsync($"api/notes/note/{noteId}"); - return response.IsSuccessStatusCode; - } - - public async Task CreateReadingNotes() - { - var rnCounter = await httpClient.GetStringAsync("api/settings/GetNextReadingNotesCounter"); - var readingNotes = new ReadingNotes(rnCounter); - - var unsortedNotes = await httpClient.GetFromJsonAsync>($"api/notes/GetNotesForSummary/{rnCounter}"); - - if (unsortedNotes == null || unsortedNotes.Count == 0) - { - return readingNotes; - } - - Dictionary> sortedNotes = GroupNotesByCategory(unsortedNotes); - - readingNotes.Notes = sortedNotes; - readingNotes.Tags = readingNotes.GetAllUniqueTags(); - - return readingNotes; - } - - public async Task GetReadingNotes(string number) - { - ReadingNotes? readingNotes; - readingNotes = await httpClient.GetFromJsonAsync($"api/summary/{number}"); - return readingNotes; - } - - private Dictionary> GroupNotesByCategory(List notes) - { - var sortedNotes = new Dictionary>(); - - foreach (var note in notes) - { - var tags = note.Tags?.ToLower().Split(',') ?? Array.Empty(); - - if (string.IsNullOrEmpty(note.Category)) - { - note.Category = NoteCategories.GetCategory(tags.Length > 0 ? tags[0] : null); - } - - string category = note.Category; - if (sortedNotes.ContainsKey(category)) - { - sortedNotes[category].Add(note); - } - else - { - sortedNotes.Add(category, new List { note }); - } - } - - return sortedNotes; - } - - public async Task SaveReadingNotes(ReadingNotes readingNotes) - { - var response = await httpClient.PostAsJsonAsync("api/notes/SaveReadingNotes", readingNotes); - - string jsonURL = ((string)await response.Content.ReadAsStringAsync()).Replace("\"", ""); - - if (response.IsSuccessStatusCode && !string.IsNullOrEmpty(jsonURL)) - { - var summary = new Summary - { - PartitionKey = readingNotes.Number, - RowKey = readingNotes.Number, - Title = readingNotes.Title, - Id = readingNotes.Number, - IsGenerated = "true", - PublishedURL = readingNotes.PublishedUrl, - FileName = jsonURL - }; - - var summaryResponse = await httpClient.PostAsJsonAsync("api/summary/summary", summary); - return summaryResponse.IsSuccessStatusCode; - } - - return false; - } - - public async Task GetPost(string id) - { - var post = await httpClient.GetFromJsonAsync($"api/posts/{id}"); - return post; - } - - public async Task SavePost(Post post) - { - var response = await httpClient.PostAsJsonAsync("api/posts", post); - return response.IsSuccessStatusCode; - } - - public async Task GetSettings() - { - var settings = await httpClient.GetFromJsonAsync("api/settings"); - return settings; - } - - public async Task SaveSettings(Settings settings) - { - var response = await httpClient.PostAsJsonAsync("api/settings/SaveSettings", settings); - return response.IsSuccessStatusCode; - } - - public async Task ExtractPostDetailsAndSave(string url) - { - var requestBody = new { url = url }; - var response = await httpClient.PostAsJsonAsync($"api/posts/extractPostDetails", requestBody); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(); - } - return null; - } - - public async Task DeletePost(string id) - { - var response = await httpClient.DeleteAsync($"api/posts/{id}"); - return response.IsSuccessStatusCode; - } - - public async Task SaveReadingNotesMarkdown(string markdown, string number) - { - var request = new { Markdown = markdown }; - var response = await httpClient.PostAsJsonAsync($"api/summary/{number}/markdown", request); - return response.IsSuccessStatusCode; - } - - public async Task GetPostHtmlAsync(string postId) - { - var response = await httpClient.GetAsync($"api/posts/{postId}/html"); - if (response.StatusCode == System.Net.HttpStatusCode.NotFound) return null; - response.EnsureSuccessStatusCode(); - return await response.Content.ReadAsStringAsync(); - } - - public async Task> GetPostsModifiedAfter(DateTime modifiedAfter) - { - var encoded = System.Web.HttpUtility.UrlEncode(modifiedAfter.ToUniversalTime().ToString("O")); - var unread = await httpClient.GetFromJsonAsync>($"api/posts?modifiedAfter={encoded}") ?? new List(); - var read = await httpClient.GetFromJsonAsync>($"api/posts/read?modifiedAfter={encoded}") ?? new List(); - return unread.Concat(read).ToList(); - } - - public async Task> GetNotesModifiedAfter(DateTime modifiedAfter) - { - try - { - var encoded = System.Web.HttpUtility.UrlEncode(modifiedAfter.ToUniversalTime().ToString("O")); - var notes = await httpClient.GetFromJsonAsync>($"api/notes?modifiedAfter={encoded}"); - return notes ?? new List(); - } - catch (System.Net.Http.HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) - { - return new List(); - } - } - - public Task SyncAsync() => Task.CompletedTask; - public event EventHandler? SyncProgressChanged { add { } remove { } } - public bool IsSyncing => false; - public bool IsOffline => false; - public bool CanSync => false; -} +using System; +using System.Net.Http.Json; +using NoteBookmark.Domain; + +namespace NoteBookmark.SharedUI; + +public class PostNoteClient(HttpClient httpClient) : IDataService +{ + public async Task> GetUnreadPosts() + { + var posts = await httpClient.GetFromJsonAsync>("api/posts"); + return posts ?? new List(); + } + + public async Task> GetReadPosts() + { + var posts = await httpClient.GetFromJsonAsync>("api/posts/read"); + return posts ?? new List(); + } + + public async Task> GetSummaries() + { + var summaries = await httpClient.GetFromJsonAsync>("api/summary"); + return summaries ?? new List(); + } + + public async Task CreateNote(Note note) + { + var rnCounter = await httpClient.GetStringAsync("api/settings/GetNextReadingNotesCounter"); + note.PartitionKey = rnCounter; + var response = await httpClient.PostAsJsonAsync("api/notes/note", note); + if (!response.IsSuccessStatusCode) + { + var errorContent = await response.Content.ReadAsStringAsync(); + throw new HttpRequestException($"Server returned Bad Request: {errorContent}", null, response.StatusCode); + } + } + + public async Task GetNote(string noteId) + { + var note = await httpClient.GetFromJsonAsync($"api/notes/note/{noteId}"); + return note; + } + + public async Task UpdateNote(Note note) + { + var response = await httpClient.PutAsJsonAsync("api/notes/note", note); + return response.IsSuccessStatusCode; + } + + public async Task DeleteNote(string noteId) + { + var response = await httpClient.DeleteAsync($"api/notes/note/{noteId}"); + return response.IsSuccessStatusCode; + } + + public async Task CreateReadingNotes() + { + var rnCounter = await httpClient.GetStringAsync("api/settings/GetNextReadingNotesCounter"); + var readingNotes = new ReadingNotes(rnCounter); + + var unsortedNotes = await httpClient.GetFromJsonAsync>($"api/notes/GetNotesForSummary/{rnCounter}"); + + if (unsortedNotes == null || unsortedNotes.Count == 0) + { + return readingNotes; + } + + Dictionary> sortedNotes = GroupNotesByCategory(unsortedNotes); + + readingNotes.Notes = sortedNotes; + readingNotes.Tags = readingNotes.GetAllUniqueTags(); + + return readingNotes; + } + + public async Task GetReadingNotes(string number) + { + ReadingNotes? readingNotes; + readingNotes = await httpClient.GetFromJsonAsync($"api/summary/{number}"); + return readingNotes; + } + + private Dictionary> GroupNotesByCategory(List notes) + { + var sortedNotes = new Dictionary>(); + + foreach (var note in notes) + { + var tags = note.Tags?.ToLower().Split(',') ?? Array.Empty(); + + if (string.IsNullOrEmpty(note.Category)) + { + note.Category = NoteCategories.GetCategory(tags.Length > 0 ? tags[0] : null); + } + + string category = note.Category; + if (sortedNotes.ContainsKey(category)) + { + sortedNotes[category].Add(note); + } + else + { + sortedNotes.Add(category, new List { note }); + } + } + + return sortedNotes; + } + + public async Task SaveReadingNotes(ReadingNotes readingNotes) + { + var response = await httpClient.PostAsJsonAsync("api/notes/SaveReadingNotes", readingNotes); + + string jsonURL = ((string)await response.Content.ReadAsStringAsync()).Replace("\"", ""); + + if (response.IsSuccessStatusCode && !string.IsNullOrEmpty(jsonURL)) + { + var summary = new Summary + { + PartitionKey = readingNotes.Number, + RowKey = readingNotes.Number, + Title = readingNotes.Title, + Id = readingNotes.Number, + IsGenerated = "true", + PublishedURL = readingNotes.PublishedUrl, + FileName = jsonURL + }; + + var summaryResponse = await httpClient.PostAsJsonAsync("api/summary/summary", summary); + return summaryResponse.IsSuccessStatusCode; + } + + return false; + } + + public async Task GetPost(string id) + { + var post = await httpClient.GetFromJsonAsync($"api/posts/{id}"); + return post; + } + + public async Task SavePost(Post post) + { + var response = await httpClient.PostAsJsonAsync("api/posts", post); + return response.IsSuccessStatusCode; + } + + public async Task GetSettings() + { + var settings = await httpClient.GetFromJsonAsync("api/settings"); + return settings; + } + + public async Task SaveSettings(Settings settings) + { + var response = await httpClient.PostAsJsonAsync("api/settings/SaveSettings", settings); + return response.IsSuccessStatusCode; + } + + public async Task ExtractPostDetailsAndSave(string url) + { + var requestBody = new { url = url }; + var response = await httpClient.PostAsJsonAsync($"api/posts/extractPostDetails", requestBody); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(); + } + return null; + } + + public async Task DeletePost(string id) + { + var response = await httpClient.DeleteAsync($"api/posts/{id}"); + return response.IsSuccessStatusCode; + } + + public async Task SaveReadingNotesMarkdown(string markdown, string number) + { + var request = new { Markdown = markdown }; + var response = await httpClient.PostAsJsonAsync($"api/summary/{number}/markdown", request); + return response.IsSuccessStatusCode; + } + + public async Task GetPostHtmlAsync(string postId) + { + var response = await httpClient.GetAsync($"api/posts/{postId}/html"); + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) return null; + response.EnsureSuccessStatusCode(); + return await response.Content.ReadAsStringAsync(); + } + + public async Task> GetPostsModifiedAfter(DateTime modifiedAfter) + { + var encoded = System.Web.HttpUtility.UrlEncode(modifiedAfter.ToUniversalTime().ToString("O")); + var unread = await httpClient.GetFromJsonAsync>($"api/posts?modifiedAfter={encoded}") ?? new List(); + var read = await httpClient.GetFromJsonAsync>($"api/posts/read?modifiedAfter={encoded}") ?? new List(); + return unread.Concat(read).ToList(); + } + + /// + /// Returns the ids of every post on the server, or null when the server does not expose the endpoint. + /// + public async Task?> GetPostIds() + { + var response = await httpClient.GetAsync("api/posts/ids"); + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) return null; + response.EnsureSuccessStatusCode(); + return await response.Content.ReadFromJsonAsync>(); + } + + public async Task> GetNotesModifiedAfter(DateTime modifiedAfter) + { + try + { + var encoded = System.Web.HttpUtility.UrlEncode(modifiedAfter.ToUniversalTime().ToString("O")); + var notes = await httpClient.GetFromJsonAsync>($"api/notes?modifiedAfter={encoded}"); + return notes ?? new List(); + } + catch (System.Net.Http.HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) + { + return new List(); + } + } + + public Task SyncAsync() => Task.CompletedTask; + public event EventHandler? SyncProgressChanged { add { } remove { } } + public bool IsSyncing => false; + public bool IsOffline => false; + public bool CanSync => false; +} From e1a1537ffda102f13b30fec31af67bb08840fc2a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 15:17:36 +0000 Subject: [PATCH 2/2] Restore CRLF line endings in LocalDataService and PostNoteClient Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Kq68qA7Jv7puAF3Kc6GSFY --- .../Data/LocalDataService.cs | 598 +++++++++--------- src/NoteBookmark.SharedUI/PostNoteClient.cs | 462 +++++++------- 2 files changed, 530 insertions(+), 530 deletions(-) diff --git a/src/NoteBookmark.MauiApp/Data/LocalDataService.cs b/src/NoteBookmark.MauiApp/Data/LocalDataService.cs index bf6dfbe..4c3bd46 100644 --- a/src/NoteBookmark.MauiApp/Data/LocalDataService.cs +++ b/src/NoteBookmark.MauiApp/Data/LocalDataService.cs @@ -1,299 +1,299 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -#if !NOT_MAUI -using Microsoft.Maui.Storage; -#endif -using NoteBookmark.Domain; -using SQLite; - -namespace NoteBookmark.MauiApp.Data; - -public class LocalDataService : ILocalDataService -{ - private const string DatabaseFilename = "NoteBookmark.db3"; - - private const SQLiteOpenFlags Flags = - SQLiteOpenFlags.ReadWrite | - SQLiteOpenFlags.Create | - SQLiteOpenFlags.SharedCache; - - private readonly string _databasePath; - private SQLiteAsyncConnection? _database; - - public LocalDataService() - { -#if !NOT_MAUI - _databasePath = Path.Combine(FileSystem.AppDataDirectory, DatabaseFilename); -#else - _databasePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), DatabaseFilename); -#endif - } - - public LocalDataService(string dbPath) - { - _databasePath = dbPath; - } - - [System.Diagnostics.CodeAnalysis.MemberNotNull(nameof(_database))] - private async Task InitAsync() - { - if (_database is not null) - return; - - _database = new SQLiteAsyncConnection(_databasePath, Flags); - await _database.CreateTableAsync(); - await _database.CreateTableAsync(); - await _database.CreateTableAsync(); - await _database.CreateTableAsync(); - } - - public async Task> GetPostsAsync() - { - await InitAsync(); - var localPosts = await _database.Table() - .Where(p => !p.IsDeleted) - .ToListAsync(); - return localPosts.Select(p => p.ToDomain()).ToList(); - } - - public async Task GetPostAsync(string id) - { - await InitAsync(); - var localPost = await _database.Table() - .Where(p => p.Id == id && !p.IsDeleted) - .FirstOrDefaultAsync(); - return localPost?.ToDomain(); - } - - public async Task SavePostAsync(Post post, bool isPendingSync = false) - { - await InitAsync(); - var localPost = LocalPost.FromDomain(post, isPendingSync); - var existing = await _database.Table().Where(p => p.Id == localPost.Id).FirstOrDefaultAsync(); - - if (existing is not null) - { - localPost.IsPendingSync = isPendingSync || existing.IsPendingSync; // Preserve flag unless we are intentionally overwriting it - await _database.UpdateAsync(localPost); - } - else - { - await _database.InsertAsync(localPost); - } - } - - public async Task SavePostsAsync(IEnumerable posts) - { - await InitAsync(); - var localPosts = posts.Select(p => LocalPost.FromDomain(p)).ToList(); - - // Use RunInTransactionAsync for bulk operations - await _database.RunInTransactionAsync(conn => - { - foreach (var localPost in localPosts) - { - var existing = conn.Table().Where(p => p.Id == localPost.Id).FirstOrDefault(); - if (existing is not null) - { - localPost.IsPendingSync = existing.IsPendingSync; - conn.Update(localPost); - } - else - { - conn.Insert(localPost); - } - } - }); - } - - public async Task> GetNotesAsync() - { - await InitAsync(); - var localNotes = await _database.Table() - .Where(n => !n.IsDeleted) - .ToListAsync(); - return localNotes.Select(n => n.ToDomain()).ToList(); - } - - public async Task GetNoteAsync(string rowKey) - { - await InitAsync(); - var localNote = await _database.Table() - .Where(n => n.RowKey == rowKey && !n.IsDeleted) - .FirstOrDefaultAsync(); - return localNote?.ToDomain(); - } - - public async Task SaveNoteAsync(Note note, bool isPendingSync = false) - { - await InitAsync(); - var localNote = LocalNote.FromDomain(note, isPendingSync); - var existing = await _database.Table().Where(n => n.RowKey == localNote.RowKey).FirstOrDefaultAsync(); - - if (existing is not null) - { - localNote.IsPendingSync = isPendingSync || existing.IsPendingSync; - await _database.UpdateAsync(localNote); - } - else - { - await _database.InsertAsync(localNote); - } - } - - public async Task DeleteNoteAsync(string rowKey, bool isPendingSync = false) - { - await InitAsync(); - var existing = await _database.Table().Where(n => n.RowKey == rowKey).FirstOrDefaultAsync(); - if (existing is not null) - { - existing.IsDeleted = true; - existing.IsPendingSync = isPendingSync || existing.IsPendingSync; - existing.DateModified = DateTime.UtcNow; - await _database.UpdateAsync(existing); - } - } - - public async Task DeletePostAsync(string id, bool isPendingSync = false) - { - await InitAsync(); - var existing = await _database.Table().Where(p => p.Id == id).FirstOrDefaultAsync(); - if (existing is not null) - { - existing.IsDeleted = true; - existing.IsPendingSync = isPendingSync || existing.IsPendingSync; - existing.DateModified = DateTime.UtcNow; - await _database.UpdateAsync(existing); - } - } - - public async Task RemovePostsAsync(IEnumerable ids) - { - await InitAsync(); - var idList = ids.ToList(); - if (idList.Count == 0) - return; - - await _database.RunInTransactionAsync(conn => - { - foreach (var id in idList) - { - conn.Delete(id); - } - }); - } - - public async Task> GetSummariesAsync() - { - await InitAsync(); - var localSummaries = await _database.Table() - .Where(s => !s.IsDeleted) - .ToListAsync(); - return localSummaries.Select(s => s.ToDomain()).ToList(); - } - - public async Task SaveSummariesAsync(IEnumerable summaries) - { - await InitAsync(); - var localSummaries = summaries.Select(s => LocalSummary.FromDomain(s)).ToList(); - - await _database.RunInTransactionAsync(conn => - { - foreach (var localSummary in localSummaries) - { - var existing = conn.Table().Where(s => s.RowKey == localSummary.RowKey).FirstOrDefault(); - if (existing is not null) - { - localSummary.IsPendingSync = existing.IsPendingSync; - conn.Update(localSummary); - } - else - { - conn.Insert(localSummary); - } - } - }); - } - - public async Task GetSettingsAsync() - { - await InitAsync(); - var localSettings = await _database.Table().FirstOrDefaultAsync(); - return localSettings?.ToDomain(); - } - - public async Task SaveSettingsAsync(Settings settings) - { - await InitAsync(); - var localSettings = LocalSettings.FromDomain(settings); - var existing = await _database.Table().Where(s => s.RowKey == localSettings.RowKey).FirstOrDefaultAsync(); - - if (existing is not null) - { - localSettings.IsPendingSync = existing.IsPendingSync; - await _database.UpdateAsync(localSettings); - } - else - { - await _database.InsertAsync(localSettings); - } - } - - public async Task> GetPendingSyncPostsAsync() - { - await InitAsync(); - var pending = await _database.Table() - .Where(p => p.IsPendingSync) - .ToListAsync(); - return pending.Select(p => p.ToDomain()).ToList(); - } - - public async Task> GetPendingSyncNotesAsync() - { - await InitAsync(); - var pending = await _database.Table() - .Where(n => n.IsPendingSync) - .ToListAsync(); - return pending.Select(n => n.ToDomain()).ToList(); - } - - public async Task MarkSyncedAsync(string id, bool isPost) - { - await InitAsync(); - if (isPost) - { - var existing = await _database.Table().Where(p => p.Id == id).FirstOrDefaultAsync(); - if (existing is not null) - { - if (existing.IsDeleted) - { - await _database.DeleteAsync(existing); - } - else - { - existing.IsPendingSync = false; - await _database.UpdateAsync(existing); - } - } - } - else - { - var existing = await _database.Table().Where(n => n.RowKey == id).FirstOrDefaultAsync(); - if (existing is not null) - { - if (existing.IsDeleted) - { - await _database.DeleteAsync(existing); - } - else - { - existing.IsPendingSync = false; - await _database.UpdateAsync(existing); - } - } - } - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +#if !NOT_MAUI +using Microsoft.Maui.Storage; +#endif +using NoteBookmark.Domain; +using SQLite; + +namespace NoteBookmark.MauiApp.Data; + +public class LocalDataService : ILocalDataService +{ + private const string DatabaseFilename = "NoteBookmark.db3"; + + private const SQLiteOpenFlags Flags = + SQLiteOpenFlags.ReadWrite | + SQLiteOpenFlags.Create | + SQLiteOpenFlags.SharedCache; + + private readonly string _databasePath; + private SQLiteAsyncConnection? _database; + + public LocalDataService() + { +#if !NOT_MAUI + _databasePath = Path.Combine(FileSystem.AppDataDirectory, DatabaseFilename); +#else + _databasePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), DatabaseFilename); +#endif + } + + public LocalDataService(string dbPath) + { + _databasePath = dbPath; + } + + [System.Diagnostics.CodeAnalysis.MemberNotNull(nameof(_database))] + private async Task InitAsync() + { + if (_database is not null) + return; + + _database = new SQLiteAsyncConnection(_databasePath, Flags); + await _database.CreateTableAsync(); + await _database.CreateTableAsync(); + await _database.CreateTableAsync(); + await _database.CreateTableAsync(); + } + + public async Task> GetPostsAsync() + { + await InitAsync(); + var localPosts = await _database.Table() + .Where(p => !p.IsDeleted) + .ToListAsync(); + return localPosts.Select(p => p.ToDomain()).ToList(); + } + + public async Task GetPostAsync(string id) + { + await InitAsync(); + var localPost = await _database.Table() + .Where(p => p.Id == id && !p.IsDeleted) + .FirstOrDefaultAsync(); + return localPost?.ToDomain(); + } + + public async Task SavePostAsync(Post post, bool isPendingSync = false) + { + await InitAsync(); + var localPost = LocalPost.FromDomain(post, isPendingSync); + var existing = await _database.Table().Where(p => p.Id == localPost.Id).FirstOrDefaultAsync(); + + if (existing is not null) + { + localPost.IsPendingSync = isPendingSync || existing.IsPendingSync; // Preserve flag unless we are intentionally overwriting it + await _database.UpdateAsync(localPost); + } + else + { + await _database.InsertAsync(localPost); + } + } + + public async Task SavePostsAsync(IEnumerable posts) + { + await InitAsync(); + var localPosts = posts.Select(p => LocalPost.FromDomain(p)).ToList(); + + // Use RunInTransactionAsync for bulk operations + await _database.RunInTransactionAsync(conn => + { + foreach (var localPost in localPosts) + { + var existing = conn.Table().Where(p => p.Id == localPost.Id).FirstOrDefault(); + if (existing is not null) + { + localPost.IsPendingSync = existing.IsPendingSync; + conn.Update(localPost); + } + else + { + conn.Insert(localPost); + } + } + }); + } + + public async Task> GetNotesAsync() + { + await InitAsync(); + var localNotes = await _database.Table() + .Where(n => !n.IsDeleted) + .ToListAsync(); + return localNotes.Select(n => n.ToDomain()).ToList(); + } + + public async Task GetNoteAsync(string rowKey) + { + await InitAsync(); + var localNote = await _database.Table() + .Where(n => n.RowKey == rowKey && !n.IsDeleted) + .FirstOrDefaultAsync(); + return localNote?.ToDomain(); + } + + public async Task SaveNoteAsync(Note note, bool isPendingSync = false) + { + await InitAsync(); + var localNote = LocalNote.FromDomain(note, isPendingSync); + var existing = await _database.Table().Where(n => n.RowKey == localNote.RowKey).FirstOrDefaultAsync(); + + if (existing is not null) + { + localNote.IsPendingSync = isPendingSync || existing.IsPendingSync; + await _database.UpdateAsync(localNote); + } + else + { + await _database.InsertAsync(localNote); + } + } + + public async Task DeleteNoteAsync(string rowKey, bool isPendingSync = false) + { + await InitAsync(); + var existing = await _database.Table().Where(n => n.RowKey == rowKey).FirstOrDefaultAsync(); + if (existing is not null) + { + existing.IsDeleted = true; + existing.IsPendingSync = isPendingSync || existing.IsPendingSync; + existing.DateModified = DateTime.UtcNow; + await _database.UpdateAsync(existing); + } + } + + public async Task DeletePostAsync(string id, bool isPendingSync = false) + { + await InitAsync(); + var existing = await _database.Table().Where(p => p.Id == id).FirstOrDefaultAsync(); + if (existing is not null) + { + existing.IsDeleted = true; + existing.IsPendingSync = isPendingSync || existing.IsPendingSync; + existing.DateModified = DateTime.UtcNow; + await _database.UpdateAsync(existing); + } + } + + public async Task RemovePostsAsync(IEnumerable ids) + { + await InitAsync(); + var idList = ids.ToList(); + if (idList.Count == 0) + return; + + await _database.RunInTransactionAsync(conn => + { + foreach (var id in idList) + { + conn.Delete(id); + } + }); + } + + public async Task> GetSummariesAsync() + { + await InitAsync(); + var localSummaries = await _database.Table() + .Where(s => !s.IsDeleted) + .ToListAsync(); + return localSummaries.Select(s => s.ToDomain()).ToList(); + } + + public async Task SaveSummariesAsync(IEnumerable summaries) + { + await InitAsync(); + var localSummaries = summaries.Select(s => LocalSummary.FromDomain(s)).ToList(); + + await _database.RunInTransactionAsync(conn => + { + foreach (var localSummary in localSummaries) + { + var existing = conn.Table().Where(s => s.RowKey == localSummary.RowKey).FirstOrDefault(); + if (existing is not null) + { + localSummary.IsPendingSync = existing.IsPendingSync; + conn.Update(localSummary); + } + else + { + conn.Insert(localSummary); + } + } + }); + } + + public async Task GetSettingsAsync() + { + await InitAsync(); + var localSettings = await _database.Table().FirstOrDefaultAsync(); + return localSettings?.ToDomain(); + } + + public async Task SaveSettingsAsync(Settings settings) + { + await InitAsync(); + var localSettings = LocalSettings.FromDomain(settings); + var existing = await _database.Table().Where(s => s.RowKey == localSettings.RowKey).FirstOrDefaultAsync(); + + if (existing is not null) + { + localSettings.IsPendingSync = existing.IsPendingSync; + await _database.UpdateAsync(localSettings); + } + else + { + await _database.InsertAsync(localSettings); + } + } + + public async Task> GetPendingSyncPostsAsync() + { + await InitAsync(); + var pending = await _database.Table() + .Where(p => p.IsPendingSync) + .ToListAsync(); + return pending.Select(p => p.ToDomain()).ToList(); + } + + public async Task> GetPendingSyncNotesAsync() + { + await InitAsync(); + var pending = await _database.Table() + .Where(n => n.IsPendingSync) + .ToListAsync(); + return pending.Select(n => n.ToDomain()).ToList(); + } + + public async Task MarkSyncedAsync(string id, bool isPost) + { + await InitAsync(); + if (isPost) + { + var existing = await _database.Table().Where(p => p.Id == id).FirstOrDefaultAsync(); + if (existing is not null) + { + if (existing.IsDeleted) + { + await _database.DeleteAsync(existing); + } + else + { + existing.IsPendingSync = false; + await _database.UpdateAsync(existing); + } + } + } + else + { + var existing = await _database.Table().Where(n => n.RowKey == id).FirstOrDefaultAsync(); + if (existing is not null) + { + if (existing.IsDeleted) + { + await _database.DeleteAsync(existing); + } + else + { + existing.IsPendingSync = false; + await _database.UpdateAsync(existing); + } + } + } + } +} diff --git a/src/NoteBookmark.SharedUI/PostNoteClient.cs b/src/NoteBookmark.SharedUI/PostNoteClient.cs index 8eb6494..1debee1 100644 --- a/src/NoteBookmark.SharedUI/PostNoteClient.cs +++ b/src/NoteBookmark.SharedUI/PostNoteClient.cs @@ -1,231 +1,231 @@ -using System; -using System.Net.Http.Json; -using NoteBookmark.Domain; - -namespace NoteBookmark.SharedUI; - -public class PostNoteClient(HttpClient httpClient) : IDataService -{ - public async Task> GetUnreadPosts() - { - var posts = await httpClient.GetFromJsonAsync>("api/posts"); - return posts ?? new List(); - } - - public async Task> GetReadPosts() - { - var posts = await httpClient.GetFromJsonAsync>("api/posts/read"); - return posts ?? new List(); - } - - public async Task> GetSummaries() - { - var summaries = await httpClient.GetFromJsonAsync>("api/summary"); - return summaries ?? new List(); - } - - public async Task CreateNote(Note note) - { - var rnCounter = await httpClient.GetStringAsync("api/settings/GetNextReadingNotesCounter"); - note.PartitionKey = rnCounter; - var response = await httpClient.PostAsJsonAsync("api/notes/note", note); - if (!response.IsSuccessStatusCode) - { - var errorContent = await response.Content.ReadAsStringAsync(); - throw new HttpRequestException($"Server returned Bad Request: {errorContent}", null, response.StatusCode); - } - } - - public async Task GetNote(string noteId) - { - var note = await httpClient.GetFromJsonAsync($"api/notes/note/{noteId}"); - return note; - } - - public async Task UpdateNote(Note note) - { - var response = await httpClient.PutAsJsonAsync("api/notes/note", note); - return response.IsSuccessStatusCode; - } - - public async Task DeleteNote(string noteId) - { - var response = await httpClient.DeleteAsync($"api/notes/note/{noteId}"); - return response.IsSuccessStatusCode; - } - - public async Task CreateReadingNotes() - { - var rnCounter = await httpClient.GetStringAsync("api/settings/GetNextReadingNotesCounter"); - var readingNotes = new ReadingNotes(rnCounter); - - var unsortedNotes = await httpClient.GetFromJsonAsync>($"api/notes/GetNotesForSummary/{rnCounter}"); - - if (unsortedNotes == null || unsortedNotes.Count == 0) - { - return readingNotes; - } - - Dictionary> sortedNotes = GroupNotesByCategory(unsortedNotes); - - readingNotes.Notes = sortedNotes; - readingNotes.Tags = readingNotes.GetAllUniqueTags(); - - return readingNotes; - } - - public async Task GetReadingNotes(string number) - { - ReadingNotes? readingNotes; - readingNotes = await httpClient.GetFromJsonAsync($"api/summary/{number}"); - return readingNotes; - } - - private Dictionary> GroupNotesByCategory(List notes) - { - var sortedNotes = new Dictionary>(); - - foreach (var note in notes) - { - var tags = note.Tags?.ToLower().Split(',') ?? Array.Empty(); - - if (string.IsNullOrEmpty(note.Category)) - { - note.Category = NoteCategories.GetCategory(tags.Length > 0 ? tags[0] : null); - } - - string category = note.Category; - if (sortedNotes.ContainsKey(category)) - { - sortedNotes[category].Add(note); - } - else - { - sortedNotes.Add(category, new List { note }); - } - } - - return sortedNotes; - } - - public async Task SaveReadingNotes(ReadingNotes readingNotes) - { - var response = await httpClient.PostAsJsonAsync("api/notes/SaveReadingNotes", readingNotes); - - string jsonURL = ((string)await response.Content.ReadAsStringAsync()).Replace("\"", ""); - - if (response.IsSuccessStatusCode && !string.IsNullOrEmpty(jsonURL)) - { - var summary = new Summary - { - PartitionKey = readingNotes.Number, - RowKey = readingNotes.Number, - Title = readingNotes.Title, - Id = readingNotes.Number, - IsGenerated = "true", - PublishedURL = readingNotes.PublishedUrl, - FileName = jsonURL - }; - - var summaryResponse = await httpClient.PostAsJsonAsync("api/summary/summary", summary); - return summaryResponse.IsSuccessStatusCode; - } - - return false; - } - - public async Task GetPost(string id) - { - var post = await httpClient.GetFromJsonAsync($"api/posts/{id}"); - return post; - } - - public async Task SavePost(Post post) - { - var response = await httpClient.PostAsJsonAsync("api/posts", post); - return response.IsSuccessStatusCode; - } - - public async Task GetSettings() - { - var settings = await httpClient.GetFromJsonAsync("api/settings"); - return settings; - } - - public async Task SaveSettings(Settings settings) - { - var response = await httpClient.PostAsJsonAsync("api/settings/SaveSettings", settings); - return response.IsSuccessStatusCode; - } - - public async Task ExtractPostDetailsAndSave(string url) - { - var requestBody = new { url = url }; - var response = await httpClient.PostAsJsonAsync($"api/posts/extractPostDetails", requestBody); - if (response.IsSuccessStatusCode) - { - return await response.Content.ReadFromJsonAsync(); - } - return null; - } - - public async Task DeletePost(string id) - { - var response = await httpClient.DeleteAsync($"api/posts/{id}"); - return response.IsSuccessStatusCode; - } - - public async Task SaveReadingNotesMarkdown(string markdown, string number) - { - var request = new { Markdown = markdown }; - var response = await httpClient.PostAsJsonAsync($"api/summary/{number}/markdown", request); - return response.IsSuccessStatusCode; - } - - public async Task GetPostHtmlAsync(string postId) - { - var response = await httpClient.GetAsync($"api/posts/{postId}/html"); - if (response.StatusCode == System.Net.HttpStatusCode.NotFound) return null; - response.EnsureSuccessStatusCode(); - return await response.Content.ReadAsStringAsync(); - } - - public async Task> GetPostsModifiedAfter(DateTime modifiedAfter) - { - var encoded = System.Web.HttpUtility.UrlEncode(modifiedAfter.ToUniversalTime().ToString("O")); - var unread = await httpClient.GetFromJsonAsync>($"api/posts?modifiedAfter={encoded}") ?? new List(); - var read = await httpClient.GetFromJsonAsync>($"api/posts/read?modifiedAfter={encoded}") ?? new List(); - return unread.Concat(read).ToList(); - } - - /// - /// Returns the ids of every post on the server, or null when the server does not expose the endpoint. - /// - public async Task?> GetPostIds() - { - var response = await httpClient.GetAsync("api/posts/ids"); - if (response.StatusCode == System.Net.HttpStatusCode.NotFound) return null; - response.EnsureSuccessStatusCode(); - return await response.Content.ReadFromJsonAsync>(); - } - - public async Task> GetNotesModifiedAfter(DateTime modifiedAfter) - { - try - { - var encoded = System.Web.HttpUtility.UrlEncode(modifiedAfter.ToUniversalTime().ToString("O")); - var notes = await httpClient.GetFromJsonAsync>($"api/notes?modifiedAfter={encoded}"); - return notes ?? new List(); - } - catch (System.Net.Http.HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) - { - return new List(); - } - } - - public Task SyncAsync() => Task.CompletedTask; - public event EventHandler? SyncProgressChanged { add { } remove { } } - public bool IsSyncing => false; - public bool IsOffline => false; - public bool CanSync => false; -} +using System; +using System.Net.Http.Json; +using NoteBookmark.Domain; + +namespace NoteBookmark.SharedUI; + +public class PostNoteClient(HttpClient httpClient) : IDataService +{ + public async Task> GetUnreadPosts() + { + var posts = await httpClient.GetFromJsonAsync>("api/posts"); + return posts ?? new List(); + } + + public async Task> GetReadPosts() + { + var posts = await httpClient.GetFromJsonAsync>("api/posts/read"); + return posts ?? new List(); + } + + public async Task> GetSummaries() + { + var summaries = await httpClient.GetFromJsonAsync>("api/summary"); + return summaries ?? new List(); + } + + public async Task CreateNote(Note note) + { + var rnCounter = await httpClient.GetStringAsync("api/settings/GetNextReadingNotesCounter"); + note.PartitionKey = rnCounter; + var response = await httpClient.PostAsJsonAsync("api/notes/note", note); + if (!response.IsSuccessStatusCode) + { + var errorContent = await response.Content.ReadAsStringAsync(); + throw new HttpRequestException($"Server returned Bad Request: {errorContent}", null, response.StatusCode); + } + } + + public async Task GetNote(string noteId) + { + var note = await httpClient.GetFromJsonAsync($"api/notes/note/{noteId}"); + return note; + } + + public async Task UpdateNote(Note note) + { + var response = await httpClient.PutAsJsonAsync("api/notes/note", note); + return response.IsSuccessStatusCode; + } + + public async Task DeleteNote(string noteId) + { + var response = await httpClient.DeleteAsync($"api/notes/note/{noteId}"); + return response.IsSuccessStatusCode; + } + + public async Task CreateReadingNotes() + { + var rnCounter = await httpClient.GetStringAsync("api/settings/GetNextReadingNotesCounter"); + var readingNotes = new ReadingNotes(rnCounter); + + var unsortedNotes = await httpClient.GetFromJsonAsync>($"api/notes/GetNotesForSummary/{rnCounter}"); + + if (unsortedNotes == null || unsortedNotes.Count == 0) + { + return readingNotes; + } + + Dictionary> sortedNotes = GroupNotesByCategory(unsortedNotes); + + readingNotes.Notes = sortedNotes; + readingNotes.Tags = readingNotes.GetAllUniqueTags(); + + return readingNotes; + } + + public async Task GetReadingNotes(string number) + { + ReadingNotes? readingNotes; + readingNotes = await httpClient.GetFromJsonAsync($"api/summary/{number}"); + return readingNotes; + } + + private Dictionary> GroupNotesByCategory(List notes) + { + var sortedNotes = new Dictionary>(); + + foreach (var note in notes) + { + var tags = note.Tags?.ToLower().Split(',') ?? Array.Empty(); + + if (string.IsNullOrEmpty(note.Category)) + { + note.Category = NoteCategories.GetCategory(tags.Length > 0 ? tags[0] : null); + } + + string category = note.Category; + if (sortedNotes.ContainsKey(category)) + { + sortedNotes[category].Add(note); + } + else + { + sortedNotes.Add(category, new List { note }); + } + } + + return sortedNotes; + } + + public async Task SaveReadingNotes(ReadingNotes readingNotes) + { + var response = await httpClient.PostAsJsonAsync("api/notes/SaveReadingNotes", readingNotes); + + string jsonURL = ((string)await response.Content.ReadAsStringAsync()).Replace("\"", ""); + + if (response.IsSuccessStatusCode && !string.IsNullOrEmpty(jsonURL)) + { + var summary = new Summary + { + PartitionKey = readingNotes.Number, + RowKey = readingNotes.Number, + Title = readingNotes.Title, + Id = readingNotes.Number, + IsGenerated = "true", + PublishedURL = readingNotes.PublishedUrl, + FileName = jsonURL + }; + + var summaryResponse = await httpClient.PostAsJsonAsync("api/summary/summary", summary); + return summaryResponse.IsSuccessStatusCode; + } + + return false; + } + + public async Task GetPost(string id) + { + var post = await httpClient.GetFromJsonAsync($"api/posts/{id}"); + return post; + } + + public async Task SavePost(Post post) + { + var response = await httpClient.PostAsJsonAsync("api/posts", post); + return response.IsSuccessStatusCode; + } + + public async Task GetSettings() + { + var settings = await httpClient.GetFromJsonAsync("api/settings"); + return settings; + } + + public async Task SaveSettings(Settings settings) + { + var response = await httpClient.PostAsJsonAsync("api/settings/SaveSettings", settings); + return response.IsSuccessStatusCode; + } + + public async Task ExtractPostDetailsAndSave(string url) + { + var requestBody = new { url = url }; + var response = await httpClient.PostAsJsonAsync($"api/posts/extractPostDetails", requestBody); + if (response.IsSuccessStatusCode) + { + return await response.Content.ReadFromJsonAsync(); + } + return null; + } + + public async Task DeletePost(string id) + { + var response = await httpClient.DeleteAsync($"api/posts/{id}"); + return response.IsSuccessStatusCode; + } + + public async Task SaveReadingNotesMarkdown(string markdown, string number) + { + var request = new { Markdown = markdown }; + var response = await httpClient.PostAsJsonAsync($"api/summary/{number}/markdown", request); + return response.IsSuccessStatusCode; + } + + public async Task GetPostHtmlAsync(string postId) + { + var response = await httpClient.GetAsync($"api/posts/{postId}/html"); + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) return null; + response.EnsureSuccessStatusCode(); + return await response.Content.ReadAsStringAsync(); + } + + public async Task> GetPostsModifiedAfter(DateTime modifiedAfter) + { + var encoded = System.Web.HttpUtility.UrlEncode(modifiedAfter.ToUniversalTime().ToString("O")); + var unread = await httpClient.GetFromJsonAsync>($"api/posts?modifiedAfter={encoded}") ?? new List(); + var read = await httpClient.GetFromJsonAsync>($"api/posts/read?modifiedAfter={encoded}") ?? new List(); + return unread.Concat(read).ToList(); + } + + /// + /// Returns the ids of every post on the server, or null when the server does not expose the endpoint. + /// + public async Task?> GetPostIds() + { + var response = await httpClient.GetAsync("api/posts/ids"); + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) return null; + response.EnsureSuccessStatusCode(); + return await response.Content.ReadFromJsonAsync>(); + } + + public async Task> GetNotesModifiedAfter(DateTime modifiedAfter) + { + try + { + var encoded = System.Web.HttpUtility.UrlEncode(modifiedAfter.ToUniversalTime().ToString("O")); + var notes = await httpClient.GetFromJsonAsync>($"api/notes?modifiedAfter={encoded}"); + return notes ?? new List(); + } + catch (System.Net.Http.HttpRequestException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) + { + return new List(); + } + } + + public Task SyncAsync() => Task.CompletedTask; + public event EventHandler? SyncProgressChanged { add { } remove { } } + public bool IsSyncing => false; + public bool IsOffline => false; + public bool CanSync => false; +}