From 20d11c0302c6b37c83ccd2e415761697bc6b2b51 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Tue, 3 Nov 2026 17:00:39 +0300 Subject: [PATCH 1/5] feat: client SDK for .NET and Node --- .../workflows/bitween-api-cicd-gateway.yml | 4 +- .github/workflows/dotnet-pr-checks.yml | 8 + .github/workflows/sdk-release.yml | 42 ++- README.md | 2 + SW.Bitween.Client/BitweenClient.cs | 162 +++++++++++ SW.Bitween.Client/BitweenClientException.cs | 59 ++++ SW.Bitween.Client/ExchangeQuery.cs | 175 ++++++++++++ SW.Bitween.Client/Exchanges.cs | 139 +++++++++ SW.Bitween.Client/SW.Bitween.Client.csproj | 35 +++ SW.Bitween.HttpTests/BitweenClientTests.cs | 140 +++++++++ .../SW.Bitween.HttpTests.csproj | 1 + SW.Bitween.sln | 14 + docs/architecture.md | 5 +- docs/client-sdk.md | 151 ++++++++++ sdk/node-client/README.md | 20 ++ sdk/node-client/package.json | 12 + sdk/node-client/src/index.d.ts | 194 +++++++++++++ sdk/node-client/src/index.js | 269 ++++++++++++++++++ sdk/node-client/test/client.test.js | 137 +++++++++ 19 files changed, 1562 insertions(+), 7 deletions(-) create mode 100644 SW.Bitween.Client/BitweenClient.cs create mode 100644 SW.Bitween.Client/BitweenClientException.cs create mode 100644 SW.Bitween.Client/ExchangeQuery.cs create mode 100644 SW.Bitween.Client/Exchanges.cs create mode 100644 SW.Bitween.Client/SW.Bitween.Client.csproj create mode 100644 SW.Bitween.HttpTests/BitweenClientTests.cs create mode 100644 docs/client-sdk.md create mode 100644 sdk/node-client/README.md create mode 100644 sdk/node-client/package.json create mode 100644 sdk/node-client/src/index.d.ts create mode 100644 sdk/node-client/src/index.js create mode 100644 sdk/node-client/test/client.test.js diff --git a/.github/workflows/bitween-api-cicd-gateway.yml b/.github/workflows/bitween-api-cicd-gateway.yml index e3709224..dc3ea880 100644 --- a/.github/workflows/bitween-api-cicd-gateway.yml +++ b/.github/workflows/bitween-api-cicd-gateway.yml @@ -176,10 +176,12 @@ jobs: container-registry: 'docker.io' image-name: 'simplify9/bitween' - # The SDK and the adapter contract package (SimplyWorks.Bitween.Adapters) adapters reference. + # The SDK, the adapter contract package (SimplyWorks.Bitween.Adapters) adapters reference, and + # the client (SimplyWorks.Bitween.Client) clients' own programs call Bitween with. nuget-projects: >- SW.Bitween.Sdk/SW.Bitween.Sdk.csproj SW.Bitween.Adapters/SW.Bitween.Adapters.csproj + SW.Bitween.Client/SW.Bitween.Client.csproj run-tests: 'true' # Integration tests run against real Postgres/RabbitMQ/MailHog via # Testcontainers - self-contained, no extra CI services needed, since diff --git a/.github/workflows/dotnet-pr-checks.yml b/.github/workflows/dotnet-pr-checks.yml index 06025989..7cd480bf 100644 --- a/.github/workflows/dotnet-pr-checks.yml +++ b/.github/workflows/dotnet-pr-checks.yml @@ -43,6 +43,14 @@ jobs: --no-build --configuration Release --logger "trx;LogFileName=unit-tests.trx" --results-directory TestResults + # SimplyWorks.Bitween.Client against a real Bitween (Testcontainers): clients build their own + # screens on it, so a change that breaks it is stopped here rather than found after they upgrade. + - run: > + dotnet test SW.Bitween.HttpTests/SW.Bitween.HttpTests.csproj + --no-build --configuration Release + --filter "FullyQualifiedName~BitweenClientTests" + --logger "trx;LogFileName=client-tests.trx" + --results-directory TestResults - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: failure() with: diff --git a/.github/workflows/sdk-release.yml b/.github/workflows/sdk-release.yml index 58af9e68..8effee63 100644 --- a/.github/workflows/sdk-release.yml +++ b/.github/workflows/sdk-release.yml @@ -1,11 +1,12 @@ name: SDK release # Publishes the adapter contract for Python to PyPI as simplyworks-bitween and for JavaScript and -# TypeScript to npm as @simplyworks/bitween. Never run by hand or by a tag: the release run on +# TypeScript to npm as @simplyworks/bitween, and the client for JavaScript and TypeScript to npm as +# @simplyworks/bitween-client. Never run by hand or by a tag: the release run on # releases/r10.0 (bitween-api-cicd-gateway.yml) calls it with the version it computed, so they carry -# Bitween's version. Only a release version is published. Each is tested first against -# SW-Serverless's SDK from its default branch, and a version already on its registry is left alone, -# so a failed run can simply be run again. +# Bitween's version. Only a release version is published. Each is tested first (the adapter +# contracts against SW-Serverless's SDK from its default branch), and a version already on its +# registry is left alone, so a failed run can simply be run again. on: workflow_call: inputs: @@ -93,3 +94,36 @@ jobs: else npm publish --access public fi + + node-client: + if: ${{ !contains(inputs.version, '-') }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdk/node-client + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + registry-url: "https://registry.npmjs.org" + - name: Version + id: version + env: + VERSION: ${{ inputs.version }} + run: | + v="$VERSION" + npm version "$v" --no-git-tag-version --allow-same-version + echo "value=$v" >> "$GITHUB_OUTPUT" + - name: Test + run: node --test + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: | + name=$(node -p "require('./package.json').name") + if npm view "$name@${{ steps.version.outputs.value }}" version >/dev/null 2>&1; then + echo "$name@${{ steps.version.outputs.value }} is already on npm." + else + npm publish --access public + fi diff --git a/README.md b/README.md index 753a9b9d..4cbc0ee9 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ The database schema is created on first start. A new database refuses to start w | [Adapters](docs/adapters.md) | Every built-in adapter and its properties, custom adapters in .NET, Python and JavaScript, publishing, the adapter editor and the source viewer | | [Adapter contract](docs/adapter-contract.md) | What Bitween calls on an adapter of each kind and what it passes, in every language | | [The bitween CLI](docs/cli.md) | Signing in from a terminal, and writing, building, checking and publishing adapters | +| [Client SDK](docs/client-sdk.md) | Exchanges from your own program, in .NET or JavaScript: search, read files, retry, bulk retry and export | | [Mapping](docs/mapping.md) | The rules-based mapper and the legacy Scriban JSON mapper | | [Data sources](docs/data-sources.md) | Long-lived connections: enabling them, placement, health and resource limits | | [External brokers](docs/external-brokers.md) | Reading from and publishing to a customer's RabbitMQ or Amazon SQS | @@ -146,6 +147,7 @@ Deeper reference written alongside the features: | `SW.Bitween.Adapters` | The adapter contract and its .NET interfaces, published to NuGet as `SimplyWorks.Bitween.Adapters` | | `SW.Bitween.Adapters.Tooling` | Bitween's adapters on SW-Serverless's tooling: the contract, templates and Python and Node packages the CLI and the adapter editor use | | `SW.Bitween.Cli` | The `bitween` command | +| `SW.Bitween.Client`, `sdk/node-client` | The client SDK, for exchanges from your own program: `SimplyWorks.Bitween.Client` on NuGet and `@simplyworks/bitween-client` on npm | | `sdk/python`, `sdk/node` | The adapter contract for Python (`simplyworks-bitween`) and for JavaScript and TypeScript (`@simplyworks/bitween`) | | `SW.Bitween.PgSql`, `SW.Bitween.MySql`, `SW.Bitween.MsSql` | Provider-specific database contexts and migrations | | `SW.Bitween.Sample*` | Sample custom adapters, including a resident handler | diff --git a/SW.Bitween.Client/BitweenClient.cs b/SW.Bitween.Client/BitweenClient.cs new file mode 100644 index 00000000..ecea2e41 --- /dev/null +++ b/SW.Bitween.Client/BitweenClient.cs @@ -0,0 +1,162 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace SW.Bitween.Client; + +/// +/// One Bitween's API, as a service account. The account's token is traded for a session on the first +/// call, and traded again whenever the session runs out or is refused, so the token is all a caller +/// hands over. Keep one for the life of the program: it can be used from many threads at once, and +/// they share one session. What the account may do is what its roles allow: exchanges.view to +/// read, and exchanges.operate to retry. +/// +/// +/// +/// using var bitween = new BitweenClient("https://bitween.example.com", "bwt_..."); +/// var failed = await bitween.Exchanges.SearchAsync(new ExchangeQuery { Status = ExchangeStatus.Failed }); +/// +/// +public sealed class BitweenClient : IDisposable +{ + internal static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web) + { + Converters = { new JsonStringEnumConverter() }, + }; + + readonly HttpClient http; + readonly bool ownsHttp; + readonly string token; + readonly SemaphoreSlim signingIn = new(1, 1); + volatile Session? session; + + /// The Bitween's address, such as https://bitween.example.com. + /// A service account's token, starting bwt_. + public BitweenClient(string url, string token) + : this(new HttpClient { BaseAddress = new Uri(url.TrimEnd('/') + "/"), Timeout = TimeSpan.FromMinutes(5) }, token) + { + ownsHttp = true; + } + + /// + /// With an of the caller's, whose is the + /// Bitween's address, ending in / when it has a path. Disposing the client leaves it open. + /// + public BitweenClient(HttpClient http, string token) + { + ArgumentNullException.ThrowIfNull(http); + if (http.BaseAddress == null) throw new ArgumentException("The HttpClient needs a BaseAddress: the Bitween's address.", nameof(http)); + ArgumentException.ThrowIfNullOrWhiteSpace(token); + this.http = http; + this.token = token.Trim(); + Exchanges = new Exchanges(this); + ScheduledRetries = new ScheduledRetries(this); + } + + /// Search exchanges, read their files, and retry or export them. + public Exchanges Exchanges { get; } + + /// The automatic retries waiting to run, and running one now. + public ScheduledRetries ScheduledRetries { get; } + + internal async Task GetAsync(string path, CancellationToken cancellation) + { + using var response = await SendAsync(() => new HttpRequestMessage(HttpMethod.Get, path), + HttpCompletionOption.ResponseContentRead, cancellation); + return (await response.Content.ReadFromJsonAsync(Json, cancellation))!; + } + + internal async Task PostAsync(string path, object body, CancellationToken cancellation) + { + using var response = await SendAsync(() => Post(path, body), HttpCompletionOption.ResponseContentRead, cancellation); + return (await response.Content.ReadFromJsonAsync(Json, cancellation))!; + } + + internal async Task PostAsync(string path, object body, CancellationToken cancellation) + { + using var _ = await SendAsync(() => Post(path, body), HttpCompletionOption.ResponseContentRead, cancellation); + } + + /// The answer's body as it comes, for a file too big to hold: the caller disposes it. + internal async Task PostForStreamAsync(string path, object body, CancellationToken cancellation) + { + var response = await SendAsync(() => Post(path, body), HttpCompletionOption.ResponseHeadersRead, cancellation); + return await response.Content.ReadAsStreamAsync(cancellation); + } + + static HttpRequestMessage Post(string path, object body) => + new(HttpMethod.Post, path) { Content = JsonContent.Create(body, body.GetType(), options: Json) }; + + async Task SendAsync(Func make, HttpCompletionOption completion, + CancellationToken cancellation) + { + var used = await SessionAsync(null, cancellation); + var response = await SendOnceAsync(make(), used, completion, cancellation); + if (response.StatusCode == HttpStatusCode.Unauthorized) + { + // Ended before its time, as by the account's roles changing: sign in again once and try again. + response.Dispose(); + used = await SessionAsync(used, cancellation); + response = await SendOnceAsync(make(), used, completion, cancellation); + } + + if (response.IsSuccessStatusCode) return response; + using (response) throw await BitweenClientException.FromAsync(response, cancellation); + } + + async Task SendOnceAsync(HttpRequestMessage request, Session with, + HttpCompletionOption completion, CancellationToken cancellation) + { + using (request) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", with.Jwt); + return await http.SendAsync(request, completion, cancellation); + } + } + + /// + /// The session to send: the one held, unless it is about to end or is , + /// when the token is traded for a new one. Calls that find it so together trade it once between them. + /// + async Task SessionAsync(Session? refused, CancellationToken cancellation) + { + var held = session; + if (Usable(held, refused)) return held!; + + await signingIn.WaitAsync(cancellation); + try + { + held = session; + if (Usable(held, refused)) return held!; + + using var response = await http.PostAsJsonAsync("api/accounts/servicetoken", new { token }, Json, cancellation); + if (!response.IsSuccessStatusCode) throw await BitweenClientException.FromAsync(response, cancellation); + var answer = await response.Content.ReadFromJsonAsync(Json, cancellation); + if (string.IsNullOrEmpty(answer?.Jwt)) + throw new BitweenClientException("Bitween answered the sign-in without a session; is this a Bitween?", response.StatusCode); + + // Renewed a minute early, so no call goes out on a session that ends on its way. + session = new Session(answer.Jwt, DateTimeOffset.UtcNow.AddSeconds(Math.Max(0, answer.ExpiresIn - 60))); + return session; + } + finally + { + signingIn.Release(); + } + } + + static bool Usable(Session? held, Session? refused) => + held != null && held != refused && DateTimeOffset.UtcNow < held.RenewAfter; + + public void Dispose() + { + if (ownsHttp) http.Dispose(); + signingIn.Dispose(); + } + + sealed record Session(string Jwt, DateTimeOffset RenewAfter); + + sealed record SignIn(string Jwt, int ExpiresIn); +} diff --git a/SW.Bitween.Client/BitweenClientException.cs b/SW.Bitween.Client/BitweenClientException.cs new file mode 100644 index 00000000..bc862019 --- /dev/null +++ b/SW.Bitween.Client/BitweenClientException.cs @@ -0,0 +1,59 @@ +using System.Net; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace SW.Bitween.Client; + +/// What a Bitween refused, and why. +public class BitweenClientException(string message, HttpStatusCode status, IReadOnlyDictionary? errors = null) + : Exception(message) +{ + public HttpStatusCode Status { get; } = status; + + /// + /// Bitween's reasons by code, such as ALREADY_RETRIED or DATABASE_TIMED_OUT, each with + /// its messages. Empty when it gave none. + /// + public IReadOnlyDictionary Errors { get; } = errors ?? new Dictionary(); + + /// The first of ' codes, or null when there are none. + public string? Code => Errors.Keys.FirstOrDefault(); + + internal static async Task FromAsync(HttpResponseMessage response, CancellationToken cancellation) + { + var body = await response.Content.ReadAsStringAsync(cancellation); + var errors = ErrorsIn(body); + var messages = errors.SelectMany(e => e.Value).Where(m => !string.IsNullOrWhiteSpace(m)).ToList(); + var message = messages.Count > 0 ? string.Join(" ", messages) : response.StatusCode switch + { + HttpStatusCode.Unauthorized => "Bitween refused the service token, or the session made with it.", + HttpStatusCode.Forbidden => "The service account's roles don't have the permission this needs.", + HttpStatusCode.NotFound => "Bitween has no such thing.", + _ => $"Bitween answered {(int)response.StatusCode}{(string.IsNullOrWhiteSpace(body) ? "" : ": " + body.Trim())}", + }; + return new BitweenClientException(message, response.StatusCode, errors); + } + + /// + /// Validation answers as { CODE: [messages] }, and problem details with errors in that + /// shape. Problem details without them say only the status, so they give no reasons. + /// + static Dictionary ErrorsIn(string body) + { + var errors = new Dictionary(); + try + { + if (JsonNode.Parse(body) is not JsonObject answer) return errors; + var reasons = answer["errors"] as JsonObject ?? (answer.ContainsKey("status") ? new JsonObject() : answer); + foreach (var (code, value) in reasons) + errors[code] = value switch + { + JsonArray list => list.Select(m => m?.ToString() ?? "").ToArray(), + null => [], + _ => [value.ToString()], + }; + } + catch (JsonException) { } + return errors; + } +} diff --git a/SW.Bitween.Client/ExchangeQuery.cs b/SW.Bitween.Client/ExchangeQuery.cs new file mode 100644 index 00000000..77bf9b41 --- /dev/null +++ b/SW.Bitween.Client/ExchangeQuery.cs @@ -0,0 +1,175 @@ +namespace SW.Bitween.Client; + +/// An exchange's status, as the Exchanges page's tabs divide them. +public enum ExchangeStatus +{ + /// Still running: no result yet. + Processing = 0, + + /// Finished, and the response (if any) was not marked bad. + Success = 1, + + /// Finished, but the response was marked bad. + BadResponse = 2, + + Failed = 3, +} + +/// +/// Which exchanges, filtered the way the Exchanges page filters them. What is left unset matches any; +/// everything set has to hold. +/// +public class ExchangeQuery +{ + public ExchangeStatus? Status { get; set; } + + /// Started at or after this. + public DateTimeOffset? From { get; set; } + + /// Started at or before this. + public DateTimeOffset? To { get; set; } + + /// Any of these partners'. + public IReadOnlyList? PartnerIds { get; set; } + + /// Run by any of these subscriptions. + public IReadOnlyList? SubscriptionIds { get; set; } + + /// Of any of these information types (documents, in the API). + public IReadOnlyList? InformationTypeIds { get; set; } + + /// + /// These exchanges, with the retry of each and the exchanges each rolled up, as the Exchanges page's + /// id search finds them. is the one exchange alone. + /// + public IReadOnlyList? Ids { get; set; } + + /// The retry of this exchange: an exchange has at most one. + public string? RetryFor { get; set; } + + public string? CorrelationId { get; set; } + + /// Only the newest attempt of each retry chain. + public bool LatestOnly { get; set; } + + /// The error contains this text, ignoring case. + public string? ErrorContains { get; set; } + + /// + /// Promoted properties by key: the value contains the text, ignoring case, or with a null text, + /// the exchange promoted the key at all. Every one has to hold. + /// + public IDictionary? PromotedProperties { get; set; } + + /// + /// Filters in Bitween's own form, Field:Rule:Value, for what the properties above don't + /// cover: IsRetry:1:true, say. See Exchanges in the API reference. + /// + public IList Filters { get; } = new List(); + + /// The filters as Bitween's search takes them, each one Field:Rule:Value. + internal List ToFilters() + { + var filters = new List(); + // Rule 1 is "is"; 9 is "is any of", with its values typed and joined. + void OneOf(string field, IReadOnlyList? values) + { + if (values is not { Count: > 0 }) return; + filters.Add(values.Count == 1 ? $"{field}:1:{values[0]}" : $"{field}:9:numeric|{string.Join("|", values)}"); + } + + if (Status != null) filters.Add($"StatusFilter:1:{(int)Status}"); + OneOf("PartnerId", PartnerIds); + OneOf("SubscriptionId", SubscriptionIds); + OneOf("DocumentId", InformationTypeIds); + if (Ids is { Count: > 0 }) filters.Add($"Id:4:text|{string.Join("|", Ids)}"); + if (!string.IsNullOrWhiteSpace(RetryFor)) filters.Add($"RetryFor:1:{RetryFor.Trim()}"); + if (!string.IsNullOrWhiteSpace(CorrelationId)) filters.Add($"CorrelationId:1:{CorrelationId.Trim()}"); + if (LatestOnly) filters.Add("LatestOnly:1:true"); + if (!string.IsNullOrWhiteSpace(ErrorContains)) filters.Add($"ErrorContains:4:{ErrorContains.Trim()}"); + foreach (var (key, text) in PromotedProperties ?? new Dictionary()) + filters.Add(string.IsNullOrWhiteSpace(text) ? $"PromotedProperty:1:{key}" : $"PromotedProperty:4:{key}:{text.Trim()}"); + // Rules 6 and 8, at or after and at or before: the range rule, 21, can't be sent a time + // Postgres will compare (the Exchanges page found the same). + if (From != null) filters.Add($"StartedOn:6:{Time(From.Value)}"); + if (To != null) filters.Add($"StartedOn:8:{Time(To.Value)}"); + filters.AddRange(Filters); + return filters; + } + + internal static string Time(DateTimeOffset time) => time.UtcDateTime.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); + + /// Filters, paging and the rest as a query string, every value escaped. + internal static string QueryString(IEnumerable filters, params (string Name, string Value)[] more) => + string.Join("&", filters.Select(f => $"filter={Uri.EscapeDataString(f)}") + .Concat(more.Select(p => $"{p.Name}={Uri.EscapeDataString(p.Value)}"))); +} + +/// +/// Which exchanges a bulk retry or an export covers: picked by id, or every one a query matches, +/// however many pages that comes to, with some left out. +/// +public sealed class ExchangeSelection +{ + ExchangeSelection(List? ids, string? filter, List? excludeIds) + { + Ids = ids; + Filter = filter; + ExcludeIds = excludeIds; + } + + internal List? Ids { get; } + internal string? Filter { get; } + internal List? ExcludeIds { get; } + + /// These exchanges. + public static ExchangeSelection Of(IEnumerable ids) => new(ids.ToList(), null, null); + + /// Every exchange matches, but those in . + public static ExchangeSelection Matching(ExchangeQuery query, IEnumerable? except = null) + { + var filters = query.ToFilters(); + // An empty filter reads as no filter, and the selection would fall back to its ids, which are + // none. Every attempt, not only the newest, matches everything without narrowing it. + if (filters.Count == 0) filters.Add("LatestOnly:1:false"); + return new(null, ExchangeQuery.QueryString(filters), except?.ToList()); + } +} + +/// Which automatic retries, filtered the way the Scheduled retries page filters them. +public class ScheduledRetryQuery +{ + public int? SubscriptionId { get; set; } + + /// Of this information type (document, in the API). + public int? InformationTypeId { get; set; } + + /// The error of the attempt it follows contains this text. + public string? ErrorContains { get; set; } + + /// Due at or after this. + public DateTimeOffset? From { get; set; } + + /// Due at or before this. + public DateTimeOffset? To { get; set; } + + internal List ToFilters() + { + var filters = new List(); + if (SubscriptionId != null) filters.Add($"SubscriptionId:1:{SubscriptionId}"); + if (InformationTypeId != null) filters.Add($"DocumentId:1:{InformationTypeId}"); + if (!string.IsNullOrWhiteSpace(ErrorContains)) filters.Add($"Exception:4:{ErrorContains.Trim()}"); + if (From != null) filters.Add($"On:6:{ExchangeQuery.Time(From.Value)}"); + if (To != null) filters.Add($"On:8:{ExchangeQuery.Time(To.Value)}"); + return filters; + } +} + +/// One page of a search, and how many match in all. +public sealed class Page +{ + public List Result { get; set; } = []; + + /// How many match. An exchange search stops counting at 10,001, meaning more than 10,000. + public int TotalCount { get; set; } +} diff --git a/SW.Bitween.Client/Exchanges.cs b/SW.Bitween.Client/Exchanges.cs new file mode 100644 index 00000000..4528ccad --- /dev/null +++ b/SW.Bitween.Client/Exchanges.cs @@ -0,0 +1,139 @@ +using SW.Bitween.Model; + +namespace SW.Bitween.Client; + +/// A Bitween's exchanges: what the Exchanges page shows and does. +public sealed class Exchanges +{ + readonly BitweenClient bitween; + + internal Exchanges(BitweenClient bitween) => this.bitween = bitween; + + /// + /// A page of the exchanges matches, newest first, and how many match: + /// counts from 0, and has exchanges. With + /// false the total is left out, as 0, and the page comes back sooner. + /// + public Task> SearchAsync(ExchangeQuery? query = null, int page = 0, int size = 25, bool count = true, + CancellationToken cancellation = default) + { + var filters = (query ?? new ExchangeQuery()).ToFilters(); + var paging = new List<(string, string)> { ("page", page.ToString()), ("size", size.ToString()) }; + if (!count) paging.Add(("count", "false")); + return bitween.GetAsync>($"api/xchanges?{ExchangeQuery.QueryString(filters, paging.ToArray())}", cancellation); + } + + /// One exchange, or null when there is none by that id. + public async Task GetAsync(string id, CancellationToken cancellation = default) + { + // The id search also finds the exchange's retry, which is newer, and the exchanges it rolled up, + // which are older: newest first, the exchange itself is first or second. + var found = await SearchAsync(new ExchangeQuery { Ids = [id] }, 0, 2, false, cancellation); + return found.Result.FirstOrDefault(x => x.Id == id); + } + + /// + /// One of an exchange's files, by the key its row gives: , + /// or . + /// + public async Task ReadFileAsync(string key, CancellationToken cancellation = default) + { + var file = await bitween.GetAsync($"api/bitweendocs?documentKey={Uri.EscapeDataString(key)}", cancellation); + return file.Data; + } + + /// Every attempt in the retry chain belongs to, oldest first. + public Task GetRetryTreeAsync(string id, CancellationToken cancellation = default) => + bitween.GetAsync($"api/xchanges/retrytree?id={Uri.EscapeDataString(id)}", cancellation); + + /// + /// Runs an exchange again from its input file, and returns the new attempt's id. With + /// , the subscription's current settings are used rather than those the + /// exchange ran with. An exchange is retried at most once: one that has been is refused with + /// ALREADY_RETRIED, so retry its newest attempt instead. One with an automatic retry booked + /// is refused with AUTO_RETRY_SCHEDULED; run that with + /// instead. + /// + public async Task RetryAsync(string id, bool reset = false, string reason = "Manual retry", + CancellationToken cancellation = default) + { + await bitween.PostAsync($"api/xchanges/{Uri.EscapeDataString(id)}/retry", + new XchangeRetry { Reason = reason, Reset = reset }, cancellation); + // Answered with nothing: the new attempt is the one that says it retries this. + var retry = await SearchAsync(new ExchangeQuery { RetryFor = id }, 0, 1, false, cancellation); + return retry.Result.FirstOrDefault()?.Id; + } + + /// + /// What would do with the same selection, changing nothing: how many + /// will run, the attempts standing in for those already retried, and those skipped with why. + /// + public Task PreviewBulkRetryAsync(ExchangeSelection selection, bool reset = false, + CancellationToken cancellation = default) => + bitween.PostAsync("api/xchanges/bulkretrypreview", BulkRetry(selection, "Bulk retry", reset), cancellation); + + /// + /// Retries a selection, each chain once from its newest attempt, and says what it did. It carries + /// out at most (500) at a time: past that the plan comes + /// back with set and nothing done. To retry more, take + /// from the preview and retry them with + /// , that many at a time. + /// + public Task BulkRetryAsync(ExchangeSelection selection, bool reset = false, + string reason = "Bulk retry", CancellationToken cancellation = default) => + bitween.PostAsync("api/xchanges/bulkretry", BulkRetry(selection, reason, reset), cancellation); + + /// + /// How many exchanges would take, and how many of them are older than + /// storage keeps files, so would come without them. Changes nothing. + /// + public Task CheckExportAsync(ExchangeSelection selection, CancellationToken cancellation = default) => + bitween.PostAsync("api/xchanges/export/check", Export(selection), cancellation); + + /// + /// The selection's files as one zip, a folder for each exchange and missing.txt naming the + /// files storage no longer has, read as it downloads. At most 500 exchanges; more is refused with + /// TOO_MANY. Dispose the stream when done with it. + /// + public Task ExportAsync(ExchangeSelection selection, CancellationToken cancellation = default) => + bitween.PostForStreamAsync("api/xchanges/export", Export(selection), cancellation); + + static XchangeBulkRetry BulkRetry(ExchangeSelection selection, string reason, bool reset) => new() + { + Ids = selection.Ids!, + Filter = selection.Filter!, + ExcludeIds = selection.ExcludeIds!, + Reason = reason, + Reset = reset, + }; + + static XchangeFilesExport Export(ExchangeSelection selection) => new() + { + Ids = selection.Ids, + Filter = selection.Filter, + ExcludeIds = selection.ExcludeIds, + }; + + sealed record FileAnswer(string Data, string Key); +} + +/// A Bitween's automatic retries waiting to run: what the Scheduled retries page shows and does. +public sealed class ScheduledRetries +{ + readonly BitweenClient bitween; + + internal ScheduledRetries(BitweenClient bitween) => this.bitween = bitween; + + /// A page of the automatic retries waiting to run, soonest first, and how many there are. + public Task> SearchAsync(ScheduledRetryQuery? query = null, int page = 0, int size = 25, + CancellationToken cancellation = default) + { + var filters = (query ?? new ScheduledRetryQuery()).ToFilters(); + return bitween.GetAsync>( + $"api/delayedretries?{ExchangeQuery.QueryString(filters, ("page", page.ToString()), ("size", size.ToString()))}", cancellation); + } + + /// Runs the automatic retry booked for exchange now, rather than when it's due. + public Task RunNowAsync(string id, CancellationToken cancellation = default) => + bitween.PostAsync($"api/delayedretries/{Uri.EscapeDataString(id)}/runnow", new DelayedRetryRunNow(), cancellation); +} diff --git a/SW.Bitween.Client/SW.Bitween.Client.csproj b/SW.Bitween.Client/SW.Bitween.Client.csproj new file mode 100644 index 00000000..0dbe74cf --- /dev/null +++ b/SW.Bitween.Client/SW.Bitween.Client.csproj @@ -0,0 +1,35 @@ + + + + + net10.0 + enable + enable + SimplyWorks.Bitween.Client + SimplyWorks.Bitween.Client + Bitween's exchanges from your own program: search, read files, retry, bulk retry and export, as a service account. + Simplify9 + MIT + https://github.com/simplify9/Bitween + https://github.com/simplify9/Bitween + icon.png + SW.Bitween.Client + + true + $(NoWarn);CS1591 + + + + + + + + + True + + + + + diff --git a/SW.Bitween.HttpTests/BitweenClientTests.cs b/SW.Bitween.HttpTests/BitweenClientTests.cs new file mode 100644 index 00000000..cac28036 --- /dev/null +++ b/SW.Bitween.HttpTests/BitweenClientTests.cs @@ -0,0 +1,140 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http.Json; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Client; +using Xunit; + +namespace SW.Bitween.HttpTests; + +/// +/// SimplyWorks.Bitween.Client against a real Bitween: what clients build their own screens on. A change +/// to the exchange endpoints that breaks the client breaks these, so it is found here and not by a +/// client after they upgrade. +/// +[Collection("Http")] +public class BitweenClientTests(HttpFixture fixture) +{ + /// A gateway whose partner has made exchanges, each sent its own body. + async Task<(int PartnerId, string[] Bodies)> ExchangesAsync(int calls) + { + var (admin, urlName, _, partnerId, _) = await Api.GatewayAsync(fixture); + var key = await Api.NewKeyAsync(admin); + await Api.SetKeysAsync(admin, partnerId, ("primary", key)); + var bodies = Enumerable.Range(1, calls).Select(i => $"{{\"order\":\"{Guid.NewGuid():N}\",\"line\":{i}}}").ToArray(); + foreach (var body in bodies) + using (var answer = await Api.CallSyncAsync(fixture, urlName, key, body)) + Assert.Equal(HttpStatusCode.OK, answer.StatusCode); + return (partnerId, bodies); + } + + /// A client signed in as a new service account whose one role has . + async Task ClientAsync(params string[] permissions) + { + using var admin = await fixture.AdminAsync(); + var roleId = await Api.CreateAsync(admin, "/api/roles", new { name = Api.Unique("client role"), permissions }); + var accountId = await Api.CreateAsync(admin, "/api/serviceaccounts", new { name = Api.Unique("client"), roleIds = new[] { roleId } }); + var issued = await Api.Json(await admin.PostAsJsonAsync($"/api/serviceaccounts/{accountId}/issueToken", + new { name = "client", expiresInDays = 30 })); + return new BitweenClient(fixture.Client(), (string)issued["token"]!); + } + + [Fact] + public async Task Searches_reads_and_retries_exchanges_as_the_Exchanges_page_does() + { + var (partnerId, bodies) = await ExchangesAsync(2); + using var bitween = await ClientAsync("exchanges.view", "exchanges.operate"); + var partners = new ExchangeQuery { PartnerIds = [partnerId] }; + + var page = await bitween.Exchanges.SearchAsync(partners); + Assert.Equal(2, page.TotalCount); + Assert.Equal(2, page.Result.Count); + Assert.All(page.Result, x => Assert.Equal(partnerId, x.PartnerId)); + Assert.Single((await bitween.Exchanges.SearchAsync(partners, page: 1, size: 1)).Result); + Assert.Equal(0, (await bitween.Exchanges.SearchAsync(partners, count: false)).TotalCount); + + // Status and time are filters the server is particular about: each has to come through as it reads them. + Assert.Equal(2, (await bitween.Exchanges.SearchAsync(new ExchangeQuery { PartnerIds = [partnerId], Status = ExchangeStatus.Success })).TotalCount); + Assert.Equal(0, (await bitween.Exchanges.SearchAsync(new ExchangeQuery { PartnerIds = [partnerId], Status = ExchangeStatus.Failed })).TotalCount); + var now = DateTimeOffset.UtcNow; + Assert.Equal(2, (await bitween.Exchanges.SearchAsync(new ExchangeQuery { PartnerIds = [partnerId], From = now.AddHours(-1), To = now.AddHours(1) })).TotalCount); + Assert.Equal(0, (await bitween.Exchanges.SearchAsync(new ExchangeQuery { PartnerIds = [partnerId], From = now.AddHours(1) })).TotalCount); + + var inputs = await Task.WhenAll(page.Result.Select(x => bitween.Exchanges.ReadFileAsync(x.InputKey))); + Assert.Equal(bodies.Order(), inputs.Order()); + var first = page.Result[Array.IndexOf(inputs, bodies[0])]; + Assert.Equal(first.Id, (await bitween.Exchanges.GetAsync(first.Id))!.Id); + Assert.Null(await bitween.Exchanges.GetAsync(Guid.NewGuid().ToString("N"))); + + var retryId = await bitween.Exchanges.RetryAsync(first.Id); + Assert.NotNull(retryId); + Assert.NotEqual(first.Id, retryId); + // The id search finds the retry too; asked for one exchange, it is that one. + Assert.Equal(first.Id, (await bitween.Exchanges.GetAsync(first.Id))!.Id); + Assert.Equal(first.Id, (await bitween.Exchanges.GetAsync(retryId!))!.RetryFor); + + var tree = await bitween.Exchanges.GetRetryTreeAsync(retryId!); + Assert.Equal(first.Id, tree.RootId); + Assert.Equal([first.Id, retryId], tree.Nodes.Select(n => n.Id).ToArray()); + + var again = await Assert.ThrowsAsync(() => bitween.Exchanges.RetryAsync(first.Id)); + Assert.Equal(HttpStatusCode.BadRequest, again.Status); + Assert.Equal("ALREADY_RETRIED", again.Code); + } + + [Fact] + public async Task Bulk_retries_and_exports_a_selection() + { + var (partnerId, _) = await ExchangesAsync(3); + using var bitween = await ClientAsync("exchanges.view", "exchanges.operate"); + var partners = new ExchangeQuery { PartnerIds = [partnerId] }; + var ids = (await bitween.Exchanges.SearchAsync(partners)).Result.Select(x => x.Id).ToList(); + // A bulk retry leaves out what succeeded, so these failed. + await fixture.InDbAsync(db => db.Set().Where(r => ids.Contains(r.Id)) + .ExecuteUpdateAsync(s => s.SetProperty(r => r.Success, false).SetProperty(r => r.Exception, "Refused by the client test"))); + var failed = new ExchangeQuery { PartnerIds = [partnerId], Status = ExchangeStatus.Failed, ErrorContains = "client TEST" }; + Assert.Equal(3, (await bitween.Exchanges.SearchAsync(failed)).TotalCount); + + var preview = await bitween.Exchanges.PreviewBulkRetryAsync(ExchangeSelection.Matching(failed, except: [ids[0]])); + Assert.Equal(2, preview.Selected); + Assert.Equal(2, preview.WillRetry); + Assert.Equal(3, (await bitween.Exchanges.SearchAsync(partners)).TotalCount); + + var plan = await bitween.Exchanges.BulkRetryAsync(ExchangeSelection.Of(ids.Take(2))); + Assert.Equal(2, plan.WillRetry); + Assert.Equal(5, (await bitween.Exchanges.SearchAsync(partners)).TotalCount); + Assert.Equal(3, (await bitween.Exchanges.SearchAsync(new ExchangeQuery { PartnerIds = [partnerId], LatestOnly = true })).TotalCount); + + var check = await bitween.Exchanges.CheckExportAsync(ExchangeSelection.Of(ids)); + Assert.Equal(3, check.Count); + Assert.Equal(0, check.WithoutFiles); + + await using var zip = await bitween.Exchanges.ExportAsync(ExchangeSelection.Of(ids)); + using var copy = new MemoryStream(); + await zip.CopyToAsync(copy); + using var archive = new ZipArchive(copy, ZipArchiveMode.Read); + Assert.All(ids, id => Assert.Contains(archive.Entries, e => e.FullName.Contains(id))); + } + + [Fact] + public async Task Says_what_Bitween_refused_and_why() + { + var (partnerId, _) = await ExchangesAsync(1); + using var viewer = await ClientAsync("exchanges.view"); + var id = (await viewer.Exchanges.SearchAsync(new ExchangeQuery { PartnerIds = [partnerId] })).Result.Single().Id; + + var forbidden = await Assert.ThrowsAsync(() => viewer.Exchanges.RetryAsync(id)); + Assert.Equal(HttpStatusCode.Forbidden, forbidden.Status); + Assert.Equal(HttpStatusCode.Forbidden, (await Assert.ThrowsAsync(() => viewer.ScheduledRetries.RunNowAsync(id))).Status); + Assert.NotNull((await viewer.ScheduledRetries.SearchAsync(new ScheduledRetryQuery { From = DateTimeOffset.UtcNow })).Result); + + using var stranger = new BitweenClient(fixture.Client(), "bwt_" + new string('x', 40)); + var refused = await Assert.ThrowsAsync(() => stranger.Exchanges.SearchAsync()); + Assert.Equal(HttpStatusCode.BadRequest, refused.Status); + Assert.StartsWith("This token isn't valid", refused.Message); + } +} diff --git a/SW.Bitween.HttpTests/SW.Bitween.HttpTests.csproj b/SW.Bitween.HttpTests/SW.Bitween.HttpTests.csproj index 085b81e5..a720bed3 100644 --- a/SW.Bitween.HttpTests/SW.Bitween.HttpTests.csproj +++ b/SW.Bitween.HttpTests/SW.Bitween.HttpTests.csproj @@ -25,6 +25,7 @@ +