diff --git a/build/containers-scenarios.yml b/build/containers-scenarios.yml index 7239ba2cb..cedd393d4 100644 --- a/build/containers-scenarios.yml +++ b/build/containers-scenarios.yml @@ -110,12 +110,22 @@ parameters: arguments: --scenario fortunes_vertx --property scenario=FortunesVertx condition: Math.round(Date.now() / 43200000) % 10 == 9 # once every 10 half-days - - displayName: Json NTex - arguments: --scenario json_ntex --property scenario=JsonNTex - condition: Math.round(Date.now() / 43200000) % 10 == 8 # once every 10 half-days - - displayName: Fortunes NTex - arguments: --scenario fortunes_ntex --property scenario=FortunesNTex - condition: Math.round(Date.now() / 43200000) % 10 == 9 # once every 10 half-days + # Json NTex and Fortunes NTex are disabled: their docker builds + # (ntex-plt.dockerfile and ntex-db.dockerfile in TechEmpower/ + # FrameworkBenchmarks) all fail because the ntex workspace has + # `tokio-postgres` as a top-level dep on the fafhrd91/postgres + # ntex-3 fork (commit fbc7a17), which is no longer compatible + # with newer ntex-bytes resolved from TechEmpower's caret-versioned + # `ntex-bytes = "1.5"`. TechEmpower/FrameworkBenchmarks is archived + # (https://github.com/TechEmpower/FrameworkBenchmarks), so no fix + # is coming upstream. See build/frameworks-scenarios.yml and + # build/frameworks-database-scenarios.yml for the matching disables. + # - displayName: Json NTex + # arguments: --scenario json_ntex --property scenario=JsonNTex + # condition: Math.round(Date.now() / 43200000) % 10 == 8 # once every 10 half-days + # - displayName: Fortunes NTex + # arguments: --scenario fortunes_ntex --property scenario=FortunesNTex + # condition: Math.round(Date.now() / 43200000) % 10 == 9 # once every 10 half-days steps: - ${{ each scenario in parameters.scenarios }}: diff --git a/build/frameworks-database-scenarios.yml b/build/frameworks-database-scenarios.yml index eecc4a4ef..0443fa6b2 100644 --- a/build/frameworks-database-scenarios.yml +++ b/build/frameworks-database-scenarios.yml @@ -52,9 +52,22 @@ parameters: arguments: --scenario json_gin --load.variables.connections 512 --property scenario=JsonGin # Ntex (Rust) - - - displayName: Ntex Fortunes - arguments: --scenario fortunes_ntex --load.variables.connections 512 --property scenario=FortunesNtex + # + # Ntex Fortunes is disabled: the docker build for ntex-db.dockerfile + # is broken because the TechEmpower ntex workspace has `tokio-postgres` + # as a top-level dependency pointing at the fafhrd91/postgres `ntex-3` + # fork (frozen at commit fbc7a17). That fork no longer compiles against + # the latest ntex-bytes (1.6+) that cargo resolves from TechEmpower's + # caret-versioned `ntex-bytes = "1.5"`, so every binary in the + # workspace - including ntex-db - fails with E0308 mismatched-types + # errors on `BytesMut` vs `BytePages`. + # TechEmpower/FrameworkBenchmarks is now archived + # (https://github.com/TechEmpower/FrameworkBenchmarks), so a real fix + # upstream is unlikely and not worth chasing. See + # build/frameworks-scenarios.yml for the matching Ntex Plaintext and + # Ntex Json disables. + # - displayName: Ntex Fortunes + # arguments: --scenario fortunes_ntex --load.variables.connections 512 --property scenario=FortunesNtex # Vertx (Java) diff --git a/build/frameworks-scenarios.yml b/build/frameworks-scenarios.yml index d77970000..24cf73263 100644 --- a/build/frameworks-scenarios.yml +++ b/build/frameworks-scenarios.yml @@ -41,11 +41,24 @@ parameters: # Scenarios are in frameworks-database-scenarios.yml since they require the db server # Ntex (Rust) - - - displayName: Ntex Plaintext - arguments: --scenario plaintext_ntex --load.variables.connections 1024 --property scenario=PlaintextNtex - - displayName: Ntex Json - arguments: --scenario json_ntex --load.variables.connections 512 --property scenario=JsonNtex +# +# Ntex Plaintext and Ntex Json are disabled: the docker build for +# ntex-plt.dockerfile is broken because the TechEmpower ntex +# workspace has `tokio-postgres` as a top-level dependency pointing +# at the fafhrd91/postgres `ntex-3` fork (frozen at commit fbc7a17). +# That fork no longer compiles against the latest ntex-bytes (1.6+) +# that cargo resolves from TechEmpower's caret-versioned +# `ntex-bytes = "1.5"`, so every binary in the workspace - including +# ntex-plt - fails with E0308 mismatched-types errors on `BytesMut` +# vs `BytePages`. TechEmpower/FrameworkBenchmarks is now archived +# (https://github.com/TechEmpower/FrameworkBenchmarks), so a real +# fix upstream is unlikely. See build/frameworks-database-scenarios.yml +# for the matching Ntex Fortunes disable. + + # - displayName: Ntex Plaintext + # arguments: --scenario plaintext_ntex --load.variables.connections 1024 --property scenario=PlaintextNtex + # - displayName: Ntex Json + # arguments: --scenario json_ntex --load.variables.connections 512 --property scenario=JsonNtex # Vertx (Java) diff --git a/build/sslstream-scenarios.yml b/build/sslstream-scenarios.yml index d572af348..28a623b15 100644 --- a/build/sslstream-scenarios.yml +++ b/build/sslstream-scenarios.yml @@ -124,7 +124,7 @@ steps: useDataContractSerializer: "false" messageBody: | { - "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 ) && false", + "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 )", "name": "crank", "args": [ "${{ parameters.tfm }} --client.options.collectCounters true --command-line-property --table SystemNet_TlsBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --description \"${{ t.displayName }} ${{ version.displayName }} ${{ s.displayName }} ${{ c.displayName }} ${{ res.displayName }} $(System.JobDisplayName)\" ${{ parameters.arguments }} --no-metadata --no-measurements ${{ t.arguments }} ${{ s.arguments }} ${{ version.arguments }} ${{ res.arguments }} ${{ c.arguments }}" ] } @@ -145,7 +145,7 @@ steps: useDataContractSerializer: "false" messageBody: | { - "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 ) && false", + "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 )", "name": "crank", "args": [ "${{ parameters.tfm }} --client.options.collectCounters true --command-line-property --table SystemNet_TlsBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --description \"${{ t.displayName }} ${{ s.displayName }} ${{ c.displayName }} $(System.JobDisplayName)\" ${{ parameters.arguments }} --no-metadata --no-measurements ${{ t.arguments }} ${{ s.arguments }} ${{ c.arguments }}" ] } @@ -169,7 +169,7 @@ steps: useDataContractSerializer: "false" messageBody: | { - "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 ) && false", + "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 )", "name": "crank", "args": [ "${{ parameters.tfm }} --client.options.collectCounters true --command-line-property --table SystemNet_TlsBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --description \"${{ t.displayName }} ${{ version.displayName }} ${{ s.displayName }} ${{ sendbuf.displayName }} ${{ recvbuf.displayName }} ${{ c.displayName }} $(System.JobDisplayName)\" ${{ parameters.arguments }} --no-metadata --no-measurements ${{ t.arguments }} ${{ s.arguments }} ${{ version.arguments }} ${{ sendbuf.arguments }} ${{ recvbuf.arguments }} ${{ c.arguments }}" ] } @@ -192,7 +192,7 @@ steps: useDataContractSerializer: "false" messageBody: | { - "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 ) && false", + "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 )", "name": "crank", "args": [ "${{ parameters.tfm }} --client.options.collectCounters true --command-line-property --table SystemNet_TlsBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --description \"${{ t.displayName }} ${{ s.displayName }} ${{ sendbuf.displayName }} ${{ recvbuf.displayName }} ${{ c.displayName }} $(System.JobDisplayName)\" ${{ parameters.arguments }} --no-metadata --no-measurements ${{ t.arguments }} ${{ s.arguments }} ${{ sendbuf.arguments }} ${{ recvbuf.arguments }} ${{ c.arguments }}" ] } diff --git a/src/System/Net/Benchmarks/Common/Server/BenchmarkServer.cs b/src/System/Net/Benchmarks/Common/Server/BenchmarkServer.cs index b683515e6..62447d1b9 100644 --- a/src/System/Net/Benchmarks/Common/Server/BenchmarkServer.cs +++ b/src/System/Net/Benchmarks/Common/Server/BenchmarkServer.cs @@ -1,12 +1,20 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.Collections.Concurrent; + namespace System.Net.Benchmarks; internal abstract class BenchmarkServer : BenchmarkApp where TListener : IListener where TOptions : IBenchmarkServerOptions, new() { + // Maximum time to wait for in-flight accepted-connection tasks to drain after + // cancellation before letting the listener dispose. Keep this comfortably under + // BenchmarkApp.s_shutdownDeadline so the outer deadline still has room to fire + // if disposal itself stalls. + private static readonly TimeSpan s_drainTimeout = TimeSpan.FromSeconds(10); + private static int s_errors; protected abstract Task ListenAsync(TOptions options, CancellationToken ct); @@ -19,17 +27,29 @@ protected override async Task RunBenchmarkAsync(TOptions options, CancellationTo await using var listener = await ListenAsync(options, ct); Log(GetReadyStateText(listener)); + // Track per-connection tasks so we can drain them on cancellation before + // the listener disposes. Without this, `await using var listener` can race + // against in-flight TLS/QUIC connections — SslStream.DisposeAsync attempts + // to send a close_notify alert, and if the underlying socket is being torn + // down concurrently, the dispose can stall long enough to strand the port + // for the next benchmark run. + var inflight = new ConcurrentDictionary(); try { while (!ct.IsCancellationRequested) { var accepted = await listener.AcceptAsync(ct).ConfigureAwait(false); - _ = Task.Run(() => ProcessAcceptedNoThrowAsync(accepted, options, ct), ct); + var task = Task.Run(() => ProcessAcceptedNoThrowAsync(accepted, options, ct), ct); + inflight[task] = 0; + _ = task.ContinueWith(static (t, state) => ((ConcurrentDictionary)state!).TryRemove(t, out _), + inflight, TaskScheduler.Default); } } catch (OperationCanceledException e) when (e.CancellationToken == ct) { } finally { + await DrainAsync(inflight).ConfigureAwait(false); + if (s_errors > 0) { LogMetric(MetricName.Errors, "Errors", s_errors, "n0"); @@ -38,6 +58,21 @@ protected override async Task RunBenchmarkAsync(TOptions options, CancellationTo Log("Exiting..."); } + private static async Task DrainAsync(ConcurrentDictionary inflight) + { + if (inflight.IsEmpty) + { + return; + } + + var pending = Task.WhenAll(inflight.Keys); + var completed = await Task.WhenAny(pending, Task.Delay(s_drainTimeout)).ConfigureAwait(false); + if (completed != pending) + { + Log($"Drain timeout ({s_drainTimeout.TotalSeconds:n0}s) reached with {inflight.Count} in-flight connection task(s) still running; continuing shutdown."); + } + } + private async Task ProcessAcceptedNoThrowAsync(TAcceptResult accepted, TOptions options, CancellationToken ct) { try diff --git a/src/System/Net/Benchmarks/Common/Shared/BenchmarkApp.cs b/src/System/Net/Benchmarks/Common/Shared/BenchmarkApp.cs index 2552c1ccc..51af919df 100644 --- a/src/System/Net/Benchmarks/Common/Shared/BenchmarkApp.cs +++ b/src/System/Net/Benchmarks/Common/Shared/BenchmarkApp.cs @@ -3,11 +3,17 @@ using System.CommandLine; using System.CommandLine.Parsing; +using System.Runtime.InteropServices; namespace System.Net.Benchmarks; public abstract class BenchmarkApp where TOptions : new() { + // Maximum time to wait for graceful shutdown after cancellation is signaled + // before forcing process exit. Crank sends SIGTERM, waits 5s, then SIGINT, + // so we keep our deadline well under any subsequent SIGKILL. + private static readonly TimeSpan s_shutdownDeadline = TimeSpan.FromSeconds(15); + private static bool s_appStarted; protected static CancellationTokenSource GlobalCts { get; } = new(); @@ -66,6 +72,26 @@ private async Task RunAsyncInternal(TOptions options) GlobalCts.Cancel(); }; + // Crank's Linux agent stops jobs with SIGTERM first (5 s grace) then SIGINT. + // Without an explicit SIGTERM handler, the .NET runtime's default behavior + // terminates the process after a short ProcessExit window and any in-flight + // TLS/QUIC connections never observe cancellation. That can leave sockets + // in a half-broken state, stranding the listen port for the next benchmark. + // Register SIGTERM/SIGQUIT here so GlobalCts.Cancel runs the same shutdown + // path on Linux as Ctrl+C does on Windows. + using var sigTerm = PosixSignalRegistration.Create(PosixSignal.SIGTERM, static ctx => + { + Log("SIGTERM received..."); + ctx.Cancel = true; + GlobalCts.Cancel(); + }); + using var sigQuit = PosixSignalRegistration.Create(PosixSignal.SIGQUIT, static ctx => + { + Log("SIGQUIT received..."); + ctx.Cancel = true; + GlobalCts.Cancel(); + }); + // NOTE: // It is better for metrics to be registered with some delay to ensure the metadata is collected. // Event listener is started (shortly) after the benchmark app starts, so it might miss the registration event. @@ -76,9 +102,15 @@ private async Task RunAsyncInternal(TOptions options) LogHelper.LogMetric("env/processorcount", Environment.ProcessorCount); }); + // Run the benchmark, but enforce a hard shutdown deadline once cancellation + // has been requested. If RunBenchmarkAsync hangs in disposal (for example a + // stuck SslStream close_notify or QuicConnection drain), Environment.Exit + // guarantees we release the port for the next run instead of waiting for + // crank's SIGKILL fallback. try { - await RunBenchmarkAsync(options, GlobalCts.Token); + var benchmarkTask = RunBenchmarkAsync(options, GlobalCts.Token); + await WaitWithShutdownDeadlineAsync(benchmarkTask).ConfigureAwait(false); } catch (Exception e) { @@ -87,4 +119,39 @@ private async Task RunAsyncInternal(TOptions options) } return 0; // on success, returning 0 explicitly; otherwise RootCommand (?) will not report unhandled exceptions to crank as failures } + + private static async Task WaitWithShutdownDeadlineAsync(Task benchmarkTask) + { + // Fast path: benchmark finished (or threw) before cancellation was ever requested. + await Task.WhenAny(benchmarkTask, WaitForCancellationAsync(GlobalCts.Token)).ConfigureAwait(false); + + if (benchmarkTask.IsCompleted) + { + await benchmarkTask.ConfigureAwait(false); + return; + } + + // Cancellation has been signaled. Give the benchmark a bounded time to wind + // down gracefully, then force-exit so we don't strand the port between runs. + var completed = await Task.WhenAny(benchmarkTask, Task.Delay(s_shutdownDeadline)).ConfigureAwait(false); + if (completed == benchmarkTask) + { + await benchmarkTask.ConfigureAwait(false); + return; + } + + Log($"Shutdown deadline ({s_shutdownDeadline.TotalSeconds:n0}s) exceeded after cancellation; forcing process exit."); + Environment.Exit(0); + } + + private static Task WaitForCancellationAsync(CancellationToken ct) + { + if (ct.IsCancellationRequested) + { + return Task.CompletedTask; + } + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + ct.Register(static state => ((TaskCompletionSource)state!).TrySetResult(), tcs); + return tcs.Task; + } }