* Deflate: encoder ~1.4-2x faster at equal or better ratio, decoder faster than zlib-ng

Compression.Benchmarks/DeflateBenchmarks.cs

@@ -0,0 +1,79 @@
+#pragma warning disable CS1591
+
+using BenchmarkDotNet.Attributes;
+using Compression.Core.Deflate;
+using Bcl = System.IO.Compression;
+
+namespace Compression.Benchmarks;
+
+/// <summary>
+/// The DEFLATE building block against the platform's zlib (zlib-ng in .NET 9+) at the same
+/// zlib level, on every <see cref="DeflateCorpus"/> kind. The platform codec appears here only
+/// as the yardstick; product code never uses it.
+/// </summary>
+/// <remarks>
+/// Both decoders inflate the same bitstream — ours, at the level under test — so the decode
+/// figures compare decoders rather than encoders. Each corpus is <see cref="DeflateCorpus.Size"/>
+/// bytes: MB/s is that size over the mean.
+/// </remarks>
+[Config(typeof(InProcessConfig))]
+public class DeflateBenchmarks {
+
+  private byte[] _data = null!;
+  private byte[] _compressed = null!;
+  private readonly MemoryStream _sink = new();
+
+  [ParamsSource(nameof(Corpora))]
+  public string Corpus { get; set; } = null!;
+
+  [Params(1, 6, 9)]
+  public int Level { get; set; }
+
+  [Params("Ours", "Platform")]
+  public string Codec { get; set; } = null!;
+
+  public static IEnumerable<string> Corpora => DeflateCorpus.Kinds;
+
+  [GlobalSetup]
+  public void Setup() {
+    this._data = DeflateCorpus.Get(this.Corpus);
+    this._compressed = DeflateCompressor.Compress(this._data, (DeflateCompressionLevel)this.Level);
+  }
+
+  [Benchmark]
+  public long Compress() {
+    this._sink.Position = 0;
+    this._sink.SetLength(0);
+    if (this.Codec == "Ours") {
+      var compressor = new DeflateCompressor(this._sink, (DeflateCompressionLevel)this.Level);
+      compressor.Write(this._data);
+      compressor.Finish();
+    } else
+      using (var deflate = new Bcl.DeflateStream(this._sink, new Bcl.ZLibCompressionOptions { CompressionLevel = this.Level }, leaveOpen: true))
+        deflate.Write(this._data);
+
+    return this._sink.Length;
+  }
+
+  [Benchmark]
+  public int Decompress() {
+    // Both stream into the same 64 KB buffer, so neither pays for growing a whole-output array.
+    var buffer = this._buffer;
+    using var input = new MemoryStream(this._compressed);
+    var total = 0;
+    int n;
+    if (this.Codec == "Ours") {
+      var inflate = new DeflateDecompressor(input);
+      while ((n = inflate.Decompress(buffer)) > 0)
+        total += n;
+    } else {
+      using var inflate = new Bcl.DeflateStream(input, Bcl.CompressionMode.Decompress);
+      while ((n = inflate.Read(buffer)) > 0)
+        total += n;
+    }
+
+    return total;
+  }
+
+  private readonly byte[] _buffer = new byte[1 << 16];
+}

Compression.Benchmarks/DeflateCorpus.cs

@@ -0,0 +1,151 @@
+#pragma warning disable CS1591
+
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace Compression.Benchmarks;
+
+/// <summary>
+/// The inputs the DEFLATE benchmarks run on, one per kind of data a general-purpose codec
+/// meets. Every kind is built from files every machine running the benchmark has — this
+/// repository's own sources and the installed .NET runtime — so no corpus is checked in.
+/// </summary>
+/// <remarks>
+/// The bytes therefore differ between checkouts and runtime versions: compare figures taken
+/// in the same run (ours against the platform's zlib on the same input), not across machines.
+/// </remarks>
+public static class DeflateCorpus {
+
+  /// <summary>Size of every corpus in bytes.</summary>
+  public const int Size = 8 << 20;
+
+  /// <summary>The corpus kinds.</summary>
+  public static readonly string[] Kinds = ["Code", "Prose", "Binary", "Compressed", "Repetitive"];
+
+  private static readonly Dictionary<string, byte[]> Cache = new(StringComparer.Ordinal);
+
+  /// <summary>Gets the corpus named <paramref name="kind"/>, built on first use.</summary>
+  /// <remarks>
+  /// With the environment variable <c>DEFLATE_CORPUS_DIR</c> set, each corpus is read from that
+  /// directory and only built (and saved there) when missing, so a before/after comparison
+  /// across two checkouts runs on the same bytes even though the sources differ.
+  /// </remarks>
+  public static byte[] Get(string kind) {
+    lock (Cache) {
+      if (Cache.TryGetValue(kind, out var data))
+        return data;
+
+      var saved = Environment.GetEnvironmentVariable("DEFLATE_CORPUS_DIR") is { Length: > 0 } directory
+        ? Path.Combine(directory, kind + ".bin")
+        : null;
+      if (saved != null && File.Exists(saved))
+        return Cache[kind] = File.ReadAllBytes(saved);
+
+      Cache[kind] = data = kind switch {
+        "Code" => SourceCode(),
+        "Prose" => Prose(),
+        "Binary" => Binary(),
+        "Compressed" => Compressed(),
+        "Repetitive" => Repetitive(),
+        _ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null),
+      };
+      if (saved != null) {
+        Directory.CreateDirectory(Path.GetDirectoryName(saved)!);
+        File.WriteAllBytes(saved, data);
+      }
+
+      return data;
+    }
+  }
+
+  /// <summary>C# sources of this repository, in ordinal path order.</summary>
+  private static byte[] SourceCode() {
+    var root = RepositoryRoot();
+    string[] skipped = ["obj", "bin", ".claude"];
+    var files = Directory.EnumerateFiles(root, "*.cs", SearchOption.AllDirectories)
+      .Where(f => !Path.GetRelativePath(root, f).Split(Path.DirectorySeparatorChar).Any(skipped.Contains));
+    return Concatenate(files);
+  }
+
+  /// <summary>English API documentation: the XML doc files of the .NET reference pack, falling back to this repository's Markdown.</summary>
+  private static byte[] Prose() {
+    var dotnetRoot = Path.GetFullPath(Path.Combine(RuntimeEnvironment.GetRuntimeDirectory(), "..", "..", ".."));
+    var version = Path.GetFileName(RuntimeEnvironment.GetRuntimeDirectory().TrimEnd(Path.DirectorySeparatorChar));
+    var refDir = Path.Combine(dotnetRoot, "packs", "Microsoft.NETCore.App.Ref", version, "ref", $"net{Environment.Version.Major}.0");
+    if (Directory.Exists(refDir))
+      return Concatenate(Directory.EnumerateFiles(refDir, "*.xml"));
+
+    return Concatenate(Directory.EnumerateFiles(RepositoryRoot(), "*.md", SearchOption.AllDirectories));
+  }
+
+  /// <summary>Executable code and metadata: the managed assemblies of the running runtime.</summary>
+  private static byte[] Binary() => Concatenate(Directory.EnumerateFiles(RuntimeEnvironment.GetRuntimeDirectory(), "*.dll"));
+
+  /// <summary>Data that is already compressed: the source corpus run through DEFLATE once.</summary>
+  private static byte[] Compressed() {
+    var once = Core.Deflate.DeflateCompressor.Compress(Get("Code"), Core.Deflate.DeflateCompressionLevel.Best);
+    var data = new byte[Size];
+    for (var offset = 0; offset < Size; offset += once.Length)
+      once.AsSpan(0, Math.Min(once.Length, Size - offset)).CopyTo(data.AsSpan(offset));
+    return data;
+  }
+
+  /// <summary>Long runs and short repeating records, as in sparse images and logs.</summary>
+  private static byte[] Repetitive() {
+    var data = new byte[Size];
+    var record = Encoding.ASCII.GetBytes("2026-10-09T12:00:00Z INFO request served status=200 bytes=");
+    var rng = new Random(1);
+    var position = 0;
+    while (position < Size) {
+      if (rng.Next(4) == 0) {
+        // A run of one value, as in zero-filled sectors.
+        var run = Math.Min(rng.Next(1, 16384), Size - position);
+        data.AsSpan(position, run).Fill((byte)(rng.Next(2) == 0 ? 0 : 0xFF));
+        position += run;
+        continue;
+      }
+
+      var line = Encoding.ASCII.GetBytes($"{Encoding.ASCII.GetString(record)}{rng.Next(1000, 1100)}\n");
+      var n = Math.Min(line.Length, Size - position);
+      line.AsSpan(0, n).CopyTo(data.AsSpan(position));
+      position += n;
+    }
+
+    return data;
+  }
+
+  private static byte[] Concatenate(IEnumerable<string> files) {
+    var data = new byte[Size];
+    var position = 0;
+    var sorted = files.Order(StringComparer.Ordinal).ToArray();
+    if (sorted.Length == 0)
+      throw new InvalidOperationException("No files found to build the corpus from.");
+
+    // Wrap around when the files are smaller than the corpus; the repeat is far beyond the window.
+    while (position < Size) {
+      var before = position;
+      foreach (var file in sorted) {
+        var bytes = File.ReadAllBytes(file);
+        var n = Math.Min(bytes.Length, Size - position);
+        bytes.AsSpan(0, n).CopyTo(data.AsSpan(position));
+        position += n;
+        if (position == Size)
+          break;
+      }
+
+      if (position == before)
+        throw new InvalidOperationException("The files to build the corpus from are all empty.");
+    }
+
+    return data;
+  }
+
+  private static string RepositoryRoot() {
+    foreach (var start in new[] { AppContext.BaseDirectory, Environment.CurrentDirectory })
+      for (var directory = new DirectoryInfo(start); directory != null; directory = directory.Parent)
+        if (File.Exists(Path.Combine(directory.FullName, "CompressionWorkbench.slnx")))
+          return directory.FullName;
+
+    throw new InvalidOperationException("Run the benchmarks from inside the repository checkout.");
+  }
+}

Compression.Benchmarks/DeflateThroughputTable.cs

@@ -0,0 +1,82 @@
+#pragma warning disable CS1591
+
+using System.Diagnostics;
+using Compression.Core.Deflate;
+using Bcl = System.IO.Compression;
+
+namespace Compression.Benchmarks;
+
+/// <summary>
+/// A quick ratio and throughput table for the DEFLATE building block against the platform's
+/// zlib, for the edit-measure loop where a full BenchmarkDotNet run takes too long. Each cell
+/// is the best of several timed runs after a warm-up; <see cref="DeflateBenchmarks"/> is the
+/// statistically careful version of the same measurement.
+/// </summary>
+/// <remarks>Run with <c>dotnet run -c Release -- deflate-table [runs] [corpus...]</c>.</remarks>
+public static class DeflateThroughputTable {
+
+  public static void Run(string[] args) {
+    var runs = args.Length > 0 && int.TryParse(args[0], out var r) ? r : 5;
+    var kinds = args.Length > 1 ? args[1..] : DeflateCorpus.Kinds;
+    var buffer = new byte[1 << 16];
+    var sink = new MemoryStream();
+
+    Console.WriteLine("| Corpus | Level | Ratio ours | Ratio zlib | Compress ours MB/s | Compress zlib MB/s | Inflate ours MB/s | Inflate zlib MB/s |");
+    Console.WriteLine("|---|---:|---:|---:|---:|---:|---:|---:|");
+    foreach (var kind in kinds) {
+      var data = DeflateCorpus.Get(kind);
+      foreach (var level in new[] { 1, 6, 9 }) {
+        var ours = DeflateCompressor.Compress(data, (DeflateCompressionLevel)level);
+        if (!DeflateDecompressor.Decompress(ours).AsSpan().SequenceEqual(data))
+          throw new InvalidOperationException($"{kind} level {level} does not round-trip.");
+
+        var platform = PlatformCompress(data, level, sink);
+
+        var compressOurs = Best(runs, () => {
+          sink.Position = 0;
+          sink.SetLength(0);
+          var compressor = new DeflateCompressor(sink, (DeflateCompressionLevel)level);
+          compressor.Write(data);
+          compressor.Finish();
+        });
+        var compressPlatform = Best(runs, () => PlatformCompress(data, level, sink));
+        var inflateOurs = Best(runs, () => {
+          var inflate = new DeflateDecompressor(new MemoryStream(ours));
+          while (inflate.Decompress(buffer) > 0) { }
+        });
+        var inflatePlatform = Best(runs, () => {
+          using var inflate = new Bcl.DeflateStream(new MemoryStream(ours), Bcl.CompressionMode.Decompress);
+          while (inflate.Read(buffer) > 0) { }
+        });
+
+        Console.WriteLine($"| {kind} | {level} | {Percent(ours.Length, data.Length)} | {Percent(platform, data.Length)} | "
+                          + $"{Speed(data.Length, compressOurs)} | {Speed(data.Length, compressPlatform)} | "
+                          + $"{Speed(data.Length, inflateOurs)} | {Speed(data.Length, inflatePlatform)} |");
+      }
+    }
+  }
+
+  private static long PlatformCompress(byte[] data, int level, MemoryStream sink) {
+    sink.Position = 0;
+    sink.SetLength(0);
+    using (var deflate = new Bcl.DeflateStream(sink, new Bcl.ZLibCompressionOptions { CompressionLevel = level }, leaveOpen: true))
+      deflate.Write(data);
+    return sink.Length;
+  }
+
+  private static double Best(int runs, Action action) {
+    action();
+    var best = double.MaxValue;
+    for (var i = 0; i < runs; ++i) {
+      var watch = Stopwatch.StartNew();
+      action();
+      best = Math.Min(best, watch.Elapsed.TotalSeconds);
+    }
+
+    return best;
+  }
+
+  private static string Percent(long part, long whole) => FormattableString.Invariant($"{100.0 * part / whole:F2}%");
+
+  private static string Speed(long bytes, double seconds) => FormattableString.Invariant($"{bytes / seconds / (1 << 20):F0}");
+}

Compression.Benchmarks/Program.cs

@@ -1,5 +1,11 @@
 #pragma warning disable CS1591
 
 using BenchmarkDotNet.Running;
+using Compression.Benchmarks;
+
+if (args is ["deflate-table", .. var rest]) {
+  DeflateThroughputTable.Run(rest);
+  return;
+}
 
 BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);

Compression.Core/Deflate/DeflateCompressor.cs

@@ -63,7 +63,9 @@ public DeflateCompressor(Stream output, DeflateCompressionLevel level = DeflateC
   /// <param name="level">The compression level.</param>
   /// <returns>The DEFLATE compressed data.</returns>
   public static byte[] Compress(ReadOnlySpan<byte> data, DeflateCompressionLevel level = DeflateCompressionLevel.Default) {
-    using var ms = new MemoryStream();
+    // Half the input holds most compressible data in one go; a MemoryStream growing from empty
+    // reallocates and copies the output at every doubling before the final copy out.
+    using var ms = new MemoryStream((int)Math.Min(data.Length / 2L + 64, Array.MaxLength));
     var compressor = new DeflateCompressor(ms, level);
     compressor.Write(data);
     compressor.Finish();