diff --git a/Compression.Benchmarks/DeflateBenchmarks.cs b/Compression.Benchmarks/DeflateBenchmarks.cs new file mode 100644 index 000000000..4d3f932cd --- /dev/null +++ b/Compression.Benchmarks/DeflateBenchmarks.cs @@ -0,0 +1,79 @@ +#pragma warning disable CS1591 + +using BenchmarkDotNet.Attributes; +using Compression.Core.Deflate; +using Bcl = System.IO.Compression; + +namespace Compression.Benchmarks; + +/// +/// The DEFLATE building block against the platform's zlib (zlib-ng in .NET 9+) at the same +/// zlib level, on every kind. The platform codec appears here only +/// as the yardstick; product code never uses it. +/// +/// +/// Both decoders inflate the same bitstream — ours, at the level under test — so the decode +/// figures compare decoders rather than encoders. Each corpus is +/// bytes: MB/s is that size over the mean. +/// +[Config(typeof(InProcessConfig))] +public class DeflateBenchmarks { + + private byte[] _data = null!; + private byte[] _compressed = null!; + private readonly MemoryStream _sink = new(); + + [ParamsSource(nameof(Corpora))] + public string Corpus { get; set; } = null!; + + [Params(1, 6, 9)] + public int Level { get; set; } + + [Params("Ours", "Platform")] + public string Codec { get; set; } = null!; + + public static IEnumerable Corpora => DeflateCorpus.Kinds; + + [GlobalSetup] + public void Setup() { + this._data = DeflateCorpus.Get(this.Corpus); + this._compressed = DeflateCompressor.Compress(this._data, (DeflateCompressionLevel)this.Level); + } + + [Benchmark] + public long Compress() { + this._sink.Position = 0; + this._sink.SetLength(0); + if (this.Codec == "Ours") { + var compressor = new DeflateCompressor(this._sink, (DeflateCompressionLevel)this.Level); + compressor.Write(this._data); + compressor.Finish(); + } else + using (var deflate = new Bcl.DeflateStream(this._sink, new Bcl.ZLibCompressionOptions { CompressionLevel = this.Level }, leaveOpen: true)) + deflate.Write(this._data); + + return this._sink.Length; + } + + [Benchmark] + public int Decompress() { + // Both stream into the same 64 KB buffer, so neither pays for growing a whole-output array. + var buffer = this._buffer; + using var input = new MemoryStream(this._compressed); + var total = 0; + int n; + if (this.Codec == "Ours") { + var inflate = new DeflateDecompressor(input); + while ((n = inflate.Decompress(buffer)) > 0) + total += n; + } else { + using var inflate = new Bcl.DeflateStream(input, Bcl.CompressionMode.Decompress); + while ((n = inflate.Read(buffer)) > 0) + total += n; + } + + return total; + } + + private readonly byte[] _buffer = new byte[1 << 16]; +} diff --git a/Compression.Benchmarks/DeflateCorpus.cs b/Compression.Benchmarks/DeflateCorpus.cs new file mode 100644 index 000000000..71f6ea603 --- /dev/null +++ b/Compression.Benchmarks/DeflateCorpus.cs @@ -0,0 +1,151 @@ +#pragma warning disable CS1591 + +using System.Runtime.InteropServices; +using System.Text; + +namespace Compression.Benchmarks; + +/// +/// The inputs the DEFLATE benchmarks run on, one per kind of data a general-purpose codec +/// meets. Every kind is built from files every machine running the benchmark has — this +/// repository's own sources and the installed .NET runtime — so no corpus is checked in. +/// +/// +/// The bytes therefore differ between checkouts and runtime versions: compare figures taken +/// in the same run (ours against the platform's zlib on the same input), not across machines. +/// +public static class DeflateCorpus { + + /// Size of every corpus in bytes. + public const int Size = 8 << 20; + + /// The corpus kinds. + public static readonly string[] Kinds = ["Code", "Prose", "Binary", "Compressed", "Repetitive"]; + + private static readonly Dictionary Cache = new(StringComparer.Ordinal); + + /// Gets the corpus named , built on first use. + /// + /// With the environment variable DEFLATE_CORPUS_DIR set, each corpus is read from that + /// directory and only built (and saved there) when missing, so a before/after comparison + /// across two checkouts runs on the same bytes even though the sources differ. + /// + public static byte[] Get(string kind) { + lock (Cache) { + if (Cache.TryGetValue(kind, out var data)) + return data; + + var saved = Environment.GetEnvironmentVariable("DEFLATE_CORPUS_DIR") is { Length: > 0 } directory + ? Path.Combine(directory, kind + ".bin") + : null; + if (saved != null && File.Exists(saved)) + return Cache[kind] = File.ReadAllBytes(saved); + + Cache[kind] = data = kind switch { + "Code" => SourceCode(), + "Prose" => Prose(), + "Binary" => Binary(), + "Compressed" => Compressed(), + "Repetitive" => Repetitive(), + _ => throw new ArgumentOutOfRangeException(nameof(kind), kind, null), + }; + if (saved != null) { + Directory.CreateDirectory(Path.GetDirectoryName(saved)!); + File.WriteAllBytes(saved, data); + } + + return data; + } + } + + /// C# sources of this repository, in ordinal path order. + private static byte[] SourceCode() { + var root = RepositoryRoot(); + string[] skipped = ["obj", "bin", ".claude"]; + var files = Directory.EnumerateFiles(root, "*.cs", SearchOption.AllDirectories) + .Where(f => !Path.GetRelativePath(root, f).Split(Path.DirectorySeparatorChar).Any(skipped.Contains)); + return Concatenate(files); + } + + /// English API documentation: the XML doc files of the .NET reference pack, falling back to this repository's Markdown. + private static byte[] Prose() { + var dotnetRoot = Path.GetFullPath(Path.Combine(RuntimeEnvironment.GetRuntimeDirectory(), "..", "..", "..")); + var version = Path.GetFileName(RuntimeEnvironment.GetRuntimeDirectory().TrimEnd(Path.DirectorySeparatorChar)); + var refDir = Path.Combine(dotnetRoot, "packs", "Microsoft.NETCore.App.Ref", version, "ref", $"net{Environment.Version.Major}.0"); + if (Directory.Exists(refDir)) + return Concatenate(Directory.EnumerateFiles(refDir, "*.xml")); + + return Concatenate(Directory.EnumerateFiles(RepositoryRoot(), "*.md", SearchOption.AllDirectories)); + } + + /// Executable code and metadata: the managed assemblies of the running runtime. + private static byte[] Binary() => Concatenate(Directory.EnumerateFiles(RuntimeEnvironment.GetRuntimeDirectory(), "*.dll")); + + /// Data that is already compressed: the source corpus run through DEFLATE once. + private static byte[] Compressed() { + var once = Core.Deflate.DeflateCompressor.Compress(Get("Code"), Core.Deflate.DeflateCompressionLevel.Best); + var data = new byte[Size]; + for (var offset = 0; offset < Size; offset += once.Length) + once.AsSpan(0, Math.Min(once.Length, Size - offset)).CopyTo(data.AsSpan(offset)); + return data; + } + + /// Long runs and short repeating records, as in sparse images and logs. + private static byte[] Repetitive() { + var data = new byte[Size]; + var record = Encoding.ASCII.GetBytes("2026-10-09T12:00:00Z INFO request served status=200 bytes="); + var rng = new Random(1); + var position = 0; + while (position < Size) { + if (rng.Next(4) == 0) { + // A run of one value, as in zero-filled sectors. + var run = Math.Min(rng.Next(1, 16384), Size - position); + data.AsSpan(position, run).Fill((byte)(rng.Next(2) == 0 ? 0 : 0xFF)); + position += run; + continue; + } + + var line = Encoding.ASCII.GetBytes($"{Encoding.ASCII.GetString(record)}{rng.Next(1000, 1100)}\n"); + var n = Math.Min(line.Length, Size - position); + line.AsSpan(0, n).CopyTo(data.AsSpan(position)); + position += n; + } + + return data; + } + + private static byte[] Concatenate(IEnumerable files) { + var data = new byte[Size]; + var position = 0; + var sorted = files.Order(StringComparer.Ordinal).ToArray(); + if (sorted.Length == 0) + throw new InvalidOperationException("No files found to build the corpus from."); + + // Wrap around when the files are smaller than the corpus; the repeat is far beyond the window. + while (position < Size) { + var before = position; + foreach (var file in sorted) { + var bytes = File.ReadAllBytes(file); + var n = Math.Min(bytes.Length, Size - position); + bytes.AsSpan(0, n).CopyTo(data.AsSpan(position)); + position += n; + if (position == Size) + break; + } + + if (position == before) + throw new InvalidOperationException("The files to build the corpus from are all empty."); + } + + return data; + } + + private static string RepositoryRoot() { + foreach (var start in new[] { AppContext.BaseDirectory, Environment.CurrentDirectory }) + for (var directory = new DirectoryInfo(start); directory != null; directory = directory.Parent) + if (File.Exists(Path.Combine(directory.FullName, "CompressionWorkbench.slnx"))) + return directory.FullName; + + throw new InvalidOperationException("Run the benchmarks from inside the repository checkout."); + } +} diff --git a/Compression.Benchmarks/DeflateThroughputTable.cs b/Compression.Benchmarks/DeflateThroughputTable.cs new file mode 100644 index 000000000..9d6d8340e --- /dev/null +++ b/Compression.Benchmarks/DeflateThroughputTable.cs @@ -0,0 +1,82 @@ +#pragma warning disable CS1591 + +using System.Diagnostics; +using Compression.Core.Deflate; +using Bcl = System.IO.Compression; + +namespace Compression.Benchmarks; + +/// +/// A quick ratio and throughput table for the DEFLATE building block against the platform's +/// zlib, for the edit-measure loop where a full BenchmarkDotNet run takes too long. Each cell +/// is the best of several timed runs after a warm-up; is the +/// statistically careful version of the same measurement. +/// +/// Run with dotnet run -c Release -- deflate-table [runs] [corpus...]. +public static class DeflateThroughputTable { + + public static void Run(string[] args) { + var runs = args.Length > 0 && int.TryParse(args[0], out var r) ? r : 5; + var kinds = args.Length > 1 ? args[1..] : DeflateCorpus.Kinds; + var buffer = new byte[1 << 16]; + var sink = new MemoryStream(); + + Console.WriteLine("| Corpus | Level | Ratio ours | Ratio zlib | Compress ours MB/s | Compress zlib MB/s | Inflate ours MB/s | Inflate zlib MB/s |"); + Console.WriteLine("|---|---:|---:|---:|---:|---:|---:|---:|"); + foreach (var kind in kinds) { + var data = DeflateCorpus.Get(kind); + foreach (var level in new[] { 1, 6, 9 }) { + var ours = DeflateCompressor.Compress(data, (DeflateCompressionLevel)level); + if (!DeflateDecompressor.Decompress(ours).AsSpan().SequenceEqual(data)) + throw new InvalidOperationException($"{kind} level {level} does not round-trip."); + + var platform = PlatformCompress(data, level, sink); + + var compressOurs = Best(runs, () => { + sink.Position = 0; + sink.SetLength(0); + var compressor = new DeflateCompressor(sink, (DeflateCompressionLevel)level); + compressor.Write(data); + compressor.Finish(); + }); + var compressPlatform = Best(runs, () => PlatformCompress(data, level, sink)); + var inflateOurs = Best(runs, () => { + var inflate = new DeflateDecompressor(new MemoryStream(ours)); + while (inflate.Decompress(buffer) > 0) { } + }); + var inflatePlatform = Best(runs, () => { + using var inflate = new Bcl.DeflateStream(new MemoryStream(ours), Bcl.CompressionMode.Decompress); + while (inflate.Read(buffer) > 0) { } + }); + + Console.WriteLine($"| {kind} | {level} | {Percent(ours.Length, data.Length)} | {Percent(platform, data.Length)} | " + + $"{Speed(data.Length, compressOurs)} | {Speed(data.Length, compressPlatform)} | " + + $"{Speed(data.Length, inflateOurs)} | {Speed(data.Length, inflatePlatform)} |"); + } + } + } + + private static long PlatformCompress(byte[] data, int level, MemoryStream sink) { + sink.Position = 0; + sink.SetLength(0); + using (var deflate = new Bcl.DeflateStream(sink, new Bcl.ZLibCompressionOptions { CompressionLevel = level }, leaveOpen: true)) + deflate.Write(data); + return sink.Length; + } + + private static double Best(int runs, Action action) { + action(); + var best = double.MaxValue; + for (var i = 0; i < runs; ++i) { + var watch = Stopwatch.StartNew(); + action(); + best = Math.Min(best, watch.Elapsed.TotalSeconds); + } + + return best; + } + + private static string Percent(long part, long whole) => FormattableString.Invariant($"{100.0 * part / whole:F2}%"); + + private static string Speed(long bytes, double seconds) => FormattableString.Invariant($"{bytes / seconds / (1 << 20):F0}"); +} diff --git a/Compression.Benchmarks/Program.cs b/Compression.Benchmarks/Program.cs index c656f325e..908c98fce 100644 --- a/Compression.Benchmarks/Program.cs +++ b/Compression.Benchmarks/Program.cs @@ -1,5 +1,11 @@ #pragma warning disable CS1591 using BenchmarkDotNet.Running; +using Compression.Benchmarks; + +if (args is ["deflate-table", .. var rest]) { + DeflateThroughputTable.Run(rest); + return; +} BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); diff --git a/Compression.Core/Deflate/DeflateCompressor.cs b/Compression.Core/Deflate/DeflateCompressor.cs index 71f2e50e5..6027fa3f7 100644 --- a/Compression.Core/Deflate/DeflateCompressor.cs +++ b/Compression.Core/Deflate/DeflateCompressor.cs @@ -63,7 +63,9 @@ public DeflateCompressor(Stream output, DeflateCompressionLevel level = DeflateC /// The compression level. /// The DEFLATE compressed data. public static byte[] Compress(ReadOnlySpan data, DeflateCompressionLevel level = DeflateCompressionLevel.Default) { - using var ms = new MemoryStream(); + // Half the input holds most compressible data in one go; a MemoryStream growing from empty + // reallocates and copies the output at every doubling before the final copy out. + using var ms = new MemoryStream((int)Math.Min(data.Length / 2L + 64, Array.MaxLength)); var compressor = new DeflateCompressor(ms, level); compressor.Write(data); compressor.Finish(); diff --git a/Compression.Core/Deflate/DeflateDecompressor.cs b/Compression.Core/Deflate/DeflateDecompressor.cs index 7eab61353..640504f1d 100644 --- a/Compression.Core/Deflate/DeflateDecompressor.cs +++ b/Compression.Core/Deflate/DeflateDecompressor.cs @@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; namespace Compression.Core.Deflate; @@ -12,7 +13,13 @@ namespace Compression.Core.Deflate; /// The decoder works on a 64-bit bit accumulator refilled from a read-ahead buffer, and /// decodes Huffman symbols through two-level lookup tables (a root table indexed by the next /// few bits, with sub-tables for the rarer longer codes), writing straight into an output -/// buffer that doubles as the 32 KB history window. +/// buffer that doubles as the 32 KB history window. A table entry carries the base length or +/// distance and its extra-bit count along with the code, so a symbol costs one lookup. +/// Two loops share the work. Wherever the read-ahead and the output buffer both have +/// room for the largest step, a fast loop decodes without bounds or truncation checks — up to +/// three literals per refill, matches copied sixteen bytes at a time; at the edges a careful +/// loop decodes one symbol at a time with every check, and hands back as soon as the fast +/// loop can resume. Malformed input is rejected by both alike. /// Input is read ahead in chunks. When the stream ends and the underlying stream is /// seekable, the read-ahead is handed back by seeking, so the stream is left positioned on /// the first byte after the DEFLATE data; otherwise reports how @@ -24,12 +31,28 @@ public sealed class DeflateDecompressor { private const int MaxMatch = 258; private const int InputChunk = 16384; private const int StreamingBufferSize = WindowSize * 4; - private const int LitRootBits = 10; + private const int LitRootBits = 11; + private const int LitRootMask = (1 << LitRootBits) - 1; private const int DistRootBits = 8; + private const int DistRootMask = (1 << DistRootBits) - 1; private const int CodeLengthRootBits = 7; - private static readonly int[] StaticLitTable = BuildTable(DeflateConstants.GetStaticLiteralLengths(), LitRootBits); - private static readonly int[] StaticDistTable = BuildTable(DeflateConstants.GetStaticDistanceLengths(), DistRootBits); + // Room a match copy needs past the write position: the match, plus the overshoot of its + // last sixteen-byte step. + private const int CopySlack = MaxMatch + 16; + // The fast loop writes up to three literals or one match per step. + private const int FastOutputSlack = CopySlack + 16; + // The fast loop reads eight bytes per refill and refills once per step. + private const int FastInputSlack = 16; + + // Decode-table entry flags; see BuildTable. + private const uint Literal = 0x8000; + private const uint Exceptional = 0x4000; + private const uint SubTable = 0x2000; + private const uint EndOfBlock = 0x1000; + + private static readonly uint[] StaticLitTable = BuildTable(DeflateConstants.GetStaticLiteralLengths(), LitRootBits, TableKind.LiteralLength, null); + private static readonly uint[] StaticDistTable = BuildTable(DeflateConstants.GetStaticDistanceLengths(), DistRootBits, TableKind.Distance, null); private readonly Stream? _input; private byte[] _in; @@ -47,8 +70,10 @@ public sealed class DeflateDecompressor { private State _state; private bool _final; private int _storedLeft; - private int[] _litTable = StaticLitTable; - private int[] _distTable = StaticDistTable; + private uint[] _litTable = StaticLitTable; + private uint[] _distTable = StaticDistTable; + private uint[]? _dynamicLitTable; + private uint[]? _dynamicDistTable; private enum State { Header, Stored, Huffman, Done } @@ -85,7 +110,7 @@ private DeflateDecompressor(byte[] compressed, int outputCapacity) { /// The decompressed data. public byte[] DecompressAll() { while (this._state != State.Done) { - if (this._buf.Length - this._wpos <= MaxMatch + 8) + if (this._buf.Length - this._wpos <= CopySlack) this.Grow(); this.Step(); @@ -176,7 +201,7 @@ public int ReadRemainder(Span destination) { private void Grow() { var size = (int)Math.Min(Math.Max(this._buf.Length * 2L, StreamingBufferSize), Array.MaxLength); - if (size - this._wpos <= MaxMatch + 8) + if (size - this._wpos <= CopySlack) throw new InvalidDataException("DEFLATE output exceeds the largest supported buffer."); Array.Resize(ref this._buf, size); @@ -189,7 +214,7 @@ private void MakeRoom() { return; } - if (this._buf.Length - this._wpos > MaxMatch + 8) + if (this._buf.Length - this._wpos > CopySlack) return; var keep = Math.Min(this._wpos, WindowSize); @@ -261,13 +286,10 @@ private uint GetBits(int count) { return value; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int DecodeSymbol(int[] table, int rootBits) { + /// Decodes one symbol of a code-length table, whose codes never need a sub-table. + private int DecodeSymbol(uint[] table, int rootBits) { var entry = table[(int)this._bitBuf & ((1 << rootBits) - 1)]; - if ((entry & 0x80) != 0) - entry = table[(entry >> 8) + ((int)(this._bitBuf >> rootBits) & ((1 << (entry & 0x0F)) - 1))]; - - var length = entry & 0xFF; + var length = (int)(entry & 0xFF); if (length == 0) ThrowInvalidCode(); if (length > this._bitCnt) @@ -275,7 +297,7 @@ private int DecodeSymbol(int[] table, int rootBits) { this._bitBuf >>= length; this._bitCnt -= length; - return entry >> 8; + return (int)(entry >> 16); } // ── blocks ─────────────────────────────────────────────────────────── @@ -398,18 +420,143 @@ private void CopyStored() { } private void DecodeHuffman() { - // The hot loop keeps the bit state in locals and writes it back before anything that - // reads it from the fields. + // Alternate between the two loops: the fast one wherever it is safe, the careful one at + // the edges. Each call of either makes progress or hands back for more room. + while (true) { + if (this._wpos < this._buf.Length - FastOutputSlack && this._inEnd - this._inPos >= 2 * FastInputSlack && this.DecodeFast()) + return; + + if (!this.DecodeCareful()) + return; + } + } + + /// + /// Decodes symbols while the output buffer and the read-ahead both have room for the + /// largest step, without any per-symbol bounds or truncation check. + /// + /// + /// Inside the safe zone every refill can load eight whole bytes, so the accumulator + /// holds at least 56 bits after each one: enough for three literals (3 × 15 bits), or for a + /// length and its distance (15 + 5 + 15 + 13 = 48 bits). The next table entry is looked up + /// before a match is copied, so the lookup overlaps the copy. + /// Returns when the block ended; otherwise the careful loop + /// takes over at the edge of the zone. + /// + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + private bool DecodeFast() { var buf = this._buf; - // Copies run in whole eight-byte steps, so up to seven bytes past a match may be written; - // stopping MaxMatch + 8 short of the end keeps that inside the buffer. - var stopAt = buf.Length - MaxMatch - 8; + var stopAt = buf.Length - FastOutputSlack; + var inEnd = this._inEnd - FastInputSlack; + var lit = this._litTable; + var dist = this._distTable; + ref var litRef = ref MemoryMarshal.GetArrayDataReference(lit); + ref var distRef = ref MemoryMarshal.GetArrayDataReference(dist); + ref var input = ref MemoryMarshal.GetArrayDataReference(this._in); + ref var output = ref MemoryMarshal.GetArrayDataReference(buf); + var wpos = this._wpos; + var bitBuf = this._bitBuf; + var bitCnt = this._bitCnt; + var inPos = this._inPos; + + Refill(ref input, ref inPos, ref bitBuf, ref bitCnt); + var entry = Unsafe.Add(ref litRef, (int)bitBuf & LitRootMask); + while (wpos < stopAt && inPos <= inEnd) { + if ((entry & Literal) != 0) { + bitBuf >>= (int)entry; + bitCnt -= (int)(entry & 0xFF); + Unsafe.Add(ref output, wpos++) = (byte)(entry >> 16); + entry = Unsafe.Add(ref litRef, (int)bitBuf & LitRootMask); + if ((entry & Literal) != 0) { + bitBuf >>= (int)entry; + bitCnt -= (int)(entry & 0xFF); + Unsafe.Add(ref output, wpos++) = (byte)(entry >> 16); + entry = Unsafe.Add(ref litRef, (int)bitBuf & LitRootMask); + if ((entry & Literal) != 0) { + bitBuf >>= (int)entry; + bitCnt -= (int)(entry & 0xFF); + Unsafe.Add(ref output, wpos++) = (byte)(entry >> 16); + entry = Unsafe.Add(ref litRef, (int)bitBuf & LitRootMask); + } + } + + // The lookup above only needed root bits that are already in place; the refill + // makes room for whatever the entry turns out to be. + Refill(ref input, ref inPos, ref bitBuf, ref bitCnt); + continue; + } + + if ((entry & Exceptional) != 0) { + if ((entry & SubTable) != 0) { + entry = Unsafe.Add(ref litRef, (int)(entry >> 16) + ((int)(bitBuf >> LitRootBits) & ((1 << (int)((entry >> 8) & 0xF)) - 1))); + if ((entry & Literal) != 0) { + bitBuf >>= (int)entry; + bitCnt -= (int)(entry & 0xFF); + Unsafe.Add(ref output, wpos++) = (byte)(entry >> 16); + Refill(ref input, ref inPos, ref bitBuf, ref bitCnt); + entry = Unsafe.Add(ref litRef, (int)bitBuf & LitRootMask); + continue; + } + } + + if ((entry & Exceptional) != 0) { + if ((entry & EndOfBlock) == 0) + ThrowInvalidCode(); + + bitBuf >>= (int)entry; + bitCnt -= (int)(entry & 0xFF); + (this._bitBuf, this._bitCnt, this._inPos, this._wpos) = (bitBuf, bitCnt, inPos, wpos); + this.EndBlock(); + return true; + } + } + + // A length, then its distance; at least 56 bits are in the accumulator. + var saved = bitBuf; + bitBuf >>= (int)entry; + bitCnt -= (int)(entry & 0xFF); + var length = (int)(entry >> 16) + ExtraBits(saved, entry); + + entry = Unsafe.Add(ref distRef, (int)bitBuf & DistRootMask); + if ((entry & Exceptional) != 0) { + if ((entry & SubTable) == 0) + ThrowInvalidCode(); + entry = Unsafe.Add(ref distRef, (int)(entry >> 16) + ((int)(bitBuf >> DistRootBits) & ((1 << (int)((entry >> 8) & 0xF)) - 1))); + if ((entry & Exceptional) != 0) + ThrowInvalidCode(); + } + + saved = bitBuf; + bitBuf >>= (int)entry; + bitCnt -= (int)(entry & 0xFF); + var distance = (int)(entry >> 16) + ExtraBits(saved, entry); + if (distance > wpos) + ThrowDistanceTooFar(distance, wpos); + + Refill(ref input, ref inPos, ref bitBuf, ref bitCnt); + entry = Unsafe.Add(ref litRef, (int)bitBuf & LitRootMask); + CopyMatch(ref Unsafe.Add(ref output, wpos), distance, length); + wpos += length; + } + + (this._bitBuf, this._bitCnt, this._inPos, this._wpos) = (bitBuf, bitCnt, inPos, wpos); + return false; + } + + /// + /// Decodes one symbol at a time with every check in place: near the end of the input, + /// where a refill may come up short, and near the end of the output buffer. + /// + /// when the fast loop can take over again. + [MethodImpl(MethodImplOptions.AggressiveOptimization)] + private bool DecodeCareful() { + var buf = this._buf; + // Copies run in whole steps of up to sixteen bytes, so up to fifteen bytes past a match + // may be written; stopping MaxMatch + 16 short of the end keeps that inside the buffer. + var stopAt = buf.Length - CopySlack; + var fastStop = buf.Length - FastOutputSlack; var lit = this._litTable; var dist = this._distTable; - ref var lengthBase = ref MemoryMarshal.GetReference(DeflateConstants.LengthBase); - ref var lengthExtra = ref MemoryMarshal.GetReference(DeflateConstants.LengthExtraBits); - ref var distBase = ref MemoryMarshal.GetReference(DeflateConstants.DistanceBase); - ref var distExtra = ref MemoryMarshal.GetReference(DeflateConstants.DistanceExtraBits); ref var output = ref MemoryMarshal.GetArrayDataReference(buf); var wpos = this._wpos; var bitBuf = this._bitBuf; @@ -419,99 +566,121 @@ private void DecodeHuffman() { while (wpos < stopAt) { // 15 (length code) + 5 (extra) + 15 (distance code) + 13 (extra) = 48 bits per token. if (bitCnt < 48) { - if (this._inEnd - inPos >= 8) { - // The bits loaded above the count are the very bytes a later refill ORs in again. - bitBuf |= BinaryPrimitives.ReadUInt64LittleEndian(this._in.AsSpan(inPos)) << bitCnt; - inPos += (63 - bitCnt) >> 3; - bitCnt |= 56; - } else { + if (this._inEnd - inPos >= 8) + Refill(ref MemoryMarshal.GetArrayDataReference(this._in), ref inPos, ref bitBuf, ref bitCnt); + else { (this._bitBuf, this._bitCnt, this._inPos) = (bitBuf, bitCnt, inPos); this.RefillSlow(); (bitBuf, bitCnt, inPos) = (this._bitBuf, this._bitCnt, this._inPos); } } - var entry = lit[(int)bitBuf & ((1 << LitRootBits) - 1)]; - if ((entry & 0x80) != 0) - entry = lit[(entry >> 8) + ((int)(bitBuf >> LitRootBits) & ((1 << (entry & 0x0F)) - 1))]; + var entry = lit[(int)bitBuf & LitRootMask]; + if ((entry & SubTable) != 0) + entry = lit[(int)(entry >> 16) + ((int)(bitBuf >> LitRootBits) & ((1 << (int)((entry >> 8) & 0xF)) - 1))]; - var codeLength = entry & 0xFF; - if (codeLength == 0) + if ((entry & (Exceptional | EndOfBlock)) == Exceptional) ThrowInvalidCode(); - if (codeLength > bitCnt) + if ((int)(entry & 0xFF) > bitCnt) ThrowTruncated(); - bitBuf >>= codeLength; - bitCnt -= codeLength; - var symbol = entry >> 8; + var saved = bitBuf; + bitBuf >>= (int)entry; + bitCnt -= (int)(entry & 0xFF); + + if ((entry & Literal) != 0) { + Unsafe.Add(ref output, wpos++) = (byte)(entry >> 16); + if (wpos < fastStop && this._inEnd - inPos >= 2 * FastInputSlack) { + (this._bitBuf, this._bitCnt, this._inPos, this._wpos) = (bitBuf, bitCnt, inPos, wpos); + return true; + } - if (symbol < 256) { - Unsafe.Add(ref output, wpos++) = (byte)symbol; continue; } - if (symbol == DeflateConstants.EndOfBlock) { + if ((entry & EndOfBlock) != 0) { (this._bitBuf, this._bitCnt, this._inPos, this._wpos) = (bitBuf, bitCnt, inPos, wpos); this.EndBlock(); - return; + return false; } - symbol -= 257; - if ((uint)symbol >= 29) - ThrowInvalidCode(); - - var length = Unsafe.Add(ref lengthBase, symbol); - var extra = Unsafe.Add(ref lengthExtra, symbol); - if (extra > bitCnt) - ThrowTruncated(); - length += (int)bitBuf & ((1 << extra) - 1); - bitBuf >>= extra; - bitCnt -= extra; + var length = (int)(entry >> 16) + ExtraBits(saved, entry); - entry = dist[(int)bitBuf & ((1 << DistRootBits) - 1)]; - if ((entry & 0x80) != 0) - entry = dist[(entry >> 8) + ((int)(bitBuf >> DistRootBits) & ((1 << (entry & 0x0F)) - 1))]; - - codeLength = entry & 0xFF; - if (codeLength == 0) + entry = dist[(int)bitBuf & DistRootMask]; + if ((entry & SubTable) != 0) + entry = dist[(int)(entry >> 16) + ((int)(bitBuf >> DistRootBits) & ((1 << (int)((entry >> 8) & 0xF)) - 1))]; + if ((entry & Exceptional) != 0) ThrowInvalidCode(); - if (codeLength > bitCnt) - ThrowTruncated(); - - bitBuf >>= codeLength; - bitCnt -= codeLength; - var distSymbol = entry >> 8; - if ((uint)distSymbol >= 30) - ThrowInvalidCode(); - - var distance = Unsafe.Add(ref distBase, distSymbol); - extra = Unsafe.Add(ref distExtra, distSymbol); - if (extra > bitCnt) + if ((int)(entry & 0xFF) > bitCnt) ThrowTruncated(); - distance += (int)bitBuf & ((1 << extra) - 1); - bitBuf >>= extra; - bitCnt -= extra; + saved = bitBuf; + bitBuf >>= (int)entry; + bitCnt -= (int)(entry & 0xFF); + var distance = (int)(entry >> 16) + ExtraBits(saved, entry); if (distance > wpos) ThrowDistanceTooFar(distance, wpos); - // Bounds: wpos < stopAt, so wpos + length + 7 < buf.Length; the source lies before wpos. - ref var dst = ref Unsafe.Add(ref output, wpos); - ref var src = ref Unsafe.Add(ref output, wpos - distance); - if (distance >= 8) - // Each eight-byte step reads only bytes at least eight behind it, all already final. - for (var i = 0; i < length; i += 8) - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dst, i), Unsafe.ReadUnaligned(ref Unsafe.Add(ref src, i))); - else if (distance == 1) - Unsafe.InitBlockUnaligned(ref dst, src, (uint)length); - else - for (var i = 0; i < length; ++i) - Unsafe.Add(ref dst, i) = Unsafe.Add(ref src, i); - + CopyMatch(ref Unsafe.Add(ref output, wpos), distance, length); wpos += length; + if (wpos < fastStop && this._inEnd - inPos >= 2 * FastInputSlack) { + (this._bitBuf, this._bitCnt, this._inPos, this._wpos) = (bitBuf, bitCnt, inPos, wpos); + return true; + } } (this._bitBuf, this._bitCnt, this._inPos, this._wpos) = (bitBuf, bitCnt, inPos, wpos); + return false; + } + + /// + /// Tops the accumulator up to at least 56 bits from eight bytes at , + /// which the caller guarantees are there. + /// + /// + /// All eight bytes are ORed in but only the whole ones that fit are counted; the bits loaded + /// above the count are the very bits a later refill ORs in again, so loading them early is + /// harmless. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Refill(ref byte input, ref int inPos, ref ulong bitBuf, ref int bitCnt) { + var word = Unsafe.ReadUnaligned(ref Unsafe.Add(ref input, inPos)); + bitBuf |= (BitConverter.IsLittleEndian ? word : BinaryPrimitives.ReverseEndianness(word)) << bitCnt; + inPos += (63 - bitCnt) >> 3; + bitCnt |= 56; + } + + /// The extra bits that follow the code of in , the accumulator before the code was consumed. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ExtraBits(ulong saved, uint entry) + => (int)((saved & ~(ulong.MaxValue << (int)(entry & 0xFF))) >> (int)((entry >> 8) & 0xF)); + + /// Copies bytes from back to . + /// + /// Copies run in whole 16- or 8-byte steps and may write up to fifteen bytes past the match, + /// which the callers' slack keeps inside the buffer. Each step reads only bytes at least a + /// step behind it, all already final; shorter distances repeat a pattern and go byte by byte. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void CopyMatch(ref byte destination, int distance, int length) { + ref var source = ref Unsafe.Subtract(ref destination, distance); + if (distance >= 16) { + var i = 0; + do { + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, i), Unsafe.ReadUnaligned>(ref Unsafe.Add(ref source, i))); + i += 16; + } while (i < length); + } else if (distance >= 8) { + var i = 0; + do { + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, i), Unsafe.ReadUnaligned(ref Unsafe.Add(ref source, i))); + i += 8; + } while (i < length); + } else if (distance == 1) + Unsafe.InitBlockUnaligned(ref destination, source, (uint)length); + else + for (var i = 0; i < length; ++i) + Unsafe.Add(ref destination, i) = Unsafe.Add(ref source, i); } private void ReadDynamicTables() { @@ -521,14 +690,15 @@ private void ReadDynamicTables() { if (hlit > 286 || hdist > 30) throw new InvalidDataException($"Invalid DEFLATE dynamic header: HLIT={hlit}, HDIST={hdist}."); - var codeLengthLengths = new int[DeflateConstants.CodeLengthAlphabetSize]; + Span codeLengthLengths = stackalloc int[DeflateConstants.CodeLengthAlphabetSize]; + codeLengthLengths.Clear(); var order = DeflateConstants.CodeLengthOrder; for (var i = 0; i < hclen; ++i) codeLengthLengths[order[i]] = (int)this.GetBits(3); - var codeLengthTable = BuildTable(codeLengthLengths, CodeLengthRootBits); + var codeLengthTable = BuildTable(codeLengthLengths, CodeLengthRootBits, TableKind.Plain, null); - var lengths = new int[hlit + hdist]; + Span lengths = stackalloc int[hlit + hdist]; var index = 0; while (index < lengths.Length) { if (this._bitCnt < 16) @@ -559,28 +729,40 @@ private void ReadDynamicTables() { if (index + repeat > lengths.Length) throw new InvalidDataException("DEFLATE code lengths overrun the table."); - lengths.AsSpan(index, repeat).Fill(value); + lengths.Slice(index, repeat).Fill(value); index += repeat; } if (lengths[DeflateConstants.EndOfBlock] == 0) throw new InvalidDataException("DEFLATE block has no end-of-block code."); - this._litTable = BuildTable(lengths.AsSpan(0, hlit), LitRootBits); - this._distTable = BuildTable(lengths.AsSpan(hlit, hdist), DistRootBits); + this._litTable = this._dynamicLitTable = BuildTable(lengths[..hlit], LitRootBits, TableKind.LiteralLength, this._dynamicLitTable); + this._distTable = this._dynamicDistTable = BuildTable(lengths.Slice(hlit, hdist), DistRootBits, TableKind.Distance, this._dynamicDistTable); } + private enum TableKind { Plain, LiteralLength, Distance } + /// - /// Builds a two-level decoding table for canonical Huffman code lengths (RFC 1951 §3.2.2). + /// Builds a two-level decoding table for canonical Huffman code lengths (RFC 1951 §3.2.2), + /// reusing when it is large enough. /// /// - /// A direct entry is symbol << 8 | length; a link to a sub-table is - /// offset << 8 | 0x80 | subBits, and the sub-table is indexed by the bits after - /// the root ones. Zero marks a bit pattern no code describes — an incomplete code is - /// accepted, an over-subscribed one is not. + /// An entry is value << 16 | flags | codeLength << 8 | totalBits, + /// where totalBits counts the code and the extra bits that follow it, so consuming a symbol + /// is one shift by the entry itself. The value is the literal, the base length or distance, + /// or — in a code-length table — the symbol. A literal/length or distance table folds the + /// RFC's base and extra-bit tables in, which spares the decoder two lookups per symbol. + /// A link to a sub-table is offset << 16 | Exceptional | SubTable | subBits << 8, + /// the sub-table being indexed by the bits after the root ones. A bit pattern no code + /// describes, and the symbols the format reserves (286, 287, distance codes 30 and 31), + /// decode to a bare Exceptional: an incomplete code is accepted, an over-subscribed + /// one is not. + /// The flag layout follows libdeflate's decode-table design (MIT licence): fold the + /// base and the extra-bit count into the entry and let the low byte drive the shift. /// - private static int[] BuildTable(ReadOnlySpan lengths, int rootBits) { + private static uint[] BuildTable(ReadOnlySpan lengths, int rootBits, TableKind kind, uint[]? reuse) { Span count = stackalloc int[16]; + count.Clear(); foreach (var length in lengths) { if ((uint)length > 15) throw new InvalidDataException($"Invalid DEFLATE code length {length}."); @@ -596,6 +778,7 @@ private static int[] BuildTable(ReadOnlySpan lengths, int rootBits) { } Span next = stackalloc int[16]; + next.Clear(); var code = 0; for (var length = 1; length <= 15; ++length) { code = (code + count[length - 1]) << 1; @@ -604,7 +787,7 @@ private static int[] BuildTable(ReadOnlySpan lengths, int rootBits) { var rootSize = 1 << rootBits; var rootMask = rootSize - 1; - var reversed = new int[lengths.Length]; + Span reversed = stackalloc int[lengths.Length]; Span subBits = stackalloc int[rootSize]; subBits.Clear(); for (var symbol = 0; symbol < lengths.Length; ++symbol) { @@ -626,14 +809,19 @@ private static int[] BuildTable(ReadOnlySpan lengths, int rootBits) { size += 1 << subBits[i]; } - var table = new int[size]; + var table = reuse != null && reuse.Length >= size ? reuse : new uint[size]; + table.AsSpan(0, size).Fill(Exceptional); + for (var i = 0; i < rootSize; ++i) + if (subBits[i] > 0) + table[i] = (uint)subOffset[i] << 16 | Exceptional | SubTable | (uint)subBits[i] << 8; + for (var symbol = 0; symbol < lengths.Length; ++symbol) { var length = lengths[symbol]; if (length == 0) continue; var rev = reversed[symbol]; - var entry = (symbol << 8) | length; + var entry = Entry(kind, symbol, length); if (length <= rootBits) { for (var i = rev; i < rootSize; i += 1 << length) table[i] = entry; @@ -642,7 +830,6 @@ private static int[] BuildTable(ReadOnlySpan lengths, int rootBits) { var root = rev & rootMask; var bits = subBits[root]; - table[root] = (subOffset[root] << 8) | 0x80 | bits; for (var i = rev >> rootBits; i < 1 << bits; i += 1 << (length - rootBits)) table[subOffset[root] + i] = entry; } @@ -650,6 +837,25 @@ private static int[] BuildTable(ReadOnlySpan lengths, int rootBits) { return table; } + private static uint Entry(TableKind kind, int symbol, int length) { + var code = (uint)length << 8; + switch (kind) { + case TableKind.Plain: + return (uint)symbol << 16 | code | (uint)length; + case TableKind.LiteralLength when symbol < 256: + return (uint)symbol << 16 | Literal | code | (uint)length; + case TableKind.LiteralLength when symbol == DeflateConstants.EndOfBlock: + return Exceptional | EndOfBlock | code | (uint)length; + case TableKind.LiteralLength when symbol - 257 < 29: + return (uint)DeflateConstants.LengthBase[symbol - 257] << 16 | code | (uint)(length + DeflateConstants.LengthExtraBits[symbol - 257]); + case TableKind.Distance when symbol < 30: + return (uint)DeflateConstants.DistanceBase[symbol] << 16 | code | (uint)(length + DeflateConstants.DistanceExtraBits[symbol]); + default: + // Reserved symbols: decoding one is an error. + return Exceptional; + } + } + [DoesNotReturn] private static void ThrowDistanceTooFar(int distance, int history) => throw new InvalidDataException($"Invalid DEFLATE distance {distance}: only {history} bytes of history."); diff --git a/Compression.Core/Deflate/DeflateEncoder.cs b/Compression.Core/Deflate/DeflateEncoder.cs index 3f53dea0e..25bde041d 100644 --- a/Compression.Core/Deflate/DeflateEncoder.cs +++ b/Compression.Core/Deflate/DeflateEncoder.cs @@ -3,6 +3,7 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; namespace Compression.Core.Deflate; @@ -10,16 +11,23 @@ namespace Compression.Core.Deflate; /// Streaming DEFLATE encoder (RFC 1951) for the hash-chain levels 1–9. /// /// -/// The match search is the one RFC 1951 §4 recommends: three-byte strings are hashed -/// into chains over a 32 KB window held in a 64 KB buffer that slides by 32 KB, chains are -/// cut off after a level-dependent number of probes, and levels 4–9 defer each match by one -/// byte to see whether the next position starts a longer one ("lazy matching"). Levels 1–3 -/// take matches greedily and insert only the first strings of short matches into the hash. +/// The match search is the one RFC 1951 §4 recommends: strings are hashed into chains +/// over a 32 KB window held in a 64 KB buffer that slides by 32 KB, chains are cut off after a +/// level-dependent number of probes, and levels 4–9 defer each match by one byte to see +/// whether the next position starts a longer one ("lazy matching"). Levels 1–3 take matches +/// greedily and insert only the inner strings of short matches into the hash. +/// The chains are keyed on four bytes rather than three: a chain then holds only real +/// candidates for a match of four or more, which is most of what a three-byte chain walks past. +/// Matches of exactly three bytes come from a side table that remembers the latest position of +/// every three-byte hash, at short distances only, where they beat three literals. Runs of one +/// byte value are linked into their chain a vector at a time. (Four-byte hashing with a +/// three-byte side table is the arrangement of zlib-ng and libdeflate; no code was taken.) /// The per-level search limits (good/lazy/nice length and chain depth) are the agreed /// values of zlib's configuration table, so a level here does the same amount of work as the /// same zlib level. Each block is emitted as stored, fixed- or dynamic-Huffman, whichever is /// smallest, with length-limited code lengths from the shared deterministic builder. The -/// output depends only on the input and the level — never on the platform. +/// output depends only on the input and the level — never on the platform, nor on how the +/// input is split across calls. /// internal sealed partial class DeflateEncoder { private const int WSize = DeflateConstants.WindowSize; @@ -29,7 +37,10 @@ internal sealed partial class DeflateEncoder { private const int MinLookahead = MaxMatch + MinMatch + 1; private const int MaxDist = WSize - MinLookahead; private const int HashBits = 15; + private const int Hash3Bits = 14; private const int TooFar = 4096; + private const int GreedyMatch3Reach = 1024; + private const uint HashMultiplier = 0x9E3779B1u; private const int SymbolBufferSize = 1 << 14; private const int OutputBufferSize = 1 << 16; @@ -61,6 +72,7 @@ private static readonly (int Good, int Lazy, int Nice, int Chain, bool Greedy)[] private readonly byte[] _window = new byte[2 * WSize + MaxMatch + 8]; private readonly ushort[] _head = new ushort[1 << HashBits]; private readonly ushort[] _prev = new ushort[WSize]; + private readonly ushort[] _head3 = new ushort[1 << Hash3Bits]; private int _strStart; private int _lookahead; private int _blockStart; @@ -68,13 +80,18 @@ private static readonly (int Good, int Lazy, int Nice, int Chain, bool Greedy)[] private int _matchLength = MinMatch - 1; private bool _matchAvailable; - private readonly byte[] _symLength = new byte[SymbolBufferSize]; + // A symbol is a literal/length index — the literal, or 256 + length - MinMatch — and a + // distance, 0 for a literal. + private readonly ushort[] _symLitLen = new ushort[SymbolBufferSize]; private readonly ushort[] _symDistance = new ushort[SymbolBufferSize]; private int _symCount; private readonly long[] _litFreq = new long[DeflateConstants.LiteralLengthAlphabetSize]; private readonly long[] _distFreq = new long[DeflateConstants.DistanceAlphabetSize]; private readonly long[] _distTreeFreq = new long[DeflateConstants.DistanceAlphabetSize]; + private readonly ulong[] _litLenEntries = new ulong[256 + MaxMatch - MinMatch + 1]; + private readonly ulong[] _distEntries = new ulong[DeflateConstants.DistanceAlphabetSize + 1]; + private readonly byte[] _out = new byte[OutputBufferSize]; private int _outPos; private ulong _bitBuf; @@ -126,6 +143,9 @@ public void Finish() { } // ── match search ───────────────────────────────────────────────────── + // + // Both loops keep the scan state in locals and write it back to the fields only around + // FlushBlock and on exit, so the JIT can hold it in registers across the hot path. private void Compress(bool flush) { if (this._greedy) @@ -134,163 +154,313 @@ private void Compress(bool flush) { this.CompressLazy(flush); } + [MethodImpl(MethodImplOptions.AggressiveOptimization)] private void CompressGreedy(bool flush) { - var window = this._window; - while (true) { - if (this._lookahead < MinLookahead && (!flush || this._lookahead == 0)) - return; - - var hashHead = this._lookahead >= MinMatch ? this.Insert(this._strStart) : 0; + ref var window = ref MemoryMarshal.GetArrayDataReference(this._window); + var tables = new HashTables(this); + var symbols = new SymbolSink(this); + var strStart = this._strStart; + var lookahead = this._lookahead; + var minimum = flush ? 1 : MinLookahead; + var (chainLimit, niceLimit, maxInsert) = (this._chain, this._nice, this._lazy); + + while (lookahead >= minimum) { var matchLength = 0; - if (hashHead != 0 && this._strStart - hashHead <= MaxDist) - matchLength = this.LongestMatch(hashHead, MinMatch - 1); - - if (matchLength >= MinMatch) { - this.TallyMatch(this._strStart - this._matchStart, matchLength); - this._lookahead -= matchLength; - if (matchLength <= this._lazy && this._lookahead >= MinMatch) { - for (var i = 1; i < matchLength; ++i) - this.Insert(this._strStart + i); + var matchStart = 0; + if (lookahead >= MinMatch) { + var candidate = tables.Insert(ref window, strStart, out var candidate3); + if (candidate != 0 && strStart - candidate <= MaxDist) + matchLength = LongestMatch(ref window, ref tables.Prev, strStart, candidate, MinMatch - 1, + chainLimit, Math.Min(niceLimit, lookahead), Math.Min(MaxMatch, lookahead), ref matchStart); + + if (matchLength < MinMatch && IsMatch3(ref window, strStart, candidate3, GreedyMatch3Reach)) { + matchLength = MinMatch; + matchStart = candidate3; } + } - this._strStart += matchLength; + if (matchLength >= MinMatch) { + symbols.Match(strStart - matchStart, matchLength); + lookahead -= matchLength; + // Like zlib's fast levels, only short matches put their inner strings into the hash. + if (matchLength <= maxInsert && lookahead >= MinMatch) + tables.InsertRange(ref window, strStart + 1, strStart + matchLength); + strStart += matchLength; } else { - this.TallyLiteral(window[this._strStart]); - --this._lookahead; - ++this._strStart; + symbols.Literal(Unsafe.Add(ref window, strStart)); + --lookahead; + ++strStart; } - if (this._symCount == SymbolBufferSize) - this.FlushBlock(last: false); + if (symbols.IsFull) { + this._strStart = strStart; + symbols.Flush(this); + } } + + this._strStart = strStart; + this._lookahead = lookahead; + symbols.Store(this); } + [MethodImpl(MethodImplOptions.AggressiveOptimization)] private void CompressLazy(bool flush) { - var window = this._window; - while (true) { - if (this._lookahead < MinLookahead && (!flush || this._lookahead == 0)) - break; - - var hashHead = this._lookahead >= MinMatch ? this.Insert(this._strStart) : 0; - var prevLength = this._matchLength; - var prevMatch = this._matchStart; - this._matchLength = MinMatch - 1; - - if (hashHead != 0 && prevLength < this._lazy && this._strStart - hashHead <= MaxDist) { - this._matchLength = this.LongestMatch(hashHead, prevLength); + ref var window = ref MemoryMarshal.GetArrayDataReference(this._window); + var tables = new HashTables(this); + var symbols = new SymbolSink(this); + var strStart = this._strStart; + var lookahead = this._lookahead; + var matchStart = this._matchStart; + var matchLength = this._matchLength; + var matchAvailable = this._matchAvailable; + var minimum = flush ? 1 : MinLookahead; + var (good, lazy, niceLimit, chainLimit) = (this._good, this._lazy, this._nice, this._chain); + + while (lookahead >= minimum) { + var candidate = 0; + var candidate3 = 0; + if (lookahead >= MinMatch) + candidate = tables.Insert(ref window, strStart, out candidate3); + + var prevLength = matchLength; + var prevMatch = matchStart; + matchLength = MinMatch - 1; + + if (candidate != 0 && prevLength < lazy && strStart - candidate <= MaxDist) { + matchLength = LongestMatch(ref window, ref tables.Prev, strStart, candidate, prevLength, + prevLength >= good ? chainLimit >> 2 : chainLimit, Math.Min(niceLimit, lookahead), Math.Min(MaxMatch, lookahead), ref matchStart); // A minimal match far away costs more than the three literals it replaces. - if (this._matchLength == MinMatch && this._strStart - this._matchStart > TooFar) - this._matchLength = MinMatch - 1; + if (matchLength == MinMatch && strStart - matchStart > TooFar) + matchLength = MinMatch - 1; } - if (prevLength >= MinMatch && this._matchLength <= prevLength) { - // The match found one position back is at least as good: emit it. - var maxInsert = this._strStart + this._lookahead - MinMatch; - this.TallyMatch(this._strStart - 1 - prevMatch, prevLength); - this._lookahead -= prevLength - 1; - for (var i = prevLength - 2; i > 0; --i) - if (++this._strStart <= maxInsert) - this.Insert(this._strStart); - - this._matchAvailable = false; - this._matchLength = MinMatch - 1; - ++this._strStart; - if (this._symCount == SymbolBufferSize) - this.FlushBlock(last: false); - } else if (this._matchAvailable) { + if (matchLength < MinMatch && prevLength < MinMatch && lookahead >= MinMatch && IsMatch3(ref window, strStart, candidate3, TooFar)) { + matchLength = MinMatch; + matchStart = candidate3; + } + + if (prevLength >= MinMatch && matchLength <= prevLength) { + // The match found one position back is at least as good: emit it, and put the strings + // it covers into the hash. + symbols.Match(strStart - 1 - prevMatch, prevLength); + var end = strStart - 1 + prevLength; + tables.InsertRange(ref window, strStart + 1, Math.Min(end, strStart + lookahead - MinMatch + 1)); + lookahead -= prevLength - 1; + strStart = end; + matchAvailable = false; + matchLength = MinMatch - 1; + if (symbols.IsFull) { + this._strStart = strStart; + symbols.Flush(this); + } + } else if (matchAvailable) { // The previous position did not start a better match than this one: it is a literal. - this.TallyLiteral(window[this._strStart - 1]); - if (this._symCount == SymbolBufferSize) - this.FlushBlock(last: false); - ++this._strStart; - --this._lookahead; + symbols.Literal(Unsafe.Add(ref window, strStart - 1)); + if (symbols.IsFull) { + this._strStart = strStart; + symbols.Flush(this); + } + + ++strStart; + --lookahead; } else { - this._matchAvailable = true; - ++this._strStart; - --this._lookahead; + matchAvailable = true; + ++strStart; + --lookahead; } } - if (flush && this._matchAvailable) { - this.TallyLiteral(window[this._strStart - 1]); - this._matchAvailable = false; + if (flush && matchAvailable) { + symbols.Literal(Unsafe.Add(ref window, strStart - 1)); + matchAvailable = false; } + + this._strStart = strStart; + this._lookahead = lookahead; + this._matchStart = matchStart; + this._matchLength = matchLength; + this._matchAvailable = matchAvailable; + symbols.Store(this); } + /// + /// Whether the three bytes at repeat at + /// within . + /// + /// + /// The chains are keyed on four bytes, so a match of exactly three is found only through this + /// side table, which remembers the latest position of every three-byte hash. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int Insert(int position) { - // Multiplicative hash of the three bytes at position. The fourth byte loaded alongside - // is masked off; the window's tail slack keeps that load in bounds. - var hash = (int)((Load32(this._window, position) & 0xFFFFFF) * 0x9E3779B1u >> (32 - HashBits)); - ref var head = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(this._head), hash); - int previous = head; - Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(this._prev), position & WMask) = (ushort)previous; - head = (ushort)position; - return previous; + private static bool IsMatch3(ref byte window, int position, int candidate, int maxDistance) + => candidate != 0 + && position - candidate <= maxDistance + && ((Load32(ref window, candidate) ^ Load32(ref window, position)) & 0xFFFFFF) == 0; + + /// The hash heads and chain links, as references the hot loops keep in registers. + private readonly ref struct HashTables { + private readonly ref ushort _head; + private readonly ref ushort _head3; + public readonly ref ushort Prev; + + public HashTables(DeflateEncoder encoder) { + this._head = ref MemoryMarshal.GetArrayDataReference(encoder._head); + this._head3 = ref MemoryMarshal.GetArrayDataReference(encoder._head3); + this.Prev = ref MemoryMarshal.GetArrayDataReference(encoder._prev); + } + + /// + /// Links the string at into its chain and returns the previous + /// head (0: none), with the latest earlier position of its first three bytes in + /// . + /// + /// + /// Multiplicative hashes of the four and the three bytes at the position. The bytes past + /// the data that the load may cover are window slack, and only steer the last strings, + /// which no later string searches from. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int Insert(ref byte window, int position, out int candidate3) { + var value = Load32(ref window, position); + ref var head3 = ref Unsafe.Add(ref this._head3, (int)((value << 8) * HashMultiplier >> (32 - Hash3Bits))); + candidate3 = head3; + head3 = (ushort)position; + ref var head = ref Unsafe.Add(ref this._head, (int)(value * HashMultiplier >> (32 - HashBits))); + int previous = head; + Unsafe.Add(ref this.Prev, position & WMask) = (ushort)previous; + head = (ushort)position; + return previous; + } + + /// Inserts every string from up to, not including, . + /// + /// Inside a run of one byte value every string equals the one before it, so its chain link + /// is simply the previous position and the heads only need their final value — written once + /// the run ends instead of being read back and rewritten for every byte. + /// + public void InsertRange(ref byte window, int from, int to) { + if (from >= to) + return; + + var value = Load32(ref window, from); + ref var head3 = ref Unsafe.Add(ref this._head3, (int)((value << 8) * HashMultiplier >> (32 - Hash3Bits))); + ref var head = ref Unsafe.Add(ref this._head, (int)(value * HashMultiplier >> (32 - HashBits))); + Unsafe.Add(ref this.Prev, from & WMask) = head; + for (var position = from + 1; position < to; ++position) { + var next = Load32(ref window, position); + if (next == value) { + // Five equal bytes: a run, whose strings all equal this one until a byte breaks it. + var room = to - position; + var run = MemoryMarshal.CreateReadOnlySpan(ref Unsafe.Add(ref window, position + 3), room).IndexOfAnyExcept((byte)value); + if (run < 0) + run = room; + + this.LinkRun(position, run); + position += run - 1; + continue; + } + + head3 = (ushort)(position - 1); + head = (ushort)(position - 1); + value = next; + head3 = ref Unsafe.Add(ref this._head3, (int)((value << 8) * HashMultiplier >> (32 - Hash3Bits))); + head = ref Unsafe.Add(ref this._head, (int)(value * HashMultiplier >> (32 - HashBits))); + Unsafe.Add(ref this.Prev, position & WMask) = head; + } + + head3 = (ushort)(to - 1); + head = (ushort)(to - 1); + } + + /// Links each of the strings from to the one before it. + private readonly void LinkRun(int start, int count) { + // The links are consecutive positions, stored a vector at a time; a run is shorter than + // the window, so it wraps around the end of the chain table at most once. + var index = start & WMask; + var first = Math.Min(count, WSize - index); + FillSequence(MemoryMarshal.CreateSpan(ref Unsafe.Add(ref this.Prev, index), first), (ushort)(start - 1)); + if (first < count) + FillSequence(MemoryMarshal.CreateSpan(ref this.Prev, count - first), (ushort)(start - 1 + first)); + } + + private static void FillSequence(Span destination, ushort value) { + var i = 0; + if (Vector128.IsHardwareAccelerated && destination.Length >= Vector128.Count) { + var vector = Vector128.Create(value) + Vector128.Create((ushort)0, 1, 2, 3, 4, 5, 6, 7); + var step = Vector128.Create((ushort)Vector128.Count); + for (; i <= destination.Length - Vector128.Count; i += Vector128.Count) { + vector.CopyTo(destination[i..]); + vector += step; + } + } + + for (; i < destination.Length; ++i) + destination[i] = (ushort)(value + i); + } } /// - /// Walks the hash chain from for the longest match at the - /// current position that beats . + /// Walks the hash chain from for the longest match at + /// that beats . /// /// /// Bytes are compared eight at a time. Every read stays inside the window array: the scan /// never passes _strStart + _lookahead + 7 <= 2 * WSize + 7, candidates lie before /// the scan, and the array carries MaxMatch + 8 bytes of slack past the window. /// - private int LongestMatch(int current, int best) { - ref var window = ref MemoryMarshal.GetArrayDataReference(this._window); - ref var prev = ref MemoryMarshal.GetArrayDataReference(this._prev); - var scan = this._strStart; - var chain = this._chain; - var nice = Math.Min(this._nice, this._lookahead); - var maxLength = Math.Min(MaxMatch, this._lookahead); + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)] + private static int LongestMatch(ref byte window, ref ushort prev, int scan, int current, int best, int chain, int nice, int maxLength, ref int matchStart) { var limit = scan > MaxDist ? scan - MaxDist : 0; - if (best >= this._good) - chain >>= 2; - - var scanStart = Unsafe.ReadUnaligned(ref Unsafe.Add(ref window, scan)); - var scanEnd = Unsafe.ReadUnaligned(ref Unsafe.Add(ref window, scan + best - 1)); + var lookahead = maxLength; + ref var scanRef = ref Unsafe.Add(ref window, scan); + var scanStart = Unsafe.ReadUnaligned(ref scanRef); + var scanEnd = Unsafe.ReadUnaligned(ref Unsafe.Add(ref scanRef, best - 1)); + // The window seen from the last two bytes of the best match so far, so a candidate's + // ending bytes are one indexed load away. + ref var windowAtEnd = ref Unsafe.Add(ref window, best - 1); do { // Quick rejects: the two bytes ending at the best length so far, then the first two. - if (Unsafe.ReadUnaligned(ref Unsafe.Add(ref window, current + best - 1)) != scanEnd - || Unsafe.ReadUnaligned(ref Unsafe.Add(ref window, current)) != scanStart) + if (Unsafe.ReadUnaligned(ref Unsafe.Add(ref windowAtEnd, (nuint)(uint)current)) != scanEnd + || Unsafe.ReadUnaligned(ref Unsafe.Add(ref window, (nuint)(uint)current)) != scanStart) continue; - var length = 2; - while (true) { - var diff = Unsafe.ReadUnaligned(ref Unsafe.Add(ref window, current + length)) - ^ Unsafe.ReadUnaligned(ref Unsafe.Add(ref window, scan + length)); - if (diff != 0) { - // The first differing byte is the lowest one in memory order. - length += (BitConverter.IsLittleEndian ? BitOperations.TrailingZeroCount(diff) : BitOperations.LeadingZeroCount(diff)) >> 3; - break; - } - - length += 8; - if (length >= maxLength) - break; - } - - if (length > maxLength) - length = maxLength; - + var length = MatchLength(ref Unsafe.Add(ref window, (nuint)(uint)current), ref scanRef, maxLength); if (length > best) { - this._matchStart = current; + matchStart = current; best = length; if (length >= nice) break; - scanEnd = Unsafe.ReadUnaligned(ref Unsafe.Add(ref window, scan + best - 1)); + scanEnd = Unsafe.ReadUnaligned(ref Unsafe.Add(ref scanRef, best - 1)); + windowAtEnd = ref Unsafe.Add(ref window, best - 1); + } + } while ((current = Unsafe.Add(ref prev, (nuint)(uint)(current & WMask))) > limit && --chain != 0); + + return Math.Min(best, lookahead); + } + + /// Length of the common prefix of two strings whose first two bytes agree, capped at . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int MatchLength(ref byte candidate, ref byte scan, int maxLength) { + var length = 2; + while (true) { + var diff = Unsafe.ReadUnaligned(ref Unsafe.Add(ref candidate, length)) + ^ Unsafe.ReadUnaligned(ref Unsafe.Add(ref scan, length)); + if (diff != 0) { + // The first differing byte is the lowest one in memory order. + length += (BitConverter.IsLittleEndian ? BitOperations.TrailingZeroCount(diff) : BitOperations.LeadingZeroCount(diff)) >> 3; + return Math.Min(length, maxLength); } - } while ((current = Unsafe.Add(ref prev, current & WMask)) > limit && --chain != 0); - return Math.Min(best, this._lookahead); + length += 8; + if (length >= maxLength) + return maxLength; + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static uint Load32(byte[] array, int index) { - var value = Unsafe.ReadUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(array), index)); + private static uint Load32(ref byte window, int index) { + var value = Unsafe.ReadUnaligned(ref Unsafe.Add(ref window, index)); return BitConverter.IsLittleEndian ? value : BinaryPrimitives.ReverseEndianness(value); } @@ -302,6 +472,7 @@ private void Slide() { SlideTable(this._head); SlideTable(this._prev); + SlideTable(this._head3); } /// @@ -310,7 +481,7 @@ private void Slide() { /// /// /// max(v, WSize) - WSize is exactly that clamp for unsigned values, so whole vectors - /// go at once; both tables are a multiple of every vector width in length. + /// go at once; every table is a multiple of every vector width in length. /// private static void SlideTable(ushort[] table) { var window = new Vector(WSize); @@ -321,26 +492,52 @@ private static void SlideTable(ushort[] table) { // ── symbols ────────────────────────────────────────────────────────── - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void TallyLiteral(byte value) { - this._symLength[this._symCount] = value; - this._symDistance[this._symCount++] = 0; - ++this._litFreq[value]; - } + /// The block's symbol buffer and frequency counts, as references the hot loops keep in registers. + private ref struct SymbolSink { + private readonly ref ushort _litLen; + private readonly ref ushort _distance; + private readonly ref long _litFreq; + private readonly ref long _distFreq; + private int _count; + + public SymbolSink(DeflateEncoder encoder) { + this._litLen = ref MemoryMarshal.GetArrayDataReference(encoder._symLitLen); + this._distance = ref MemoryMarshal.GetArrayDataReference(encoder._symDistance); + this._litFreq = ref MemoryMarshal.GetArrayDataReference(encoder._litFreq); + this._distFreq = ref MemoryMarshal.GetArrayDataReference(encoder._distFreq); + this._count = encoder._symCount; + } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void TallyMatch(int distance, int length) { - this._symLength[this._symCount] = (byte)(length - MinMatch); - this._symDistance[this._symCount++] = (ushort)distance; - ++this._litFreq[257 + LengthCodes[length - MinMatch]]; - ++this._distFreq[DistanceCode(distance)]; + public readonly bool IsFull => this._count == SymbolBufferSize; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Literal(byte value) { + Unsafe.Add(ref this._litLen, this._count) = value; + Unsafe.Add(ref this._distance, this._count++) = 0; + ++Unsafe.Add(ref this._litFreq, value); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Match(int distance, int length) { + Unsafe.Add(ref this._litLen, this._count) = (ushort)(256 + length - MinMatch); + Unsafe.Add(ref this._distance, this._count++) = (ushort)distance; + ++Unsafe.Add(ref this._litFreq, 257 + LengthCodes[length - MinMatch]); + ++Unsafe.Add(ref this._distFreq, DistanceCodes[DistanceSlot(distance)]); + } + + /// Emits the buffered symbols as a block ending at the encoder's _strStart, and starts over. + public void Flush(DeflateEncoder encoder) { + encoder._symCount = this._count; + encoder.FlushBlock(last: false); + this._count = 0; + } + + public readonly void Store(DeflateEncoder encoder) => encoder._symCount = this._count; } + /// Index of in ; 0 for no distance. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int DistanceCode(int distance) { - var d = distance - 1; - return d < 256 ? DistanceCodes[d] : DistanceCodes[256 + (d >> 7)]; - } + private static int DistanceSlot(int distance) => distance <= 256 ? distance : 256 + ((distance - 1) >> 7); // ── blocks ─────────────────────────────────────────────────────────── @@ -422,39 +619,100 @@ private void EmitStored(ReadOnlySpan data, bool last) { } while (!data.IsEmpty); } + /// Writes the buffered symbols and the end-of-block code with the given codes. + /// + /// Every code is first folded into a table entry: a literal's code, or a length's code + /// together with its extra bits, in one; a distance code with its length, extra-bit count and + /// base in another. A literal looks up the distance entry of distance 0, which writes + /// nothing, so literals and matches take the same branch-free path. + /// The bit accumulator is spilled after every symbol by an unconditional eight-byte + /// store, of which only the whole bytes count: at most 7 bits stay behind, a symbol adds at + /// most 15 + 5 + 15 + 13 = 48, so the accumulator never overflows, and keeping eight bytes + /// of room in the output buffer is the only check per symbol. + /// + [MethodImpl(MethodImplOptions.AggressiveOptimization)] private void EmitSymbols(int[] litLengths, ushort[] litCodes, int[] distLengths, ushort[] distCodes) { - var lengthExtra = DeflateConstants.LengthExtraBits; + var litLen = this._litLenEntries; + for (var symbol = 0; symbol < 256; ++symbol) + litLen[symbol] = litCodes[symbol] | (ulong)litLengths[symbol] << 32; + var lengthBase = DeflateConstants.LengthBase; - var distExtra = DeflateConstants.DistanceExtraBits; - var distBase = DeflateConstants.DistanceBase; + var lengthExtra = DeflateConstants.LengthExtraBits; + for (var value = 0; value <= MaxMatch - MinMatch; ++value) { + var code = LengthCodes[value]; + var count = litLengths[257 + code]; + litLen[256 + value] = litCodes[257 + code] | (ulong)(value + MinMatch - lengthBase[code]) << count | (ulong)(count + lengthExtra[code]) << 32; + } - for (var i = 0; i < this._symCount; ++i) { - int distance = this._symDistance[i]; - int value = this._symLength[i]; - if (distance == 0) { - this.PutBits(litCodes[value], litLengths[value]); - continue; + var distances = this._distEntries; + var distBase = DeflateConstants.DistanceBase; + var distExtra = DeflateConstants.DistanceExtraBits; + for (var code = 0; code < DeflateConstants.DistanceAlphabetSize; ++code) + distances[code] = distCodes[code] | (ulong)distLengths[code] << 32 | (ulong)distExtra[code] << 40 | (ulong)distBase[code] << 48; + + ref var litLenRef = ref MemoryMarshal.GetArrayDataReference(litLen); + ref var distRef = ref MemoryMarshal.GetArrayDataReference(distances); + ref var slots = ref MemoryMarshal.GetArrayDataReference(DistanceCodes); + ref var symLitLen = ref MemoryMarshal.GetArrayDataReference(this._symLitLen); + ref var symDistance = ref MemoryMarshal.GetArrayDataReference(this._symDistance); + var output = this._out; + var room = output.Length - 8; + + // The header may have left up to 31 bits; bring that down to the at most 7 the loop expects. + if (this._outPos > room) + this.FlushOutput(); + var bitBuf = this._bitBuf; + var bitCount = this._bitCount; + var outPos = this._outPos; + Spill(output, ref outPos, ref bitBuf, ref bitCount); + + var symbols = this._symCount; + for (var i = 0; i < symbols; ++i) { + if (outPos > room) { + this._outPos = outPos; + this.FlushOutput(); + outPos = 0; } - var lengthCode = LengthCodes[value]; - var symbol = 257 + lengthCode; - var bits = (ulong)litCodes[symbol]; - var count = litLengths[symbol]; - var extra = lengthExtra[lengthCode]; - bits |= (ulong)(value + MinMatch - lengthBase[lengthCode]) << count; - count += extra; - - var distCode = DistanceCode(distance); - bits |= (ulong)distCodes[distCode] << count; - count += distLengths[distCode]; - bits |= (ulong)(distance - distBase[distCode]) << count; - count += distExtra[distCode]; - - // At most 15 + 5 + 15 + 13 = 48 bits. - this.PutBits(bits, count); + var entry = Unsafe.Add(ref litLenRef, Unsafe.Add(ref symLitLen, i)); + int distance = Unsafe.Add(ref symDistance, i); + var distEntry = Unsafe.Add(ref distRef, Unsafe.Add(ref slots, DistanceSlot(distance))); + var bits = (ulong)(uint)entry; + var count = (int)(byte)(entry >> 32); + bits |= (distEntry & 0xFFFF) << count; + count += (byte)(distEntry >> 32); + bits |= (ulong)(distance - (int)(distEntry >> 48)) << count; + count += (byte)(distEntry >> 40); + + bitBuf |= bits << bitCount; + bitCount += count; + Spill(output, ref outPos, ref bitBuf, ref bitCount); } - this.PutBits(litCodes[DeflateConstants.EndOfBlock], litLengths[DeflateConstants.EndOfBlock]); + if (outPos > room) { + this._outPos = outPos; + this.FlushOutput(); + outPos = 0; + } + + bitBuf |= (ulong)litCodes[DeflateConstants.EndOfBlock] << bitCount; + bitCount += litLengths[DeflateConstants.EndOfBlock]; + Spill(output, ref outPos, ref bitBuf, ref bitCount); + + this._bitBuf = bitBuf; + this._bitCount = bitCount; + this._outPos = outPos; + } + + /// Stores the accumulator's whole bytes; needs eight bytes of room at . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Spill(byte[] output, ref int outPos, ref ulong bitBuf, ref int bitCount) { + Unsafe.WriteUnaligned(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(output), outPos), + BitConverter.IsLittleEndian ? bitBuf : BinaryPrimitives.ReverseEndianness(bitBuf)); + var bytes = bitCount >> 3; + outPos += bytes; + bitBuf >>= bytes << 3; + bitCount &= 7; } // ── bit output ─────────────────────────────────────────────────────── @@ -532,11 +790,13 @@ private static byte[] BuildLengthCodes() { } // Distances 1–256 directly, larger ones by (distance - 1) >> 7, which is exact because every - // code above 15 spans a multiple of 128 distances. + // code above 15 spans a multiple of 128 distances. Slot 0 — a literal's distance — maps to a + // code past the alphabet, whose emit entry writes nothing. private static byte[] BuildDistanceCodes() { var table = new byte[512]; - for (var d = 0; d < 256; ++d) - table[d] = (byte)DeflateConstants.GetDistanceCode(d + 1); + table[0] = DeflateConstants.DistanceAlphabetSize; + for (var d = 1; d <= 256; ++d) + table[d] = (byte)DeflateConstants.GetDistanceCode(d); for (var i = 2; i < 256; ++i) table[256 + i] = (byte)DeflateConstants.GetDistanceCode((i << 7) + 1); return table; diff --git a/Compression.Core/README.md b/Compression.Core/README.md index a2faf4ba2..c28960650 100644 --- a/Compression.Core/README.md +++ b/Compression.Core/README.md @@ -222,7 +222,7 @@ Audio, archive and filesystem functionality lives in the separate package projec - A common API shape does not imply identical streaming, memory, or parameter semantics across every algorithm; use concrete APIs when those distinctions matter. - Some encoders intentionally implement a standards-compliant subset while decoders accept a wider format. -- Pure managed code is the design goal, not an automatic speed claim. BCL/native implementations may be faster for common algorithms on some workloads: Deflate compresses to the same ratio as the zlib-ng behind `System.IO.Compression` but at roughly half its speed at the default level, and decompresses somewhat slower. In exchange the output is identical on every platform. +- Pure managed code is the design goal, not an automatic speed claim. BCL/native implementations may be faster for common algorithms on some workloads: Deflate compresses to the same or a better ratio than the zlib-ng behind `System.IO.Compression` at every level and decompresses 10–30% faster, but compresses text 15–25% slower at levels 6–9 (zlib-ng's level 6 uses a cheaper strategy that compresses worse) and about 40% slower at level 1, where zlib-ng's output is 20–25% larger. In exchange the output is identical on every platform. `dotnet run -c Release --project Compression.Benchmarks -- deflate-table` measures it on your machine. - Do not infer a package, format, algorithm, profile, or release state from roadmap intent. Checked-in project files, the compiled registry/public API, and tests are the evidence sources. - Do not state volatile algorithm counts unless they are generated from the registry/build. - When code and prose disagree, the compiled registry/API and tests win; update the prose. diff --git a/Compression.Tests/Deflate/DeflateEngineTests.cs b/Compression.Tests/Deflate/DeflateEngineTests.cs index c83b7786b..d6c373819 100644 --- a/Compression.Tests/Deflate/DeflateEngineTests.cs +++ b/Compression.Tests/Deflate/DeflateEngineTests.cs @@ -130,8 +130,8 @@ public void Output_IsIdenticalOnEveryPlatform() { }); } - private const string PinnedFast = "A528579C874D21C6C31D70D872CE59C1B74F23E3A1EF291C5F4C005AB32AF755"; - private const string PinnedDefault = "00AE2FF7031DA36A3F48201EAEC2CC438A2736EA2E53864FC7D1C2F01E377E33"; + private const string PinnedFast = "12921C92F566F8B78357C6BB8E345701EAB6573DDB53A47F954EC806028ED980"; + private const string PinnedDefault = "0E0F24602132F44EAED9559B30B4FFD853546217F8851EEDD8338A844EE65EF4"; private const string PinnedBest = "9BF1601DEF1F86D8911FC74C10BB867A03DB3A5D831B9A82E6FFBB1DE7DCF100"; private static string Sha(byte[] data) => Convert.ToHexString(SHA256.HashData(data)); diff --git a/Compression.Tests/Deflate/DeflateFastPathTests.cs b/Compression.Tests/Deflate/DeflateFastPathTests.cs new file mode 100644 index 000000000..b5547a79a --- /dev/null +++ b/Compression.Tests/Deflate/DeflateFastPathTests.cs @@ -0,0 +1,269 @@ +using Compression.Core.Deflate; +using Bcl = System.IO.Compression; + +namespace Compression.Tests.Deflate; + +/// +/// The encoder's four-byte chains with their three-byte side table and run linking, and the +/// decoder's unchecked fast loop with its careful edge loop: round trips across every level, +/// data shape and boundary size, both directions against the platform's zlib, independence +/// from how input and output are chunked, and rejection — never a crash — of damaged input. +/// +[TestFixture] +public class DeflateFastPathTests { + + public enum Shape { Text, Random, Runs, ShortRepeats, Mixed } + + private static byte[] Make(Shape shape, int size, int seed = 11) { + var rng = new Random(seed); + var data = new byte[size]; + switch (shape) { + case Shape.Text: { + string[] words = ["the", "quick", "brown", "fox", "jumps", "over", "lazy", "dog", "deflate", "window", + "huffman", "length", "distance", "block", "stream", "header", "trailer", "checksum", "archive", "format"]; + var text = new System.Text.StringBuilder(size + 16); + while (text.Length < size) + text.Append(words[rng.Next(words.Length)]).Append(rng.Next(12) == 0 ? ".\n" : " "); + return System.Text.Encoding.ASCII.GetBytes(text.ToString(0, size)); + } + case Shape.Random: + rng.NextBytes(data); + return data; + case Shape.Runs: + // Runs of one value, long and short, with a little noise between them. + for (var i = 0; i < size;) { + var run = Math.Min(rng.Next(4) == 0 ? rng.Next(1, 1200) : rng.Next(1, 9), size - i); + data.AsSpan(i, run).Fill((byte)rng.Next(4)); + i += run; + if (i < size && rng.Next(3) == 0) + data[i++] = (byte)rng.Next(256); + } + + return data; + case Shape.ShortRepeats: { + // A small vocabulary of three-byte tokens between random bytes, as in machine code: + // plenty of matches of exactly three bytes, few longer ones. + var tokens = new byte[48][]; + for (var t = 0; t < tokens.Length; ++t) { + tokens[t] = new byte[3]; + rng.NextBytes(tokens[t]); + } + + for (var i = 0; i < size;) { + if (rng.Next(3) == 0) { + data[i++] = (byte)rng.Next(256); + continue; + } + + var token = tokens[rng.Next(tokens.Length)]; + var n = Math.Min(3, size - i); + token.AsSpan(0, n).CopyTo(data.AsSpan(i)); + i += n; + } + + return data; + } + default: { + var shapes = new[] { Shape.Text, Shape.Random, Shape.Runs, Shape.ShortRepeats }; + for (var i = 0; i < size;) { + var n = Math.Min(rng.Next(1, 40_000), size - i); + Make(shapes[rng.Next(shapes.Length)], n, rng.Next()).CopyTo(data, i); + i += n; + } + + return data; + } + } + } + + private static byte[] BclInflate(byte[] compressed) { + using var input = new MemoryStream(compressed); + using var inflate = new Bcl.DeflateStream(input, Bcl.CompressionMode.Decompress); + using var output = new MemoryStream(); + inflate.CopyTo(output); + return output.ToArray(); + } + + private static byte[] BclDeflate(byte[] data, int level) { + using var output = new MemoryStream(); + using (var deflate = new Bcl.DeflateStream(output, new Bcl.ZLibCompressionOptions { CompressionLevel = level }, leaveOpen: true)) + deflate.Write(data); + return output.ToArray(); + } + + public static IEnumerable Levels => Enumerable.Range(1, 9); + + // Empty and tiny inputs, the match limits (3, 258, MinLookahead 262), a symbol buffer's worth + // (16 Ki), the farthest distance the encoder emits (32768 - 262) and the window, its + // double (where the buffer slides) and past that. + public static IEnumerable Sizes => [0, 1, 2, 3, 4, 5, 257, 258, 259, 262, 263, 16383, 16384, 16385, + 32505, 32506, 32507, 32767, 32768, 32769, 65535, 65536, 65537, 98305, 200_003]; + + // ── round trips ─────────────────────────────────────────────────────── + + [Test, Category("RoundTrip"), Category("Boundary")] + public void EveryLevelShapeAndSize_RoundTrips_ThroughBothDecoders( + [ValueSource(nameof(Levels))] int level, + [Values] Shape shape, + [ValueSource(nameof(Sizes))] int size) { + var data = Make(shape, size); + + var compressed = DeflateCompressor.Compress(data, (DeflateCompressionLevel)level); + + Assert.That(DeflateDecompressor.Decompress(compressed), Is.EqualTo(data), "our decoder"); + Assert.That(BclInflate(compressed), Is.EqualTo(data), "platform zlib"); + } + + [Test, Category("ThemVsUs"), Category("RoundTrip")] + public void PlatformZlibOutput_AtEveryLevel_IsReadByOurDecoder( + [Values(1, 2, 6, 9)] int zlibLevel, + [Values] Shape shape) { + // zlib-ng's level 1 writes fixed-Huffman blocks, the others dynamic ones with their own + // block boundaries and code-length choices. + var data = Make(shape, 300_000, seed: 5); + + var compressed = BclDeflate(data, zlibLevel); + + Assert.That(DeflateDecompressor.Decompress(compressed), Is.EqualTo(data)); + } + + [Test, Category("RoundTrip"), Category("EdgeCase")] + public void SkewedAlphabets_ForceSubTables_RoundTrip([Values(1, 6, 9)] int level) { + // Literal frequencies halving from one value to the next give codes of up to 15 bits, + // past the decoder's 11-bit root table; distances spread over many orders of magnitude + // do the same for the 8-bit distance root. + var rng = new Random(3); + using var buffer = new MemoryStream(); + while (buffer.Length < 400_000) { + var value = 0; + while (value < 40 && rng.Next(2) == 0) + ++value; + buffer.WriteByte((byte)value); + + if (rng.Next(16) == 0 && buffer.Length > 40_000) { + var distance = 1 + (int)Math.Min(32_000, Math.Pow(2, rng.NextDouble() * 15)); + var length = rng.Next(3, 40); + for (var i = 0; i < length; ++i) + buffer.WriteByte(buffer.GetBuffer()[buffer.Length - distance]); + } + } + + var data = buffer.ToArray(); + foreach (var compressed in new[] { DeflateCompressor.Compress(data, (DeflateCompressionLevel)level), BclDeflate(data, level) }) + Assert.That(DeflateDecompressor.Decompress(compressed), Is.EqualTo(data)); + } + + // ── determinism ─────────────────────────────────────────────────────── + + [Test, Category("HappyPath")] + public void Output_DoesNotDependOnWriteSplits( + [Values(1, 3, 4, 6, 9)] int level, + [Values(Shape.Runs, Shape.ShortRepeats, Shape.Mixed)] Shape shape, + [Values(1, 5, 4096, 65543)] int chunk) { + var data = Make(shape, 150_000, seed: 8); + var whole = DeflateCompressor.Compress(data, (DeflateCompressionLevel)level); + + using var output = new MemoryStream(); + var compressor = new DeflateCompressor(output, (DeflateCompressionLevel)level); + for (var offset = 0; offset < data.Length; offset += chunk) + compressor.Write(data.AsSpan(offset, Math.Min(chunk, data.Length - offset))); + compressor.Finish(); + + Assert.That(output.ToArray(), Is.EqualTo(whole)); + } + + [Test, Category("ThemVsUs")] + public void Ratio_NotWorseThanPlatformZlib_AtTheSameLevel([Values(6, 9)] int level, [Values(Shape.Text, Shape.ShortRepeats)] Shape shape) { + // Levels 6 and 9 run zlib's own search limits, so they should land where zlib-ng does or + // better; a percent of slack absorbs the two encoders' different block boundaries. + var data = Make(shape, 500_000, seed: 2); + + var ours = DeflateCompressor.Compress(data, (DeflateCompressionLevel)level).Length; + var platform = BclDeflate(data, level).Length; + + Assert.That(ours, Is.LessThanOrEqualTo(platform * 1.01), $"ours {ours} vs zlib-ng {platform}"); + } + + // ── decoder chunking ────────────────────────────────────────────────── + + [Test, Category("EdgeCase")] + public void Decoder_AnyInputAndOutputChunking_GivesTheSameBytes( + [Values(1, 13, 4096)] int maxRead, + [Values(1, 300, 65536)] int outputChunk) { + var data = Make(Shape.Mixed, 180_000, seed: 4); + var compressed = DeflateCompressor.Compress(data, DeflateCompressionLevel.Default); + + var decoder = new DeflateDecompressor(new ChunkedStream(compressed, maxRead)); + var output = new byte[data.Length + 1]; + var total = 0; + int n; + while ((n = decoder.Decompress(output.AsSpan(total, Math.Min(outputChunk, output.Length - total)))) > 0) + total += n; + + Assert.That(output.AsSpan(0, total).ToArray(), Is.EqualTo(data)); + } + + // ── decoder hardening ───────────────────────────────────────────────── + + [Test, Category("ErrorHandling")] + public void Decoder_DamagedStreams_AreRejectedOrDecoded_NeverCrash([Values(1, 6)] int level, [Values(Shape.Text, Shape.Mixed)] Shape shape) { + // Flipped bits anywhere — header, tables, symbols — must end in InvalidDataException or + // EndOfStreamException (or, by chance, some output), never an out-of-range access. + var data = Make(shape, 40_000, seed: 6); + var compressed = DeflateCompressor.Compress(data, (DeflateCompressionLevel)level); + var rng = new Random(level * 31 + (int)shape); + + for (var trial = 0; trial < 600; ++trial) { + var damaged = (byte[])compressed.Clone(); + for (var flips = rng.Next(1, 4); flips > 0; --flips) + damaged[rng.Next(damaged.Length)] ^= (byte)(1 << rng.Next(8)); + + try { + DeflateDecompressor.Decompress(damaged); + } catch (InvalidDataException) { + } catch (EndOfStreamException) { + } + } + } + + [Test, Category("ErrorHandling")] + public void Decoder_EveryTruncation_IsRejectedOrAPrefix() { + var data = Make(Shape.Mixed, 6_000, seed: 9); + var compressed = DeflateCompressor.Compress(data, DeflateCompressionLevel.Default); + + for (var length = 0; length < compressed.Length; ++length) { + byte[] decoded; + try { + decoded = DeflateDecompressor.Decompress(compressed.AsSpan(0, length)); + } catch (EndOfStreamException) { + continue; + } catch (InvalidDataException) { + continue; + } + + // A stream cut exactly between two blocks reads as ending there; whatever is decoded + // must be the start of the original, never garbage. + Assert.That(data.AsSpan().StartsWith(decoded), Is.True, $"a {length}-byte prefix decoded to something else"); + } + } + + /// A forward-only stream that hands out at most maxRead bytes per read. + private sealed class ChunkedStream(byte[] data, int maxRead) : Stream { + private int _position; + public override int Read(byte[] buffer, int offset, int count) { + var n = Math.Min(Math.Min(count, maxRead), data.Length - this._position); + Array.Copy(data, this._position, buffer, offset, n); + this._position += n; + return n; + } + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override void Flush() { } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } +} diff --git a/docs/screenshots/archive-browser.png b/docs/screenshots/archive-browser.png index 45f9f057b..b122a5b70 100644 Binary files a/docs/screenshots/archive-browser.png and b/docs/screenshots/archive-browser.png differ diff --git a/screenshots/main-window.png b/screenshots/main-window.png index da3089a3a..b122a5b70 100644 Binary files a/screenshots/main-window.png and b/screenshots/main-window.png differ