From 17a46a16d367c4270707cd703b3141330e2da4cc Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 16 Oct 2026 18:39:38 +0300 Subject: [PATCH 01/11] feat(encoding): long-distance matching for every strategy Long-distance matching only ran under the optimal parsers: the producer lived on the binary-tree backend, so below btopt the setting was silently ignored by every API, and the CLI refused `--long` under level 16. - The match driver owns the long-distance matcher for the frame, as upstream keeps `ldmState` in the context. The optimal parser still takes its sequences as candidates; every other strategy takes them outright (upstream `ZSTD_ldm_blockCompress`): the block is committed, handed back and committed again in pieces, the backend matches each gap between matches as its own block, a match body is indexed only at its tail, and the repeat offsets move as upstream moves them. - What a match may reach is fixed against the block end, one window back or the whole dictionary while the window has not passed it (`ZSTD_window_enforceMaxDist` on the chunk end). - A dictionary loaded under long-distance matching is indexed into its table (`ZSTD_ldm_fillHashTable` from `ZSTD_loadDictionaryContent`); the primed snapshot keeps that table, keyed on its parameters, so a frame restored from it writes what a freshly primed one writes. - A block is no longer written off by the raw-skip sample while the matcher is on: its matches are external to the block. - CLI: `--long` runs at every level, and `--patch-from` turns long mode on by upstream's rule (`FIO_adjustParamsForPatchFromMode`: `fileWindowLog > ZSTD_cycleLog(chainLog, strategy)`). A 61 MiB patch against a 60 MiB random reference at the default level drops from 62.6 MB to 76 KB (upstream 72 KB) and round-trips through both decoders. --- README.md | 6 +- zstd/src/bin/structured-zstd/main.rs | 61 +-- zstd/src/bin/structured-zstd/tests.rs | 128 +++-- zstd/src/encoding/bt/ldm_helper_tests.rs | 59 --- zstd/src/encoding/bt/mod.rs | 137 +----- zstd/src/encoding/dfast/mod.rs | 32 ++ zstd/src/encoding/hc/generator.rs | 25 - zstd/src/encoding/hc/optimal.rs | 22 - zstd/src/encoding/ldm/mod.rs | 126 +++-- zstd/src/encoding/ldm/params.rs | 2 + zstd/src/encoding/ldm/table.rs | 25 +- zstd/src/encoding/levels/fastest.rs | 5 +- .../encoding/match_generator/dict_prime.rs | 59 ++- zstd/src/encoding/match_generator/mod.rs | 451 +++++++++++++----- zstd/src/encoding/match_table/storage.rs | 42 ++ zstd/src/encoding/mod.rs | 7 + zstd/src/encoding/parameters.rs | 6 +- zstd/src/encoding/row/mod.rs | 36 ++ zstd/src/encoding/simple/fast_matcher.rs | 43 ++ zstd/src/tests/parameters_test.rs | 145 ++++++ 20 files changed, 924 insertions(+), 493 deletions(-) diff --git a/README.md b/README.md index bd4332f38..5df36702e 100644 --- a/README.md +++ b/README.md @@ -99,9 +99,9 @@ does, with the window stopped at 30 (the widest this build writes), and upstream's layout. `--target-compressed-block-size` does take effect: it bounds what goes into a block, so blocks flush sooner. `--long` means `--long=27`, as upstream -documents. It needs level 16 or above, where long-distance matching actually -runs — below that it is refused rather than accepted as a wider window and -nothing else. A window is never declared larger than the source can fill, so a +documents, and runs long-distance matching at every level. `--patch-from` +turns it on itself when the input outgrows what the level's match finder +reaches, by upstream's rule. A window is never declared larger than the source can fill, so a small file compressed with `--long` does not ask its decoders to reserve 128 MiB. Decoding takes windows up to 128 MiB by default; a wider frame (from `--long=28` and up) is refused with upstream's message, and decodes with diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index bc87a4827..0ca1b6492 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -360,11 +360,6 @@ const MIN_BENCH_BLOCK_SIZE: u64 = 32; /// Window log a bare `--long` selects, as upstream documents (128 MiB). const DEFAULT_LONG_WINDOW_LOG: u32 = 27; -/// Lowest level whose matcher actually runs long-distance matching. Below it -/// the encoder uses a strategy that carries no long-distance producer, so -/// `--long` would be a wider window and nothing else. -const MIN_LONG_LEVEL: i32 = 16; - /// Which spellings of a size suffix [`read_leading_number`] reads. #[derive(Clone, Copy, PartialEq, Eq)] enum SuffixCase { @@ -1764,29 +1759,6 @@ fn parse_args_into( opts.bench_start = opts.bench_start.min(CLI_MAX_LEVEL_WITHOUT_ULTRA); opts.bench_end = opts.bench_end.min(CLI_MAX_LEVEL_WITHOUT_ULTRA); } - // Long-distance matching runs on the optimal parser here, so below it the - // flag would widen the window and never run the matcher it names. Settled - // after the whole command line, since the level may follow the flag — and - // read from the benchmark range when there is one, because those are the - // levels that will actually compress. The whole range runs with the flag, - // so its lowest level is the one that has to carry the matcher. - let long_level = if opts.bench { - opts.bench_start.min(opts.bench_end) - } else { - opts.level - }; - // A `--zstd=strat=` override onto the optimal parser carries the matcher - // whatever the level says. - let optimal_strategy = opts - .advanced - .strategy - .is_some_and(|strategy| strategy >= Strategy::Btopt); - if opts.long && compresses(&opts) && long_level < MIN_LONG_LEVEL && !optimal_strategy { - bail!( - "--long needs level {MIN_LONG_LEVEL} or above, where long-distance \ - matching runs; at level {long_level} it would only widen the window", - ); - } // Window logs are validated here, before any file is opened, against the // mode the whole command line settled on; a knob out of its range is a // broken command line. @@ -2173,6 +2145,22 @@ fn patch_window_log(source_size: u64) -> Result { Ok(file_window_log.max(lower)) } +/// Whether a `--patch-from` compression of `source_size` bytes against a +/// `reference_size`-byte reference needs long-distance matching to reach the +/// reference: when the file's window outgrows what the level's match finder +/// cycles through, as the reference command decides it +/// (`FIO_adjustParamsForPatchFromMode`, `fileWindowLog > ZSTD_cycleLog(chainLog, +/// strategy)` on the level's parameters for those sizes, the binary-tree +/// strategies cycling one log short of their chain). +fn patch_needs_long_distance(level: i32, source_size: u64, reference_size: u64) -> bool { + let file_window_log = u64::BITS - source_size.max(1).leading_zeros(); + // A reference past the address space cannot have been read to get here. + let reference_size = usize::try_from(reference_size).unwrap_or(usize::MAX); + let params = LevelParameters::for_level(level, Some(source_size), reference_size); + let cycle_log = params.chain_log - u32::from(params.strategy >= Strategy::Btlazy2); + file_window_log > cycle_log +} + /// Whether the run reads the `-D` / `--patch-from` file it names. Listing /// walks frame headers and training builds a dictionary from its samples; /// neither consults that one. Reading it anyway would fail a listing over a @@ -2766,15 +2754,14 @@ fn run_selected(mut opts: Options) -> Result { _ => bail!("Using --patch-from with stdin requires --stream-size"), }; opts.advanced.window_log = Some(patch_window_log(source_size)?); - // Long-distance matching is what finds the reference across a - // window this wide; it runs on the optimal parser here, so it is - // switched on where that parser runs. - let optimal = opts.level >= MIN_LONG_LEVEL - || opts - .advanced - .strategy - .is_some_and(|strategy| strategy >= Strategy::Btopt); - if optimal { + let reference = opts.patch_from.as_deref().expect("set in this branch"); + let reference_size = fs::metadata(reference) + .map_err(|err| eyre!("can't stat {} : {err}", reference.display()))? + .len(); + if patch_needs_long_distance(opts.level, source_size, reference_size) { + if !opts.long { + display!(opts.verbosity, 2, "long mode automatically triggered"); + } opts.long = true; } } diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index f2a18cc61..81eb8ba2c 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -2139,33 +2139,101 @@ fn an_explicit_window_is_capped_by_a_known_source_size() { ); } -/// Long-distance matching lives on the optimal parser here, so the levels below -/// it can widen the window and still never run the matcher the flag names. -/// Taking `--long` at those levels would report success for work that did not -/// happen, and hand back a file compressed the ordinary way. -#[test] -fn long_is_refused_at_levels_that_cannot_run_it() { - assert!(parse(&["-3", "--long", "in.txt"]).is_err()); - assert!(parse(&["--long", "-15", "in.txt"]).is_err()); - // From the optimal parser up, the matcher is there to run. - assert!(parse(&["-16", "--long", "in.txt"]).is_ok()); +/// `--long` runs long-distance matching at every level, as upstream's does: its +/// own tests compress with `--long` at levels 1 and 3 (`tests/playTests.sh`). +#[test] +fn long_is_taken_at_every_level() { + for level in 1..=19 { + let flag = format!("-{level}"); + assert!( + parse(&[&flag, "--long", "in.txt"]).unwrap().long, + "level {level}" + ); + } + assert!(parse(&["--fast=3", "--long", "in.txt"]).unwrap().long); assert!(parse(&["--ultra", "-22", "--long=27", "in.txt"]).is_ok()); - // Decompression takes the flag as the window hint it is; nothing to run. assert!(parse(&["-d", "--long", "in.txt.zst"]).is_ok()); - // Benchmarking compresses the levels `-b`/`-e` name, not the one `-N` set, - // so those are the levels the flag has to be true of. - assert!(parse(&["-b16", "--long", "in.txt"]).is_ok()); - assert!(parse(&["-b16", "-e19", "--long", "in.txt"]).is_ok()); - assert!(parse(&["-b3", "--long", "in.txt"]).is_err()); - // The whole range runs with the flag, so a range that starts below the - // matcher is refused even when it ends above it. - assert!(parse(&["-b3", "-e19", "--long", "in.txt"]).is_err()); - // And a benchmark range is what counts, not a level that was also typed. - assert!(parse(&["-16", "-b3", "--long", "in.txt"]).is_err()); - // Benchmarking compresses whatever mode was asked for, so `-d` does not - // excuse the flag from naming levels that can run it. - assert!(parse(&["-d", "-b3", "--long", "in.txt"]).is_err()); - assert!(parse(&["-d", "-b16", "--long", "in.txt"]).is_ok()); + assert!(parse(&["-b3", "-e19", "--long", "in.txt"]).is_ok()); + assert!(parse(&["-3", "--long", "--zstd=strat=6", "in.txt"]).is_ok()); +} + +/// Past the window a level keeps, `--long` is what finds a repeat: a copy laid +/// out in shuffled pieces up to 4 MiB behind, beyond level 3's 2 MiB window, +/// costs next to nothing with the flag and nearly its whole size without it. +#[test] +fn long_finds_a_far_repeat_at_a_low_level() { + const PIECE: usize = 4096; + let mut state = 0x5EED_0569u64; + // Six-bit symbols: compressible enough that no block is written off, and + // short local matches stay rare. + let half: Vec = (0..4 << 20) + .map(|_| { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + ((state >> 33) & 63) as u8 + }) + .collect(); + let pieces = half.len() / PIECE; + let mut data = half.clone(); + for i in 0..pieces { + let piece = (i * 37) % pieces; + data.extend_from_slice(&half[piece * PIECE..(piece + 1) * PIECE]); + } + let compress = |long: bool| { + let mut frame = Vec::new(); + compress_stream( + data.as_slice(), + &mut frame, + &FrameSettings { + level: 3, + long, + long_window_log: long.then_some(DEFAULT_LONG_WINDOW_LOG), + ..FrameSettings::default() + }, + &mut no_dict(), + ) + .expect("compressing succeeds"); + frame + }; + let without = compress(false); + let with = compress(true); + assert!( + with.len() * 3 < without.len() * 2, + "--long at level 3: {} bytes against {} without it", + with.len(), + without.len() + ); + assert_eq!(decoded(&with).unwrap(), data); +} + +/// `--patch-from` turns long mode on itself when the input outgrows what the +/// level's match finder cycles through, by upstream's rule +/// (`FIO_adjustParamsForPatchFromMode`: `fileWindowLog > +/// ZSTD_cycleLog(chainLog, strategy)`, the binary-tree strategies cycling one +/// log short of their chain). +#[test] +fn patch_from_needs_long_mode_where_the_matcher_cannot_cycle_the_input() { + // Level 3 on 61 MiB against 60 MiB: dfast, chainLog 16, file window log 26. + assert!(patch_needs_long_distance(3, 61 << 20, 60 << 20)); + // A small patch fits the chain the level already has. + assert!(!patch_needs_long_distance(3, 2000, 4000)); + // The binary tree cycles one log short: a file window equal to the chain + // log is past it at btlazy2 (level 13), within it at lazy2 (level 12). + let reference = 1u64 << 20; + for (level, binary_tree) in [(12, false), (13, true)] { + let chain_log = + LevelParameters::for_level(level, Some(1 << 30), reference as usize).chain_log; + let size = 1u64 << (chain_log - 1); + let params = LevelParameters::for_level(level, Some(size), reference as usize); + assert_eq!(params.chain_log, chain_log, "level {level}: one size tier"); + assert_eq!(params.strategy >= Strategy::Btlazy2, binary_tree); + assert_eq!( + patch_needs_long_distance(level, size, reference), + binary_tree, + "level {level}" + ); + } } /// The window log has a supported range; a value outside it has to fail rather @@ -6154,16 +6222,6 @@ fn max_is_refused_on_a_32_bit_target() { assert!(err.to_string().contains("32-bit"), "{err}"); } -/// `--long` below level 16 is refused because the matcher does not run there, -/// unless `--zstd=strat=` moves the level onto a parser where it does. -#[test] -fn a_strategy_override_onto_the_optimal_parser_admits_long() { - assert!(parse(&["-3", "--long", "f"]).is_err()); - assert!(parse(&["-3", "--long", "--zstd=strat=7", "f"]).is_ok()); - assert!(parse(&["-3", "--long", "--zstd=strat=9", "f"]).is_ok()); - assert!(parse(&["-3", "--long", "--zstd=strat=6", "f"]).is_err()); -} - /// `--[no-]compress-literals` decides whether literal sections are /// entropy-coded: forced raw, a literal-heavy input compresses worse than the /// level's default; forced on at a negative level, where the default is raw, diff --git a/zstd/src/encoding/bt/ldm_helper_tests.rs b/zstd/src/encoding/bt/ldm_helper_tests.rs index d3696d37d..e951921e2 100644 --- a/zstd/src/encoding/bt/ldm_helper_tests.rs +++ b/zstd/src/encoding/bt/ldm_helper_tests.rs @@ -32,65 +32,6 @@ fn store(size: usize) -> HcRawSeqStore { } } -/// Regression test for PR #139 Copilot review — threads #1/#2: -/// `prepare_ldm_candidates` must translate the absolute -/// `current_abs_start` (`history_abs_start + window_size - -/// current_len`) into a slice index inside the supplied -/// `history` slice. Before the fix, the code passed -/// `current_abs_start` directly as `block_start` to -/// `LdmProducer::generate_into`, which would index out of -/// bounds (or into the wrong byte range) as soon as -/// `history_abs_start != 0` after window eviction. -/// -/// We construct a `BtMatcher` with an active `LdmProducer` -/// whose table starts empty, hand it a small `history` slice, -/// and call `prepare_ldm_candidates` with absolute coordinates -/// shifted by a non-zero `history_abs_start`. The call must -/// complete without panic — under the pre-fix code it would -/// panic on the `history[block_start..block_end]` slice access -/// inside `generate_into`. -#[test] -#[cfg(feature = "ldm")] -fn prepare_ldm_candidates_translates_absolute_positions_to_slice_indices() { - use crate::encoding::ldm::{LdmProducer, params::LdmParams}; - - let mut bt = BtMatcher::new(); - // Activate the producer with hand-tuned small params — - // the upstream zstd btultra2 defaults (`with_window_and_strategy(27, 9)`) - // would allocate `1 << 23 = 8M` entries × 8 bytes = 64 MiB - // for a table this test never actually populates beyond a - // handful of entries. Build a 10-bit hash log instead so - // the table is ~8 KiB — keeps the suite stable under - // parallel nextest on 32-bit / memory-tight CI shards. - bt.ldm_producer = Some(LdmProducer::new(LdmParams { - window_log: 27, - hash_log: 10, - hash_rate_log: 4, - min_match_length: 32, - bucket_size_log: 4, - })); - - // 256 bytes of arbitrary content — enough for the gear - // hash to make progress against the upstream zstd btultra2 - // `min_match_length = 32`. - let history: alloc::vec::Vec = (0u8..=255).collect(); - - // Simulate a frame whose window has already evicted some - // bytes: `history_abs_start = 1024`, and the current block - // covers absolute `[1024, 1280)` which maps to slice - // `[0, 256)` inside `history`. - let history_abs_start = 1024usize; - let current_abs_start = 1024usize; - let current_len = history.len(); - - // Pre-fix: would panic with `slice index out of bounds` - // on `history[1024..1280]`. Post-fix: translates to - // `[0..256]` and runs cleanly. - bt.prepare_ldm_candidates(&history, history_abs_start, current_abs_start, current_len); - // Cross-block fixture isn't engineered to emit a match — - // the assertion is simply that the call succeeded. -} - #[test] fn ldm_skip_raw_seq_store_bytes_walks_whole_sequences() { let bt = bt_with_seqs(vec![raw(2, 3, 100), raw(4, 1, 200), raw(0, 5, 50)]); diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs index f4579e557..6774f6282 100644 --- a/zstd/src/encoding/bt/mod.rs +++ b/zstd/src/encoding/bt/mod.rs @@ -22,8 +22,6 @@ use alloc::vec::Vec; use core::mem::MaybeUninit; use super::cost_model::{HC_MAX_LIT, HcOptState, HcOptimalCostProfile}; -#[cfg(feature = "ldm")] -use super::ldm::LdmProducer; use super::opt::ldm::{HcOptLdmState, HcRawSeq, HcRawSeqStore}; use super::opt::types::{HcOptimalNode, HcOptimalPlanBuffers, HcOptimalSequence, MatchCandidate}; @@ -77,28 +75,10 @@ pub(crate) struct BtMatcher { pub(crate) opt_lit_price_stamp: u32, pub(crate) opt_ml_price_stamp: u32, /// Long-distance match (LDM) candidates seeded into the optimal - /// parser. Built per-block during `start_matching_optimal` and - /// drained as the parser advances. + /// parser: the driver's long-distance matcher fills them for each block + /// before the parse, which drains them as it advances. Empty while the + /// frame runs without it. pub(crate) ldm_sequences: Vec, - /// LDM producer — `None` while LDM is opt-out (current - /// default) so the table allocation only happens for callers - /// that opt in. The producer is **never auto-constructed**: - /// every `CompressionLevel` preset leaves this field as - /// `None`, matching upstream `libzstd.so.1` where - /// `ZSTD_compress(..., level)` never enables LDM. The - /// opt-in surface (Rust parameter API, see #27) builds an - /// `LdmProducer` from caller-supplied params and assigns it - /// here at frame setup time; `prepare_ldm_candidates` only - /// consumes the field, it does not construct it. See - /// [`super::ldm::LdmProducer`]. - /// - /// Gated behind the `ldm` feature (which implies `hash`, since the - /// producer's per-window XXH64 hashing needs the optional `twox-hash` - /// dependency). Without it the field disappears and the - /// `prepare_ldm_candidates` body shrinks to an `ldm_sequences.clear()` - /// stub. - #[cfg(feature = "ldm")] - pub(crate) ldm_producer: Option, } impl BtMatcher { @@ -189,29 +169,21 @@ impl BtMatcher { opt_lit_price_stamp: 0, opt_ml_price_stamp: 0, ldm_sequences: Vec::new(), - #[cfg(feature = "ldm")] - ldm_producer: None, } } - /// Heap bytes the optimal-parser scratch buffers and the optional LDM - /// producer hold. The fixed-size price arrays and `opt_state` are inline - /// (counted by the owner's `size_of`), so only the `Vec` fields contribute. + /// Heap bytes the optimal-parser scratch buffers hold. The fixed-size + /// price arrays and `opt_state` are inline (counted by the owner's + /// `size_of`), so only the `Vec` fields contribute. pub(crate) fn heap_size(&self) -> usize { - let scratch = self.opt_nodes_scratch.len() * core::mem::size_of::() + self.opt_nodes_scratch.len() * core::mem::size_of::() + self.opt_node_prices_scratch.len() * core::mem::size_of::() + self.opt_candidates_scratch.capacity() * core::mem::size_of::() + self.opt_store_scratch.capacity() * core::mem::size_of::() + (self.opt_segment_plan_scratch.capacity() + self.opt_seed_plan_scratch.capacity()) * core::mem::size_of::() + self.opt_price_arena.len() * core::mem::size_of::<[u32; 2]>() - + self.ldm_sequences.capacity() * core::mem::size_of::(); - // The LDM producer is only present under the `ldm` feature. - #[cfg(feature = "ldm")] - let ldm = self.ldm_producer.as_ref().map_or(0, |p| p.heap_size()); - #[cfg(not(feature = "ldm"))] - let ldm = 0; - scratch + ldm + + self.ldm_sequences.capacity() * core::mem::size_of::() } /// Per-frame reset — clears scratch buffers, resets cost model, @@ -236,10 +208,6 @@ impl BtMatcher { self.opt_lit_price_generation = [0; HC_MAX_LIT + 1]; self.opt_lit_price_stamp = 0; self.ldm_sequences.clear(); - #[cfg(feature = "ldm")] - if let Some(producer) = self.ldm_producer.as_mut() { - producer.clear(); - } } /// Upstream zstd parity: `ZSTD_optLdm_skipRawSeqStoreBytes`. Fast-forward the @@ -405,95 +373,6 @@ impl BtMatcher { result } - /// Upstream zstd parity: `ZSTD_ldm_blockCompress` seeds external - /// long-distance match candidates here when `enableLdm == - /// ZSTD_ps_enable`. The default Rust encoder still keeps LDM - /// disabled (`ldm_producer = None`); when an external caller - /// opts in (#18 Phase 5 wiring — see #27 for the parameter - /// surface), the producer is delegated to via - /// [`LdmProducer::generate_into`]. - /// - /// While `ldm_producer.is_none()` the behaviour matches the - /// pre-Phase-5 stub: a defensive clear of [`Self::ldm_sequences`] - /// so cross-frame carry-over is impossible if a producer is - /// activated mid-session. - /// - /// # Coordinate convention (PR #139 review feedback) - /// - /// The producer operates entirely in **absolute stream - /// coordinates** (so its bucket-table entries remain valid - /// across window evictions — entries inserted by an earlier - /// view of the frame are filtered by the staleness check - /// `entry.offset < history_abs_start`, i.e. inclusive lower - /// bound: entries at exactly `history_abs_start` survive, - /// see `ldm::search::FindBestMatchInputs::lowest_index_abs`). - /// The caller is expected to pass: - /// - /// * `live_history` — the contiguous *live* slice of the - /// per-frame `MatchTable::history` (`&history[history_start..]`). - /// `live_history[0]` corresponds to absolute position - /// `history_abs_start`. - /// * `history_abs_start` — absolute stream position of - /// `live_history[0]`. - /// * `current_abs_start` / `current_len` — absolute span of - /// the block to scan. - /// - /// `prepare_ldm_candidates` forwards these absolute - /// coordinates straight to [`LdmProducer::generate_into`]; the - /// abs→slice translation happens inside the producer only at - /// the moment of `live_history[..]` indexing. - pub(crate) fn prepare_ldm_candidates( - &mut self, - live_history: &[u8], - history_abs_start: usize, - current_abs_start: usize, - current_len: usize, - ) { - self.ldm_sequences.clear(); - #[cfg(feature = "ldm")] - if let Some(producer) = self.ldm_producer.as_mut() { - debug_assert!(current_abs_start >= history_abs_start); - // MatchTable invariant: `live_history.len() == - // window_size`, and `current_abs_start = - // history_abs_start + window_size − current_len` - // (match_generator.rs around line 1330). The two - // sides below must coincide; using `min(...)` would - // silently truncate the scanned range and mask an - // invariant violation. Raw `+` is safe — the frame- - // level `check_stream_abs_headroom` - // (`match_table/storage.rs:50`) guarantees - // `history_abs_start + window_size + - // STREAM_ABS_HEADROOM ≤ usize::MAX`. - debug_assert_eq!( - current_abs_start + current_len, - history_abs_start + live_history.len(), - "MatchTable invariant violation: current block range \ - `[current_abs_start, +current_len)` must coincide with the \ - live-history end (window_size == live_history.len())" - ); - let block_end_abs = current_abs_start + current_len; - producer.generate_into( - live_history, - history_abs_start, - current_abs_start, - block_end_abs, - &mut self.ldm_sequences, - ); - } - #[cfg(not(feature = "ldm"))] - { - // Without the `ldm` feature `LdmProducer` is not compiled, so - // `live_history` / `history_abs_start` / `current_abs_start` / - // `current_len` would otherwise be unused. - let _ = ( - live_history, - history_abs_start, - current_abs_start, - current_len, - ); - } - } - /// Upstream zstd parity: `ZSTD_storeSeq` — encode `actual_offset` into the /// upstream zstd's compact offset base (1/2/3 for rep slots, otherwise /// `actual_offset + 3`) and update the rolling `reps` window in diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index e1f511bc2..e1c6305e3 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -902,6 +902,38 @@ impl DfastMatchGenerator { self.insert_positions(current_abs_start, current_abs_end); } + /// Hand the last committed block of `len` bytes back to the uncommitted + /// input, undoing only the bookkeeping of [`Self::commit_block`]: what it + /// evicted stays evicted, so committing the same bytes again in pieces + /// evicts nothing more. + #[cfg(feature = "ldm")] + pub(crate) fn uncommit_block(&mut self, len: usize) { + assert_eq!(self.window_blocks.pop_back(), Some(len)); + self.window_size -= len; + self.uncommitted_len += len; + } + + /// Index the last committed block as the body of a long-distance match: + /// only its tail, as upstream fills the fast tables after one + /// (`ZSTD_ldm_limitTableUpdate` + `ZSTD_ldm_fillFastTables`), since the + /// match itself is never searched. + #[cfg(feature = "ldm")] + pub(crate) fn skip_long_match(&mut self) { + self.ensure_hash_tables(); + let current_len = self.window_blocks.back().copied().unwrap_or(0); + let end = self.history_abs_start + self.window_size; + let from = end - current_len.min(crate::encoding::ldm::LONG_MATCH_INDEXED_TAIL); + self.insert_positions(from, end); + } + + /// Carry a long-distance match the driver emitted into the repeat + /// offsets, as the encoder will code it. + #[cfg(feature = "ldm")] + pub(crate) fn note_long_match(&mut self, offset: usize, literal_len: usize) { + let _ = + encode_offset_with_history(offset as u32, literal_len as u32, &mut self.offset_hist); + } + /// Upstream zstd `ZSTD_dictMatchState` attach (dfast): hash the dictionary block at /// the front of history into the SEPARATE immutable [`Self::dict`] tables /// (long+short), once, instead of re-priming the live tables every frame diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs index 8dd007e00..0944fe913 100644 --- a/zstd/src/encoding/hc/generator.rs +++ b/zstd/src/encoding/hc/generator.rs @@ -1476,31 +1476,6 @@ impl HcMatchGenerator { } } - /// Install (or clear) the long-distance-match producer (#27). Only - /// the BT backend owns an `ldm_producer` slot; on the HC (lazy) - /// backend the producer is dropped because there is no optimal-parser - /// candidate buffer to seed. Call after [`Self::reset`]. - #[cfg(feature = "ldm")] - pub(crate) fn set_ldm_producer(&mut self, producer: Option) { - if let HcBackend::Bt(bt) = &mut self.backend { - bt.ldm_producer = producer; - } - } - - /// Move the LDM producer out of the BT backend, leaving `None`. Used by the - /// dictionary snapshot path: the producer carries no dictionary state (LDM - /// is not dict-primed; its hash table is empty at capture), so it is not - /// retained in the snapshot — the working frame's freshly-reset producer is - /// reinstated on restore instead. - #[cfg(feature = "ldm")] - pub(crate) fn take_ldm_producer(&mut self) -> Option { - if let HcBackend::Bt(bt) = &mut self.backend { - bt.ldm_producer.take() - } else { - None - } - } - pub(crate) fn reset(&mut self) { self.table.reset(); if let HcBackend::Bt(bt) = &mut self.backend { diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index 0fe271aad..6a296bb7e 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -1295,28 +1295,6 @@ impl HcMatchGenerator { self.table .backfill_boundary_positions(current_abs_start, current_abs_end); self.table.next_to_update3 = hash3_start_cursor; - // Borrow split: `prepare_ldm_candidates` needs immutable - // access to the live history (the post-`history_start` - // slice of `self.table.history`) while it mutates the LDM - // bucket table owned by `self.backend.bt_mut()`. Both live - // in disjoint fields of `Self`, so we capture the slice + - // its base before reaching for `bt_mut()`. - // - // The producer operates in absolute stream coordinates - // throughout; `live_history[0]` corresponds to absolute - // `history_abs_start` (upstream zstd `base + dictLimit`), and the - // abs→slice translation happens inside the producer at - // each `live_history[..]` access. Passing the full - // `history` Vec would index into the dead prefix (the - // bytes already retired past `history_start`). - let live_history = self.table.live_history(); - let history_abs_start = self.table.history_abs_start; - self.backend.bt_mut().prepare_ldm_candidates( - live_history, - history_abs_start, - current_abs_start, - current_len, - ); // Take the five DP scratch buffers ONCE for both the seed pass and the // main pass (upstream shares one workspace across btultra2's two diff --git a/zstd/src/encoding/ldm/mod.rs b/zstd/src/encoding/ldm/mod.rs index a918ef04f..af868a9af 100644 --- a/zstd/src/encoding/ldm/mod.rs +++ b/zstd/src/encoding/ldm/mod.rs @@ -15,32 +15,19 @@ //! `strategy` (`ZSTD_ldm_adjustParameters` parity). //! //! Aggregate [`LdmProducer`] holds the rolling-hash state, the -//! bucket table, and the per-call scratch buffers. The downstream -//! consumer (`bt::ldm_sequences`) was plumbed during Phase 1 -//! (#119) — Phase 5 swaps the `prepare_ldm_candidates` no-op stub -//! for a real producer that fills that buffer. +//! bucket table, and the per-call scratch buffers. The match driver owns +//! one per frame, as upstream keeps `ldmState` in the context: the optimal +//! parser takes its sequences as candidates, every other strategy takes them +//! outright and searches only the literals between them +//! (`ZSTD_ldm_blockCompress`). //! -//! # Activation policy (distro-grade parity) +//! # Activation policy //! -//! `LdmProducer` is **never activated automatically by a -//! [`super::CompressionLevel`] preset.** This mirrors upstream -//! `libzstd.so.1` behaviour: `ZSTD_compress(..., level)` keeps -//! LDM off at every standard level (1..22). Activation is an -//! explicit user opt-in: -//! -//! * upstream — `ZSTD_CCtx_setParameter(cctx, -//! ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)`; -//! * `zstd` CLI — `zstd --long[=N]`; -//! * Rust port (this crate) — the Phase-5 surface ships an -//! `ldm_producer: Option` field on `BtMatcher` that -//! stays `None` by default. The forthcoming parameter-API issue -//! (#27) plugs into that field; the C ABI work in #126 / #127 -//! wires `ZSTD_c_enableLongDistanceMatching` through the same -//! surface. -//! -//! Therefore the `level22_sequences_match_upstream zstd_on_corpus_proxy` -//! ratio gate continues to compare two LDM-OFF outputs (ours vs -//! upstream `ZSTD_compress(..., 22)`) — byte-parity invariant. +//! No [`super::CompressionLevel`] preset turns it on, as upstream +//! `ZSTD_compress(..., level)` keeps LDM off at every level. It is an +//! explicit opt-in: the parameter API +//! (`enable_long_distance_matching`), `ZSTD_c_enableLongDistanceMatching` +//! through the C ABI, and `zstd --long[=N]`. //! //! Upstream zstd parity anchors: //! * `lib/compress/zstd_ldm.c` v1.5.7 @@ -49,19 +36,6 @@ //! table reproduced verbatim in [`gear_hash::GEAR_TAB`] to preserve //! byte-for-byte split-point compatibility. -// Phase 5 of #111 ships the full LDM producer (gear hash + params -// + bucket table + verify/extend search + aggregate driver) in -// two commits. By distro-parity design (see the module docs -// above), no `CompressionLevel` preset activates the producer — -// `BtMatcher::ldm_producer` stays `None` until #27 (Rust -// parameter API) or #126/#127 (C ABI) plug the user opt-in. -// Several `pub(crate)` accessors (`LDM_HASHLOG_MIN/MAX`, -// `params::bounded`, `bucket_mask`) are therefore reachable only -// via the unit test suite until the parameter API lands; the -// `#![allow(dead_code)]` matches the same transitional marker in -// `bt/mod.rs` introduced during Phase 1. -#![allow(dead_code)] - use alloc::vec; use alloc::vec::Vec; @@ -84,6 +58,12 @@ use table::LdmHashTable; /// (`zstd_ldm.c:315`: `XXH64(split, minMatchLength, 0)`). const LDM_XXH64_SEED: u64 = 0; +/// How much of a long-distance match's body the strategy's own tables index: +/// the match is taken without being searched, and upstream catches its tables +/// up on no more than this much of it (`ZSTD_ldm_limitTableUpdate`, +/// `zstd_ldm.c:335`, brings `nextToUpdate` to within 512 of the match end). +pub(crate) const LONG_MATCH_INDEXED_TAIL: usize = 512; + /// LDM sequence producer — owns the rolling-hash state, bucket /// table, and scratch buffers needed to scan an input block and /// emit a stream of [`HcRawSeq`] candidates consumed by the @@ -95,7 +75,6 @@ const LDM_XXH64_SEED: u64 = 0; /// [`Self::clear`] only when starting a new frame (so the /// long-range history accumulated across blocks is preserved /// within a frame, mirroring upstream zstd's `ldmState_t` lifecycle). -#[derive(Clone)] pub(crate) struct LdmProducer { /// Parameter set this producer was built with. Used by the /// split walker (next commit) to honour `min_match_length` / @@ -112,6 +91,26 @@ pub(crate) struct LdmProducer { splits_scratch: Vec, } +impl Clone for LdmProducer { + fn clone(&self) -> Self { + Self { + params: self.params, + hash_state: self.hash_state, + hash_table: self.hash_table.clone(), + splits_scratch: self.splits_scratch.clone(), + } + } + + /// Into the table already held (see [`LdmHashTable`]'s `clone_from`); the + /// split scratch is per call, so its contents are not carried. + fn clone_from(&mut self, source: &Self) { + self.params = source.params; + self.hash_state = source.hash_state; + self.hash_table.clone_from(&source.hash_table); + self.splits_scratch.resize(source.splits_scratch.len(), 0); + } +} + impl LdmProducer { /// Build a fresh producer for the given parameter set. /// @@ -132,12 +131,6 @@ impl LdmProducer { } } - /// Re-derive parameters from a `(window_log, strategy)` pair - /// using [`LdmParams::adjust_for`]. Convenience wrapper. - pub(crate) fn with_window_and_strategy(window_log: u32, strategy: u32) -> Self { - Self::new(LdmParams::adjust_for(window_log, strategy)) - } - /// Heap bytes this producer owns: the LDM hash table and the split-scratch. pub(crate) fn heap_size(&self) -> usize { self.hash_table.heap_size() + self.splits_scratch.capacity() * core::mem::size_of::() @@ -153,6 +146,51 @@ impl LdmProducer { ); } + /// Index `[start_abs, end_abs)` of `live_history` (whose first byte sits + /// at `history_abs_start`) without searching it: a dictionary loaded under + /// long-distance matching (upstream zstd `ZSTD_ldm_fillHashTable`, + /// `zstd_ldm.c:289-327`). Leaves the rolling hash for the next + /// [`Self::generate_into`], which re-seeds it anyway. + pub(crate) fn fill_table( + &mut self, + live_history: &[u8], + history_abs_start: usize, + start_abs: usize, + end_abs: usize, + ) { + debug_assert!(history_abs_start <= start_abs && start_abs <= end_abs); + debug_assert!(end_abs <= history_abs_start + live_history.len()); + let min_match = self.params.min_match_length as usize; + let hash_id_mask = self.hash_table.bucket_mask(); + let mut hash_state = GearHashState::new(min_match, self.params.hash_rate_log); + let mut ip_abs = start_abs; + while ip_abs < end_abs { + let chunk = &live_history[ip_abs - history_abs_start..end_abs - history_abs_start]; + let (hashed, num_splits) = + gear_hash::feed(&mut hash_state, chunk, &mut self.splits_scratch); + for &split in &self.splits_scratch[..num_splits] { + // A split closes a `min_match`-byte window; one that would open + // before the range is not a window of it. + if ip_abs + split < start_abs + min_match { + continue; + } + let split_abs = ip_abs + split - min_match; + let split_idx = split_abs - history_abs_start; + let mut hasher = XxHash64::with_seed(LDM_XXH64_SEED); + hasher.write(&live_history[split_idx..split_idx + min_match]); + let xxhash = hasher.finish(); + self.hash_table.ensure_room_for(split_abs); + self.hash_table.insert_absolute( + (xxhash as u32) & hash_id_mask, + split_abs, + (xxhash >> 32) as u32, + ); + } + // `feed` consumes at least one byte of a non-empty chunk. + ip_abs += hashed.max(1); + } + } + /// Read-only view of the parameter set for diagnostics / tests. pub(crate) fn params(&self) -> LdmParams { self.params diff --git a/zstd/src/encoding/ldm/params.rs b/zstd/src/encoding/ldm/params.rs index d2ae33ff4..67048d46d 100644 --- a/zstd/src/encoding/ldm/params.rs +++ b/zstd/src/encoding/ldm/params.rs @@ -145,11 +145,13 @@ impl LdmParams { } /// Hash-table size in entries: `1 << hash_log`. + #[cfg(test)] pub(crate) const fn hash_table_entries(&self) -> usize { 1usize << self.hash_log } /// Per-bucket slot count: `1 << bucket_size_log`. + #[cfg(test)] pub(crate) const fn bucket_slots(&self) -> usize { 1usize << self.bucket_size_log } diff --git a/zstd/src/encoding/ldm/table.rs b/zstd/src/encoding/ldm/table.rs index 0251e70a4..ffb65a19e 100644 --- a/zstd/src/encoding/ldm/table.rs +++ b/zstd/src/encoding/ldm/table.rs @@ -91,7 +91,6 @@ const REBASE_GUARD_BAND: u32 = 1u32 << 30; /// [`LdmHashTable::insert_absolute`] / [`LdmHashTable::resolve`] /// helpers so absolute-position support extends past `u32::MAX` /// transparently — same pattern as `MatchTable`'s chain rebase. -#[derive(Clone)] pub(crate) struct LdmHashTable { entries: Vec, bucket_offsets: Vec, @@ -110,6 +109,29 @@ pub(crate) struct LdmHashTable { position_base: usize, } +impl Clone for LdmHashTable { + fn clone(&self) -> Self { + Self { + entries: self.entries.clone(), + bucket_offsets: self.bucket_offsets.clone(), + effective_bucket_log: self.effective_bucket_log, + bucket_mask: self.bucket_mask, + position_base: self.position_base, + } + } + + /// Into the buffers already held: a restored dictionary snapshot copies + /// the table every frame, and allocating it again each time would cost + /// more than the copy. + fn clone_from(&mut self, source: &Self) { + self.entries.clone_from(&source.entries); + self.bucket_offsets.clone_from(&source.bucket_offsets); + self.effective_bucket_log = source.effective_bucket_log; + self.bucket_mask = source.bucket_mask; + self.position_base = source.position_base; + } +} + impl LdmHashTable { /// Allocate a fresh table for the given parameters. /// @@ -288,6 +310,7 @@ impl LdmHashTable { /// Absolute stream position that corresponds to the +1 /// relative-offset bias — i.e. an entry with `offset == 1` /// refers to absolute byte `position_base`. + #[cfg(test)] pub(crate) const fn position_base(&self) -> usize { self.position_base } diff --git a/zstd/src/encoding/levels/fastest.rs b/zstd/src/encoding/levels/fastest.rs index 3bc259d89..e6aa56280 100644 --- a/zstd/src/encoding/levels/fastest.rs +++ b/zstd/src/encoding/levels/fastest.rs @@ -88,7 +88,10 @@ pub(crate) fn compress_block_encoded( // the dictionary. The classifier's own sample is safe in a way this one is // not — it measures the block against itself, and a repeat is what the grid // then catches. - let dict_rejects_raw = dict_active; + // + // Long-distance matches are external to the block in the same way, and no + // sample sees them either: upstream searches every block for them. + let dict_rejects_raw = dict_active || state.matcher.matches_beyond_window_tables(); // The level and the window decide this for the whole frame, so it also says // whether the grid is worth keeping at all: nothing it records can be acted // on where no block may go out raw, and recording there would take its diff --git a/zstd/src/encoding/match_generator/dict_prime.rs b/zstd/src/encoding/match_generator/dict_prime.rs index b8de6a385..1657b9c37 100644 --- a/zstd/src/encoding/match_generator/dict_prime.rs +++ b/zstd/src/encoding/match_generator/dict_prime.rs @@ -101,6 +101,8 @@ impl MatchGeneratorDriver { if dict_content.is_empty() { return; } + #[cfg(feature = "ldm")] + let ldm_dict_start = self.ldm_committed_end; // Dictionary bytes should stay addressable until produced frame output // itself exceeds the live window size. We bump `max_window_size` @@ -215,6 +217,24 @@ impl MatchGeneratorDriver { committed_dict_budget += end - start; start = end; } + // The long-distance matcher indexes the dictionary as upstream does when + // it loads one (`ZSTD_ldm_fillHashTable` from + // `ZSTD_loadDictionaryContent`), so a frame can reach it from as far as + // the window allows. + #[cfg(feature = "ldm")] + { + self.ldm_dict_end = self.ldm_committed_end; + if let Some(producer) = self.ldm.as_mut() { + let window = self.storage.committed_window(); + let window_start = self.ldm_committed_end - window.len(); + producer.fill_table( + window, + window_start, + ldm_dict_start.max(window_start), + self.ldm_committed_end, + ); + } + } // Derive `granted_retained_budget` directly from the two real // bounds — bytes actually committed and bytes the cap allows @@ -395,20 +415,12 @@ impl MatchGeneratorDriver { if tree_in_snapshot { tables.clone_from(&snapshot_tables); } - // The snapshot does not retain the LDM producer (it holds no - // dict state; see `capture_primed_dictionary`). Carry over the - // frame's freshly-reset producer, built this frame by `reset` - // with the same params the snapshot key pins. - #[cfg(feature = "ldm")] - let fresh_ldm = live.take_ldm_producer(); *live = snap.clone(); live.table.tables = tables; live.table.history = history; if !tree_in_snapshot { live.table.ensure_tables(); } - #[cfg(feature = "ldm")] - live.set_ldm_producer(fresh_ldm); snap.table.tables = snapshot_tables; snap.table.history = snapshot_history; } @@ -419,6 +431,16 @@ impl MatchGeneratorDriver { } } self.dictionary_retained_budget = budget; + // The restored window is the dictionary, the frame's first bytes, and + // the long-distance matcher's table holds it as priming left it. + #[cfg(feature = "ldm")] + { + self.ldm_committed_end = self.storage.committed_window().len(); + self.ldm_dict_end = self.ldm_committed_end; + if let (Some(live), Some(snapshot)) = (self.ldm.as_mut(), self.primed_ldm.as_ref()) { + live.clone_from(snapshot); + } + } true } @@ -436,6 +458,13 @@ impl MatchGeneratorDriver { fast_tagged, ldm, }; + // The key pins the parameters, so a restore copies into a table of the + // same shape. + #[cfg(feature = "ldm")] + match (self.primed_ldm.as_mut(), self.ldm.as_ref()) { + (Some(snapshot), Some(live)) => snapshot.clone_from(live), + (_, live) => self.primed_ldm = live.cloned(), + } // CDict-equivalent retained state. A binary-tree level in ATTACH mode // decouples the dictionary into `dms` (the upstream zstd `dictMatchState`); its // live hash / chain / hash3 tables carry NO dict entries @@ -468,24 +497,15 @@ impl MatchGeneratorDriver { let tables = core::mem::take(&mut hc.table.tables); let chain_off = core::mem::take(&mut hc.table.chain_off); let hash3_off = core::mem::take(&mut hc.table.hash3_off); - // The LDM producer carries no dictionary state (LDM is not - // dict-primed; its hash table is empty at capture), so it is not - // retained either — `restore` reinstates the frame's freshly-reset - // producer. Take it out so the clone does not duplicate its table. - #[cfg(feature = "ldm")] - let ldm_producer = hc.take_ldm_producer(); - // Clone the dict-state-only storage (live tables now empty Vecs, - // LDM producer detached). + // Clone the dict-state-only storage (live tables now empty Vecs). let snapshot = self.storage.clone(); - // Move the live tables (and LDM producer) back into the working storage. + // Move the live tables back into the working storage. let MatcherStorage::HashChain(hc) = &mut self.storage else { unreachable!("storage variant is stable across the take/put"); }; hc.table.tables = tables; hc.table.chain_off = chain_off; hc.table.hash3_off = hash3_off; - #[cfg(feature = "ldm")] - hc.set_ldm_producer(ldm_producer); self.primed = Some((snapshot, self.dictionary_retained_budget, key)); } else { self.primed = Some((self.storage.clone(), self.dictionary_retained_budget, key)); @@ -494,6 +514,7 @@ impl MatchGeneratorDriver { pub(super) fn invalidate_primed_dictionary_impl(&mut self) { self.primed = None; + self.primed_ldm = Default::default(); // Drop the Fast-backend CDict-equivalent table cache too: it is keyed // to the dictionary being removed / replaced. Left in place, the next // same-params `reset` would retain it and the kernel would probe a diff --git a/zstd/src/encoding/match_generator/mod.rs b/zstd/src/encoding/match_generator/mod.rs index ec4a4169a..e43b87fce 100644 --- a/zstd/src/encoding/match_generator/mod.rs +++ b/zstd/src/encoding/match_generator/mod.rs @@ -5,7 +5,7 @@ //! //! The task here is to efficiently find matches in the already encoded data for the current suffix of the not yet encoded data. -#[cfg(feature = "bench-internals")] +#[cfg(any(feature = "bench-internals", feature = "ldm"))] use alloc::vec::Vec; // SIMD/CRC intrinsics now live in `crate::encoding::fastpath::*` where they // sit under per-CPU `#[target_feature]` umbrellas; no architecture-specific @@ -239,6 +239,77 @@ impl MatcherStorage { } } + /// The committed window of the owned history, oldest byte first. + #[cfg(feature = "ldm")] + fn committed_window(&self) -> &[u8] { + match self { + Self::Simple(m) => m.committed_window(), + Self::Dfast(m) => m.live_history(), + Self::Row(m) => m.live_history(), + Self::HashChain(m) => m.table.live_history(), + } + } + + /// Length of the last committed block. + #[cfg(feature = "ldm")] + fn last_block_len(&self) -> usize { + match self { + Self::Simple(m) => m.last_committed_space().len(), + Self::Dfast(m) => m.get_last_space().len(), + Self::Row(m) => m.get_last_space().len(), + Self::HashChain(m) => m.table.get_last_space().len(), + } + } + + /// See the backends' `uncommit_block`. + #[cfg(feature = "ldm")] + fn uncommit_block(&mut self, len: usize) { + match self { + Self::Simple(m) => m.uncommit_block(len), + Self::Dfast(m) => m.uncommit_block(len), + Self::Row(m) => m.uncommit_block(len), + Self::HashChain(m) => m.table.uncommit_block(len), + } + } + + /// Commit `len` bytes of a block handed back by [`Self::uncommit_block`]. + /// The whole block fit the window when it was first committed, so a piece + /// of it evicts nothing. + #[cfg(feature = "ldm")] + fn commit_piece(&mut self, len: usize) { + match self { + Self::Simple(m) => { + let evicted = m.commit_block(len); + debug_assert_eq!(evicted, 0, "a recommitted piece evicted window bytes"); + } + Self::Dfast(m) => m.commit_block(len), + Self::Row(m) => m.commit_block(len), + Self::HashChain(m) => m.table.commit_block(len), + } + } + + /// See the backends' `skip_long_match`. + #[cfg(feature = "ldm")] + fn skip_long_match(&mut self) { + match self { + Self::Simple(m) => m.skip_long_match(), + Self::Dfast(m) => m.skip_long_match(), + Self::Row(m) => m.skip_long_match(), + Self::HashChain(m) => m.table.skip_long_match(), + } + } + + /// See the backends' `note_long_match`. + #[cfg(feature = "ldm")] + fn note_long_match(&mut self, offset: usize, literal_len: usize) { + match self { + Self::Simple(m) => m.note_long_match(offset, literal_len), + Self::Dfast(m) => m.note_long_match(offset, literal_len), + Self::Row(m) => m.note_long_match(offset, literal_len), + Self::HashChain(m) => m.table.note_long_match(offset, literal_len), + } + } + /// [`super::strategy::BackendTag`] family of the active variant. fn backend(&self) -> super::strategy::BackendTag { use super::strategy::BackendTag; @@ -327,17 +398,12 @@ pub struct MatchGeneratorDriver { reset_dict_attach_ok: bool, // Hint-resolved matcher shape from the last `reset`: the [`LevelParams`], the // active backend's applied Dfast/Row hash-table width (`0` for HC/Fast), the - // Fast attach-vs-copy mode, the Fast slot format, and the active LDM override - // (#27). Combined with the frame's level into the [`PrimedKey`] that keys - // the primed snapshot, so it is only restored into a reset that resolved the - // identical matcher AND LDM configuration. `None` before the first `reset`. - reset_shape: Option<( - LevelParams, - usize, - bool, - bool, - Option, - )>, + // Fast attach-vs-copy mode, the Fast slot format and the long-distance + // matcher's parameters. Combined with the frame's level into the + // [`PrimedKey`] that keys the primed snapshot, so it is only restored into a + // reset that resolved the identical matcher. `None` before the first + // `reset`. + reset_shape: Option<(LevelParams, usize, bool, bool, LdmShape)>, // One-shot borrowed block range `[start, end)` staged by the borrowed // Fast frame path (`set_borrowed_block`) for the NEXT // `start_matching` / `skip_matching_with_hint`. `Some` routes that @@ -357,6 +423,10 @@ pub struct MatchGeneratorDriver { /// is only restored into a reset that produces the same matcher — see /// `restore_primed_dictionary`. primed: Option<(MatcherStorage, usize, PrimedKey)>, + /// The long-distance matcher as `prime_with_dictionary` left it, with the + /// dictionary indexed, captured and restored together with + /// [`Self::primed`]; its parameters are in the key. + primed_ldm: LdmSnapshot, /// Where the tables live when the driver is reset on its own through /// [`Matcher::reset`]; inside a compression context they live in the /// context's workspace instead and this stays empty. @@ -365,6 +435,27 @@ pub struct MatchGeneratorDriver { /// place rather than copied into the history. Decided at the reset, /// because it decides how large the history is laid out. frame_in_place: bool, + /// Long-distance matcher, `Some` while the frame runs with it on. It + /// belongs to the driver rather than to a backend, as upstream keeps + /// `ldmState` beside the match state: every strategy reads the same one. + #[cfg(feature = "ldm")] + ldm: Option, + /// The long-distance sequences found for the block being matched. + #[cfg(feature = "ldm")] + ldm_sequences: Vec, + /// Stream position of the committed end, in the coordinates the + /// long-distance matcher keeps its table in: the frame's first byte + /// (its dictionary's, when it has one) is `0`. + #[cfg(feature = "ldm")] + ldm_committed_end: usize, + /// Where the frame's dictionary ends in those coordinates, `0` without + /// one: what a match may still reach while the window has not passed it. + #[cfg(feature = "ldm")] + ldm_dict_end: usize, + /// Length of a block the long-distance matcher recommitted in pieces, so + /// [`Matcher::get_last_space`] reports the whole block, not its last piece. + #[cfg(feature = "ldm")] + ldm_split_block: Option, } /// Identity of the matcher configuration a primed snapshot was captured under: @@ -417,17 +508,24 @@ struct PrimedKey { /// frame from how full the input will leave the table, so two frames of one /// geometry can differ in it; a restore carries the snapshot's format in. fast_tagged: bool, - /// Fine-grained LDM override (#27) active at capture time. The - /// snapshot's cloned `storage` carries `BtMatcher::ldm_producer`, - /// which is configured from this override; restoring a snapshot - /// captured under a different LDM configuration (enable flip or - /// changed knobs) would reinstate a stale producer. `params` already - /// pins `window_log` / `strategy_tag` (the rest of the producer's - /// identity), so folding the override completes the LDM identity. - /// `None` = LDM off, matching `ParamOverrides::ldm`. - ldm: Option, + /// The long-distance matcher's parameters, `None` without one. A primed + /// snapshot carries its table with the dictionary indexed in it, which a + /// frame with other parameters could not use. + ldm: LdmShape, } +/// [`PrimedKey::ldm`]: nothing to key on in a build without the matcher. +#[cfg(feature = "ldm")] +type LdmShape = Option; +#[cfg(not(feature = "ldm"))] +type LdmShape = (); + +/// The long-distance matcher as a primed snapshot keeps it. +#[cfg(feature = "ldm")] +type LdmSnapshot = Option; +#[cfg(not(feature = "ldm"))] +type LdmSnapshot = (); + /// Whether a configured HashChain matcher attaches the dictionary for a frame /// of `size_log` (see [`MatchGeneratorDriver::hc_dict_attach_mode`]). fn hc_attaches_dictionary(hc: &HcMatchGenerator, size_log: Option) -> bool { @@ -634,8 +732,19 @@ impl MatchGeneratorDriver { dictionary_size_hint: None, borrowed_pending: None, primed: None, + primed_ldm: Default::default(), own_workspace: crate::encoding::workspace::Workspace::new(), frame_in_place: false, + #[cfg(feature = "ldm")] + ldm: None, + #[cfg(feature = "ldm")] + ldm_sequences: Vec::new(), + #[cfg(feature = "ldm")] + ldm_committed_end: 0, + #[cfg(feature = "ldm")] + ldm_dict_end: 0, + #[cfg(feature = "ldm")] + ldm_split_block: None, } } @@ -1108,7 +1217,13 @@ impl Matcher for MatchGeneratorDriver { .primed .as_ref() .map_or(0, |(storage, _, _)| storage.heap_size()); - self.storage.heap_size() + snapshot + self.own_workspace.heap_bytes() + #[cfg(feature = "ldm")] + let ldm = self.ldm.as_ref().map_or(0, |p| p.heap_size()) + + self.primed_ldm.as_ref().map_or(0, |p| p.heap_size()) + + self.ldm_sequences.capacity() * core::mem::size_of::(); + #[cfg(not(feature = "ldm"))] + let ldm = 0; + self.storage.heap_size() + snapshot + self.own_workspace.heap_bytes() + ldm } fn clear_param_overrides(&mut self) { @@ -1247,6 +1362,14 @@ impl Matcher for MatchGeneratorDriver { hint, ); } + // The long-distance matcher's parameters for the frame, `None` without one. + #[cfg(feature = "ldm")] + let frame_ldm: LdmShape = self + .param_overrides + .and_then(|ov| ov.ldm) + .map(|ldm_ov| crate::encoding::levels::config::frame_ldm_params(¶ms, &ldm_ov)); + #[cfg(not(feature = "ldm"))] + let frame_ldm: LdmShape = (); // A dictionary frame's hash-chain / binary-tree widths are the CDict's // (`resolve_level_params_with_dict`): verbatim when the dictionary is // copied into the live tables (`ZSTD_resetCCtx_byCopyingCDict` builds @@ -1392,9 +1515,16 @@ impl Matcher for MatchGeneratorDriver { // taking it again. The kernels keep positions in `u32`, counting a // dictionary ahead of the slice, so a longer one is copied. let dict_len = dict_hint.map_or(0, |sizes| sizes.content); + // The long-distance matcher splits a block into pieces through the + // owned history, so its frames keep the input there. + #[cfg(feature = "ldm")] + let long_distance = self.param_overrides.is_some_and(|ov| ov.ldm.is_some()); + #[cfg(not(feature = "ldm"))] + let long_distance = false; self.frame_in_place = match workspace.ingest() { crate::encoding::workspace::IngestPlan::Slice(len) => { (dict_hint.is_none() || fast_attach) + && !long_distance && self.borrowed_supported() && len .checked_add(dict_len) @@ -1447,8 +1577,8 @@ impl Matcher for MatchGeneratorDriver { // the table contents and bias wholesale, so the reset's // full-table memset would be thrown away. The // key components mirror `reset_shape` below: Simple leaves - // `resolved_table_bits` 0, never carries an LDM override, - // and `fast_attach` is false in copy mode by construction. + // `resolved_table_bits` 0, and `fast_attach` is false in copy + // mode by construction. // The slot format is the one component this reset has yet to // choose, so it travels with the carry: the reset leaves the // table only if it chooses the snapshot's format, which is @@ -1469,7 +1599,7 @@ impl Matcher for MatchGeneratorDriver { table_bits: 0, fast_attach: false, fast_tagged: captured.fast_tagged, - ldm: None, + ldm: frame_ldm, } }) .map(|(_, _, captured)| captured.fast_tagged); @@ -1722,41 +1852,22 @@ impl Matcher for MatchGeneratorDriver { hc.reset(); } } - // LDM wiring (#27): attach (or clear) the long-distance-match - // producer on the optimal (BT) backend. LDM is the only - // back-reference path that crosses the regular window, so it - // only has a home on the `BtMatcher`; non-BT strategies drop the - // producer. Built AFTER `hc.reset()` because `BtMatcher::reset` - // clears an existing producer's table but does not null the - // slot — installing here gives the new frame a fresh producer. + // The long-distance matcher for the frame, whichever strategy runs it. + // The same parameters keep the table allocation and only clear it, as + // upstream reuses its `ldmState_t`. #[cfg(feature = "ldm")] { - // Resolve the derived LDM params first (immutable borrow of the - // overrides), then reuse the existing producer's allocation below. - let derived_ldm = self - .param_overrides - .as_ref() - .and_then(|ov| ov.ldm) - .map(|ldm_ov| crate::encoding::levels::config::frame_ldm_params(¶ms, &ldm_ov)); - if let MatcherStorage::HashChain(hc) = &mut self.storage { - // Reuse the existing producer's hash-table allocation when the - // derived params are unchanged: only `clear()` (re-zero the - // table + re-seed the rolling hash, no allocation) is needed for - // the new frame. A params change (or the first frame) forces a - // fresh `LdmProducer::new`. On the reused-encoder compress-dict - // path this avoids re-allocating the LDM hash table (large at - // btultra2) every frame — upstream zstd reuses its `ldmState_t` - // the same way. `clear()` is mandatory here for correctness - // regardless of what `BtMatcher::reset` did to the old table. - let producer = derived_ldm.map(|p| match hc.take_ldm_producer() { - Some(mut existing) if existing.params() == p => { - existing.clear(); - existing - } - _ => super::ldm::LdmProducer::new(p), - }); - hc.set_ldm_producer(producer); - } + self.ldm = frame_ldm.map(|p| match self.ldm.take() { + Some(mut existing) if existing.params() == p => { + existing.clear(); + existing + } + _ => super::ldm::LdmProducer::new(p), + }); + // A dictionary the reset kept in place is the frame's first bytes. + self.ldm_committed_end = self.storage.committed_window().len(); + self.ldm_dict_end = self.ldm_committed_end; + self.ldm_split_block = None; } // Record the resolved matcher shape for the primed-snapshot key. Captured // here (post-resolution, after the test-only param override) so the key @@ -1768,19 +1879,9 @@ impl Matcher for MatchGeneratorDriver { // resolved matcher geometry instead, so folding the Fast bit into their // key would over-key identical resolved shapes. `fast_attach` is the // decision `prime_with_dictionary` makes from the same `reset_size_log`. - // The LDM override is part of the snapshot identity ONLY on the - // optimal (BinaryTree) path: that is the only backend whose cloned - // `storage` carries a `BtMatcher::ldm_producer`. On Fast / Dfast / - // Row and lazy-HashChain resets the producer slot does not exist, - // so folding the override there would over-key the snapshot and - // force needless re-primes when LDM is toggled. Gated like - // `fast_attach` (a key bit only participates where it changes the - // cloned matcher shape). - let active_ldm = if matches!(params.search, super::strategy::SearchMethod::BinaryTree) { - self.param_overrides.and_then(|ov| ov.ldm) - } else { - None - }; + // The long-distance matcher's parameters are part of it on every + // backend: the snapshot keeps the matcher's table with the dictionary + // indexed, so a restored frame writes what a freshly primed one does. // The Fast slot format this reset chose; only the Simple backend has one. let fast_tagged = matches!(&self.storage, MatcherStorage::Simple(m) if m.slots_tagged()); self.reset_shape = Some(( @@ -1788,7 +1889,7 @@ impl Matcher for MatchGeneratorDriver { resolved_table_bits, fast_attach, fast_tagged, - active_ldm, + frame_ldm, )); // Everything is laid out in `workspace` now. A driver reset on its own // before it joined a context still holds the workspace it used then, @@ -1864,7 +1965,20 @@ impl Matcher for MatchGeneratorDriver { self.reported_window_size as u64 } + fn matches_beyond_window_tables(&self) -> bool { + #[cfg(feature = "ldm")] + return self.ldm.is_some(); + #[cfg(not(feature = "ldm"))] + false + } + fn get_last_space(&mut self) -> &[u8] { + // A block matched in pieces is still one block to the encoder. + #[cfg(feature = "ldm")] + if let Some(len) = self.ldm_split_block { + let window = self.storage.committed_window(); + return &window[window.len() - len..]; + } match &self.storage { MatcherStorage::Simple(m) => m.last_committed_space(), MatcherStorage::Dfast(m) => m.get_last_space(), @@ -1909,6 +2023,11 @@ impl Matcher for MatchGeneratorDriver { /// matches older than the window the frame header reports, which encodes /// an offset no decoder can resolve. fn commit_filled(&mut self, len: usize) { + #[cfg(feature = "ldm")] + { + self.ldm_committed_end += len; + self.ldm_split_block = None; + } // The window backends evict whole blocks inside `commit_block`, // subtracting each evicted block from `window_size` and then adding // `len`, so `pre + len - post` is exactly what was evicted and never @@ -1941,7 +2060,6 @@ impl Matcher for MatchGeneratorDriver { } fn start_matching(&mut self, mut handle_sequence: impl FnMut(Sequence)) { - use super::strategy::{self, StrategyTag}; // Borrowed one-shot Fast path: if the frame driver staged a // block range via `set_borrowed_block`, scan it in place against // the borrowed window instead of the owned committed block. Only @@ -1990,50 +2108,25 @@ impl Matcher for MatchGeneratorDriver { } return; } - // Decoupled parse×search dispatch (fires once per block). The - // search axis (`self.search`) picks the candidate-finding backend; - // the parse axis (greedy vs lazy depth) is carried by the - // backend's runtime `lazy_depth`, set per level at `reset()`. - // The two are independent, so any parse can run on any search - // backend. The `BinaryTree` arm still selects the opt `Strategy` - // ZST off `strategy_tag` so `compress_block::` keeps its - // const-folded optimal-parser monomorphisation. - use super::strategy::SearchMethod; - match self.search { - SearchMethod::Fast => { - self.simple_mut().start_matching(&mut handle_sequence); - } - SearchMethod::DoubleFast => { - self.dfast_matcher_mut() - .start_matching(&mut handle_sequence); - } - SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => { - // One upstream `lazy_generic` body for greedy (depth 0: a - // repcode hit is stored without a search, no lookahead), lazy - // / lazy2 (`lazy_depth` 1 / 2) and btlazy2 (depth 2 over the - // binary-tree finder); the finder (rows / chain / tree) was - // resolved at `configure`. - self.row_matcher_mut().start_matching(&mut handle_sequence); - } - SearchMethod::HashChain => { - // Greedy/lazy/lazy2 all flow through the lazy parser; it - // reads `hc.lazy_depth` (0 = greedy commit). - self.hc_matcher_mut() - .start_matching_lazy(&mut handle_sequence); + #[cfg(feature = "ldm")] + if self.ldm.is_some() { + // The optimal parser weighs long-distance matches as candidates; + // below it they are taken outright (upstream + // `ZSTD_ldm_blockCompress`, `zstd_ldm.c:685-749`). + if matches!(self.search, super::strategy::SearchMethod::BinaryTree) { + self.generate_long_distance_sequences(); + let MatcherStorage::HashChain(hc) = &mut self.storage else { + unreachable!("the optimal parser runs on the hash-chain backend"); + }; + core::mem::swap( + &mut self.ldm_sequences, + &mut hc.backend.bt_mut().ldm_sequences, + ); + } else if self.start_matching_long_distance(&mut handle_sequence) { + return; } - SearchMethod::BinaryTree => match self.strategy_tag { - StrategyTag::BtOpt => self.compress_block::(&mut handle_sequence), - StrategyTag::BtUltra => { - self.compress_block::(&mut handle_sequence) - } - StrategyTag::BtUltra2 => { - self.compress_block::(&mut handle_sequence) - } - _ => unreachable!( - "SearchMethod::BinaryTree requires an optimal strategy tag (BtOpt/BtUltra/BtUltra2)" - ), - }, } + self.start_matching_owned(&mut handle_sequence); } fn skip_matching(&mut self) { @@ -2084,6 +2177,134 @@ impl Matcher for MatchGeneratorDriver { } } +impl MatchGeneratorDriver { + /// Match the last committed block of the owned history with the frame's + /// backend. + fn start_matching_owned(&mut self, handle_sequence: &mut impl FnMut(Sequence)) { + use super::strategy::{self, StrategyTag}; + // Decoupled parse×search dispatch (fires once per block). The + // search axis (`self.search`) picks the candidate-finding backend; + // the parse axis (greedy vs lazy depth) is carried by the + // backend's runtime `lazy_depth`, set per level at `reset()`. + // The two are independent, so any parse can run on any search + // backend. The `BinaryTree` arm still selects the opt `Strategy` + // ZST off `strategy_tag` so `compress_block::` keeps its + // const-folded optimal-parser monomorphisation. + use super::strategy::SearchMethod; + match self.search { + SearchMethod::Fast => { + self.simple_mut().start_matching(handle_sequence); + } + SearchMethod::DoubleFast => { + self.dfast_matcher_mut().start_matching(handle_sequence); + } + SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => { + // One upstream `lazy_generic` body for greedy (depth 0: a + // repcode hit is stored without a search, no lookahead), lazy + // / lazy2 (`lazy_depth` 1 / 2) and btlazy2 (depth 2 over the + // binary-tree finder); the finder (rows / chain / tree) was + // resolved at `configure`. + self.row_matcher_mut().start_matching(handle_sequence); + } + SearchMethod::HashChain => { + // Greedy/lazy/lazy2 all flow through the lazy parser; it + // reads `hc.lazy_depth` (0 = greedy commit). + self.hc_matcher_mut().start_matching_lazy(handle_sequence); + } + SearchMethod::BinaryTree => match self.strategy_tag { + StrategyTag::BtOpt => self.compress_block::(handle_sequence), + StrategyTag::BtUltra => self.compress_block::(handle_sequence), + StrategyTag::BtUltra2 => self.compress_block::(handle_sequence), + _ => unreachable!( + "SearchMethod::BinaryTree requires an optimal strategy tag (BtOpt/BtUltra/BtUltra2)" + ), + }, + } + } + + /// Find the long-distance sequences of the last committed block into + /// [`Self::ldm_sequences`], and return the block's length. + /// + /// What a match may reach is fixed against the block's END, as upstream + /// enforces the distance once per chunk before generating + /// (`ZSTD_window_enforceMaxDist` on `chunkEnd`, `zstd_ldm.c:585`): one + /// window back from it, or the whole dictionary while the window has not + /// yet passed the dictionary's end. Every sequence then holds an offset + /// valid at its last byte, wherever in the block it lies. + #[cfg(feature = "ldm")] + fn generate_long_distance_sequences(&mut self) -> usize { + let block_len = self.storage.last_block_len(); + self.ldm_sequences.clear(); + let Some(producer) = self.ldm.as_mut() else { + return block_len; + }; + let block_end = self.ldm_committed_end; + let window = self.reported_window_size; + let reach_floor = if self.ldm_dict_end != 0 && block_end <= window + self.ldm_dict_end { + 0 + } else { + block_end.saturating_sub(window) + }; + let live = self.storage.committed_window(); + let live_start = block_end - live.len(); + let floor = reach_floor.max(live_start); + producer.generate_into( + &live[floor - live_start..], + floor, + block_end - block_len, + block_end, + &mut self.ldm_sequences, + ); + block_len + } + + /// Take the block's long-distance matches outright and match only the + /// literals between them with the frame's backend (upstream + /// `ZSTD_ldm_blockCompress` below btopt). The block is handed back to the + /// uncommitted input and committed again in pieces, so each gap is the + /// backend's whole block and each match body is indexed without being + /// searched. Returns `false` when the block has no long-distance match, + /// leaving it committed for the ordinary search. + #[cfg(feature = "ldm")] + fn start_matching_long_distance(&mut self, handle_sequence: &mut impl FnMut(Sequence)) -> bool { + let block_len = self.generate_long_distance_sequences(); + if self.ldm_sequences.is_empty() { + return false; + } + let sequences = core::mem::take(&mut self.ldm_sequences); + self.storage.uncommit_block(block_len); + let mut consumed = 0usize; + for seq in &sequences { + // The gap's trailing literals become the long match's own. + let mut gap_literals = 0usize; + if seq.lit_length != 0 { + self.storage.commit_piece(seq.lit_length); + self.start_matching_owned(&mut |sequence| match sequence { + Sequence::Literals { len } => gap_literals = len, + triple => handle_sequence(triple), + }); + } + self.storage.commit_piece(seq.match_length); + self.storage.skip_long_match(); + handle_sequence(Sequence::Triple { + literal_len: gap_literals, + offset: seq.offset, + match_len: seq.match_length, + }); + self.storage.note_long_match(seq.offset, gap_literals); + consumed += seq.lit_length + seq.match_length; + } + let tail = block_len - consumed; + if tail != 0 { + self.storage.commit_piece(tail); + self.start_matching_owned(handle_sequence); + } + self.ldm_sequences = sequences; + self.ldm_split_block = Some(block_len); + true + } +} + impl MatchGeneratorDriver { /// Monomorphised optimal-parser entry point. Only the `BinaryTree` /// search arm of [`Matcher::start_matching`] routes here, selecting diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index 80e266b86..944b1a9fe 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -2401,6 +2401,48 @@ impl MatchTable { self.next_to_update3 = self.next_to_update3.max(committed_end); } + /// Hand the last committed block of `len` bytes back to the uncommitted + /// input, undoing only the bookkeeping of [`Self::commit_block`]: what it + /// evicted stays evicted, so committing the same bytes again in pieces + /// evicts nothing more. + #[cfg(feature = "ldm")] + pub(crate) fn uncommit_block(&mut self, len: usize) { + assert_eq!(self.chunk_lens.pop_back(), Some(len)); + self.window_size -= len; + self.uncommitted_len += len; + } + + /// Index the last committed block as the body of a long-distance match on + /// the lazy chain: only its tail, which is what upstream's lazy search + /// catches up on after one (`ZSTD_ldm_limitTableUpdate` moves + /// `nextToUpdate` to it), since the match itself is never searched. + #[cfg(feature = "ldm")] + pub(crate) fn skip_long_match(&mut self) { + debug_assert!( + !self.uses_bt, + "the optimal parser takes LDM matches as candidates" + ); + self.ensure_tables(); + let (start, len) = self.current_block_range(); + let end = start + len; + self.backfill_boundary_positions(start, end); + self.insert_positions( + end - len.min(crate::encoding::ldm::LONG_MATCH_INDEXED_TAIL), + end, + ); + } + + /// Carry a long-distance match the driver emitted into the repeat + /// offsets, as the encoder will code it. + #[cfg(feature = "ldm")] + pub(crate) fn note_long_match(&mut self, offset: usize, literal_len: usize) { + let _ = crate::encoding::blocks::encode_offset_with_history( + offset as u32, + literal_len as u32, + &mut self.offset_hist, + ); + } + pub(crate) fn skip_matching(&mut self, incompressible_hint: Option) { self.ensure_tables(); let (current_abs_start, current_len) = self.current_block_range(); diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index add752c44..b6aed0189 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -613,6 +613,13 @@ pub trait Matcher { fn block_samples_match_dict(&self, _block: &[u8]) -> bool { false } + /// Whether this matcher finds matches beyond what its regular tables + /// reach (long-distance matching). A block that looks incompressible can + /// then still be a repeat from far back, so the encoder searches every + /// block instead of writing one off on a sample. Defaults to `false`. + fn matches_beyond_window_tables(&self) -> bool { + false + } /// Heap bytes this matcher's allocations hold (tables, history, scratch), /// excluding the inline struct itself. Lets a context report its true /// footprint via `ZSTD_sizeof_CCtx`. Defaults to `0` for custom matchers. diff --git a/zstd/src/encoding/parameters.rs b/zstd/src/encoding/parameters.rs index cc4f7616c..ef7a68fcd 100644 --- a/zstd/src/encoding/parameters.rs +++ b/zstd/src/encoding/parameters.rs @@ -40,9 +40,9 @@ //! never enables LDM — even at level 22. It is activated either by //! [`CompressionParametersBuilder::enable_long_distance_matching`] or by any of //! the `ldm_*` setters, which each imply `enable_long_distance_matching(true)`. -//! When enabled, the LDM producer attaches to the optimal (`btopt` / `btultra` -//! / `btultra2`) match-finder; pair it with an optimal [`Strategy`] (or a level -//! ≥ 16) for it to take effect. +//! It works under every [`Strategy`]: the optimal parsers weigh its matches as +//! candidates, the others take them outright and search only the bytes between +//! them, as upstream does. A dictionary loaded under it is indexed for it too. use crate::encoding::CompressionLevel; diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 12665f9bb..2f46a8cb6 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -4012,6 +4012,42 @@ impl RowMatchGenerator { crate::encoding::fastpath::scalar::common_prefix_len_ptr ); + /// Hand the last committed block of `len` bytes back to the uncommitted + /// input, undoing only the bookkeeping of [`Self::commit_block`]: what it + /// evicted stays evicted, so committing the same bytes again in pieces + /// evicts nothing more. + #[cfg(feature = "ldm")] + pub(crate) fn uncommit_block(&mut self, len: usize) { + assert_eq!(self.chunk_lens.pop_back(), Some(len)); + self.window_size -= len; + self.uncommitted_len += len; + } + + /// Take the last committed block as the body of a long-distance match: + /// nothing in it is indexed now, and the next block resumes indexing its + /// tail, as upstream's `ZSTD_ldm_limitTableUpdate` leaves `nextToUpdate`. + #[cfg(feature = "ldm")] + pub(crate) fn skip_long_match(&mut self) { + let (start, len) = self.current_block_range(); + self.skip_matching_with_hint(None); + let end = start + len; + self.lazy_next_to_update = end - len.min(crate::encoding::ldm::LONG_MATCH_INDEXED_TAIL); + } + + /// Carry a long-distance match the driver emitted into the repeat offsets: + /// the lazy parse's carried pair takes it as the newest (upstream + /// `ZSTD_ldm_blockCompress` shifts `rep`), the encoder-side history as the + /// encoder will code it. + #[cfg(feature = "ldm")] + pub(crate) fn note_long_match(&mut self, offset: usize, literal_len: usize) { + let newest = self + .lazy_reps + .map_or(self.offset_hist[0] as usize, |reps| reps[0]); + self.lazy_reps = Some([offset, newest]); + let _ = + encode_offset_with_history(offset as u32, literal_len as u32, &mut self.offset_hist); + } + pub(crate) fn skip_matching_with_hint(&mut self, incompressible_hint: Option) { match self.row_log { 4 => self.skip_matching_with_hint_rl::<4>(incompressible_hint), diff --git a/zstd/src/encoding/simple/fast_matcher.rs b/zstd/src/encoding/simple/fast_matcher.rs index 1efb1705c..d6d3a15d9 100644 --- a/zstd/src/encoding/simple/fast_matcher.rs +++ b/zstd/src/encoding/simple/fast_matcher.rs @@ -1653,6 +1653,49 @@ impl FastKernelMatcher { } } + /// The committed window, oldest byte first. + #[cfg(feature = "ldm")] + pub(crate) fn committed_window(&self) -> &[u8] { + self.history_bytes() + } + + /// Hand the staged block of `len` bytes back to the uncommitted input, + /// undoing only the bookkeeping of [`Self::commit_block`]: what it evicted + /// stays evicted, so committing the same bytes again in pieces evicts + /// nothing more. + #[cfg(feature = "ldm")] + pub(crate) fn uncommit_block(&mut self, len: usize) { + assert_eq!(self.staged_block.take(), Some(len)); + self.uncommitted_len += len; + } + + /// Index the staged block as the body of a long-distance match: only its + /// tail, the way upstream fills the fast table after one + /// (`ZSTD_ldm_limitTableUpdate` + `ZSTD_fillHashTable`, every third + /// position), since the match itself is never searched. + #[cfg(feature = "ldm")] + pub(crate) fn skip_long_match(&mut self) { + const FILL_STEP: usize = 3; + let block_start = self.take_staged_block(); + let end = self.committed_len(); + let from = end - (end - block_start).min(crate::encoding::ldm::LONG_MATCH_INDEXED_TAIL); + self.prime_hash_table_for_range_stepped(from, FILL_STEP); + } + + /// Carry a long-distance match the driver emitted into the repeat + /// offsets: the kernel's pair shifts it in as the newest (upstream + /// `ZSTD_ldm_blockCompress`), the encoder-side history takes it as the + /// encoder will code it. + #[cfg(feature = "ldm")] + pub(crate) fn note_long_match(&mut self, offset: usize, literal_len: usize) { + self.rep = [offset as u32, self.rep[0]]; + let _ = crate::encoding::blocks::encode_offset_with_history( + offset as u32, + literal_len as u32, + &mut self.offset_hist, + ); + } + /// Commit one slice of a COPY-mode dictionary: append it to history, index /// it into the LIVE table with upstream's dictionary fill, and carry the /// dictionary boundary. diff --git a/zstd/src/tests/parameters_test.rs b/zstd/src/tests/parameters_test.rs index c90fff909..57d1eb6fc 100644 --- a/zstd/src/tests/parameters_test.rs +++ b/zstd/src/tests/parameters_test.rs @@ -251,6 +251,151 @@ fn ldm_custom_knobs_round_trip() { assert_eq!(decode(&compress_with_parameters(&data, ¶ms)), data); } +/// Long-distance matching runs under every strategy, not only the optimal +/// ones: a random half repeated at a distance the strategy's own tables +/// cannot reach is found by LDM alone, so the second copy costs next to +/// nothing. Upstream applies LDM below btopt through `ZSTD_ldm_blockCompress`. +/// Six-bit symbols keep every block searched (a full-entropy block would be +/// written off by the raw skip, whose repeat grid could find the copy instead) +/// while leaving short local matches rare. The copy is laid out in shuffled +/// 4 KiB pieces, so each piece sits at an offset of its own and one lucky hit +/// cannot carry the rest through the repeat code. +#[cfg(feature = "ldm")] +#[test] +fn ldm_finds_far_repeat_under_every_strategy() { + const PIECE: usize = 4096; + let half: Vec = generate_data(0x5EED_0569, 512 * 1024) + .into_iter() + .map(|byte| byte & 63) + .collect(); + let pieces = half.len() / PIECE; + let mut data = half.clone(); + for i in 0..pieces { + // 37 is coprime with the piece count, so this visits every piece once. + let piece = (i * 37) % pieces; + data.extend_from_slice(&half[piece * PIECE..(piece + 1) * PIECE]); + } + for strategy in [ + Strategy::Fast, + Strategy::Dfast, + Strategy::Greedy, + Strategy::Lazy, + Strategy::Lazy2, + Strategy::Btlazy2, + Strategy::Btopt, + ] { + // Tables far smaller than the distance, so nothing but LDM reaches the + // first copy. + let builder = || { + CompressionParameters::builder(CompressionLevel::Level(3)) + .strategy(strategy) + .window_log(22) + .hash_log(12) + .chain_log(12) + }; + let off = compress_with_parameters(&data, &builder().build().unwrap()); + let on = compress_with_parameters( + &data, + &builder() + .enable_long_distance_matching(true) + .build() + .unwrap(), + ); + // Each copy costs about three quarters of its size in literals; the + // second one costs nearly that again without LDM (a few pieces are + // still found by chance) and almost nothing with it. + assert!( + off.len() > data.len() * 13 / 20, + "{strategy:?}: the regular finder already reached the repeat ({} of {})", + off.len(), + data.len(), + ); + assert!( + on.len() < half.len() * 8 / 10, + "{strategy:?}: LDM did not find the repeat ({} of {}, {} without LDM)", + on.len(), + data.len(), + off.len(), + ); + assert_eq!(decode(&on), data, "{strategy:?}: LDM frame round-trip"); + } +} + +/// Long-distance matching reaches into a dictionary, under every strategy: +/// the dictionary is indexed into its table when it is loaded (upstream +/// `ZSTD_ldm_fillHashTable` from `ZSTD_loadDictionaryContent`), so a source +/// made of shuffled pieces of a large dictionary costs next to nothing. A +/// second frame on the same compressor restores the primed dictionary and must +/// write what a freshly primed compressor writes. +#[cfg(feature = "ldm")] +#[test] +fn ldm_reaches_into_the_dictionary_under_every_strategy() { + use crate::decoding::{Dictionary, FrameDecoder}; + + const PIECE: usize = 4096; + const DICT_ID: u32 = 0x0569_D1C7; + let dict: Vec = generate_data(0xD1C7_0569, 256 * 1024) + .into_iter() + .map(|byte| byte & 63) + .collect(); + let pieces = dict.len() / PIECE; + let mut source = Vec::new(); + for i in 0..pieces { + let piece = (i * 37) % pieces; + source.extend_from_slice(&dict[piece * PIECE..(piece + 1) * PIECE]); + } + for strategy in [ + Strategy::Fast, + Strategy::Dfast, + Strategy::Greedy, + Strategy::Lazy2, + Strategy::Btlazy2, + Strategy::Btopt, + ] { + let params = CompressionParameters::builder(CompressionLevel::Level(3)) + .strategy(strategy) + .window_log(22) + .hash_log(12) + .chain_log(12) + .enable_long_distance_matching(true) + .build() + .unwrap(); + let compressor = || { + let mut compressor: FrameCompressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_parameters(¶ms); + compressor + .set_dictionary(Dictionary::from_raw_content(DICT_ID, dict.clone()).unwrap()) + .unwrap(); + compressor + }; + let mut warm = compressor(); + let first = warm.compress_independent_frame(source.as_slice()); + let restored = warm.compress_independent_frame(source.as_slice()); + let fresh = compressor().compress_independent_frame(source.as_slice()); + assert!( + first.len() < source.len() / 10, + "{strategy:?}: LDM did not reach into the dictionary ({} of {})", + first.len(), + source.len(), + ); + assert_eq!( + first, fresh, + "{strategy:?}: a fresh compressor wrote another frame" + ); + assert_eq!( + restored, fresh, + "{strategy:?}: the restored dictionary wrote another frame" + ); + let mut decoder = FrameDecoder::new(); + decoder + .add_dict(Dictionary::from_raw_content(DICT_ID, dict.clone()).unwrap()) + .unwrap(); + let mut decoded = Vec::with_capacity(source.len()); + decoder.decode_all_to_vec(&restored, &mut decoded).unwrap(); + assert_eq!(decoded, source, "{strategy:?}: round-trip"); + } +} + /// `FrameCompressor::set_parameters` on the streaming path reuses one /// compressor across frames and produces decodable output. #[test] From 5fd4ee0cfca34c3edd9b78423d126435159fbc06 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 16 Oct 2026 19:06:52 +0300 Subject: [PATCH 02/11] perf(encoding): index only the end of a long dictionary A dictionary was indexed into the match-finder tables from its first byte, however long it was. Tables of that size keep at most a window of entries anyway, so the indexing of a large reference was pure cost: priming a 60 MiB `--patch-from` reference at the default level indexed all of it densely, a third of the run. Upstream indexes only the end of a dictionary the tables can hold, `MIN(MAX(hashLog + 3, chainLog + 1), 31)` bytes (`ZSTD_loadDictionaryContent`, zstd_compress.c:4987), while the whole of it stays in the window. The driver now sets that floor for the prime and every backend's fill starts from it: the copied and attached tables of the fast, double-fast, row, chain and tree finders, and the dictionary match states. A dictionary within the reach writes the same frames as before (levels 1-19 over three inputs and three dictionaries up to 125 KB, md5 equal). A 930 KB dictionary moves a few frames by a handful of bytes, smaller in all but one (+2 bytes on 449 KB). The 61 MiB patch against a 60 MiB reference is byte-identical and takes 750 ms of task-clock instead of 1170 on the x86 bench host. --- zstd/src/encoding/dfast/mod.rs | 28 ++++++++++++++--- zstd/src/encoding/levels/config.rs | 30 +++++++++++++++++++ .../encoding/match_generator/dict_prime.rs | 11 +++++++ zstd/src/encoding/match_generator/mod.rs | 10 +++++++ zstd/src/encoding/match_table/storage.rs | 29 ++++++++++++++++-- zstd/src/encoding/row/mod.rs | 21 +++++++++++-- zstd/src/encoding/simple/fast_matcher.rs | 19 ++++++++++-- 7 files changed, 136 insertions(+), 12 deletions(-) diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index e1c6305e3..04afd1d04 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -96,6 +96,9 @@ pub(crate) struct DfastMatchGenerator { pub(crate) history_start: usize, pub(crate) history_abs_start: usize, pub(crate) offset_hist: [u32; 3], + /// While a dictionary is primed: how far into it the tables start + /// indexing, `0` for all of it (see [`Self::set_dictionary_index_floor`]). + dict_index_floor: usize, // Storage: single `u32` per bucket — upstream zstd-parity overwrite-on- // collision. Each slot holds a +1-biased relative position // (`(abs_pos - position_base + 1) as u32`); `DFAST_EMPTY_SLOT = 0` @@ -304,6 +307,7 @@ impl DfastMatchGenerator { history_start: 0, history_abs_start: 0, offset_hist: [1, 4, 8], + dict_index_floor: 0, tables: Table::empty(), tables_fresh: false, tagged: false, @@ -883,6 +887,12 @@ impl DfastMatchGenerator { } } + /// Index a dictionary being primed only from `floor` bytes into it, `0` + /// for all of it; the driver sets it for the prime and clears it after. + pub(crate) fn set_dictionary_index_floor(&mut self, floor: usize) { + self.dict_index_floor = floor; + } + pub(crate) fn skip_matching_dense(&mut self) { self.ensure_hash_tables(); let current_len = self.window_blocks.back().copied().unwrap_or(0); @@ -891,11 +901,15 @@ impl DfastMatchGenerator { // committed yet, so none to seed. return; } - let current_abs_start = self.history_abs_start + self.window_size - current_len; - let current_abs_end = current_abs_start + current_len; + let current_abs_end = self.history_abs_start + self.window_size; + // A dictionary being primed sits at the front of the history, so its + // index floor counts from there. + let floor = self.history_abs_start + self.dict_index_floor; + let current_abs_start = (current_abs_end - current_len).max(floor.min(current_abs_end)); let backfill_start = current_abs_start .saturating_sub(Self::BOUNDARY_DENSE_TAIL_LEN) - .max(self.history_abs_start); + .max(self.history_abs_start) + .max(floor); if backfill_start < current_abs_start { self.insert_positions(backfill_start, current_abs_start); } @@ -1055,7 +1069,13 @@ impl DfastMatchGenerator { // or `7+1` dict chunking), so gate the early return on `backfill_floor`, // not `start_concat` — otherwise those seam inserts are dropped and the // attached dict tables stay incomplete. - let backfill_floor = start_concat.saturating_sub(Self::BOUNDARY_DENSE_TAIL_LEN); + // Nothing below the index floor is indexed, seam included: the chunk + // before it was not. + let floor = self.dict_index_floor.min(end_concat); + let backfill_floor = start_concat + .max(floor) + .saturating_sub(Self::BOUNDARY_DENSE_TAIL_LEN) + .max(floor); if backfill_floor >= short_safe_end { return; } diff --git a/zstd/src/encoding/levels/config.rs b/zstd/src/encoding/levels/config.rs index 3ea6451a2..1ff642e00 100644 --- a/zstd/src/encoding/levels/config.rs +++ b/zstd/src/encoding/levels/config.rs @@ -226,6 +226,36 @@ impl LevelParams { pub(crate) fn pre_split(&self) -> Option { Some(pre_split_for(self.strategy_tag, self.lazy_depth)) } + + /// `log2` of how much of a dictionary's end the match-finder tables index + /// when it is loaded: a longer dictionary is indexed only that far from its + /// end, since tables this size could not keep more of it anyway (upstream + /// `ZSTD_loadDictionaryContent`, `zstd_compress.c:4987`: + /// `MIN(MAX(hashLog + 3, chainLog + 1), 31)`). Every byte of it stays in + /// the window. + pub(crate) fn dictionary_reach_log(&self) -> u32 { + use crate::encoding::strategy::SearchMethod; + // The active backend's config: an override may leave the level row's + // own beside it. + let logs = match self.search { + SearchMethod::Fast => self.fast.map(|fast| (fast.hash_log as usize, 0)), + SearchMethod::DoubleFast => self + .dfast + .map(|dfast| (dfast.long_hash_log as usize, dfast.short_hash_log as usize)), + SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => { + self.row.map(|row| (row.hash_bits, row.chain_log)) + } + SearchMethod::HashChain | SearchMethod::BinaryTree => { + self.hc.map(|hc| (hc.hash_log, hc.chain_log)) + } + }; + // The reset synthesises the active config when a row lacks it, so a + // missing one only means a table-less level: nothing to index. + let Some((hash_log, chain_log)) = logs else { + return 0; + }; + (hash_log + 3).max(chain_log + 1).min(31) as u32 + } } /// The pre-splitter level for an effective strategy (the tag plus, for the diff --git a/zstd/src/encoding/match_generator/dict_prime.rs b/zstd/src/encoding/match_generator/dict_prime.rs index 1657b9c37..97f1b06e4 100644 --- a/zstd/src/encoding/match_generator/dict_prime.rs +++ b/zstd/src/encoding/match_generator/dict_prime.rs @@ -103,6 +103,15 @@ impl MatchGeneratorDriver { } #[cfg(feature = "ldm")] let ldm_dict_start = self.ldm_committed_end; + // The match-finder tables index only as much of the dictionary's end as + // they can hold (upstream `ZSTD_loadDictionaryContent`, + // `zstd_compress.c:4987`); the whole of it is committed all the same, + // so every byte stays in reach of an offset. + let reach_log = self + .reset_shape + .map_or(31, |(params, ..)| params.dictionary_reach_log()); + self.storage + .set_dictionary_index_floor(dict_content.len().saturating_sub(1usize << reach_log)); // Dictionary bytes should stay addressable until produced frame output // itself exceeds the live window size. We bump `max_window_size` @@ -331,6 +340,8 @@ impl MatchGeneratorDriver { super::super::strategy::BackendTag::Row => self.row_matcher_mut().mark_dict_primed(), _ => {} } + // The dictionary is indexed; later inserts are the input's own. + self.storage.set_dictionary_index_floor(0); } pub(super) fn restore_primed_dictionary_impl( diff --git a/zstd/src/encoding/match_generator/mod.rs b/zstd/src/encoding/match_generator/mod.rs index e43b87fce..f728ab9b1 100644 --- a/zstd/src/encoding/match_generator/mod.rs +++ b/zstd/src/encoding/match_generator/mod.rs @@ -239,6 +239,16 @@ impl MatcherStorage { } } + /// See the backends' `set_dictionary_index_floor`. + fn set_dictionary_index_floor(&mut self, floor: usize) { + match self { + Self::Simple(m) => m.set_dictionary_index_floor(floor), + Self::Dfast(m) => m.set_dictionary_index_floor(floor), + Self::Row(m) => m.set_dictionary_index_floor(floor), + Self::HashChain(m) => m.table.set_dictionary_index_floor(floor), + } + } + /// The committed window of the owned history, oldest byte first. #[cfg(feature = "ldm")] fn committed_window(&self) -> &[u8] { diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index 944b1a9fe..8d98dd944 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -261,6 +261,9 @@ pub(crate) struct MatchTable { pub(crate) hash3_log: usize, pub(crate) next_to_update3: usize, pub(crate) skip_insert_until_abs: usize, + /// While a dictionary is primed: how far into it the tables start + /// indexing, `0` for all of it (see [`Self::set_dictionary_index_floor`]). + pub(crate) dict_index_floor: usize, pub(crate) dictionary_limit_abs: Option, pub(crate) dictionary_primed_for_frame: bool, /// Sticky across resets: set once a dictionary is primed into this table @@ -360,6 +363,7 @@ impl Clone for MatchTable { hash3_log: self.hash3_log, next_to_update3: self.next_to_update3, skip_insert_until_abs: self.skip_insert_until_abs, + dict_index_floor: self.dict_index_floor, dictionary_limit_abs: self.dictionary_limit_abs, dictionary_primed_for_frame: self.dictionary_primed_for_frame, dictionary_active: self.dictionary_active, @@ -407,6 +411,7 @@ impl Clone for MatchTable { self.hash3_log = source.hash3_log; self.next_to_update3 = source.next_to_update3; self.skip_insert_until_abs = source.skip_insert_until_abs; + self.dict_index_floor = source.dict_index_floor; self.dictionary_limit_abs = source.dictionary_limit_abs; self.dictionary_primed_for_frame = source.dictionary_primed_for_frame; self.dictionary_active = source.dictionary_active; @@ -541,6 +546,7 @@ impl MatchTable { hash3_log: HC3_HASH_LOG, next_to_update3: 0, skip_insert_until_abs: 0, + dict_index_floor: 0, dictionary_limit_abs: None, dictionary_primed_for_frame: false, dictionary_active: false, @@ -984,6 +990,7 @@ impl MatchTable { // generous fixed depth keeps the tree well-ordered without the live // searchLog cap. Mirrors upstream zstd `ZSTD_insertBt1` nbCompares. const DMS_BUILD_DEPTH: usize = 1 << 9; + let floor = self.dict_index_floor; // 2 children per dict position: [smaller, larger]. build_dms!( self, @@ -993,7 +1000,7 @@ impl MatchTable { read, 2, |concat, hash_table, chain_table, dms_hash_log, region| { - let mut current = 0usize; + let mut current = floor; while current + read <= region { let h = Self::hash_position_at(concat, current, dms_hash_log, mls); // Insert `current` as the new root; splay the old tree into @@ -1061,6 +1068,7 @@ impl MatchTable { /// chain. Cached across frames via `DictAttach::is_primed`. pub(crate) fn prime_dms_hc(&mut self, region_len: usize) { let mls = HC_MIN_MATCH_LEN; + let floor = self.dict_index_floor; // Single `next` link per dict position (HC4 chain, not the BT's 2/pos). build_dms!( self, @@ -1070,7 +1078,7 @@ impl MatchTable { mls, 1, |concat, hash_table, chain_table, dms_hash_log, region| { - let mut current = 0usize; + let mut current = floor; while current + mls <= region { let h = Self::hash_position_at(concat, current, dms_hash_log, mls); // Prepend `current` to its bucket chain (upstream zstd @@ -2401,6 +2409,19 @@ impl MatchTable { self.next_to_update3 = self.next_to_update3.max(committed_end); } + /// Index a dictionary being primed only from `floor` bytes into it, `0` + /// for all of it; the driver sets it for the prime and clears it after. + /// The tree's insertion cursors start from the floor too, since a copied + /// dictionary is threaded through them. + pub(crate) fn set_dictionary_index_floor(&mut self, floor: usize) { + self.dict_index_floor = floor; + if floor != 0 { + let floor_abs = self.history_abs_start + floor; + self.skip_insert_until_abs = self.skip_insert_until_abs.max(floor_abs); + self.next_to_update3 = self.next_to_update3.max(floor_abs); + } + } + /// Hand the last committed block of `len` bytes back to the uncommitted /// input, undoing only the bookkeeping of [`Self::commit_block`]: what it /// evicted stays evicted, so committing the same bytes again in pieces @@ -2447,6 +2468,10 @@ impl MatchTable { self.ensure_tables(); let (current_abs_start, current_len) = self.current_block_range(); let current_abs_end = current_abs_start + current_len; + // A dictionary being primed sits at the front of the history and is + // indexed only from its floor on. + let floor = (self.history_abs_start + self.dict_index_floor).min(current_abs_end); + let current_abs_start = current_abs_start.max(floor); self.backfill_boundary_positions(current_abs_start, current_abs_end); if self.uses_bt { if incompressible_hint == Some(true) { diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 2f46a8cb6..6ffdad0cd 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -2419,6 +2419,9 @@ pub(crate) struct RowMatchGenerator { /// stores it raw and rotates, so the encoder-side history and /// upstream's parse-side reps legitimately differ. lazy_reps: Option<[usize; 2]>, + /// While a dictionary is primed: how far into it the tables start + /// indexing, `0` for all of it (see [`Self::set_dictionary_index_floor`]). + dict_index_floor: usize, pub(crate) lazy_depth: u8, /// Cached fastpath kernel for the per-candidate `common_prefix_len` /// compare, resolved once per matcher so the rep probe skips the @@ -2547,6 +2550,7 @@ impl RowMatchGenerator { row_hash_mls: ROW_MIN_MATCH_LEN.clamp(4, 6) as u32, lazy_next_to_update: 0, lazy_reps: None, + dict_index_floor: 0, lazy_depth: 1, cpl_kernel: crate::encoding::fastpath::select_kernel(), rows_len: 0, @@ -4012,6 +4016,12 @@ impl RowMatchGenerator { crate::encoding::fastpath::scalar::common_prefix_len_ptr ); + /// Index a dictionary being primed only from `floor` bytes into it, `0` + /// for all of it; the driver sets it for the prime and clears it after. + pub(crate) fn set_dictionary_index_floor(&mut self, floor: usize) { + self.dict_index_floor = floor; + } + /// Hand the last committed block of `len` bytes back to the uncommitted /// input, undoing only the bookkeeping of [`Self::commit_block`]: what it /// evicted stays evicted, so committing the same bytes again in pieces @@ -4474,6 +4484,7 @@ impl RowMatchGenerator { bt: &mut [u32], content: *const u8, concat_len: usize, + start: usize, index_base: usize, hash_log: usize, chain_log: usize, @@ -4481,7 +4492,7 @@ impl RowMatchGenerator { mls: u32, ) { let target = concat_len.saturating_sub(8); - let mut idx = 0usize; + let mut idx = start; while idx < target { // SAFETY: `idx + 8 <= concat_len`; the tree holds only earlier // dictionary positions (this loop's own inserts). @@ -4526,6 +4537,7 @@ impl RowMatchGenerator { hc_chain, base.add(history_start), concat_len, + self.dict_index_floor, index_base, hash_log, chain_log, @@ -4557,6 +4569,7 @@ impl RowMatchGenerator { } let base = self.history.as_ptr(); let history_start = self.history_start; + let floor = self.dict_index_floor; let dict = self.dict.table_mut_or_init(|| RowDictTables { heads: Vec::new(), positions: Vec::new(), @@ -4581,6 +4594,7 @@ impl RowMatchGenerator { &mut dict.hc_chain, base.add(history_start), concat_len, + floor, BT_IDX_BASE, hash_log, chain_log, @@ -4644,7 +4658,8 @@ impl RowMatchGenerator { // tables. The index cursor carries across the dictionary's // slices; `mark_dict_primed` moves it to the dictionary end. self.dict.set_region_len(concat_len); - let from = self.lazy_next_to_update.max(hist_start) - hist_start; + let from = + (self.lazy_next_to_update.max(hist_start) - hist_start).max(self.dict_index_floor); if from < indexable_end { let scan = self.scan_ctx(); if self.finder == LazyFinder::Chain { @@ -4709,7 +4724,7 @@ impl RowMatchGenerator { if self.dict.is_primed() { return; } - let from = self.dict.next_to_update(); + let from = self.dict.next_to_update().max(self.dict_index_floor); if from >= indexable_end { return; } diff --git a/zstd/src/encoding/simple/fast_matcher.rs b/zstd/src/encoding/simple/fast_matcher.rs index d6d3a15d9..4b9aa4563 100644 --- a/zstd/src/encoding/simple/fast_matcher.rs +++ b/zstd/src/encoding/simple/fast_matcher.rs @@ -341,6 +341,9 @@ pub(crate) struct FastKernelMatcher { /// the slices one dictionary is committed in so the phase stays continuous. /// See [`Self::prime_hash_table_for_dict_copy`]. Reset with the table. dict_copy_fill_next: usize, + /// While a dictionary is primed: how far into it the tables start + /// indexing, `0` for all of it (see [`Self::set_dictionary_index_floor`]). + dict_index_floor: usize, } impl Clone for FastKernelMatcher { @@ -367,6 +370,7 @@ impl Clone for FastKernelMatcher { dict_table_hash_log: self.dict_table_hash_log, loaded_dict_end: self.loaded_dict_end, dict_copy_fill_next: self.dict_copy_fill_next, + dict_index_floor: self.dict_index_floor, } } @@ -395,6 +399,7 @@ impl Clone for FastKernelMatcher { self.dict_resident = source.dict_resident; self.loaded_dict_end = source.loaded_dict_end; self.dict_copy_fill_next = source.dict_copy_fill_next; + self.dict_index_floor = source.dict_index_floor; } } @@ -632,6 +637,7 @@ impl FastKernelMatcher { dict_table_hash_log: None, loaded_dict_end: 0, dict_copy_fill_next: 0, + dict_index_floor: 0, } } @@ -1653,6 +1659,12 @@ impl FastKernelMatcher { } } + /// Index a dictionary being primed only from `floor` bytes into it, `0` + /// for all of it; the driver sets it for the prime and clears it after. + pub(crate) fn set_dictionary_index_floor(&mut self, floor: usize) { + self.dict_index_floor = floor; + } + /// The committed window, oldest byte first. #[cfg(feature = "ldm")] pub(crate) fn committed_window(&self) -> &[u8] { @@ -1935,7 +1947,7 @@ impl FastKernelMatcher { return; } let last_hashable = history_len - HASH_READ_SIZE; - let fill_start = self.dict_copy_fill_next; + let fill_start = self.dict_copy_fill_next.max(self.dict_index_floor); if fill_start > last_hashable { return; } @@ -2261,7 +2273,8 @@ impl FastKernelMatcher { "dict fill origin {} ran ahead of slice start {range_start}", self.dict.next_to_update(), ); - if self.dict.next_to_update() > last_hashable { + // Nothing below the index floor is indexed. + if self.dict.next_to_update().max(self.dict_index_floor) > last_hashable { return; } // Size the dict table to the CDict geometry of the WHOLE dictionary @@ -2295,7 +2308,7 @@ impl FastKernelMatcher { self.dict.invalidate(); self.dict.set_region_len(history_len); } - let fill_start = self.dict.next_to_update(); + let fill_start = self.dict.next_to_update().max(self.dict_index_floor); self.dict .table_mut_or_init(|| FastHashTable::new(hash_log, mls)); let base = self.history.as_ptr(); From f7a839e56b6fe58f0f61630124094a363ffb9fc4 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 16 Oct 2026 19:37:56 +0300 Subject: [PATCH 03/11] feat(encoding): keep long-distance matches whole across block cuts The block splitter reads byte statistics only. Over input that a long match covers it still cuts, since a sparse sample of random bytes looks different from the next one, and every cut costs a block whose one sequence carries the same match on: a 61 MiB patch came out as 7605 blocks of a single sequence each. - The long-distance matches are now searched ahead of the blocks and kept between them, as upstream keeps a raw sequence store across blocks: each block takes the matches covering it, the one crossing its end cut there and carried into the next, a piece shorter than the level's minMatch left as literals (`maybeSplitSequence`, `ZSTD_ldm_skipSequences`). - Before a cut, both frame loops ask the matcher through the new `Matcher::settle_block_cut`; the driver searches the window the splitter looked at and moves a cut that falls inside a long match to the match's end. The 61 MiB patch against a 60 MiB reference drops from 76127 bytes to 5386 (upstream: 71906 by default, 13163 single-threaded) and round-trips through both decoders. Without long-distance matching nothing changes (corpus totals at levels 1-19 equal). With `--long` the corpus comes out smaller than upstream single-threaded `--long` at every level but 1, where it is 0.02% larger. --- zstd/src/encoding/dfast/mod.rs | 6 + zstd/src/encoding/frame_compressor.rs | 7 + zstd/src/encoding/ldm/mod.rs | 1 + zstd/src/encoding/ldm/store.rs | 132 ++++++++++++++++ zstd/src/encoding/ldm/store/tests.rs | 104 +++++++++++++ zstd/src/encoding/levels/config.rs | 16 ++ .../encoding/match_generator/long_distance.rs | 133 +++++++++++++++++ zstd/src/encoding/match_generator/mod.rs | 141 +++++++----------- zstd/src/encoding/match_table/storage.rs | 6 + zstd/src/encoding/mod.rs | 10 ++ zstd/src/encoding/row/mod.rs | 6 + zstd/src/encoding/simple/fast_matcher.rs | 6 + zstd/src/encoding/streaming_encoder.rs | 14 +- zstd/src/tests/parameters_test.rs | 36 +++++ 14 files changed, 524 insertions(+), 94 deletions(-) create mode 100644 zstd/src/encoding/ldm/store.rs create mode 100644 zstd/src/encoding/ldm/store/tests.rs create mode 100644 zstd/src/encoding/match_generator/long_distance.rs diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index 04afd1d04..c20bbc51e 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -916,6 +916,12 @@ impl DfastMatchGenerator { self.insert_positions(current_abs_start, current_abs_end); } + /// The committed window followed by the input read but not yet committed. + #[cfg(feature = "ldm")] + pub(crate) fn window_with_pending(&self) -> &[u8] { + &self.history[self.history_start..] + } + /// Hand the last committed block of `len` bytes back to the uncommitted /// input, undoing only the bookkeeping of [`Self::commit_block`]: what it /// evicted stays evicted, so committing the same bytes again in pieces diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 032f4a93f..5f5660f8d 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -2741,6 +2741,13 @@ impl FrameCompressor { block_capacity, savings, ); + // The splitter sees byte statistics only; a cut it proposes + // inside a match the matcher already knows of is moved out. + let split_at = if split_at < available { + self.state.matcher.settle_block_cut(available, split_at) + } else { + split_at + }; if split_at < available { block_len = split_at; last_block = false; diff --git a/zstd/src/encoding/ldm/mod.rs b/zstd/src/encoding/ldm/mod.rs index af868a9af..b53fc65a5 100644 --- a/zstd/src/encoding/ldm/mod.rs +++ b/zstd/src/encoding/ldm/mod.rs @@ -47,6 +47,7 @@ use super::opt::ldm::HcRawSeq; pub(crate) mod gear_hash; pub(crate) mod params; pub(crate) mod search; +pub(crate) mod store; pub(crate) mod table; use gear_hash::{GearHashState, LDM_BATCH_SIZE}; diff --git a/zstd/src/encoding/ldm/store.rs b/zstd/src/encoding/ldm/store.rs new file mode 100644 index 000000000..c89d415f4 --- /dev/null +++ b/zstd/src/encoding/ldm/store.rs @@ -0,0 +1,132 @@ +//! Long-distance matches searched ahead of the blocks that consume them. +//! +//! The search runs over whatever input is known, which can reach past the +//! block about to be compressed: the block splitter asks where long matches +//! lie before it cuts. Each block then takes the matches covering it, the one +//! crossing its end cut there and carried into the next block, as upstream +//! carries a raw sequence store across blocks (`maybeSplitSequence` and +//! `ZSTD_ldm_skipSequences`, `zstd_ldm.c:607-666`). + +use alloc::vec::Vec; + +use crate::encoding::opt::ldm::HcRawSeq; + +/// One match found by the search, in stream positions. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +struct LongMatch { + start: usize, + end: usize, + offset: usize, +} + +#[derive(Default)] +pub(crate) struct LdmSequenceStore { + /// Found and not yet taken past, in stream order. + matches: Vec, + /// Index of the first match a block has not taken to its end. + pos: usize, + /// Stream position the next block starts at. + taken_end: usize, + /// Stream position the input is searched up to. + searched_end: usize, +} + +impl LdmSequenceStore { + /// Forget everything, with input from `at` on unsearched and untaken. + pub(crate) fn reset(&mut self, at: usize) { + self.matches.clear(); + self.pos = 0; + self.taken_end = at; + self.searched_end = at; + } + + /// Stream position the input is searched up to. + pub(crate) fn searched_end(&self) -> usize { + self.searched_end + } + + /// Stream position the next block starts at. + pub(crate) fn taken_end(&self) -> usize { + self.taken_end + } + + /// Heap bytes the matches hold. + pub(crate) fn heap_size(&self) -> usize { + self.matches.capacity() * core::mem::size_of::() + } + + /// Record the sequences a search of `[searched_end, to)` found, literal + /// lengths counted from the start of that range. + pub(crate) fn searched(&mut self, found: &[HcRawSeq], to: usize) { + let mut at = self.searched_end; + for seq in found { + let start = at + seq.lit_length; + let end = start + seq.match_length; + self.matches.push(LongMatch { + start, + end, + offset: seq.offset, + }); + at = end; + } + debug_assert!(at <= to, "a sequence reaches past the searched range"); + self.searched_end = to; + } + + /// `cut` bytes past the next block's start, moved out of any long match it + /// falls inside: to the match end, or to `window` when the match runs past + /// it. Cutting inside one buys a block whose first sequence repeats the + /// offset the previous one ended on. + pub(crate) fn cut_outside_matches(&self, window: usize, cut: usize) -> usize { + let cut_at = self.taken_end + cut; + for found in &self.matches[self.pos..] { + if found.start >= cut_at { + break; + } + if cut_at < found.end { + return (found.end - self.taken_end).min(window); + } + } + cut + } + + /// Take the matches of the next `len` bytes into `out` as sequences, + /// literal lengths counted from the block start. A match crossing the + /// block end is cut there, the rest carried into the next block; a piece + /// shorter than `min_match`, on either side of the cut, is left as + /// literals. + pub(crate) fn take(&mut self, len: usize, min_match: usize, out: &mut Vec) { + out.clear(); + let block_end = self.taken_end + len; + let mut at = self.taken_end; + while let Some(&found) = self.matches.get(self.pos) { + if found.start >= block_end { + break; + } + let start = found.start.max(self.taken_end); + let end = found.end.min(block_end); + if end - start >= min_match { + out.push(HcRawSeq { + lit_length: start - at, + offset: found.offset, + match_length: end - start, + }); + at = end; + } + if found.end > block_end { + break; + } + self.pos += 1; + } + self.taken_end = block_end; + // Taken matches go once they are the bulk of the vector, so a search + // far ahead of a long stream holds only what is still to come. + if self.pos > self.matches.len() / 2 { + self.matches.drain(..self.pos); + self.pos = 0; + } + } +} + +#[cfg(test)] +mod tests; diff --git a/zstd/src/encoding/ldm/store/tests.rs b/zstd/src/encoding/ldm/store/tests.rs new file mode 100644 index 000000000..87ab7a4ed --- /dev/null +++ b/zstd/src/encoding/ldm/store/tests.rs @@ -0,0 +1,104 @@ +use alloc::vec; +use alloc::vec::Vec; + +use super::LdmSequenceStore; +use crate::encoding::opt::ldm::HcRawSeq; + +fn seq(lit_length: usize, offset: usize, match_length: usize) -> HcRawSeq { + HcRawSeq { + lit_length, + offset, + match_length, + } +} + +fn triples(out: &[HcRawSeq]) -> Vec<(usize, usize, usize)> { + out.iter() + .map(|s| (s.lit_length, s.offset, s.match_length)) + .collect() +} + +/// A block takes its matches with literals counted from its own start, and a +/// later search's literal lengths count from where the previous one stopped. +#[test] +fn blocks_take_their_matches_relative_to_their_start() { + let mut store = LdmSequenceStore::default(); + store.reset(1000); + store.searched(&[seq(10, 500, 100), seq(20, 700, 64)], 1300); + store.searched(&[seq(50, 900, 80)], 1600); + let mut out = Vec::new(); + store.take(200, 4, &mut out); + // [1000, 1200): 10 literals, a 100-byte match, 20 literals, a 64-byte one. + assert_eq!(triples(&out), vec![(10, 500, 100), (20, 700, 64)]); + store.take(400, 4, &mut out); + // The third match starts at 1300 + 50, 150 bytes into this block. + assert_eq!(triples(&out), vec![(150, 900, 80)]); + assert_eq!(store.taken_end(), 1600); +} + +/// A match crossing a block end is cut there and its rest carried into the +/// next block at the same offset; a piece under `min_match` stays literals, +/// on either side. +#[test] +fn a_match_crossing_the_block_end_is_carried() { + let mut store = LdmSequenceStore::default(); + store.reset(0); + store.searched(&[seq(10, 300, 200)], 1000); + let mut out = Vec::new(); + store.take(110, 4, &mut out); + assert_eq!(triples(&out), vec![(10, 300, 100)]); + store.take(500, 4, &mut out); + assert_eq!(triples(&out), vec![(0, 300, 100)]); + + // Cut two bytes into the match: the head is under `min_match` and stays + // literals, the tail is taken. + let mut store = LdmSequenceStore::default(); + store.reset(0); + store.searched(&[seq(10, 300, 200)], 1000); + store.take(12, 4, &mut out); + assert!(out.is_empty()); + store.take(500, 4, &mut out); + assert_eq!(triples(&out), vec![(0, 300, 198)]); + + // And a short tail is literals of the next block. + let mut store = LdmSequenceStore::default(); + store.reset(0); + store.searched(&[seq(10, 300, 200), seq(30, 400, 50)], 1000); + store.take(208, 4, &mut out); + assert_eq!(triples(&out), vec![(10, 300, 198)]); + store.take(500, 4, &mut out); + assert_eq!(triples(&out), vec![(32, 400, 50)]); +} + +/// A cut inside a match moves to the match end, or to the window end when the +/// match runs past it; a cut between matches stays. +#[test] +fn a_cut_is_moved_out_of_a_match() { + let mut store = LdmSequenceStore::default(); + store.reset(100); + store.searched(&[seq(10, 300, 200), seq(100, 700, 5000)], 10_000); + // Inside the first match [110, 310): moved to its end, 210 from the start. + assert_eq!(store.cut_outside_matches(8192, 150), 210); + // Between the matches: kept. + assert_eq!(store.cut_outside_matches(8192, 250), 250); + // Inside the second, which runs past the window: the whole window. + assert_eq!(store.cut_outside_matches(4096, 1000), 4096); + // On a match's first byte is not inside it. + assert_eq!(store.cut_outside_matches(8192, 10), 10); +} + +/// Taken matches are dropped once they are the bulk of the store, without +/// moving what is still to come. +#[test] +fn taken_matches_are_dropped() { + let mut store = LdmSequenceStore::default(); + store.reset(0); + let found: Vec = (0..100).map(|_| seq(10, 64, 90)).collect(); + store.searched(&found, 10_000); + let mut out = Vec::new(); + store.take(6000, 4, &mut out); + assert_eq!(out.len(), 60); + store.take(4000, 4, &mut out); + assert_eq!(out.len(), 40); + assert_eq!(triples(&out)[0], (10, 64, 90)); +} diff --git a/zstd/src/encoding/levels/config.rs b/zstd/src/encoding/levels/config.rs index 1ff642e00..54d51e031 100644 --- a/zstd/src/encoding/levels/config.rs +++ b/zstd/src/encoding/levels/config.rs @@ -256,6 +256,22 @@ impl LevelParams { }; (hash_log + 3).max(chain_log + 1).min(31) as u32 } + + /// The level's upstream `cParams.minMatch`: the shortest match its search + /// accepts, and so the shortest piece of a long-distance match a block cut + /// leaves worth keeping (`maybeSplitSequence`, `zstd_ldm.c:644`). + #[cfg(feature = "ldm")] + pub(crate) fn min_match(&self) -> usize { + use crate::encoding::strategy::SearchMethod; + let configured = match self.search { + SearchMethod::Fast => self.fast.map(|fast| fast.mls as usize), + SearchMethod::DoubleFast => Some(crate::encoding::match_generator::DFAST_MIN_MATCH_LEN), + SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => self.row.map(|row| row.mls), + SearchMethod::HashChain => Some(crate::encoding::hc::HC_MIN_MATCH_LEN), + SearchMethod::BinaryTree => self.hc.map(|hc| hc.search_mls), + }; + configured.unwrap_or(crate::encoding::cost_model::HC_FORMAT_MINMATCH) + } } /// The pre-splitter level for an effective strategy (the tag plus, for the diff --git a/zstd/src/encoding/match_generator/long_distance.rs b/zstd/src/encoding/match_generator/long_distance.rs new file mode 100644 index 000000000..45ac017bb --- /dev/null +++ b/zstd/src/encoding/match_generator/long_distance.rs @@ -0,0 +1,133 @@ +//! Long-distance matching in the [`MatchGeneratorDriver`]: the search ahead of +//! the blocks, the matches each block takes, and the block matched around them +//! below the optimal parser. +//! +//! A child module so it reaches the driver's private long-distance state and +//! storage directly. + +use super::{MatchGeneratorDriver, Sequence}; + +impl MatchGeneratorDriver { + /// Search the input from where the last search stopped up to `to`, a + /// stream position at most the end of the input read so far. + /// + /// What a match may reach is fixed against the END of the range, as + /// upstream enforces the distance once per chunk before generating + /// (`ZSTD_window_enforceMaxDist` on `chunkEnd`, `zstd_ldm.c:585`): one + /// window back from it, or the whole dictionary while the window has not + /// yet passed the dictionary's end. Every match then holds an offset valid + /// at its last byte, wherever in the range it lies. + fn search_long_distance_to(&mut self, to: usize) { + let Some(producer) = self.ldm.as_mut() else { + return; + }; + let from = self.ldm_store.searched_end(); + if from >= to { + return; + } + let window = self.reported_window_size; + let reach_floor = if self.ldm_dict_end != 0 && to <= window + self.ldm_dict_end { + 0 + } else { + to.saturating_sub(window) + }; + let live_start = self.ldm_committed_end - self.storage.committed_window().len(); + let live = self.storage.window_with_pending(); + let floor = reach_floor.max(live_start); + debug_assert!(floor <= from, "a search range wider than the window"); + self.ldm_found.clear(); + producer.generate_into( + &live[floor - live_start..], + floor, + from, + to, + &mut self.ldm_found, + ); + self.ldm_store.searched(&self.ldm_found, to); + } + + /// Line the matches up with the next block starting at `at`: anything + /// kept for another position (a prime, a reset, a skipped search) is gone. + fn align_long_distance_store(&mut self, at: usize) { + if self.ldm_store.taken_end() != at { + self.ldm_store.reset(at); + } + } + + /// The matches of the last committed block, into [`Self::ldm_block`], + /// searched first as far as the block reaches. Returns the block's length. + pub(super) fn take_long_distance_block(&mut self) -> usize { + let block_len = self.storage.last_block_len(); + let block_end = self.ldm_committed_end; + self.align_long_distance_store(block_end - block_len); + self.search_long_distance_to(block_end); + let min_match = self.reset_shape.map_or( + crate::encoding::cost_model::HC_FORMAT_MINMATCH, + |(params, ..)| params.min_match(), + ); + self.ldm_store + .take(block_len, min_match, &mut self.ldm_block); + block_len + } + + /// See [`crate::encoding::Matcher::settle_block_cut`]: the next `window` + /// uncommitted bytes are searched now, and the cut moved out of any long + /// match. + pub(super) fn settle_long_distance_cut(&mut self, window: usize, cut: usize) -> usize { + if self.ldm.is_none() { + return cut; + } + let start = self.ldm_committed_end; + self.align_long_distance_store(start); + self.search_long_distance_to(start + window); + self.ldm_store.cut_outside_matches(window, cut) + } + + /// Take the block's long-distance matches outright and match only the + /// literals between them with the frame's backend (upstream + /// `ZSTD_ldm_blockCompress` below btopt). The block is handed back to the + /// uncommitted input and committed again in pieces, so each gap is the + /// backend's whole block and each match body is indexed without being + /// searched. Returns `false` when the block has no long-distance match, + /// leaving it committed for the ordinary search. + pub(super) fn start_matching_long_distance( + &mut self, + handle_sequence: &mut impl FnMut(Sequence), + ) -> bool { + let block_len = self.take_long_distance_block(); + if self.ldm_block.is_empty() { + return false; + } + let sequences = core::mem::take(&mut self.ldm_block); + self.storage.uncommit_block(block_len); + let mut consumed = 0usize; + for seq in &sequences { + // The gap's trailing literals become the long match's own. + let mut gap_literals = 0usize; + if seq.lit_length != 0 { + self.storage.commit_piece(seq.lit_length); + self.start_matching_owned(&mut |sequence| match sequence { + Sequence::Literals { len } => gap_literals = len, + triple => handle_sequence(triple), + }); + } + self.storage.commit_piece(seq.match_length); + self.storage.skip_long_match(); + handle_sequence(Sequence::Triple { + literal_len: gap_literals, + offset: seq.offset, + match_len: seq.match_length, + }); + self.storage.note_long_match(seq.offset, gap_literals); + consumed += seq.lit_length + seq.match_length; + } + let tail = block_len - consumed; + if tail != 0 { + self.storage.commit_piece(tail); + self.start_matching_owned(handle_sequence); + } + self.ldm_block = sequences; + self.ldm_split_block = Some(block_len); + true + } +} diff --git a/zstd/src/encoding/match_generator/mod.rs b/zstd/src/encoding/match_generator/mod.rs index f728ab9b1..4b32b31b0 100644 --- a/zstd/src/encoding/match_generator/mod.rs +++ b/zstd/src/encoding/match_generator/mod.rs @@ -161,6 +161,10 @@ use super::hc::generator::HcMatchGenerator; // can reach the driver's private `primed` / `reset_shape` state directly; the // `Matcher` trait's dict entry points forward to its inherent `*_impl` helpers. mod dict_prime; +// Long-distance matching: the search ahead of the blocks and the blocks +// matched around its matches. A child module for the same private access. +#[cfg(feature = "ldm")] +mod long_distance; // `Strategy` and `StrategyTag` live in `crate::encoding::strategy`. // The driver carries a `StrategyTag` field set at `reset()` and @@ -260,6 +264,17 @@ impl MatcherStorage { } } + /// The committed window followed by the input read but not committed. + #[cfg(feature = "ldm")] + fn window_with_pending(&self) -> &[u8] { + match self { + Self::Simple(m) => m.window_with_pending(), + Self::Dfast(m) => m.window_with_pending(), + Self::Row(m) => m.window_with_pending(), + Self::HashChain(m) => m.table.window_with_pending(), + } + } + /// Length of the last committed block. #[cfg(feature = "ldm")] fn last_block_len(&self) -> usize { @@ -450,9 +465,15 @@ pub struct MatchGeneratorDriver { /// `ldmState` beside the match state: every strategy reads the same one. #[cfg(feature = "ldm")] ldm: Option, - /// The long-distance sequences found for the block being matched. + /// The long-distance matches searched ahead of the blocks. + #[cfg(feature = "ldm")] + ldm_store: super::ldm::store::LdmSequenceStore, + /// What the last search found, before the store takes it. + #[cfg(feature = "ldm")] + ldm_found: Vec, + /// The long-distance sequences of the block being matched. #[cfg(feature = "ldm")] - ldm_sequences: Vec, + ldm_block: Vec, /// Stream position of the committed end, in the coordinates the /// long-distance matcher keeps its table in: the frame's first byte /// (its dictionary's, when it has one) is `0`. @@ -748,7 +769,11 @@ impl MatchGeneratorDriver { #[cfg(feature = "ldm")] ldm: None, #[cfg(feature = "ldm")] - ldm_sequences: Vec::new(), + ldm_store: Default::default(), + #[cfg(feature = "ldm")] + ldm_found: Vec::new(), + #[cfg(feature = "ldm")] + ldm_block: Vec::new(), #[cfg(feature = "ldm")] ldm_committed_end: 0, #[cfg(feature = "ldm")] @@ -1230,7 +1255,9 @@ impl Matcher for MatchGeneratorDriver { #[cfg(feature = "ldm")] let ldm = self.ldm.as_ref().map_or(0, |p| p.heap_size()) + self.primed_ldm.as_ref().map_or(0, |p| p.heap_size()) - + self.ldm_sequences.capacity() * core::mem::size_of::(); + + self.ldm_store.heap_size() + + (self.ldm_found.capacity() + self.ldm_block.capacity()) + * core::mem::size_of::(); #[cfg(not(feature = "ldm"))] let ldm = 0; self.storage.heap_size() + snapshot + self.own_workspace.heap_bytes() + ldm @@ -1982,6 +2009,16 @@ impl Matcher for MatchGeneratorDriver { false } + fn settle_block_cut(&mut self, window: usize, cut: usize) -> usize { + #[cfg(feature = "ldm")] + return self.settle_long_distance_cut(window, cut); + #[cfg(not(feature = "ldm"))] + { + let _ = window; + cut + } + } + fn get_last_space(&mut self) -> &[u8] { // A block matched in pieces is still one block to the encoder. #[cfg(feature = "ldm")] @@ -2124,14 +2161,11 @@ impl Matcher for MatchGeneratorDriver { // below it they are taken outright (upstream // `ZSTD_ldm_blockCompress`, `zstd_ldm.c:685-749`). if matches!(self.search, super::strategy::SearchMethod::BinaryTree) { - self.generate_long_distance_sequences(); + self.take_long_distance_block(); let MatcherStorage::HashChain(hc) = &mut self.storage else { unreachable!("the optimal parser runs on the hash-chain backend"); }; - core::mem::swap( - &mut self.ldm_sequences, - &mut hc.backend.bt_mut().ldm_sequences, - ); + core::mem::swap(&mut self.ldm_block, &mut hc.backend.bt_mut().ldm_sequences); } else if self.start_matching_long_distance(&mut handle_sequence) { return; } @@ -2169,6 +2203,13 @@ impl Matcher for MatchGeneratorDriver { } return; } + // A block written out unsearched is still searched for long matches, + // as upstream searches every block it compresses, and its matches are + // taken past. + #[cfg(feature = "ldm")] + if self.ldm.is_some() { + self.take_long_distance_block(); + } match self.active_backend() { super::strategy::BackendTag::Simple => { self.simple_mut() @@ -2231,88 +2272,6 @@ impl MatchGeneratorDriver { }, } } - - /// Find the long-distance sequences of the last committed block into - /// [`Self::ldm_sequences`], and return the block's length. - /// - /// What a match may reach is fixed against the block's END, as upstream - /// enforces the distance once per chunk before generating - /// (`ZSTD_window_enforceMaxDist` on `chunkEnd`, `zstd_ldm.c:585`): one - /// window back from it, or the whole dictionary while the window has not - /// yet passed the dictionary's end. Every sequence then holds an offset - /// valid at its last byte, wherever in the block it lies. - #[cfg(feature = "ldm")] - fn generate_long_distance_sequences(&mut self) -> usize { - let block_len = self.storage.last_block_len(); - self.ldm_sequences.clear(); - let Some(producer) = self.ldm.as_mut() else { - return block_len; - }; - let block_end = self.ldm_committed_end; - let window = self.reported_window_size; - let reach_floor = if self.ldm_dict_end != 0 && block_end <= window + self.ldm_dict_end { - 0 - } else { - block_end.saturating_sub(window) - }; - let live = self.storage.committed_window(); - let live_start = block_end - live.len(); - let floor = reach_floor.max(live_start); - producer.generate_into( - &live[floor - live_start..], - floor, - block_end - block_len, - block_end, - &mut self.ldm_sequences, - ); - block_len - } - - /// Take the block's long-distance matches outright and match only the - /// literals between them with the frame's backend (upstream - /// `ZSTD_ldm_blockCompress` below btopt). The block is handed back to the - /// uncommitted input and committed again in pieces, so each gap is the - /// backend's whole block and each match body is indexed without being - /// searched. Returns `false` when the block has no long-distance match, - /// leaving it committed for the ordinary search. - #[cfg(feature = "ldm")] - fn start_matching_long_distance(&mut self, handle_sequence: &mut impl FnMut(Sequence)) -> bool { - let block_len = self.generate_long_distance_sequences(); - if self.ldm_sequences.is_empty() { - return false; - } - let sequences = core::mem::take(&mut self.ldm_sequences); - self.storage.uncommit_block(block_len); - let mut consumed = 0usize; - for seq in &sequences { - // The gap's trailing literals become the long match's own. - let mut gap_literals = 0usize; - if seq.lit_length != 0 { - self.storage.commit_piece(seq.lit_length); - self.start_matching_owned(&mut |sequence| match sequence { - Sequence::Literals { len } => gap_literals = len, - triple => handle_sequence(triple), - }); - } - self.storage.commit_piece(seq.match_length); - self.storage.skip_long_match(); - handle_sequence(Sequence::Triple { - literal_len: gap_literals, - offset: seq.offset, - match_len: seq.match_length, - }); - self.storage.note_long_match(seq.offset, gap_literals); - consumed += seq.lit_length + seq.match_length; - } - let tail = block_len - consumed; - if tail != 0 { - self.storage.commit_piece(tail); - self.start_matching_owned(handle_sequence); - } - self.ldm_sequences = sequences; - self.ldm_split_block = Some(block_len); - true - } } impl MatchGeneratorDriver { diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index 8d98dd944..30685c6ae 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -2422,6 +2422,12 @@ impl MatchTable { } } + /// The committed window followed by the input read but not yet committed. + #[cfg(feature = "ldm")] + pub(crate) fn window_with_pending(&self) -> &[u8] { + &self.history[self.history_start..] + } + /// Hand the last committed block of `len` bytes back to the uncommitted /// input, undoing only the bookkeeping of [`Self::commit_block`]: what it /// evicted stays evicted, so committing the same bytes again in pieces diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index b6aed0189..4fe52caff 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -620,6 +620,16 @@ pub trait Matcher { fn matches_beyond_window_tables(&self) -> bool { false } + /// Where to cut the next block, given the cut the block splitter + /// proposes `cut` bytes into the `window` bytes of + /// [`uncommitted_input`](Self::uncommitted_input) it looked at. A matcher + /// that already knows a match crossing the cut moves it out of the match, + /// at most to `window`; cutting inside one only makes the next block + /// start by repeating it. Defaults to `cut`. + fn settle_block_cut(&mut self, window: usize, cut: usize) -> usize { + let _ = window; + cut + } /// Heap bytes this matcher's allocations hold (tables, history, scratch), /// excluding the inline struct itself. Lets a context report its true /// footprint via `ZSTD_sizeof_CCtx`. Defaults to `0` for custom matchers. diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 6ffdad0cd..eee8a2c39 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -4022,6 +4022,12 @@ impl RowMatchGenerator { self.dict_index_floor = floor; } + /// The committed window followed by the input read but not yet committed. + #[cfg(feature = "ldm")] + pub(crate) fn window_with_pending(&self) -> &[u8] { + &self.history[self.history_start..] + } + /// Hand the last committed block of `len` bytes back to the uncommitted /// input, undoing only the bookkeeping of [`Self::commit_block`]: what it /// evicted stays evicted, so committing the same bytes again in pieces diff --git a/zstd/src/encoding/simple/fast_matcher.rs b/zstd/src/encoding/simple/fast_matcher.rs index 4b9aa4563..84b194a39 100644 --- a/zstd/src/encoding/simple/fast_matcher.rs +++ b/zstd/src/encoding/simple/fast_matcher.rs @@ -1671,6 +1671,12 @@ impl FastKernelMatcher { self.history_bytes() } + /// The committed window followed by the input read but not yet committed. + #[cfg(feature = "ldm")] + pub(crate) fn window_with_pending(&self) -> &[u8] { + &self.history[..] + } + /// Hand the staged block of `len` bytes back to the uncommitted input, /// undoing only the bookkeeping of [`Self::commit_block`]: what it evicted /// stays evicted, so committing the same bytes again in pieces evicts diff --git a/zstd/src/encoding/streaming_encoder.rs b/zstd/src/encoding/streaming_encoder.rs index cb4cbabeb..3ab885e4b 100644 --- a/zstd/src/encoding/streaming_encoder.rs +++ b/zstd/src/encoding/streaming_encoder.rs @@ -1103,19 +1103,27 @@ impl CompressionContext { /// `remaining` is the input still to come as far as the splitter knows: /// a full block again while writes continue, the buffered bytes at the /// end of the frame. - fn pre_split_len(&self, block_capacity: usize, remaining: usize) -> usize { + fn pre_split_len(&mut self, block_capacity: usize, remaining: usize) -> usize { if matches!(self.compression_level, CompressionLevel::Uncompressed) { return self.pending_len(); } let pending = self.state.matcher.uncommitted_input(); - crate::encoding::frame_compressor::optimal_block_size_with( + let window = pending.len(); + let cut = crate::encoding::frame_compressor::optimal_block_size_with( self.state.pre_split.map(usize::from), pending, remaining, block_capacity, self.savings, ) - .min(pending.len()) + .min(window); + // The splitter sees byte statistics only; a cut it proposes inside a + // match the matcher already knows of is moved out. + if cut < window { + self.state.matcher.settle_block_cut(window, cut) + } else { + cut + } } /// Emit the pending block once it is full, cut where the pre-splitter diff --git a/zstd/src/tests/parameters_test.rs b/zstd/src/tests/parameters_test.rs index 57d1eb6fc..a5824112d 100644 --- a/zstd/src/tests/parameters_test.rs +++ b/zstd/src/tests/parameters_test.rs @@ -321,6 +321,42 @@ fn ldm_finds_far_repeat_under_every_strategy() { } } +/// The block splitter does not cut inside a long-distance match: a repeat of +/// random data is one match per block, and a cut through it would only buy a +/// block whose single sequence carries the same match on. Both frame loops +/// ask, the one-shot one and the stream's. +#[cfg(feature = "ldm")] +#[test] +fn ldm_matches_are_not_cut_by_the_block_splitter() { + use crate::encoding::StreamingEncoder; + use crate::io::Write; + + let half = generate_data(0xC07_0569, 1 << 20); + let mut data = half.clone(); + data.extend_from_slice(&half); + let params = CompressionParameters::builder(CompressionLevel::Level(3)) + .window_log(22) + .enable_long_distance_matching(true) + .build() + .unwrap(); + let one_shot = compress_with_parameters(&data, ¶ms); + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(3)); + encoder.set_parameters(¶ms).unwrap(); + encoder.write_all(&data).unwrap(); + let streamed = encoder.finish().unwrap(); + for (path, frame) in [("one-shot", &one_shot), ("stream", &streamed)] { + // The first copy goes out raw; the second is eight blocks of one + // sequence each, where a cut at every splitter boundary would make it + // about a hundred. + assert!( + frame.len() < half.len() + 512, + "{path}: the repeat cost {} bytes", + frame.len() - half.len() + ); + assert_eq!(decode(frame), data, "{path}: round-trip"); + } +} + /// Long-distance matching reaches into a dictionary, under every strategy: /// the dictionary is indexed into its table when it is loaded (upstream /// `ZSTD_ldm_fillHashTable` from `ZSTD_loadDictionaryContent`), so a source From 23b9fe52fcddcf9185af496a1fc20a51391d1e91 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 16 Oct 2026 20:46:00 +0300 Subject: [PATCH 04/11] fix(cli): take --memory as the decode window, as upstream does Upstream sets the decoder's window limit to the -M value itself (`ZSTD_DCtx_setMaxWindowSize(dctx, memLimit)` in fileio.c). We counted the decoder's other buffers and the dictionary against the limit too, so a frame with a 2^29 window was refused under `--memory=512MB`, and a 2^28 one under `--memory=256MB`: both are decode cases of the upstream `--long` test script. - Under -M the window ceiling is the limit, clamped to the decoder's range; the dictionary no longer enters it. - The `--memory` a refused frame names is the window rounded up, which now decodes it; the benchmark keeps its own accounting. --- zstd/src/bin/structured-zstd/main.rs | 49 +++++++--------------- zstd/src/bin/structured-zstd/tests.rs | 60 +++++++++++++-------------- 2 files changed, 44 insertions(+), 65 deletions(-) diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index 0ca1b6492..920c017cc 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -188,11 +188,8 @@ struct Options { /// `--trace FILE`: where one CSV line per frame is appended. trace: Option, /// The widest window decoding takes; see [`decode_window_ceiling`]. Set - /// once the dictionary's size is known. + /// once the command line is settled. decode_window: u64, - /// What `-M` counts besides the window; see [`decode_memory_overhead`]. - /// Set with [`Self::decode_window`]. - decode_memory_overhead: Option, /// A patch lifted the decode window to its widest, and no `-M` (`-M0` /// included) came after to set a limit of its own. patch_window_lifted: bool, @@ -495,26 +492,17 @@ fn parse_memory_limit(text: &str) -> Result> { /// What the decoder holds BESIDES the window, rounded well up: the literal and /// block buffers (a block is capped at 128 KiB each), the sequence storage, -/// the Huffman and FSE tables, and the tool's own I/O buffers. Counted because -/// `-M` promises total memory, not one allocation: a limit equal to the window -/// alone is one we would break. +/// the Huffman and FSE tables, and the tool's own I/O buffers. A limit that +/// cannot cover these and the default window together is refused up front. const DECODER_AUXILIARY_BYTES: u64 = 1 << 20; -/// What `-M` counts besides the window: the decoder's other buffers and the -/// dictionary (`dictionary_bytes`), held as read and again as parsed. `None` -/// past `u64`, which no limit covers. -fn decode_memory_overhead(dictionary_bytes: u64) -> Option { - dictionary_bytes - .checked_mul(2) - .and_then(|dictionary| dictionary.checked_add(DECODER_AUXILIARY_BYTES)) -} - /// The widest window this run decodes, handed to the decoder as its ceiling. /// -/// Under `-M` the window may have what the promise leaves once -/// [`decode_memory_overhead`] is counted; the limit was already checked to -/// leave at least the default. `-M` comes first because a patch keeps it too: -/// `--patch-apply` drops a `-M` given before it, so one still set came after, +/// Under `-M` the window is the limit, as the reference sets it +/// (`ZSTD_DCtx_setMaxWindowSize(dctx, memLimit)`, fileio.c): scripts written +/// for it decode a 2^29 window under `--memory=512MB`, which a limit counting +/// the other buffers too would refuse. `-M` comes first because a patch keeps +/// it too: `--patch-apply` drops a `-M` given before it, so one still set came after, /// and replaces the lifted limit as a later `-M` replaces it in the reference /// command. Otherwise a patch whose window no later `-M` (`-M0` included) has /// limited again reaches back across its whole reference, so it is decoded @@ -524,14 +512,11 @@ fn decode_memory_overhead(dictionary_bytes: u64) -> Option { /// failing those the decoder's default stands. A log below the default does /// not lower it: the reference would then refuse frames its own default /// accepts, which serves nobody. -fn decode_window_ceiling(opts: &Options, dictionary_bytes: u64) -> u64 { +fn decode_window_ceiling(opts: &Options) -> u64 { use structured_zstd::decoding::{MAX_DECODER_WINDOW_SIZE, MAXIMUM_ALLOWED_WINDOW_SIZE}; if let Some(limit) = opts.memory_limit { - return decode_memory_overhead(dictionary_bytes) - .and_then(|held| limit.checked_sub(held)) - .unwrap_or(0) - .clamp(MAXIMUM_ALLOWED_WINDOW_SIZE, MAX_DECODER_WINDOW_SIZE); + return limit.clamp(MAXIMUM_ALLOWED_WINDOW_SIZE, MAX_DECODER_WINDOW_SIZE); } if opts.patch_window_lifted { return MAX_DECODER_WINDOW_SIZE; @@ -1184,7 +1169,6 @@ fn parse_args_into( console: FakeConsole::default(), trace: None, decode_window: structured_zstd::decoding::MAXIMUM_ALLOWED_WINDOW_SIZE, - decode_memory_overhead: decode_memory_overhead(0), patch_window_lifted: false, bench_separately: false, block_size: None, @@ -2768,9 +2752,7 @@ fn run_selected(mut opts: Options) -> Result { } let dict_bytes = load_dictionary(&opts)?; - let dictionary_bytes = dict_bytes.as_ref().map_or(0, |bytes| bytes.len() as u64); - opts.decode_window = decode_window_ceiling(&opts, dictionary_bytes); - opts.decode_memory_overhead = decode_memory_overhead(dictionary_bytes); + opts.decode_window = decode_window_ceiling(&opts); // `-b` benchmarks compression/decompression across levels instead of // producing output files; handle it before the streaming flow. It takes the @@ -5616,8 +5598,9 @@ struct DecodeSettings { pass_through: bool, /// The decoder's window ceiling; see [`decode_window_ceiling`]. max_window: u64, - /// What `-M` counts besides the window, for the `--memory` a refusal - /// names; see [`decode_memory_overhead`]. + /// What the `--memory` a refusal names adds to the window: nothing for a + /// plain decode, whose `-M` is the window, and what a benchmark holds + /// besides it for one (see [`bench_decode_settings`]). memory_overhead: Option, } @@ -5627,7 +5610,7 @@ impl Default for DecodeSettings { verify_checksum: true, pass_through: false, max_window: structured_zstd::decoding::MAXIMUM_ALLOWED_WINDOW_SIZE, - memory_overhead: decode_memory_overhead(0), + memory_overhead: Some(0), } } } @@ -5646,7 +5629,7 @@ impl DecodeSettings { pass_through: opts.mode != Mode::Test && opts.pass_through.unwrap_or(opts.force && to_stdout), max_window: opts.decode_window, - memory_overhead: opts.decode_memory_overhead, + memory_overhead: Some(0), } } diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index 81eb8ba2c..dc20015df 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -2506,8 +2506,10 @@ fn test_and_tuning_options_are_accepted() { /// The decoder's window ceiling follows the command line as the reference's /// does: a patch reaches across its whole reference, `--long=N` and /// `--zstd=wlog=N` raise the ceiling to their window (the latter winning, as -/// it wins when compressing), `-M` gives what its promise leaves after the -/// other buffers and the dictionary, and nothing named keeps the default. +/// it wins when compressing), `-M` is the window itself as the reference +/// takes it (`ZSTD_DCtx_setMaxWindowSize(dctx, memLimit)`, so its own tests +/// decode a 2^29 window under `--memory=512MB`), and nothing named keeps the +/// default. /// Nothing lowers it below the default. A `-M` after `--patch-apply` is a /// promise the patch keeps too, as the reference's later `-M` replaces the /// limit `--patch-apply` lifted. @@ -2515,46 +2517,43 @@ fn test_and_tuning_options_are_accepted() { fn the_decode_window_follows_the_command_line() { use structured_zstd::decoding::{MAX_DECODER_WINDOW_SIZE, MAXIMUM_ALLOWED_WINDOW_SIZE}; - let ceiling = - |args: &[&str], dictionary: u64| decode_window_ceiling(&parse(args).unwrap(), dictionary); - assert_eq!(ceiling(&["-d", "f.zst"], 0), MAXIMUM_ALLOWED_WINDOW_SIZE); - assert_eq!(ceiling(&["-d", "--long=28", "f.zst"], 0), 1 << 28); - assert_eq!(ceiling(&["-d", "--zstd=wlog=29", "f.zst"], 0), 1 << 29); + let ceiling = |args: &[&str]| decode_window_ceiling(&parse(args).unwrap()); + assert_eq!(ceiling(&["-d", "f.zst"]), MAXIMUM_ALLOWED_WINDOW_SIZE); + assert_eq!(ceiling(&["-d", "--long=28", "f.zst"]), 1 << 28); + assert_eq!(ceiling(&["-d", "--zstd=wlog=29", "f.zst"]), 1 << 29); assert_eq!( - ceiling(&["-d", "--long=27", "--zstd=wlog=29", "f.zst"], 0), + ceiling(&["-d", "--long=27", "--zstd=wlog=29", "f.zst"]), 1 << 29 ); assert_eq!( - ceiling(&["-d", "--long=20", "f.zst"], 0), + ceiling(&["-d", "--long=20", "f.zst"]), MAXIMUM_ALLOWED_WINDOW_SIZE ); assert_eq!( - ceiling(&["--patch-apply=ref", "f.zst"], 0), + ceiling(&["--patch-apply=ref", "f.zst"]), MAX_DECODER_WINDOW_SIZE ); assert_eq!( - ceiling(&["--patch-apply=ref", "-M512MB", "f.zst"], 1 << 20), - (512 << 20) - (1 << 20) - (2 << 20) + ceiling(&["--patch-apply=ref", "-M512MB", "f.zst"]), + 512 << 20 ); // A later `-M0` asks for the default limit again, as zero replaces the // reference command's limit (zstdcli.c:1572-1577); an earlier one is // overridden by the patch like any other `-M`. assert_eq!( - ceiling(&["--patch-apply=ref", "-M0", "f.zst"], 0), + ceiling(&["--patch-apply=ref", "-M0", "f.zst"]), MAXIMUM_ALLOWED_WINDOW_SIZE ); assert_eq!( - ceiling(&["--patch-apply=ref", "--long=28", "-M0", "f.zst"], 0), + ceiling(&["--patch-apply=ref", "--long=28", "-M0", "f.zst"]), 1 << 28 ); assert_eq!( - ceiling(&["-M0", "--patch-apply=ref", "f.zst"], 0), + ceiling(&["-M0", "--patch-apply=ref", "f.zst"]), MAX_DECODER_WINDOW_SIZE ); - assert_eq!( - ceiling(&["-d", "-M512MB", "f.zst"], 1 << 20), - (512 << 20) - (1 << 20) - (2 << 20) - ); + assert_eq!(ceiling(&["-d", "-M512MB", "f.zst"]), 512 << 20); + assert_eq!(ceiling(&["-d", "--memory=1GB", "f.zst"]), 1 << 30); } /// A benchmark holds its input, frames, decoded copy, encoder and dictionaries @@ -2566,7 +2565,7 @@ fn a_benchmark_window_leaves_room_for_what_the_benchmark_holds() { use structured_zstd::decoding::MAXIMUM_ALLOWED_WINDOW_SIZE; let mut opts = parse(&["-b19", "-M600MB", "--long=29", "f"]).unwrap(); - opts.decode_window = decode_window_ceiling(&opts, 0); + opts.decode_window = decode_window_ceiling(&opts); let held = 400u64 << 20; assert_eq!( bench_decode_settings(&opts, Some(held)).max_window, @@ -2579,7 +2578,7 @@ fn a_benchmark_window_leaves_room_for_what_the_benchmark_holds() { MAXIMUM_ALLOWED_WINDOW_SIZE ); let mut free = parse(&["-b19", "--long=29", "f"]).unwrap(); - free.decode_window = decode_window_ceiling(&free, 0); + free.decode_window = decode_window_ceiling(&free); assert_eq!(bench_decode_settings(&free, None).max_window, 1 << 29); } @@ -2600,29 +2599,24 @@ fn decoding_takes_the_decoders_widest_window_log() { let wlog = format!("--zstd=wlog={widest}"); for flag in [&long, &wlog] { let opts = parse(&["-d", flag, "f.zst"]).unwrap(); - assert_eq!(decode_window_ceiling(&opts, 0), MAX_DECODER_WINDOW_SIZE); + assert_eq!(decode_window_ceiling(&opts), MAX_DECODER_WINDOW_SIZE); assert!(parse(&[flag, "f"]).is_err(), "{flag} compressing"); } assert!(parse(&["-d", &format!("--long={}", widest + 1), "f.zst"]).is_err()); } /// A frame past the ceiling is refused with the reference command's message, -/// naming the flags that would decode it (fileio.c:2674-2684). -/// The `--memory` named covers the window and what `-M` counts besides it, the -/// decoder's other buffers and the dictionary, so it is a limit that decodes -/// the frame. +/// naming the flags that would decode it (fileio.c:2674-2684): the `--memory` +/// is the window in whole MiB, which is the limit that decodes it. #[test] fn a_window_past_the_ceiling_names_the_flag_that_decodes_it() { assert_eq!( - window_too_large_message(1 << 28, 1 << 27, Some(DECODER_AUXILIARY_BYTES)), + window_too_large_message(1 << 28, 1 << 27, Some(0)), "Decoding error (36) : Frame requires too much memory for decoding\n\ Window size larger than maximum : 268435456 > 134217728\n\ - Use --long=28 or --memory=257MB" - ); - assert!( - window_too_large_message(1 << 40, 1 << 27, Some(DECODER_AUXILIARY_BYTES)) - .contains("ZSTD_WINDOWLOG_MAX") + Use --long=28 or --memory=256MB" ); + assert!(window_too_large_message(1 << 40, 1 << 27, Some(0)).contains("ZSTD_WINDOWLOG_MAX")); } /// A frame from `zstd --long=28` does not decode with the default ceiling and @@ -2648,6 +2642,8 @@ fn a_wide_window_decodes_with_long_or_memory() { .split_whitespace() .find(|word| word.starts_with("--memory=")) .expect("the refusal names a --memory"); + // The window itself, as upstream names and takes it. + assert_eq!(named, "--memory=256MB"); for flag in ["--long=28", "-M512MB", named] { assert_eq!( run(parse(&["-d", "-qq", "-f", flag, input, "-o", output]).unwrap()).unwrap(), From 378062819be91cb45d6157c9f5f0620945f0a8a8 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Fri, 16 Oct 2026 20:46:11 +0300 Subject: [PATCH 05/11] docs(readme): describe the project as independent Replace the fork paragraph with an Origins section: the project is developed on its own, and the ruzstd code it started from stays credited to its author under the MIT License, whose notice is kept in NOTICE. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5df36702e..6a1852cc8 100644 --- a/README.md +++ b/README.md @@ -420,9 +420,9 @@ upstream `main`, so the link target is correct for forks too — fork consumers should point readers at the upstream registry rather than maintain divergent copies. --> -## Project relationship +## Origins -Maintained fork of [KillingSpark/zstd-rs](https://github.com/KillingSpark/zstd-rs) (ruzstd) by [Dmitry Prudnikov](https://github.com/polaz). We sync periodically with upstream but maintain an independent development trajectory focused on the [CoordiNode](https://github.com/structured-world/coordinode) database engine's per-label dictionary needs. +structured-zstd is an independent project by [Dmitry Prudnikov](https://github.com/polaz), developed for the [CoordiNode](https://github.com/structured-world/coordinode) database engine and its per-label dictionary workloads. It started from the code base of [ruzstd](https://github.com/KillingSpark/zstd-rs) by Moritz Borcherding; that code is used under the MIT License, whose notice is kept in [NOTICE](https://github.com/structured-world/structured-zstd/blob/main/NOTICE). ## Support the project From 4090d7399cb1934be43f35330b9bad3a463b849f Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 17 Oct 2026 00:28:27 +0300 Subject: [PATCH 06/11] fix(cli): decide patch long mode on the matcher that runs `--patch-from` turned long mode on from the bare level's chain and strategy, while a `--zstd=` override changes the matcher that actually runs. A short `clog=` at a high level left the reference out of reach with long mode off, and the patch came out as literals. The cycle check now reads the overridden chain and strategy. The reference command reads the bare level here; following the effective matcher only differs where its patch would miss the reference. --- zstd/src/bin/structured-zstd/main.rs | 25 ++++++++++----- zstd/src/bin/structured-zstd/tests.rs | 44 +++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index 920c017cc..05f1f60a4 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -2131,17 +2131,26 @@ fn patch_window_log(source_size: u64) -> Result { /// Whether a `--patch-from` compression of `source_size` bytes against a /// `reference_size`-byte reference needs long-distance matching to reach the -/// reference: when the file's window outgrows what the level's match finder -/// cycles through, as the reference command decides it -/// (`FIO_adjustParamsForPatchFromMode`, `fileWindowLog > ZSTD_cycleLog(chainLog, -/// strategy)` on the level's parameters for those sizes, the binary-tree -/// strategies cycling one log short of their chain). -fn patch_needs_long_distance(level: i32, source_size: u64, reference_size: u64) -> bool { +/// reference: when the file's window outgrows what the match finder cycles +/// through, by the reference command's rule (`FIO_adjustParamsForPatchFromMode`, +/// `fileWindowLog > ZSTD_cycleLog(chainLog, strategy)`, the binary-tree +/// strategies cycling one log short of their chain). The chain and strategy +/// are the ones the frame runs, the level's for those sizes under any +/// `--zstd=` override; the reference command reads the bare level's and +/// leaves an overridden short chain without long mode. +fn patch_needs_long_distance( + level: i32, + advanced: &AdvancedParams, + source_size: u64, + reference_size: u64, +) -> bool { let file_window_log = u64::BITS - source_size.max(1).leading_zeros(); // A reference past the address space cannot have been read to get here. let reference_size = usize::try_from(reference_size).unwrap_or(usize::MAX); let params = LevelParameters::for_level(level, Some(source_size), reference_size); - let cycle_log = params.chain_log - u32::from(params.strategy >= Strategy::Btlazy2); + let chain_log = advanced.chain_log.unwrap_or(params.chain_log); + let strategy = advanced.strategy.unwrap_or(params.strategy); + let cycle_log = chain_log - u32::from(strategy >= Strategy::Btlazy2); file_window_log > cycle_log } @@ -2742,7 +2751,7 @@ fn run_selected(mut opts: Options) -> Result { let reference_size = fs::metadata(reference) .map_err(|err| eyre!("can't stat {} : {err}", reference.display()))? .len(); - if patch_needs_long_distance(opts.level, source_size, reference_size) { + if patch_needs_long_distance(opts.level, &opts.advanced, source_size, reference_size) { if !opts.long { display!(opts.verbosity, 2, "long mode automatically triggered"); } diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index dc20015df..81974ed8b 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -2214,10 +2214,16 @@ fn long_finds_a_far_repeat_at_a_low_level() { /// log short of their chain). #[test] fn patch_from_needs_long_mode_where_the_matcher_cannot_cycle_the_input() { + let level_only = AdvancedParams::default(); // Level 3 on 61 MiB against 60 MiB: dfast, chainLog 16, file window log 26. - assert!(patch_needs_long_distance(3, 61 << 20, 60 << 20)); + assert!(patch_needs_long_distance( + 3, + &level_only, + 61 << 20, + 60 << 20 + )); // A small patch fits the chain the level already has. - assert!(!patch_needs_long_distance(3, 2000, 4000)); + assert!(!patch_needs_long_distance(3, &level_only, 2000, 4000)); // The binary tree cycles one log short: a file window equal to the chain // log is past it at btlazy2 (level 13), within it at lazy2 (level 12). let reference = 1u64 << 20; @@ -2229,13 +2235,45 @@ fn patch_from_needs_long_mode_where_the_matcher_cannot_cycle_the_input() { assert_eq!(params.chain_log, chain_log, "level {level}: one size tier"); assert_eq!(params.strategy >= Strategy::Btlazy2, binary_tree); assert_eq!( - patch_needs_long_distance(level, size, reference), + patch_needs_long_distance(level, &level_only, size, reference), binary_tree, "level {level}" ); } } +/// The rule reads the matcher that will run: a `--zstd=` chain or strategy +/// override that leaves the reference out of the level's cycle turns long mode +/// on, and one that brings it in keeps it off. The reference command reads the +/// bare level and so writes a patch of literals for the first case. +#[test] +fn patch_from_long_mode_follows_the_overridden_matcher() { + let reference = 1u64 << 20; + let size = 4u64 << 20; + let params = LevelParameters::for_level(19, Some(size), reference as usize); + let file_window_log = u64::BITS - size.leading_zeros(); + assert!( + params.chain_log > file_window_log, + "level 19 cycles the input by itself" + ); + let level_only = AdvancedParams::default(); + assert!(!patch_needs_long_distance(19, &level_only, size, reference)); + let short_chain = AdvancedParams { + chain_log: Some(16), + ..AdvancedParams::default() + }; + assert!(patch_needs_long_distance(19, &short_chain, size, reference)); + // Level 1 runs fast with a short chain; a binary tree with a long one + // cycles the input. + assert!(patch_needs_long_distance(1, &level_only, size, reference)); + let long_tree = AdvancedParams { + chain_log: Some(file_window_log + 1), + strategy: Some(Strategy::Btopt), + ..AdvancedParams::default() + }; + assert!(!patch_needs_long_distance(1, &long_tree, size, reference)); +} + /// The window log has a supported range; a value outside it has to fail rather /// than be quietly replaced by a working one, or the caller believes in a /// window the frame does not have. From c1f62c292dfa423f766feb969f5bd343da743e15 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 17 Oct 2026 00:28:35 +0300 Subject: [PATCH 07/11] fix(encoding): keep a resident dictionary in long-distance reach - A dictionary a reused context keeps resident between small or unsized frames had its long-distance table cleared by the reset and never filled again, so every frame after the first lost its long matches into the dictionary. The resident path now indexes the dictionary into the cleared table, as priming it would; a test runs three frames on one context against a fresh encoder under every strategy. - The primed snapshot no longer keeps a copy of the long-distance table: the restore path indexes the restored dictionary instead, as upstream fills the table each frame it loads a dictionary (`ZSTD_ldm_fillHashTable`). The context held two copies of its largest table, which the workspace estimates did not count, and copying it cost more than the fill. With no long-distance state in the snapshot its key drops the matcher's parameters, and a restore across a long-distance switch writes what a fresh compressor writes. - Gaps between long matches, borrowed scans and plain blocks report through one sink type, so each backend's scan is compiled once per caller instead of twice. The library's LLVM IR had grown by 64%; it is back within 0.3% of main, and the test build on CI's macOS runner had tripled. --- zstd/src/encoding/ldm/mod.rs | 20 --- zstd/src/encoding/ldm/table.rs | 23 --- .../encoding/match_generator/dict_prime.rs | 45 +++--- .../encoding/match_generator/long_distance.rs | 13 +- zstd/src/encoding/match_generator/mod.rs | 86 +++++------ zstd/src/encoding/match_generator/tests.rs | 49 ------ zstd/src/tests/parameters_test.rs | 142 ++++++++++++++++++ 7 files changed, 206 insertions(+), 172 deletions(-) diff --git a/zstd/src/encoding/ldm/mod.rs b/zstd/src/encoding/ldm/mod.rs index b53fc65a5..84c74c42f 100644 --- a/zstd/src/encoding/ldm/mod.rs +++ b/zstd/src/encoding/ldm/mod.rs @@ -92,26 +92,6 @@ pub(crate) struct LdmProducer { splits_scratch: Vec, } -impl Clone for LdmProducer { - fn clone(&self) -> Self { - Self { - params: self.params, - hash_state: self.hash_state, - hash_table: self.hash_table.clone(), - splits_scratch: self.splits_scratch.clone(), - } - } - - /// Into the table already held (see [`LdmHashTable`]'s `clone_from`); the - /// split scratch is per call, so its contents are not carried. - fn clone_from(&mut self, source: &Self) { - self.params = source.params; - self.hash_state = source.hash_state; - self.hash_table.clone_from(&source.hash_table); - self.splits_scratch.resize(source.splits_scratch.len(), 0); - } -} - impl LdmProducer { /// Build a fresh producer for the given parameter set. /// diff --git a/zstd/src/encoding/ldm/table.rs b/zstd/src/encoding/ldm/table.rs index ffb65a19e..b66549257 100644 --- a/zstd/src/encoding/ldm/table.rs +++ b/zstd/src/encoding/ldm/table.rs @@ -109,29 +109,6 @@ pub(crate) struct LdmHashTable { position_base: usize, } -impl Clone for LdmHashTable { - fn clone(&self) -> Self { - Self { - entries: self.entries.clone(), - bucket_offsets: self.bucket_offsets.clone(), - effective_bucket_log: self.effective_bucket_log, - bucket_mask: self.bucket_mask, - position_base: self.position_base, - } - } - - /// Into the buffers already held: a restored dictionary snapshot copies - /// the table every frame, and allocating it again each time would cost - /// more than the copy. - fn clone_from(&mut self, source: &Self) { - self.entries.clone_from(&source.entries); - self.bucket_offsets.clone_from(&source.bucket_offsets); - self.effective_bucket_log = source.effective_bucket_log; - self.bucket_mask = source.bucket_mask; - self.position_base = source.position_base; - } -} - impl LdmHashTable { /// Allocate a fresh table for the given parameters. /// diff --git a/zstd/src/encoding/match_generator/dict_prime.rs b/zstd/src/encoding/match_generator/dict_prime.rs index 97f1b06e4..c863f31cc 100644 --- a/zstd/src/encoding/match_generator/dict_prime.rs +++ b/zstd/src/encoding/match_generator/dict_prime.rs @@ -60,6 +60,10 @@ impl MatchGeneratorDriver { } }; self.dictionary_retained_budget = inflated.saturating_sub(base); + // The reset kept the dictionary's bytes and index but cleared the + // long-distance table, which a freshly primed frame would hold filled. + #[cfg(feature = "ldm")] + self.index_dictionary_in_place(); } pub(super) fn prime_with_dictionary_impl( @@ -358,7 +362,7 @@ impl MatchGeneratorDriver { // match this reset. Restoring it would let the encoder search past the // frame header's window (an undecodable match), so on a key mismatch we // refuse and the caller re-primes. - let Some((params, table_bits, fast_attach, fast_tagged, ldm)) = self.reset_shape else { + let Some((params, table_bits, fast_attach, fast_tagged)) = self.reset_shape else { return false; }; let key = PrimedKey { @@ -367,7 +371,6 @@ impl MatchGeneratorDriver { table_bits, fast_attach, fast_tagged, - ldm, }; let Some((snapshot, budget, captured_key)) = &mut self.primed else { return false; @@ -442,23 +445,32 @@ impl MatchGeneratorDriver { } } self.dictionary_retained_budget = budget; - // The restored window is the dictionary, the frame's first bytes, and - // the long-distance matcher's table holds it as priming left it. #[cfg(feature = "ldm")] - { - self.ldm_committed_end = self.storage.committed_window().len(); - self.ldm_dict_end = self.ldm_committed_end; - if let (Some(live), Some(snapshot)) = (self.ldm.as_mut(), self.primed_ldm.as_ref()) { - live.clone_from(snapshot); - } - } + self.index_dictionary_in_place(); true } + /// Index the dictionary the reset left in the window into the + /// long-distance matcher's cleared table, as priming it would. Upstream + /// re-indexes per frame as well: a CDict carries no long-distance table, + /// and a frame that loads the dictionary fills one + /// (`ZSTD_ldm_fillHashTable` from `ZSTD_loadDictionaryContent`). Keeping a + /// copy of the filled table instead would hold a second table, the + /// largest the frame has, and copying it costs more than the fill. + #[cfg(feature = "ldm")] + fn index_dictionary_in_place(&mut self) { + self.ldm_committed_end = self.storage.committed_window().len(); + self.ldm_dict_end = self.ldm_committed_end; + if let Some(producer) = self.ldm.as_mut() { + let window = self.storage.committed_window(); + producer.fill_table(window, 0, 0, window.len()); + } + } + pub(super) fn capture_primed_dictionary_impl(&mut self, level: super::super::CompressionLevel) { // No resolved shape means `reset` has not run for this frame — nothing // valid to key a snapshot on, so skip the capture. - let Some((params, table_bits, fast_attach, fast_tagged, ldm)) = self.reset_shape else { + let Some((params, table_bits, fast_attach, fast_tagged)) = self.reset_shape else { return; }; let key = PrimedKey { @@ -467,15 +479,7 @@ impl MatchGeneratorDriver { table_bits, fast_attach, fast_tagged, - ldm, }; - // The key pins the parameters, so a restore copies into a table of the - // same shape. - #[cfg(feature = "ldm")] - match (self.primed_ldm.as_mut(), self.ldm.as_ref()) { - (Some(snapshot), Some(live)) => snapshot.clone_from(live), - (_, live) => self.primed_ldm = live.cloned(), - } // CDict-equivalent retained state. A binary-tree level in ATTACH mode // decouples the dictionary into `dms` (the upstream zstd `dictMatchState`); its // live hash / chain / hash3 tables carry NO dict entries @@ -525,7 +529,6 @@ impl MatchGeneratorDriver { pub(super) fn invalidate_primed_dictionary_impl(&mut self) { self.primed = None; - self.primed_ldm = Default::default(); // Drop the Fast-backend CDict-equivalent table cache too: it is keyed // to the dictionary being removed / replaced. Left in place, the next // same-params `reset` would retain it and the kernel would probe a diff --git a/zstd/src/encoding/match_generator/long_distance.rs b/zstd/src/encoding/match_generator/long_distance.rs index 45ac017bb..ece570cd0 100644 --- a/zstd/src/encoding/match_generator/long_distance.rs +++ b/zstd/src/encoding/match_generator/long_distance.rs @@ -5,7 +5,7 @@ //! A child module so it reaches the driver's private long-distance state and //! storage directly. -use super::{MatchGeneratorDriver, Sequence}; +use super::{MatchGeneratorDriver, Sequence, owned_sink}; impl MatchGeneratorDriver { /// Search the input from where the last search stopped up to `to`, a @@ -103,14 +103,12 @@ impl MatchGeneratorDriver { let mut consumed = 0usize; for seq in &sequences { // The gap's trailing literals become the long match's own. - let mut gap_literals = 0usize; + let mut gap = Some(0usize); if seq.lit_length != 0 { self.storage.commit_piece(seq.lit_length); - self.start_matching_owned(&mut |sequence| match sequence { - Sequence::Literals { len } => gap_literals = len, - triple => handle_sequence(triple), - }); + self.start_matching_owned(&mut owned_sink(handle_sequence, &mut gap)); } + let gap_literals = gap.expect("the gap sink only ever writes Some"); self.storage.commit_piece(seq.match_length); self.storage.skip_long_match(); handle_sequence(Sequence::Triple { @@ -124,7 +122,8 @@ impl MatchGeneratorDriver { let tail = block_len - consumed; if tail != 0 { self.storage.commit_piece(tail); - self.start_matching_owned(handle_sequence); + let mut forward = None; + self.start_matching_owned(&mut owned_sink(handle_sequence, &mut forward)); } self.ldm_block = sequences; self.ldm_split_block = Some(block_len); diff --git a/zstd/src/encoding/match_generator/mod.rs b/zstd/src/encoding/match_generator/mod.rs index 4b32b31b0..3687b9d53 100644 --- a/zstd/src/encoding/match_generator/mod.rs +++ b/zstd/src/encoding/match_generator/mod.rs @@ -423,12 +423,11 @@ pub struct MatchGeneratorDriver { reset_dict_attach_ok: bool, // Hint-resolved matcher shape from the last `reset`: the [`LevelParams`], the // active backend's applied Dfast/Row hash-table width (`0` for HC/Fast), the - // Fast attach-vs-copy mode, the Fast slot format and the long-distance - // matcher's parameters. Combined with the frame's level into the - // [`PrimedKey`] that keys the primed snapshot, so it is only restored into a - // reset that resolved the identical matcher. `None` before the first - // `reset`. - reset_shape: Option<(LevelParams, usize, bool, bool, LdmShape)>, + // Fast attach-vs-copy mode and the Fast slot format. Combined with the + // frame's level into the [`PrimedKey`] that keys the primed snapshot, so it + // is only restored into a reset that resolved the identical matcher. `None` + // before the first `reset`. + reset_shape: Option<(LevelParams, usize, bool, bool)>, // One-shot borrowed block range `[start, end)` staged by the borrowed // Fast frame path (`set_borrowed_block`) for the NEXT // `start_matching` / `skip_matching_with_hint`. `Some` routes that @@ -448,10 +447,6 @@ pub struct MatchGeneratorDriver { /// is only restored into a reset that produces the same matcher — see /// `restore_primed_dictionary`. primed: Option<(MatcherStorage, usize, PrimedKey)>, - /// The long-distance matcher as `prime_with_dictionary` left it, with the - /// dictionary indexed, captured and restored together with - /// [`Self::primed`]; its parameters are in the key. - primed_ldm: LdmSnapshot, /// Where the tables live when the driver is reset on its own through /// [`Matcher::reset`]; inside a compression context they live in the /// context's workspace instead and this stays empty. @@ -539,24 +534,8 @@ struct PrimedKey { /// frame from how full the input will leave the table, so two frames of one /// geometry can differ in it; a restore carries the snapshot's format in. fast_tagged: bool, - /// The long-distance matcher's parameters, `None` without one. A primed - /// snapshot carries its table with the dictionary indexed in it, which a - /// frame with other parameters could not use. - ldm: LdmShape, } -/// [`PrimedKey::ldm`]: nothing to key on in a build without the matcher. -#[cfg(feature = "ldm")] -type LdmShape = Option; -#[cfg(not(feature = "ldm"))] -type LdmShape = (); - -/// The long-distance matcher as a primed snapshot keeps it. -#[cfg(feature = "ldm")] -type LdmSnapshot = Option; -#[cfg(not(feature = "ldm"))] -type LdmSnapshot = (); - /// Whether a configured HashChain matcher attaches the dictionary for a frame /// of `size_log` (see [`MatchGeneratorDriver::hc_dict_attach_mode`]). fn hc_attaches_dictionary(hc: &HcMatchGenerator, size_log: Option) -> bool { @@ -763,7 +742,6 @@ impl MatchGeneratorDriver { dictionary_size_hint: None, borrowed_pending: None, primed: None, - primed_ldm: Default::default(), own_workspace: crate::encoding::workspace::Workspace::new(), frame_in_place: false, #[cfg(feature = "ldm")] @@ -1254,7 +1232,6 @@ impl Matcher for MatchGeneratorDriver { .map_or(0, |(storage, _, _)| storage.heap_size()); #[cfg(feature = "ldm")] let ldm = self.ldm.as_ref().map_or(0, |p| p.heap_size()) - + self.primed_ldm.as_ref().map_or(0, |p| p.heap_size()) + self.ldm_store.heap_size() + (self.ldm_found.capacity() + self.ldm_block.capacity()) * core::mem::size_of::(); @@ -1401,12 +1378,10 @@ impl Matcher for MatchGeneratorDriver { } // The long-distance matcher's parameters for the frame, `None` without one. #[cfg(feature = "ldm")] - let frame_ldm: LdmShape = self + let frame_ldm = self .param_overrides .and_then(|ov| ov.ldm) .map(|ldm_ov| crate::encoding::levels::config::frame_ldm_params(¶ms, &ldm_ov)); - #[cfg(not(feature = "ldm"))] - let frame_ldm: LdmShape = (); // A dictionary frame's hash-chain / binary-tree widths are the CDict's // (`resolve_level_params_with_dict`): verbatim when the dictionary is // copied into the live tables (`ZSTD_resetCCtx_byCopyingCDict` builds @@ -1636,7 +1611,6 @@ impl Matcher for MatchGeneratorDriver { table_bits: 0, fast_attach: false, fast_tagged: captured.fast_tagged, - ldm: frame_ldm, } }) .map(|(_, _, captured)| captured.fast_tagged); @@ -1916,18 +1890,9 @@ impl Matcher for MatchGeneratorDriver { // resolved matcher geometry instead, so folding the Fast bit into their // key would over-key identical resolved shapes. `fast_attach` is the // decision `prime_with_dictionary` makes from the same `reset_size_log`. - // The long-distance matcher's parameters are part of it on every - // backend: the snapshot keeps the matcher's table with the dictionary - // indexed, so a restored frame writes what a freshly primed one does. // The Fast slot format this reset chose; only the Simple backend has one. let fast_tagged = matches!(&self.storage, MatcherStorage::Simple(m) if m.slots_tagged()); - self.reset_shape = Some(( - params, - resolved_table_bits, - fast_attach, - fast_tagged, - frame_ldm, - )); + self.reset_shape = Some((params, resolved_table_bits, fast_attach, fast_tagged)); // Everything is laid out in `workspace` now. A driver reset on its own // before it joined a context still holds the workspace it used then, // which nothing points into any more; on its own the field is empty @@ -2113,6 +2078,10 @@ impl Matcher for MatchGeneratorDriver { // the Simple backend is instrumented (the gate guarantees it), // and the stage is consumed so the next block re-stages. if let Some((block_start, block_end)) = self.borrowed_pending.take() { + // The borrowed scans share their kernels with the owned ones, so + // they report through the same sink type. + let mut forward = None; + let mut sink = owned_sink(&mut handle_sequence, &mut forward); match self.active_backend() { super::strategy::BackendTag::Simple => { let m = self.simple_mut(); @@ -2120,18 +2089,14 @@ impl Matcher for MatchGeneratorDriver { // Dict-attach borrowed scan: live matches read the // borrowed input in place, dict matches read the // committed dict prefix via the 2-segment counter. - m.start_matching_borrowed_dict( - block_start, - block_end, - &mut handle_sequence, - ); + m.start_matching_borrowed_dict(block_start, block_end, &mut sink); } else { - m.start_matching_borrowed(block_start, block_end, &mut handle_sequence); + m.start_matching_borrowed(block_start, block_end, &mut sink); } } super::strategy::BackendTag::Dfast => self .dfast_matcher_mut() - .start_matching_borrowed(block_start, block_end, &mut handle_sequence), + .start_matching_borrowed(block_start, block_end, &mut sink), super::strategy::BackendTag::Row => { // Same greedy/lazy parse split as the owned RowHash arm. let greedy = self.parse == super::strategy::ParseMode::Greedy; @@ -2139,13 +2104,13 @@ impl Matcher for MatchGeneratorDriver { block_start, block_end, greedy, - &mut handle_sequence, + &mut sink, ); } super::strategy::BackendTag::HashChain => match self.search { super::strategy::SearchMethod::HashChain => self .hc_matcher_mut() - .start_matching_lazy_borrowed(block_start, block_end, &mut handle_sequence), + .start_matching_lazy_borrowed(block_start, block_end, &mut sink), // `borrowed_supported()` keeps the optimal parsers on the // owned path and `set_borrowed_block` asserts it. other => { @@ -2170,7 +2135,8 @@ impl Matcher for MatchGeneratorDriver { return; } } - self.start_matching_owned(&mut handle_sequence); + let mut gap_literals = None; + self.start_matching_owned(&mut owned_sink(&mut handle_sequence, &mut gap_literals)); } fn skip_matching(&mut self) { @@ -2228,6 +2194,22 @@ impl Matcher for MatchGeneratorDriver { } } +/// The sink every owned match runs against: `handle_sequence`, except that a +/// block's trailing literals go to `gap_literals` when it is `Some`, as the +/// gap before a long-distance match needs them. One sink type for the plain +/// block and for those gaps compiles each backend's scan once per caller sink +/// instead of twice. +fn owned_sink<'a, F: FnMut(Sequence)>( + handle_sequence: &'a mut F, + gap_literals: &'a mut Option, +) -> impl FnMut(Sequence) + 'a { + // The guard is read only for the block's last report, never per triple. + move |sequence| match sequence { + Sequence::Literals { len } if gap_literals.is_some() => *gap_literals = Some(len), + sequence => handle_sequence(sequence), + } +} + impl MatchGeneratorDriver { /// Match the last committed block of the owned history with the frame's /// backend. diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index 9878c7b68..7b3dc79bd 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -2834,55 +2834,6 @@ fn hc_prime_with_empty_dictionary_disables_btultra2_seed_pass() { ); } -#[test] -fn primed_snapshot_not_restored_across_ldm_config_change() { - // The CDict-equivalent primed snapshot clones `storage`, which on the - // BT backend carries `BtMatcher::ldm_producer`. A snapshot captured - // under one LDM configuration must NOT be restored into a reset that - // resolved a different LDM configuration (else the restored producer - // is stale). `PrimedKey` must fold the LDM override into the key so - // such a restore is refused and the caller re-primes. - use super::super::parameters::CompressionParameters; - - let dict = b"abcdefghabcdefghabcdefgh"; - let ldm_on = CompressionParameters::builder(CompressionLevel::Level(19)) - .enable_long_distance_matching(true) - .build() - .unwrap() - .overrides(); - let ldm_off = CompressionParameters::builder(CompressionLevel::Level(19)) - .build() - .unwrap() - .overrides(); - - let mut driver = MatchGeneratorDriver::new(1024, 1); - - // Capture a snapshot primed under LDM-on at level 19. - driver.set_param_overrides(Some(ldm_on)); - driver.reset(CompressionLevel::Level(19)); - driver.prime_with_dictionary(dict, [1, 4, 8]); - driver.capture_primed_dictionary(CompressionLevel::Level(19)); - - // Same dictionary + level, but LDM now OFF: the snapshot's LDM state - // is stale, so restore must be refused. - driver.set_param_overrides(Some(ldm_off)); - driver.reset(CompressionLevel::Level(19)); - assert!( - !driver.restore_primed_dictionary(CompressionLevel::Level(19)), - "primed snapshot restored across an LDM config change (stale producer)", - ); - - // Sanity: re-priming + capturing under LDM-off, then restoring under - // the IDENTICAL LDM-off config DOES match (the key is not over-tight). - driver.prime_with_dictionary(dict, [1, 4, 8]); - driver.capture_primed_dictionary(CompressionLevel::Level(19)); - driver.reset(CompressionLevel::Level(19)); - assert!( - driver.restore_primed_dictionary(CompressionLevel::Level(19)), - "primed snapshot not restored under identical LDM config", - ); -} - /// btultra2 parses a frame's first block twice and hides the first pass from /// the second by re-encoding the stored positions. A reused matcher keeps the /// previous frame's entries below the floor, and the re-encoding has to leave diff --git a/zstd/src/tests/parameters_test.rs b/zstd/src/tests/parameters_test.rs index a5824112d..1831e009e 100644 --- a/zstd/src/tests/parameters_test.rs +++ b/zstd/src/tests/parameters_test.rs @@ -432,6 +432,148 @@ fn ldm_reaches_into_the_dictionary_under_every_strategy() { } } +/// The primed dictionary snapshot carries no long-distance state, so it is +/// restored whether the next frame turns long-distance matching on or off, +/// and the frame matches what a freshly primed compressor writes either way. +#[cfg(feature = "ldm")] +#[test] +fn a_primed_dictionary_restores_across_a_long_distance_switch() { + use crate::decoding::Dictionary; + + const PIECE: usize = 4096; + const DICT_ID: u32 = 0x0569_D1C8; + let dict: Vec = generate_data(0xD1C7_0571, 256 * 1024) + .into_iter() + .map(|byte| byte & 63) + .collect(); + let pieces = dict.len() / PIECE; + let mut source = Vec::new(); + for i in 0..pieces { + let piece = (i * 37) % pieces; + source.extend_from_slice(&dict[piece * PIECE..(piece + 1) * PIECE]); + } + for strategy in [Strategy::Fast, Strategy::Lazy2, Strategy::Btopt] { + let params = |long: bool| { + CompressionParameters::builder(CompressionLevel::Level(3)) + .strategy(strategy) + .window_log(22) + .hash_log(12) + .chain_log(12) + .enable_long_distance_matching(long) + .build() + .unwrap() + }; + let compressor = |long: bool| { + let mut compressor: FrameCompressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_parameters(¶ms(long)); + compressor + .set_dictionary(Dictionary::from_raw_content(DICT_ID, dict.clone()).unwrap()) + .unwrap(); + compressor + }; + for first in [false, true] { + let mut warm = compressor(first); + warm.compress_independent_frame(source.as_slice()); + warm.set_parameters(¶ms(!first)); + let restored = warm.compress_independent_frame(source.as_slice()); + let fresh = compressor(!first).compress_independent_frame(source.as_slice()); + assert!( + restored == fresh, + "{strategy:?}: long-distance matching {} after {}: {} bytes, fresh {}", + !first, + first, + restored.len(), + fresh.len(), + ); + } + } +} + +/// A dictionary a reused context keeps resident between unsized frames stays +/// in reach of long-distance matching: every frame on the context writes what +/// a fresh encoder writes, the first one included. The pieces come from the +/// dictionary's start, which the regular tables leave unindexed. +#[cfg(feature = "ldm")] +#[test] +fn ldm_reaches_into_a_resident_dictionary_frame_after_frame() { + use crate::encoding::{CompressionContext, EncoderDictionary, StreamingEncoder}; + use crate::io::Write; + + const PIECE: usize = 4096; + let dict: Vec = generate_data(0xD1C7_0570, 256 * 1024) + .into_iter() + .map(|byte| byte & 63) + .collect(); + let mut source = Vec::new(); + for piece in [3, 17, 9] { + source.extend_from_slice(&dict[piece * PIECE..(piece + 1) * PIECE]); + } + let dictionary = EncoderDictionary::from_serialized_or_raw_content(&dict).unwrap(); + for strategy in [ + Strategy::Fast, + Strategy::Dfast, + Strategy::Greedy, + Strategy::Lazy2, + Strategy::Btlazy2, + Strategy::Btopt, + ] { + let params = CompressionParameters::builder(CompressionLevel::Level(3)) + .strategy(strategy) + .window_log(22) + .hash_log(12) + .chain_log(12) + .enable_long_distance_matching(true) + .build() + .unwrap(); + let mut fresh = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(3)); + fresh.set_parameters(¶ms).unwrap(); + fresh.set_encoder_dictionary(dictionary.clone()).unwrap(); + fresh.write_all(&source).unwrap(); + let expected = fresh.finish().unwrap(); + let mut without = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(3)); + without + .set_parameters( + &CompressionParameters::builder(CompressionLevel::Level(3)) + .strategy(strategy) + .window_log(22) + .hash_log(12) + .chain_log(12) + .build() + .unwrap(), + ) + .unwrap(); + without.set_encoder_dictionary(dictionary.clone()).unwrap(); + without.write_all(&source).unwrap(); + let without = without.finish().unwrap(); + // The optimal parser weighs the long matches as candidates rather than + // taking them outright, so it is held to a gain, not to a size. + assert!( + expected.len() < without.len() / 2, + "{strategy:?}: LDM did not reach into the dictionary ({} bytes, {} without)", + expected.len(), + without.len(), + ); + let mut context = CompressionContext::new(CompressionLevel::Level(3)); + context.set_parameters(¶ms).unwrap(); + context.set_encoder_dictionary(dictionary.clone()).unwrap(); + for frame_index in 0..3 { + let mut frame = Vec::new(); + let mut rest = source.as_slice(); + while !rest.is_empty() { + let taken = context.write(&mut frame, rest).unwrap(); + rest = &rest[taken..]; + } + context.finish_frame(&mut frame).unwrap(); + assert!( + frame == expected, + "{strategy:?}: frame {frame_index} on the reused context wrote {} bytes, a fresh encoder {}", + frame.len(), + expected.len(), + ); + } + } +} + /// `FrameCompressor::set_parameters` on the streaming path reuses one /// compressor across frames and produces decodable output. #[test] From 77cf644e6e646dcd89b5575890fe0d45bc6a695c Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 17 Oct 2026 00:51:34 +0300 Subject: [PATCH 08/11] perf(encoding): forward match reports field by field The sink every owned match runs through forwarded each report as the matched value whole, which the compiler copies as one 32-byte block out of the separate stores that wrote its fields. Rebuilding the report from its fields brings the encoder's instruction count back to that of the code before the shared sink (387.13M at level 9 on z000033, from 387.51M), with output identical at every level. --- zstd/src/encoding/match_generator/mod.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/zstd/src/encoding/match_generator/mod.rs b/zstd/src/encoding/match_generator/mod.rs index 3687b9d53..6c0be9155 100644 --- a/zstd/src/encoding/match_generator/mod.rs +++ b/zstd/src/encoding/match_generator/mod.rs @@ -2204,9 +2204,23 @@ fn owned_sink<'a, F: FnMut(Sequence)>( gap_literals: &'a mut Option, ) -> impl FnMut(Sequence) + 'a { // The guard is read only for the block's last report, never per triple. + // Each arm rebuilds the report from its fields: forwarding the matched + // value whole copies it as one 32-byte block out of the separate stores + // that wrote it, and that copy stalls on every triple. move |sequence| match sequence { - Sequence::Literals { len } if gap_literals.is_some() => *gap_literals = Some(len), - sequence => handle_sequence(sequence), + Sequence::Triple { + literal_len, + offset, + match_len, + } => handle_sequence(Sequence::Triple { + literal_len, + offset, + match_len, + }), + Sequence::Literals { len } => match gap_literals { + Some(gap) => *gap = len, + None => handle_sequence(Sequence::Literals { len }), + }, } } From b489c785dfbf425a7e3931470d6ccc856619e3af Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 17 Oct 2026 00:59:31 +0300 Subject: [PATCH 09/11] test(encoding): take long matches on the lazy hash-chain search The hash-chain lazy search runs only under the parse and search override, so its long-distance path (handing the block back, skipping the match body, carrying the offset into the repeat history) had no test. The new one checks that a shuffled repeat beyond the chain's reach stays literals without long-distance matching and is matched with it, and that the stream rebuilds both ways. --- zstd/src/encoding/match_generator/tests.rs | 75 ++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index 7b3dc79bd..4ca19c42e 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -131,6 +131,81 @@ fn parse_search_matrix_decoupled_roundtrips() { assert_eq!(got, data, "lazy-on-rowhash diverged"); } +/// Long-distance matches are taken outright on the lazy hash-chain search as +/// on every other non-optimal one: a repeat beyond the chain's reach is +/// matched, the stream rebuilds, and the chain carries on around the matches. +#[cfg(feature = "ldm")] +#[test] +fn hash_chain_lazy_takes_long_distance_matches() { + use super::super::parameters::CompressionParameters; + use super::super::strategy::{ParseMode, SearchMethod}; + + // Six-bit symbols from an LCG's high bits: no repeat the chain finds. + let mut state = 0x0569_C4A1u32; + let half: Vec = (0..256 * 1024) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state >> 26) as u8 + }) + .collect(); + // The copy in shuffled pieces, so one hit cannot carry the rest. + const PIECE: usize = 4096; + let pieces = half.len() / PIECE; + let mut data = half.clone(); + for i in 0..pieces { + let piece = (i * 37) % pieces; + data.extend_from_slice(&half[piece * PIECE..(piece + 1) * PIECE]); + } + // Literal bytes the parse leaves, after checking the stream rebuilds. + let literals = |long: bool| { + let overrides = CompressionParameters::builder(CompressionLevel::Level(5)) + .window_log(22) + .hash_log(12) + .chain_log(12) + .enable_long_distance_matching(long) + .build() + .unwrap() + .overrides(); + let mut driver = MatchGeneratorDriver::new(1 << 17, 8); + driver.set_param_overrides(Some(overrides)); + driver.set_config_override(SearchMethod::HashChain, ParseMode::Lazy2); + driver.reset(CompressionLevel::Level(5)); + assert_eq!(driver.search, SearchMethod::HashChain); + let mut out = Vec::with_capacity(data.len()); + let mut literals = 0usize; + let mut consumed = 0usize; + while consumed < data.len() { + let take = (data.len() - consumed).min(driver.slice_size); + let block = &data[consumed..consumed + take]; + driver.commit_input(block); + consumed += take; + let mut replay = BlockReplay::new(block); + driver.start_matching(|seq| { + literals += match seq { + Sequence::Triple { literal_len, .. } => literal_len, + Sequence::Literals { len } => len, + }; + replay.apply(&mut out, seq); + }); + } + assert_eq!( + out, data, + "long-distance matching {long}: the stream does not rebuild" + ); + literals + }; + let without = literals(false); + let with = literals(true); + assert!( + without > half.len() * 3 / 2, + "the chain reached the repeat by itself: {without} literal bytes" + ); + assert!( + with < half.len() + half.len() / 8, + "the repeat was not matched: {with} literal bytes, {without} without" + ); +} + /// The row `mls` knob (C-like `minMatch`) is respected: every accepted /// match (regular row + repcode, on the lazy parse) is at least `mls` /// bytes, and the stream still round-trips for the whole 4..=7 range. The From cfb74ef4f09741a6da2a51940953d8006df43136 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 17 Oct 2026 01:22:59 +0300 Subject: [PATCH 10/11] fix(cli): weigh patch long mode over the whole benchmark range A benchmark with `--patch-from` compresses every level from `-b` to `-e`, but long mode was decided on the first level alone. Whether a level needs it is not constant across a range: at an equal chain log btlazy2 (level 13) cannot cycle the input where lazy2 (level 12) can, so `-b12 -e13` measured level 13 without long mode, as literals. Long mode now turns on when any benchmarked level needs it; the regression test runs that range. --- zstd/src/bin/structured-zstd/main.rs | 17 ++++++++++++++++- zstd/src/bin/structured-zstd/tests.rs | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index 05f1f60a4..48751f25b 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -2154,6 +2154,21 @@ fn patch_needs_long_distance( file_window_log > cycle_log } +/// Whether a level this run compresses at needs long-distance matching for the +/// patch (see [`patch_needs_long_distance`]). A benchmark compresses every +/// level of its `-b`/`-e` range under one setting, so one level that cannot +/// cycle the input turns it on for all. +fn patch_run_needs_long_distance(opts: &Options, source_size: u64, reference_size: u64) -> bool { + let levels = if opts.bench { + opts.bench_start..=opts.bench_end + } else { + opts.level..=opts.level + }; + levels + .into_iter() + .any(|level| patch_needs_long_distance(level, &opts.advanced, source_size, reference_size)) +} + /// Whether the run reads the `-D` / `--patch-from` file it names. Listing /// walks frame headers and training builds a dictionary from its samples; /// neither consults that one. Reading it anyway would fail a listing over a @@ -2751,7 +2766,7 @@ fn run_selected(mut opts: Options) -> Result { let reference_size = fs::metadata(reference) .map_err(|err| eyre!("can't stat {} : {err}", reference.display()))? .len(); - if patch_needs_long_distance(opts.level, &opts.advanced, source_size, reference_size) { + if patch_run_needs_long_distance(&opts, source_size, reference_size) { if !opts.long { display!(opts.verbosity, 2, "long mode automatically triggered"); } diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index 81974ed8b..bdb335e07 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -2242,6 +2242,24 @@ fn patch_from_needs_long_mode_where_the_matcher_cannot_cycle_the_input() { } } +/// A benchmark compresses its whole level range, so a patch needs long mode +/// when any level of it does: lazy2 (level 12) cycles a file window equal to +/// the chain log that btlazy2 (level 13) does not, and `-b12 -e13` measures +/// both. +#[test] +fn patch_from_long_mode_covers_every_benchmarked_level() { + let reference = 1u64 << 20; + let chain_log = LevelParameters::for_level(13, Some(1 << 30), reference as usize).chain_log; + let size = 1u64 << (chain_log - 1); + let level_only = AdvancedParams::default(); + assert!(!patch_needs_long_distance(12, &level_only, size, reference)); + assert!(patch_needs_long_distance(13, &level_only, size, reference)); + let range = parse(&["-b12", "-e13", "--patch-from=ref", "in"]).unwrap(); + assert!(patch_run_needs_long_distance(&range, size, reference)); + let single = parse(&["-12", "--patch-from=ref", "in"]).unwrap(); + assert!(!patch_run_needs_long_distance(&single, size, reference)); +} + /// The rule reads the matcher that will run: a `--zstd=` chain or strategy /// override that leaves the reference out of the level's cycle turns long mode /// on, and one that brings it in keeps it off. The reference command reads the From db566facc09e3c92dc4c2e05daa844830efbd687 Mon Sep 17 00:00:00 2001 From: Dmitry Prudnikov Date: Sat, 17 Oct 2026 01:23:04 +0300 Subject: [PATCH 11/11] fix(encoding): budget long-distance sequence storage Both workspace estimates counted the long-distance table alone, while the driver also keeps the sequences it searched, its store, the block's sequences and the optimal parser's copy of them. At a short minimum match those hold several MiB, so a benchmark `-M` limit could pass a run that then exceeds it. The estimates now add five vectors of a block's worth of sequences at the minimum match, upstream's per-store bound (`ZSTD_ldm_getMaxNbSeq`); a test checks that minMatch 4 budgets at least one such vector more than minMatch 64. --- zstd/src/encoding/ldm/mod.rs | 15 ++++++ zstd/src/encoding/levels/config.rs | 18 +++---- zstd/src/encoding/levels/config/tests.rs | 60 ++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 12 deletions(-) diff --git a/zstd/src/encoding/ldm/mod.rs b/zstd/src/encoding/ldm/mod.rs index 84c74c42f..07179b8e9 100644 --- a/zstd/src/encoding/ldm/mod.rs +++ b/zstd/src/encoding/ldm/mod.rs @@ -65,6 +65,21 @@ const LDM_XXH64_SEED: u64 = 0; /// `zstd_ldm.c:335`, brings `nextToUpdate` to within 512 of the match end). pub(crate) const LONG_MATCH_INDEXED_TAIL: usize = 512; +/// Bytes a frame's long-distance matching holds under `params`: its table and +/// the sequence vectors around it. Each vector keeps at most a block's matches +/// at `min_match_length` apiece (upstream's `ZSTD_ldm_getMaxNbSeq`), and five +/// are retained: what a search found, the store, which keeps the matches a +/// block took until they are half of it, the block's own, and the optimal +/// parser's copy of those. +pub(crate) fn estimated_workspace_bytes(params: &LdmParams) -> usize { + const RETAINED_SEQUENCE_VECTORS: usize = 5; + let block = crate::common::MAX_BLOCK_SIZE as usize; + let sequences = block / params.min_match_length.max(1) as usize + 1; + let vectors = sequences * core::mem::size_of::() * RETAINED_SEQUENCE_VECTORS; + LdmHashTable::estimated_workspace_bytes(params.hash_log, params.bucket_size_log) + .saturating_add(vectors) +} + /// LDM sequence producer — owns the rolling-hash state, bucket /// table, and scratch buffers needed to scan an input block and /// emit a stream of [`HcRawSeq`] candidates consumed by the diff --git a/zstd/src/encoding/levels/config.rs b/zstd/src/encoding/levels/config.rs index 54d51e031..eff1d6309 100644 --- a/zstd/src/encoding/levels/config.rs +++ b/zstd/src/encoding/levels/config.rs @@ -993,13 +993,11 @@ pub fn estimated_compression_workspace_bytes_for_run( #[cfg(feature = "ldm")] let ldm = if long_distance_matching { let strategy = ldm_strategy_ordinal(params.strategy_tag, params.lazy_depth); - let ldm_params = crate::encoding::ldm::params::LdmParams::adjust_for( - u32::from(params.window_log), - strategy, - ); - crate::encoding::ldm::table::LdmHashTable::estimated_workspace_bytes( - ldm_params.hash_log, - ldm_params.bucket_size_log, + crate::encoding::ldm::estimated_workspace_bytes( + &crate::encoding::ldm::params::LdmParams::adjust_for( + u32::from(params.window_log), + strategy, + ), ) } else { 0 @@ -1058,11 +1056,7 @@ pub fn estimated_compression_workspace_bytes_for_parameters( apply_frame_overrides(&mut params, &overrides, dictionary.is_some(), src_size_hint); #[cfg(feature = "ldm")] let ldm = overrides.ldm.map_or(0, |ldm| { - let ldm_params = frame_ldm_params(¶ms, &ldm); - crate::encoding::ldm::table::LdmHashTable::estimated_workspace_bytes( - ldm_params.hash_log, - ldm_params.bucket_size_log, - ) + crate::encoding::ldm::estimated_workspace_bytes(&frame_ldm_params(¶ms, &ldm)) }); #[cfg(not(feature = "ldm"))] let ldm = 0; diff --git a/zstd/src/encoding/levels/config/tests.rs b/zstd/src/encoding/levels/config/tests.rs index c73d955db..68e2ba504 100644 --- a/zstd/src/encoding/levels/config/tests.rs +++ b/zstd/src/encoding/levels/config/tests.rs @@ -296,6 +296,66 @@ fn a_strategy_override_counts_only_the_backend_it_selects() { assert_eq!(estimate(&as_fast), estimate(&level_one)); } +/// Long-distance matching holds its found sequences besides its table, a +/// block's worth at the matcher's minimum match apiece in each of the vectors +/// it keeps (upstream budgets one such store, `ZSTD_ldm_getMaxNbSeq`). The +/// shortest minimum match multiplies them, and both estimates count it. +#[cfg(feature = "ldm")] +#[test] +fn long_distance_estimates_count_the_sequence_storage() { + use crate::encoding::CompressionParameters; + let with_min_match = |min_match: u32| { + CompressionParameters::builder(CompressionLevel::Level(3)) + .enable_long_distance_matching(true) + .ldm_min_match(min_match) + .build() + .unwrap() + }; + let short = + super::estimated_compression_workspace_bytes_for_parameters(&with_min_match(4), None, None); + let long = super::estimated_compression_workspace_bytes_for_parameters( + &with_min_match(64), + None, + None, + ); + // One vector of a block at four bytes a match, the least the extra + // storage can be. + let one_store = (128 * 1024 / 4) * core::mem::size_of::(); + assert!( + short >= long + one_store, + "minMatch 4 budgets {short} bytes, minMatch 64 {long}" + ); + let off = super::estimated_compression_workspace_bytes_for_run( + CompressionLevel::Level(3), + None, + None, + false, + None, + ); + let on = super::estimated_compression_workspace_bytes_for_run( + CompressionLevel::Level(3), + None, + None, + true, + None, + ); + let table_only = { + let params = super::resolve_level_params(CompressionLevel::Level(3), None); + let ldm = crate::encoding::ldm::params::LdmParams::adjust_for( + u32::from(params.window_log), + super::ldm_strategy_ordinal(params.strategy_tag, params.lazy_depth), + ); + crate::encoding::ldm::table::LdmHashTable::estimated_workspace_bytes( + ldm.hash_log, + ldm.bucket_size_log, + ) + }; + assert!( + on > off + table_only, + "the run estimate counts the table alone: {on} with, {off} without" + ); +} + /// The parameters the encoder actually runs for a (level, source size) pair /// are upstream's `ZSTD_getCParams(level, size, 0)` for that pair: strategy, /// window / hash / chain widths, search depth, minMatch and targetLength, on