diff --git a/README.md b/README.md index bd4332f38..6a1852cc8 100644 --- a/README.md +++ b/README.md @@ -99,9 +99,9 @@ does, with the window stopped at 30 (the widest this build writes), and upstream's layout. `--target-compressed-block-size` does take effect: it bounds what goes into a block, so blocks flush sooner. `--long` means `--long=27`, as upstream -documents. It needs level 16 or above, where long-distance matching actually -runs — below that it is refused rather than accepted as a wider window and -nothing else. A window is never declared larger than the source can fill, so a +documents, and runs long-distance matching at every level. `--patch-from` +turns it on itself when the input outgrows what the level's match finder +reaches, by upstream's rule. A window is never declared larger than the source can fill, so a small file compressed with `--long` does not ask its decoders to reserve 128 MiB. Decoding takes windows up to 128 MiB by default; a wider frame (from `--long=28` and up) is refused with upstream's message, and decodes with @@ -420,9 +420,9 @@ upstream `main`, so the link target is correct for forks too — fork consumers should point readers at the upstream registry rather than maintain divergent copies. --> -## Project relationship +## Origins -Maintained fork of [KillingSpark/zstd-rs](https://github.com/KillingSpark/zstd-rs) (ruzstd) by [Dmitry Prudnikov](https://github.com/polaz). We sync periodically with upstream but maintain an independent development trajectory focused on the [CoordiNode](https://github.com/structured-world/coordinode) database engine's per-label dictionary needs. +structured-zstd is an independent project by [Dmitry Prudnikov](https://github.com/polaz), developed for the [CoordiNode](https://github.com/structured-world/coordinode) database engine and its per-label dictionary workloads. It started from the code base of [ruzstd](https://github.com/KillingSpark/zstd-rs) by Moritz Borcherding; that code is used under the MIT License, whose notice is kept in [NOTICE](https://github.com/structured-world/structured-zstd/blob/main/NOTICE). ## Support the project diff --git a/zstd/src/bin/structured-zstd/main.rs b/zstd/src/bin/structured-zstd/main.rs index bc87a4827..48751f25b 100644 --- a/zstd/src/bin/structured-zstd/main.rs +++ b/zstd/src/bin/structured-zstd/main.rs @@ -188,11 +188,8 @@ struct Options { /// `--trace FILE`: where one CSV line per frame is appended. trace: Option, /// The widest window decoding takes; see [`decode_window_ceiling`]. Set - /// once the dictionary's size is known. + /// once the command line is settled. decode_window: u64, - /// What `-M` counts besides the window; see [`decode_memory_overhead`]. - /// Set with [`Self::decode_window`]. - decode_memory_overhead: Option, /// A patch lifted the decode window to its widest, and no `-M` (`-M0` /// included) came after to set a limit of its own. patch_window_lifted: bool, @@ -360,11 +357,6 @@ const MIN_BENCH_BLOCK_SIZE: u64 = 32; /// Window log a bare `--long` selects, as upstream documents (128 MiB). const DEFAULT_LONG_WINDOW_LOG: u32 = 27; -/// Lowest level whose matcher actually runs long-distance matching. Below it -/// the encoder uses a strategy that carries no long-distance producer, so -/// `--long` would be a wider window and nothing else. -const MIN_LONG_LEVEL: i32 = 16; - /// Which spellings of a size suffix [`read_leading_number`] reads. #[derive(Clone, Copy, PartialEq, Eq)] enum SuffixCase { @@ -500,26 +492,17 @@ fn parse_memory_limit(text: &str) -> Result> { /// What the decoder holds BESIDES the window, rounded well up: the literal and /// block buffers (a block is capped at 128 KiB each), the sequence storage, -/// the Huffman and FSE tables, and the tool's own I/O buffers. Counted because -/// `-M` promises total memory, not one allocation: a limit equal to the window -/// alone is one we would break. +/// the Huffman and FSE tables, and the tool's own I/O buffers. A limit that +/// cannot cover these and the default window together is refused up front. const DECODER_AUXILIARY_BYTES: u64 = 1 << 20; -/// What `-M` counts besides the window: the decoder's other buffers and the -/// dictionary (`dictionary_bytes`), held as read and again as parsed. `None` -/// past `u64`, which no limit covers. -fn decode_memory_overhead(dictionary_bytes: u64) -> Option { - dictionary_bytes - .checked_mul(2) - .and_then(|dictionary| dictionary.checked_add(DECODER_AUXILIARY_BYTES)) -} - /// The widest window this run decodes, handed to the decoder as its ceiling. /// -/// Under `-M` the window may have what the promise leaves once -/// [`decode_memory_overhead`] is counted; the limit was already checked to -/// leave at least the default. `-M` comes first because a patch keeps it too: -/// `--patch-apply` drops a `-M` given before it, so one still set came after, +/// Under `-M` the window is the limit, as the reference sets it +/// (`ZSTD_DCtx_setMaxWindowSize(dctx, memLimit)`, fileio.c): scripts written +/// for it decode a 2^29 window under `--memory=512MB`, which a limit counting +/// the other buffers too would refuse. `-M` comes first because a patch keeps +/// it too: `--patch-apply` drops a `-M` given before it, so one still set came after, /// and replaces the lifted limit as a later `-M` replaces it in the reference /// command. Otherwise a patch whose window no later `-M` (`-M0` included) has /// limited again reaches back across its whole reference, so it is decoded @@ -529,14 +512,11 @@ fn decode_memory_overhead(dictionary_bytes: u64) -> Option { /// failing those the decoder's default stands. A log below the default does /// not lower it: the reference would then refuse frames its own default /// accepts, which serves nobody. -fn decode_window_ceiling(opts: &Options, dictionary_bytes: u64) -> u64 { +fn decode_window_ceiling(opts: &Options) -> u64 { use structured_zstd::decoding::{MAX_DECODER_WINDOW_SIZE, MAXIMUM_ALLOWED_WINDOW_SIZE}; if let Some(limit) = opts.memory_limit { - return decode_memory_overhead(dictionary_bytes) - .and_then(|held| limit.checked_sub(held)) - .unwrap_or(0) - .clamp(MAXIMUM_ALLOWED_WINDOW_SIZE, MAX_DECODER_WINDOW_SIZE); + return limit.clamp(MAXIMUM_ALLOWED_WINDOW_SIZE, MAX_DECODER_WINDOW_SIZE); } if opts.patch_window_lifted { return MAX_DECODER_WINDOW_SIZE; @@ -1189,7 +1169,6 @@ fn parse_args_into( console: FakeConsole::default(), trace: None, decode_window: structured_zstd::decoding::MAXIMUM_ALLOWED_WINDOW_SIZE, - decode_memory_overhead: decode_memory_overhead(0), patch_window_lifted: false, bench_separately: false, block_size: None, @@ -1764,29 +1743,6 @@ fn parse_args_into( opts.bench_start = opts.bench_start.min(CLI_MAX_LEVEL_WITHOUT_ULTRA); opts.bench_end = opts.bench_end.min(CLI_MAX_LEVEL_WITHOUT_ULTRA); } - // Long-distance matching runs on the optimal parser here, so below it the - // flag would widen the window and never run the matcher it names. Settled - // after the whole command line, since the level may follow the flag — and - // read from the benchmark range when there is one, because those are the - // levels that will actually compress. The whole range runs with the flag, - // so its lowest level is the one that has to carry the matcher. - let long_level = if opts.bench { - opts.bench_start.min(opts.bench_end) - } else { - opts.level - }; - // A `--zstd=strat=` override onto the optimal parser carries the matcher - // whatever the level says. - let optimal_strategy = opts - .advanced - .strategy - .is_some_and(|strategy| strategy >= Strategy::Btopt); - if opts.long && compresses(&opts) && long_level < MIN_LONG_LEVEL && !optimal_strategy { - bail!( - "--long needs level {MIN_LONG_LEVEL} or above, where long-distance \ - matching runs; at level {long_level} it would only widen the window", - ); - } // Window logs are validated here, before any file is opened, against the // mode the whole command line settled on; a knob out of its range is a // broken command line. @@ -2173,6 +2129,46 @@ fn patch_window_log(source_size: u64) -> Result { Ok(file_window_log.max(lower)) } +/// Whether a `--patch-from` compression of `source_size` bytes against a +/// `reference_size`-byte reference needs long-distance matching to reach the +/// reference: when the file's window outgrows what the match finder cycles +/// through, by the reference command's rule (`FIO_adjustParamsForPatchFromMode`, +/// `fileWindowLog > ZSTD_cycleLog(chainLog, strategy)`, the binary-tree +/// strategies cycling one log short of their chain). The chain and strategy +/// are the ones the frame runs, the level's for those sizes under any +/// `--zstd=` override; the reference command reads the bare level's and +/// leaves an overridden short chain without long mode. +fn patch_needs_long_distance( + level: i32, + advanced: &AdvancedParams, + source_size: u64, + reference_size: u64, +) -> bool { + let file_window_log = u64::BITS - source_size.max(1).leading_zeros(); + // A reference past the address space cannot have been read to get here. + let reference_size = usize::try_from(reference_size).unwrap_or(usize::MAX); + let params = LevelParameters::for_level(level, Some(source_size), reference_size); + let chain_log = advanced.chain_log.unwrap_or(params.chain_log); + let strategy = advanced.strategy.unwrap_or(params.strategy); + let cycle_log = chain_log - u32::from(strategy >= Strategy::Btlazy2); + file_window_log > cycle_log +} + +/// Whether a level this run compresses at needs long-distance matching for the +/// patch (see [`patch_needs_long_distance`]). A benchmark compresses every +/// level of its `-b`/`-e` range under one setting, so one level that cannot +/// cycle the input turns it on for all. +fn patch_run_needs_long_distance(opts: &Options, source_size: u64, reference_size: u64) -> bool { + let levels = if opts.bench { + opts.bench_start..=opts.bench_end + } else { + opts.level..=opts.level + }; + levels + .into_iter() + .any(|level| patch_needs_long_distance(level, &opts.advanced, source_size, reference_size)) +} + /// Whether the run reads the `-D` / `--patch-from` file it names. Listing /// walks frame headers and training builds a dictionary from its samples; /// neither consults that one. Reading it anyway would fail a listing over a @@ -2766,24 +2762,21 @@ fn run_selected(mut opts: Options) -> Result { _ => bail!("Using --patch-from with stdin requires --stream-size"), }; opts.advanced.window_log = Some(patch_window_log(source_size)?); - // Long-distance matching is what finds the reference across a - // window this wide; it runs on the optimal parser here, so it is - // switched on where that parser runs. - let optimal = opts.level >= MIN_LONG_LEVEL - || opts - .advanced - .strategy - .is_some_and(|strategy| strategy >= Strategy::Btopt); - if optimal { + let reference = opts.patch_from.as_deref().expect("set in this branch"); + let reference_size = fs::metadata(reference) + .map_err(|err| eyre!("can't stat {} : {err}", reference.display()))? + .len(); + if patch_run_needs_long_distance(&opts, source_size, reference_size) { + if !opts.long { + display!(opts.verbosity, 2, "long mode automatically triggered"); + } opts.long = true; } } } let dict_bytes = load_dictionary(&opts)?; - let dictionary_bytes = dict_bytes.as_ref().map_or(0, |bytes| bytes.len() as u64); - opts.decode_window = decode_window_ceiling(&opts, dictionary_bytes); - opts.decode_memory_overhead = decode_memory_overhead(dictionary_bytes); + opts.decode_window = decode_window_ceiling(&opts); // `-b` benchmarks compression/decompression across levels instead of // producing output files; handle it before the streaming flow. It takes the @@ -5629,8 +5622,9 @@ struct DecodeSettings { pass_through: bool, /// The decoder's window ceiling; see [`decode_window_ceiling`]. max_window: u64, - /// What `-M` counts besides the window, for the `--memory` a refusal - /// names; see [`decode_memory_overhead`]. + /// What the `--memory` a refusal names adds to the window: nothing for a + /// plain decode, whose `-M` is the window, and what a benchmark holds + /// besides it for one (see [`bench_decode_settings`]). memory_overhead: Option, } @@ -5640,7 +5634,7 @@ impl Default for DecodeSettings { verify_checksum: true, pass_through: false, max_window: structured_zstd::decoding::MAXIMUM_ALLOWED_WINDOW_SIZE, - memory_overhead: decode_memory_overhead(0), + memory_overhead: Some(0), } } } @@ -5659,7 +5653,7 @@ impl DecodeSettings { pass_through: opts.mode != Mode::Test && opts.pass_through.unwrap_or(opts.force && to_stdout), max_window: opts.decode_window, - memory_overhead: opts.decode_memory_overhead, + memory_overhead: Some(0), } } diff --git a/zstd/src/bin/structured-zstd/tests.rs b/zstd/src/bin/structured-zstd/tests.rs index f2a18cc61..bdb335e07 100644 --- a/zstd/src/bin/structured-zstd/tests.rs +++ b/zstd/src/bin/structured-zstd/tests.rs @@ -2139,33 +2139,157 @@ fn an_explicit_window_is_capped_by_a_known_source_size() { ); } -/// Long-distance matching lives on the optimal parser here, so the levels below -/// it can widen the window and still never run the matcher the flag names. -/// Taking `--long` at those levels would report success for work that did not -/// happen, and hand back a file compressed the ordinary way. -#[test] -fn long_is_refused_at_levels_that_cannot_run_it() { - assert!(parse(&["-3", "--long", "in.txt"]).is_err()); - assert!(parse(&["--long", "-15", "in.txt"]).is_err()); - // From the optimal parser up, the matcher is there to run. - assert!(parse(&["-16", "--long", "in.txt"]).is_ok()); +/// `--long` runs long-distance matching at every level, as upstream's does: its +/// own tests compress with `--long` at levels 1 and 3 (`tests/playTests.sh`). +#[test] +fn long_is_taken_at_every_level() { + for level in 1..=19 { + let flag = format!("-{level}"); + assert!( + parse(&[&flag, "--long", "in.txt"]).unwrap().long, + "level {level}" + ); + } + assert!(parse(&["--fast=3", "--long", "in.txt"]).unwrap().long); assert!(parse(&["--ultra", "-22", "--long=27", "in.txt"]).is_ok()); - // Decompression takes the flag as the window hint it is; nothing to run. assert!(parse(&["-d", "--long", "in.txt.zst"]).is_ok()); - // Benchmarking compresses the levels `-b`/`-e` name, not the one `-N` set, - // so those are the levels the flag has to be true of. - assert!(parse(&["-b16", "--long", "in.txt"]).is_ok()); - assert!(parse(&["-b16", "-e19", "--long", "in.txt"]).is_ok()); - assert!(parse(&["-b3", "--long", "in.txt"]).is_err()); - // The whole range runs with the flag, so a range that starts below the - // matcher is refused even when it ends above it. - assert!(parse(&["-b3", "-e19", "--long", "in.txt"]).is_err()); - // And a benchmark range is what counts, not a level that was also typed. - assert!(parse(&["-16", "-b3", "--long", "in.txt"]).is_err()); - // Benchmarking compresses whatever mode was asked for, so `-d` does not - // excuse the flag from naming levels that can run it. - assert!(parse(&["-d", "-b3", "--long", "in.txt"]).is_err()); - assert!(parse(&["-d", "-b16", "--long", "in.txt"]).is_ok()); + assert!(parse(&["-b3", "-e19", "--long", "in.txt"]).is_ok()); + assert!(parse(&["-3", "--long", "--zstd=strat=6", "in.txt"]).is_ok()); +} + +/// Past the window a level keeps, `--long` is what finds a repeat: a copy laid +/// out in shuffled pieces up to 4 MiB behind, beyond level 3's 2 MiB window, +/// costs next to nothing with the flag and nearly its whole size without it. +#[test] +fn long_finds_a_far_repeat_at_a_low_level() { + const PIECE: usize = 4096; + let mut state = 0x5EED_0569u64; + // Six-bit symbols: compressible enough that no block is written off, and + // short local matches stay rare. + let half: Vec = (0..4 << 20) + .map(|_| { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + ((state >> 33) & 63) as u8 + }) + .collect(); + let pieces = half.len() / PIECE; + let mut data = half.clone(); + for i in 0..pieces { + let piece = (i * 37) % pieces; + data.extend_from_slice(&half[piece * PIECE..(piece + 1) * PIECE]); + } + let compress = |long: bool| { + let mut frame = Vec::new(); + compress_stream( + data.as_slice(), + &mut frame, + &FrameSettings { + level: 3, + long, + long_window_log: long.then_some(DEFAULT_LONG_WINDOW_LOG), + ..FrameSettings::default() + }, + &mut no_dict(), + ) + .expect("compressing succeeds"); + frame + }; + let without = compress(false); + let with = compress(true); + assert!( + with.len() * 3 < without.len() * 2, + "--long at level 3: {} bytes against {} without it", + with.len(), + without.len() + ); + assert_eq!(decoded(&with).unwrap(), data); +} + +/// `--patch-from` turns long mode on itself when the input outgrows what the +/// level's match finder cycles through, by upstream's rule +/// (`FIO_adjustParamsForPatchFromMode`: `fileWindowLog > +/// ZSTD_cycleLog(chainLog, strategy)`, the binary-tree strategies cycling one +/// log short of their chain). +#[test] +fn patch_from_needs_long_mode_where_the_matcher_cannot_cycle_the_input() { + let level_only = AdvancedParams::default(); + // Level 3 on 61 MiB against 60 MiB: dfast, chainLog 16, file window log 26. + assert!(patch_needs_long_distance( + 3, + &level_only, + 61 << 20, + 60 << 20 + )); + // A small patch fits the chain the level already has. + assert!(!patch_needs_long_distance(3, &level_only, 2000, 4000)); + // The binary tree cycles one log short: a file window equal to the chain + // log is past it at btlazy2 (level 13), within it at lazy2 (level 12). + let reference = 1u64 << 20; + for (level, binary_tree) in [(12, false), (13, true)] { + let chain_log = + LevelParameters::for_level(level, Some(1 << 30), reference as usize).chain_log; + let size = 1u64 << (chain_log - 1); + let params = LevelParameters::for_level(level, Some(size), reference as usize); + assert_eq!(params.chain_log, chain_log, "level {level}: one size tier"); + assert_eq!(params.strategy >= Strategy::Btlazy2, binary_tree); + assert_eq!( + patch_needs_long_distance(level, &level_only, size, reference), + binary_tree, + "level {level}" + ); + } +} + +/// A benchmark compresses its whole level range, so a patch needs long mode +/// when any level of it does: lazy2 (level 12) cycles a file window equal to +/// the chain log that btlazy2 (level 13) does not, and `-b12 -e13` measures +/// both. +#[test] +fn patch_from_long_mode_covers_every_benchmarked_level() { + let reference = 1u64 << 20; + let chain_log = LevelParameters::for_level(13, Some(1 << 30), reference as usize).chain_log; + let size = 1u64 << (chain_log - 1); + let level_only = AdvancedParams::default(); + assert!(!patch_needs_long_distance(12, &level_only, size, reference)); + assert!(patch_needs_long_distance(13, &level_only, size, reference)); + let range = parse(&["-b12", "-e13", "--patch-from=ref", "in"]).unwrap(); + assert!(patch_run_needs_long_distance(&range, size, reference)); + let single = parse(&["-12", "--patch-from=ref", "in"]).unwrap(); + assert!(!patch_run_needs_long_distance(&single, size, reference)); +} + +/// The rule reads the matcher that will run: a `--zstd=` chain or strategy +/// override that leaves the reference out of the level's cycle turns long mode +/// on, and one that brings it in keeps it off. The reference command reads the +/// bare level and so writes a patch of literals for the first case. +#[test] +fn patch_from_long_mode_follows_the_overridden_matcher() { + let reference = 1u64 << 20; + let size = 4u64 << 20; + let params = LevelParameters::for_level(19, Some(size), reference as usize); + let file_window_log = u64::BITS - size.leading_zeros(); + assert!( + params.chain_log > file_window_log, + "level 19 cycles the input by itself" + ); + let level_only = AdvancedParams::default(); + assert!(!patch_needs_long_distance(19, &level_only, size, reference)); + let short_chain = AdvancedParams { + chain_log: Some(16), + ..AdvancedParams::default() + }; + assert!(patch_needs_long_distance(19, &short_chain, size, reference)); + // Level 1 runs fast with a short chain; a binary tree with a long one + // cycles the input. + assert!(patch_needs_long_distance(1, &level_only, size, reference)); + let long_tree = AdvancedParams { + chain_log: Some(file_window_log + 1), + strategy: Some(Strategy::Btopt), + ..AdvancedParams::default() + }; + assert!(!patch_needs_long_distance(1, &long_tree, size, reference)); } /// The window log has a supported range; a value outside it has to fail rather @@ -2438,8 +2562,10 @@ fn test_and_tuning_options_are_accepted() { /// The decoder's window ceiling follows the command line as the reference's /// does: a patch reaches across its whole reference, `--long=N` and /// `--zstd=wlog=N` raise the ceiling to their window (the latter winning, as -/// it wins when compressing), `-M` gives what its promise leaves after the -/// other buffers and the dictionary, and nothing named keeps the default. +/// it wins when compressing), `-M` is the window itself as the reference +/// takes it (`ZSTD_DCtx_setMaxWindowSize(dctx, memLimit)`, so its own tests +/// decode a 2^29 window under `--memory=512MB`), and nothing named keeps the +/// default. /// Nothing lowers it below the default. A `-M` after `--patch-apply` is a /// promise the patch keeps too, as the reference's later `-M` replaces the /// limit `--patch-apply` lifted. @@ -2447,46 +2573,43 @@ fn test_and_tuning_options_are_accepted() { fn the_decode_window_follows_the_command_line() { use structured_zstd::decoding::{MAX_DECODER_WINDOW_SIZE, MAXIMUM_ALLOWED_WINDOW_SIZE}; - let ceiling = - |args: &[&str], dictionary: u64| decode_window_ceiling(&parse(args).unwrap(), dictionary); - assert_eq!(ceiling(&["-d", "f.zst"], 0), MAXIMUM_ALLOWED_WINDOW_SIZE); - assert_eq!(ceiling(&["-d", "--long=28", "f.zst"], 0), 1 << 28); - assert_eq!(ceiling(&["-d", "--zstd=wlog=29", "f.zst"], 0), 1 << 29); + let ceiling = |args: &[&str]| decode_window_ceiling(&parse(args).unwrap()); + assert_eq!(ceiling(&["-d", "f.zst"]), MAXIMUM_ALLOWED_WINDOW_SIZE); + assert_eq!(ceiling(&["-d", "--long=28", "f.zst"]), 1 << 28); + assert_eq!(ceiling(&["-d", "--zstd=wlog=29", "f.zst"]), 1 << 29); assert_eq!( - ceiling(&["-d", "--long=27", "--zstd=wlog=29", "f.zst"], 0), + ceiling(&["-d", "--long=27", "--zstd=wlog=29", "f.zst"]), 1 << 29 ); assert_eq!( - ceiling(&["-d", "--long=20", "f.zst"], 0), + ceiling(&["-d", "--long=20", "f.zst"]), MAXIMUM_ALLOWED_WINDOW_SIZE ); assert_eq!( - ceiling(&["--patch-apply=ref", "f.zst"], 0), + ceiling(&["--patch-apply=ref", "f.zst"]), MAX_DECODER_WINDOW_SIZE ); assert_eq!( - ceiling(&["--patch-apply=ref", "-M512MB", "f.zst"], 1 << 20), - (512 << 20) - (1 << 20) - (2 << 20) + ceiling(&["--patch-apply=ref", "-M512MB", "f.zst"]), + 512 << 20 ); // A later `-M0` asks for the default limit again, as zero replaces the // reference command's limit (zstdcli.c:1572-1577); an earlier one is // overridden by the patch like any other `-M`. assert_eq!( - ceiling(&["--patch-apply=ref", "-M0", "f.zst"], 0), + ceiling(&["--patch-apply=ref", "-M0", "f.zst"]), MAXIMUM_ALLOWED_WINDOW_SIZE ); assert_eq!( - ceiling(&["--patch-apply=ref", "--long=28", "-M0", "f.zst"], 0), + ceiling(&["--patch-apply=ref", "--long=28", "-M0", "f.zst"]), 1 << 28 ); assert_eq!( - ceiling(&["-M0", "--patch-apply=ref", "f.zst"], 0), + ceiling(&["-M0", "--patch-apply=ref", "f.zst"]), MAX_DECODER_WINDOW_SIZE ); - assert_eq!( - ceiling(&["-d", "-M512MB", "f.zst"], 1 << 20), - (512 << 20) - (1 << 20) - (2 << 20) - ); + assert_eq!(ceiling(&["-d", "-M512MB", "f.zst"]), 512 << 20); + assert_eq!(ceiling(&["-d", "--memory=1GB", "f.zst"]), 1 << 30); } /// A benchmark holds its input, frames, decoded copy, encoder and dictionaries @@ -2498,7 +2621,7 @@ fn a_benchmark_window_leaves_room_for_what_the_benchmark_holds() { use structured_zstd::decoding::MAXIMUM_ALLOWED_WINDOW_SIZE; let mut opts = parse(&["-b19", "-M600MB", "--long=29", "f"]).unwrap(); - opts.decode_window = decode_window_ceiling(&opts, 0); + opts.decode_window = decode_window_ceiling(&opts); let held = 400u64 << 20; assert_eq!( bench_decode_settings(&opts, Some(held)).max_window, @@ -2511,7 +2634,7 @@ fn a_benchmark_window_leaves_room_for_what_the_benchmark_holds() { MAXIMUM_ALLOWED_WINDOW_SIZE ); let mut free = parse(&["-b19", "--long=29", "f"]).unwrap(); - free.decode_window = decode_window_ceiling(&free, 0); + free.decode_window = decode_window_ceiling(&free); assert_eq!(bench_decode_settings(&free, None).max_window, 1 << 29); } @@ -2532,29 +2655,24 @@ fn decoding_takes_the_decoders_widest_window_log() { let wlog = format!("--zstd=wlog={widest}"); for flag in [&long, &wlog] { let opts = parse(&["-d", flag, "f.zst"]).unwrap(); - assert_eq!(decode_window_ceiling(&opts, 0), MAX_DECODER_WINDOW_SIZE); + assert_eq!(decode_window_ceiling(&opts), MAX_DECODER_WINDOW_SIZE); assert!(parse(&[flag, "f"]).is_err(), "{flag} compressing"); } assert!(parse(&["-d", &format!("--long={}", widest + 1), "f.zst"]).is_err()); } /// A frame past the ceiling is refused with the reference command's message, -/// naming the flags that would decode it (fileio.c:2674-2684). -/// The `--memory` named covers the window and what `-M` counts besides it, the -/// decoder's other buffers and the dictionary, so it is a limit that decodes -/// the frame. +/// naming the flags that would decode it (fileio.c:2674-2684): the `--memory` +/// is the window in whole MiB, which is the limit that decodes it. #[test] fn a_window_past_the_ceiling_names_the_flag_that_decodes_it() { assert_eq!( - window_too_large_message(1 << 28, 1 << 27, Some(DECODER_AUXILIARY_BYTES)), + window_too_large_message(1 << 28, 1 << 27, Some(0)), "Decoding error (36) : Frame requires too much memory for decoding\n\ Window size larger than maximum : 268435456 > 134217728\n\ - Use --long=28 or --memory=257MB" - ); - assert!( - window_too_large_message(1 << 40, 1 << 27, Some(DECODER_AUXILIARY_BYTES)) - .contains("ZSTD_WINDOWLOG_MAX") + Use --long=28 or --memory=256MB" ); + assert!(window_too_large_message(1 << 40, 1 << 27, Some(0)).contains("ZSTD_WINDOWLOG_MAX")); } /// A frame from `zstd --long=28` does not decode with the default ceiling and @@ -2580,6 +2698,8 @@ fn a_wide_window_decodes_with_long_or_memory() { .split_whitespace() .find(|word| word.starts_with("--memory=")) .expect("the refusal names a --memory"); + // The window itself, as upstream names and takes it. + assert_eq!(named, "--memory=256MB"); for flag in ["--long=28", "-M512MB", named] { assert_eq!( run(parse(&["-d", "-qq", "-f", flag, input, "-o", output]).unwrap()).unwrap(), @@ -6154,16 +6274,6 @@ fn max_is_refused_on_a_32_bit_target() { assert!(err.to_string().contains("32-bit"), "{err}"); } -/// `--long` below level 16 is refused because the matcher does not run there, -/// unless `--zstd=strat=` moves the level onto a parser where it does. -#[test] -fn a_strategy_override_onto_the_optimal_parser_admits_long() { - assert!(parse(&["-3", "--long", "f"]).is_err()); - assert!(parse(&["-3", "--long", "--zstd=strat=7", "f"]).is_ok()); - assert!(parse(&["-3", "--long", "--zstd=strat=9", "f"]).is_ok()); - assert!(parse(&["-3", "--long", "--zstd=strat=6", "f"]).is_err()); -} - /// `--[no-]compress-literals` decides whether literal sections are /// entropy-coded: forced raw, a literal-heavy input compresses worse than the /// level's default; forced on at a negative level, where the default is raw, diff --git a/zstd/src/encoding/bt/ldm_helper_tests.rs b/zstd/src/encoding/bt/ldm_helper_tests.rs index d3696d37d..e951921e2 100644 --- a/zstd/src/encoding/bt/ldm_helper_tests.rs +++ b/zstd/src/encoding/bt/ldm_helper_tests.rs @@ -32,65 +32,6 @@ fn store(size: usize) -> HcRawSeqStore { } } -/// Regression test for PR #139 Copilot review — threads #1/#2: -/// `prepare_ldm_candidates` must translate the absolute -/// `current_abs_start` (`history_abs_start + window_size - -/// current_len`) into a slice index inside the supplied -/// `history` slice. Before the fix, the code passed -/// `current_abs_start` directly as `block_start` to -/// `LdmProducer::generate_into`, which would index out of -/// bounds (or into the wrong byte range) as soon as -/// `history_abs_start != 0` after window eviction. -/// -/// We construct a `BtMatcher` with an active `LdmProducer` -/// whose table starts empty, hand it a small `history` slice, -/// and call `prepare_ldm_candidates` with absolute coordinates -/// shifted by a non-zero `history_abs_start`. The call must -/// complete without panic — under the pre-fix code it would -/// panic on the `history[block_start..block_end]` slice access -/// inside `generate_into`. -#[test] -#[cfg(feature = "ldm")] -fn prepare_ldm_candidates_translates_absolute_positions_to_slice_indices() { - use crate::encoding::ldm::{LdmProducer, params::LdmParams}; - - let mut bt = BtMatcher::new(); - // Activate the producer with hand-tuned small params — - // the upstream zstd btultra2 defaults (`with_window_and_strategy(27, 9)`) - // would allocate `1 << 23 = 8M` entries × 8 bytes = 64 MiB - // for a table this test never actually populates beyond a - // handful of entries. Build a 10-bit hash log instead so - // the table is ~8 KiB — keeps the suite stable under - // parallel nextest on 32-bit / memory-tight CI shards. - bt.ldm_producer = Some(LdmProducer::new(LdmParams { - window_log: 27, - hash_log: 10, - hash_rate_log: 4, - min_match_length: 32, - bucket_size_log: 4, - })); - - // 256 bytes of arbitrary content — enough for the gear - // hash to make progress against the upstream zstd btultra2 - // `min_match_length = 32`. - let history: alloc::vec::Vec = (0u8..=255).collect(); - - // Simulate a frame whose window has already evicted some - // bytes: `history_abs_start = 1024`, and the current block - // covers absolute `[1024, 1280)` which maps to slice - // `[0, 256)` inside `history`. - let history_abs_start = 1024usize; - let current_abs_start = 1024usize; - let current_len = history.len(); - - // Pre-fix: would panic with `slice index out of bounds` - // on `history[1024..1280]`. Post-fix: translates to - // `[0..256]` and runs cleanly. - bt.prepare_ldm_candidates(&history, history_abs_start, current_abs_start, current_len); - // Cross-block fixture isn't engineered to emit a match — - // the assertion is simply that the call succeeded. -} - #[test] fn ldm_skip_raw_seq_store_bytes_walks_whole_sequences() { let bt = bt_with_seqs(vec![raw(2, 3, 100), raw(4, 1, 200), raw(0, 5, 50)]); diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs index f4579e557..6774f6282 100644 --- a/zstd/src/encoding/bt/mod.rs +++ b/zstd/src/encoding/bt/mod.rs @@ -22,8 +22,6 @@ use alloc::vec::Vec; use core::mem::MaybeUninit; use super::cost_model::{HC_MAX_LIT, HcOptState, HcOptimalCostProfile}; -#[cfg(feature = "ldm")] -use super::ldm::LdmProducer; use super::opt::ldm::{HcOptLdmState, HcRawSeq, HcRawSeqStore}; use super::opt::types::{HcOptimalNode, HcOptimalPlanBuffers, HcOptimalSequence, MatchCandidate}; @@ -77,28 +75,10 @@ pub(crate) struct BtMatcher { pub(crate) opt_lit_price_stamp: u32, pub(crate) opt_ml_price_stamp: u32, /// Long-distance match (LDM) candidates seeded into the optimal - /// parser. Built per-block during `start_matching_optimal` and - /// drained as the parser advances. + /// parser: the driver's long-distance matcher fills them for each block + /// before the parse, which drains them as it advances. Empty while the + /// frame runs without it. pub(crate) ldm_sequences: Vec, - /// LDM producer — `None` while LDM is opt-out (current - /// default) so the table allocation only happens for callers - /// that opt in. The producer is **never auto-constructed**: - /// every `CompressionLevel` preset leaves this field as - /// `None`, matching upstream `libzstd.so.1` where - /// `ZSTD_compress(..., level)` never enables LDM. The - /// opt-in surface (Rust parameter API, see #27) builds an - /// `LdmProducer` from caller-supplied params and assigns it - /// here at frame setup time; `prepare_ldm_candidates` only - /// consumes the field, it does not construct it. See - /// [`super::ldm::LdmProducer`]. - /// - /// Gated behind the `ldm` feature (which implies `hash`, since the - /// producer's per-window XXH64 hashing needs the optional `twox-hash` - /// dependency). Without it the field disappears and the - /// `prepare_ldm_candidates` body shrinks to an `ldm_sequences.clear()` - /// stub. - #[cfg(feature = "ldm")] - pub(crate) ldm_producer: Option, } impl BtMatcher { @@ -189,29 +169,21 @@ impl BtMatcher { opt_lit_price_stamp: 0, opt_ml_price_stamp: 0, ldm_sequences: Vec::new(), - #[cfg(feature = "ldm")] - ldm_producer: None, } } - /// Heap bytes the optimal-parser scratch buffers and the optional LDM - /// producer hold. The fixed-size price arrays and `opt_state` are inline - /// (counted by the owner's `size_of`), so only the `Vec` fields contribute. + /// Heap bytes the optimal-parser scratch buffers hold. The fixed-size + /// price arrays and `opt_state` are inline (counted by the owner's + /// `size_of`), so only the `Vec` fields contribute. pub(crate) fn heap_size(&self) -> usize { - let scratch = self.opt_nodes_scratch.len() * core::mem::size_of::() + self.opt_nodes_scratch.len() * core::mem::size_of::() + self.opt_node_prices_scratch.len() * core::mem::size_of::() + self.opt_candidates_scratch.capacity() * core::mem::size_of::() + self.opt_store_scratch.capacity() * core::mem::size_of::() + (self.opt_segment_plan_scratch.capacity() + self.opt_seed_plan_scratch.capacity()) * core::mem::size_of::() + self.opt_price_arena.len() * core::mem::size_of::<[u32; 2]>() - + self.ldm_sequences.capacity() * core::mem::size_of::(); - // The LDM producer is only present under the `ldm` feature. - #[cfg(feature = "ldm")] - let ldm = self.ldm_producer.as_ref().map_or(0, |p| p.heap_size()); - #[cfg(not(feature = "ldm"))] - let ldm = 0; - scratch + ldm + + self.ldm_sequences.capacity() * core::mem::size_of::() } /// Per-frame reset — clears scratch buffers, resets cost model, @@ -236,10 +208,6 @@ impl BtMatcher { self.opt_lit_price_generation = [0; HC_MAX_LIT + 1]; self.opt_lit_price_stamp = 0; self.ldm_sequences.clear(); - #[cfg(feature = "ldm")] - if let Some(producer) = self.ldm_producer.as_mut() { - producer.clear(); - } } /// Upstream zstd parity: `ZSTD_optLdm_skipRawSeqStoreBytes`. Fast-forward the @@ -405,95 +373,6 @@ impl BtMatcher { result } - /// Upstream zstd parity: `ZSTD_ldm_blockCompress` seeds external - /// long-distance match candidates here when `enableLdm == - /// ZSTD_ps_enable`. The default Rust encoder still keeps LDM - /// disabled (`ldm_producer = None`); when an external caller - /// opts in (#18 Phase 5 wiring — see #27 for the parameter - /// surface), the producer is delegated to via - /// [`LdmProducer::generate_into`]. - /// - /// While `ldm_producer.is_none()` the behaviour matches the - /// pre-Phase-5 stub: a defensive clear of [`Self::ldm_sequences`] - /// so cross-frame carry-over is impossible if a producer is - /// activated mid-session. - /// - /// # Coordinate convention (PR #139 review feedback) - /// - /// The producer operates entirely in **absolute stream - /// coordinates** (so its bucket-table entries remain valid - /// across window evictions — entries inserted by an earlier - /// view of the frame are filtered by the staleness check - /// `entry.offset < history_abs_start`, i.e. inclusive lower - /// bound: entries at exactly `history_abs_start` survive, - /// see `ldm::search::FindBestMatchInputs::lowest_index_abs`). - /// The caller is expected to pass: - /// - /// * `live_history` — the contiguous *live* slice of the - /// per-frame `MatchTable::history` (`&history[history_start..]`). - /// `live_history[0]` corresponds to absolute position - /// `history_abs_start`. - /// * `history_abs_start` — absolute stream position of - /// `live_history[0]`. - /// * `current_abs_start` / `current_len` — absolute span of - /// the block to scan. - /// - /// `prepare_ldm_candidates` forwards these absolute - /// coordinates straight to [`LdmProducer::generate_into`]; the - /// abs→slice translation happens inside the producer only at - /// the moment of `live_history[..]` indexing. - pub(crate) fn prepare_ldm_candidates( - &mut self, - live_history: &[u8], - history_abs_start: usize, - current_abs_start: usize, - current_len: usize, - ) { - self.ldm_sequences.clear(); - #[cfg(feature = "ldm")] - if let Some(producer) = self.ldm_producer.as_mut() { - debug_assert!(current_abs_start >= history_abs_start); - // MatchTable invariant: `live_history.len() == - // window_size`, and `current_abs_start = - // history_abs_start + window_size − current_len` - // (match_generator.rs around line 1330). The two - // sides below must coincide; using `min(...)` would - // silently truncate the scanned range and mask an - // invariant violation. Raw `+` is safe — the frame- - // level `check_stream_abs_headroom` - // (`match_table/storage.rs:50`) guarantees - // `history_abs_start + window_size + - // STREAM_ABS_HEADROOM ≤ usize::MAX`. - debug_assert_eq!( - current_abs_start + current_len, - history_abs_start + live_history.len(), - "MatchTable invariant violation: current block range \ - `[current_abs_start, +current_len)` must coincide with the \ - live-history end (window_size == live_history.len())" - ); - let block_end_abs = current_abs_start + current_len; - producer.generate_into( - live_history, - history_abs_start, - current_abs_start, - block_end_abs, - &mut self.ldm_sequences, - ); - } - #[cfg(not(feature = "ldm"))] - { - // Without the `ldm` feature `LdmProducer` is not compiled, so - // `live_history` / `history_abs_start` / `current_abs_start` / - // `current_len` would otherwise be unused. - let _ = ( - live_history, - history_abs_start, - current_abs_start, - current_len, - ); - } - } - /// Upstream zstd parity: `ZSTD_storeSeq` — encode `actual_offset` into the /// upstream zstd's compact offset base (1/2/3 for rep slots, otherwise /// `actual_offset + 3`) and update the rolling `reps` window in diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs index e1f511bc2..c20bbc51e 100644 --- a/zstd/src/encoding/dfast/mod.rs +++ b/zstd/src/encoding/dfast/mod.rs @@ -96,6 +96,9 @@ pub(crate) struct DfastMatchGenerator { pub(crate) history_start: usize, pub(crate) history_abs_start: usize, pub(crate) offset_hist: [u32; 3], + /// While a dictionary is primed: how far into it the tables start + /// indexing, `0` for all of it (see [`Self::set_dictionary_index_floor`]). + dict_index_floor: usize, // Storage: single `u32` per bucket — upstream zstd-parity overwrite-on- // collision. Each slot holds a +1-biased relative position // (`(abs_pos - position_base + 1) as u32`); `DFAST_EMPTY_SLOT = 0` @@ -304,6 +307,7 @@ impl DfastMatchGenerator { history_start: 0, history_abs_start: 0, offset_hist: [1, 4, 8], + dict_index_floor: 0, tables: Table::empty(), tables_fresh: false, tagged: false, @@ -883,6 +887,12 @@ impl DfastMatchGenerator { } } + /// Index a dictionary being primed only from `floor` bytes into it, `0` + /// for all of it; the driver sets it for the prime and clears it after. + pub(crate) fn set_dictionary_index_floor(&mut self, floor: usize) { + self.dict_index_floor = floor; + } + pub(crate) fn skip_matching_dense(&mut self) { self.ensure_hash_tables(); let current_len = self.window_blocks.back().copied().unwrap_or(0); @@ -891,17 +901,59 @@ impl DfastMatchGenerator { // committed yet, so none to seed. return; } - let current_abs_start = self.history_abs_start + self.window_size - current_len; - let current_abs_end = current_abs_start + current_len; + let current_abs_end = self.history_abs_start + self.window_size; + // A dictionary being primed sits at the front of the history, so its + // index floor counts from there. + let floor = self.history_abs_start + self.dict_index_floor; + let current_abs_start = (current_abs_end - current_len).max(floor.min(current_abs_end)); let backfill_start = current_abs_start .saturating_sub(Self::BOUNDARY_DENSE_TAIL_LEN) - .max(self.history_abs_start); + .max(self.history_abs_start) + .max(floor); if backfill_start < current_abs_start { self.insert_positions(backfill_start, current_abs_start); } self.insert_positions(current_abs_start, current_abs_end); } + /// The committed window followed by the input read but not yet committed. + #[cfg(feature = "ldm")] + pub(crate) fn window_with_pending(&self) -> &[u8] { + &self.history[self.history_start..] + } + + /// Hand the last committed block of `len` bytes back to the uncommitted + /// input, undoing only the bookkeeping of [`Self::commit_block`]: what it + /// evicted stays evicted, so committing the same bytes again in pieces + /// evicts nothing more. + #[cfg(feature = "ldm")] + pub(crate) fn uncommit_block(&mut self, len: usize) { + assert_eq!(self.window_blocks.pop_back(), Some(len)); + self.window_size -= len; + self.uncommitted_len += len; + } + + /// Index the last committed block as the body of a long-distance match: + /// only its tail, as upstream fills the fast tables after one + /// (`ZSTD_ldm_limitTableUpdate` + `ZSTD_ldm_fillFastTables`), since the + /// match itself is never searched. + #[cfg(feature = "ldm")] + pub(crate) fn skip_long_match(&mut self) { + self.ensure_hash_tables(); + let current_len = self.window_blocks.back().copied().unwrap_or(0); + let end = self.history_abs_start + self.window_size; + let from = end - current_len.min(crate::encoding::ldm::LONG_MATCH_INDEXED_TAIL); + self.insert_positions(from, end); + } + + /// Carry a long-distance match the driver emitted into the repeat + /// offsets, as the encoder will code it. + #[cfg(feature = "ldm")] + pub(crate) fn note_long_match(&mut self, offset: usize, literal_len: usize) { + let _ = + encode_offset_with_history(offset as u32, literal_len as u32, &mut self.offset_hist); + } + /// Upstream zstd `ZSTD_dictMatchState` attach (dfast): hash the dictionary block at /// the front of history into the SEPARATE immutable [`Self::dict`] tables /// (long+short), once, instead of re-priming the live tables every frame @@ -1023,7 +1075,13 @@ impl DfastMatchGenerator { // or `7+1` dict chunking), so gate the early return on `backfill_floor`, // not `start_concat` — otherwise those seam inserts are dropped and the // attached dict tables stay incomplete. - let backfill_floor = start_concat.saturating_sub(Self::BOUNDARY_DENSE_TAIL_LEN); + // Nothing below the index floor is indexed, seam included: the chunk + // before it was not. + let floor = self.dict_index_floor.min(end_concat); + let backfill_floor = start_concat + .max(floor) + .saturating_sub(Self::BOUNDARY_DENSE_TAIL_LEN) + .max(floor); if backfill_floor >= short_safe_end { return; } diff --git a/zstd/src/encoding/frame_compressor.rs b/zstd/src/encoding/frame_compressor.rs index 032f4a93f..5f5660f8d 100644 --- a/zstd/src/encoding/frame_compressor.rs +++ b/zstd/src/encoding/frame_compressor.rs @@ -2741,6 +2741,13 @@ impl FrameCompressor { block_capacity, savings, ); + // The splitter sees byte statistics only; a cut it proposes + // inside a match the matcher already knows of is moved out. + let split_at = if split_at < available { + self.state.matcher.settle_block_cut(available, split_at) + } else { + split_at + }; if split_at < available { block_len = split_at; last_block = false; diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs index 8dd007e00..0944fe913 100644 --- a/zstd/src/encoding/hc/generator.rs +++ b/zstd/src/encoding/hc/generator.rs @@ -1476,31 +1476,6 @@ impl HcMatchGenerator { } } - /// Install (or clear) the long-distance-match producer (#27). Only - /// the BT backend owns an `ldm_producer` slot; on the HC (lazy) - /// backend the producer is dropped because there is no optimal-parser - /// candidate buffer to seed. Call after [`Self::reset`]. - #[cfg(feature = "ldm")] - pub(crate) fn set_ldm_producer(&mut self, producer: Option) { - if let HcBackend::Bt(bt) = &mut self.backend { - bt.ldm_producer = producer; - } - } - - /// Move the LDM producer out of the BT backend, leaving `None`. Used by the - /// dictionary snapshot path: the producer carries no dictionary state (LDM - /// is not dict-primed; its hash table is empty at capture), so it is not - /// retained in the snapshot — the working frame's freshly-reset producer is - /// reinstated on restore instead. - #[cfg(feature = "ldm")] - pub(crate) fn take_ldm_producer(&mut self) -> Option { - if let HcBackend::Bt(bt) = &mut self.backend { - bt.ldm_producer.take() - } else { - None - } - } - pub(crate) fn reset(&mut self) { self.table.reset(); if let HcBackend::Bt(bt) = &mut self.backend { diff --git a/zstd/src/encoding/hc/optimal.rs b/zstd/src/encoding/hc/optimal.rs index 0fe271aad..6a296bb7e 100644 --- a/zstd/src/encoding/hc/optimal.rs +++ b/zstd/src/encoding/hc/optimal.rs @@ -1295,28 +1295,6 @@ impl HcMatchGenerator { self.table .backfill_boundary_positions(current_abs_start, current_abs_end); self.table.next_to_update3 = hash3_start_cursor; - // Borrow split: `prepare_ldm_candidates` needs immutable - // access to the live history (the post-`history_start` - // slice of `self.table.history`) while it mutates the LDM - // bucket table owned by `self.backend.bt_mut()`. Both live - // in disjoint fields of `Self`, so we capture the slice + - // its base before reaching for `bt_mut()`. - // - // The producer operates in absolute stream coordinates - // throughout; `live_history[0]` corresponds to absolute - // `history_abs_start` (upstream zstd `base + dictLimit`), and the - // abs→slice translation happens inside the producer at - // each `live_history[..]` access. Passing the full - // `history` Vec would index into the dead prefix (the - // bytes already retired past `history_start`). - let live_history = self.table.live_history(); - let history_abs_start = self.table.history_abs_start; - self.backend.bt_mut().prepare_ldm_candidates( - live_history, - history_abs_start, - current_abs_start, - current_len, - ); // Take the five DP scratch buffers ONCE for both the seed pass and the // main pass (upstream shares one workspace across btultra2's two diff --git a/zstd/src/encoding/ldm/mod.rs b/zstd/src/encoding/ldm/mod.rs index a918ef04f..07179b8e9 100644 --- a/zstd/src/encoding/ldm/mod.rs +++ b/zstd/src/encoding/ldm/mod.rs @@ -15,32 +15,19 @@ //! `strategy` (`ZSTD_ldm_adjustParameters` parity). //! //! Aggregate [`LdmProducer`] holds the rolling-hash state, the -//! bucket table, and the per-call scratch buffers. The downstream -//! consumer (`bt::ldm_sequences`) was plumbed during Phase 1 -//! (#119) — Phase 5 swaps the `prepare_ldm_candidates` no-op stub -//! for a real producer that fills that buffer. +//! bucket table, and the per-call scratch buffers. The match driver owns +//! one per frame, as upstream keeps `ldmState` in the context: the optimal +//! parser takes its sequences as candidates, every other strategy takes them +//! outright and searches only the literals between them +//! (`ZSTD_ldm_blockCompress`). //! -//! # Activation policy (distro-grade parity) +//! # Activation policy //! -//! `LdmProducer` is **never activated automatically by a -//! [`super::CompressionLevel`] preset.** This mirrors upstream -//! `libzstd.so.1` behaviour: `ZSTD_compress(..., level)` keeps -//! LDM off at every standard level (1..22). Activation is an -//! explicit user opt-in: -//! -//! * upstream — `ZSTD_CCtx_setParameter(cctx, -//! ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)`; -//! * `zstd` CLI — `zstd --long[=N]`; -//! * Rust port (this crate) — the Phase-5 surface ships an -//! `ldm_producer: Option` field on `BtMatcher` that -//! stays `None` by default. The forthcoming parameter-API issue -//! (#27) plugs into that field; the C ABI work in #126 / #127 -//! wires `ZSTD_c_enableLongDistanceMatching` through the same -//! surface. -//! -//! Therefore the `level22_sequences_match_upstream zstd_on_corpus_proxy` -//! ratio gate continues to compare two LDM-OFF outputs (ours vs -//! upstream `ZSTD_compress(..., 22)`) — byte-parity invariant. +//! No [`super::CompressionLevel`] preset turns it on, as upstream +//! `ZSTD_compress(..., level)` keeps LDM off at every level. It is an +//! explicit opt-in: the parameter API +//! (`enable_long_distance_matching`), `ZSTD_c_enableLongDistanceMatching` +//! through the C ABI, and `zstd --long[=N]`. //! //! Upstream zstd parity anchors: //! * `lib/compress/zstd_ldm.c` v1.5.7 @@ -49,19 +36,6 @@ //! table reproduced verbatim in [`gear_hash::GEAR_TAB`] to preserve //! byte-for-byte split-point compatibility. -// Phase 5 of #111 ships the full LDM producer (gear hash + params -// + bucket table + verify/extend search + aggregate driver) in -// two commits. By distro-parity design (see the module docs -// above), no `CompressionLevel` preset activates the producer — -// `BtMatcher::ldm_producer` stays `None` until #27 (Rust -// parameter API) or #126/#127 (C ABI) plug the user opt-in. -// Several `pub(crate)` accessors (`LDM_HASHLOG_MIN/MAX`, -// `params::bounded`, `bucket_mask`) are therefore reachable only -// via the unit test suite until the parameter API lands; the -// `#![allow(dead_code)]` matches the same transitional marker in -// `bt/mod.rs` introduced during Phase 1. -#![allow(dead_code)] - use alloc::vec; use alloc::vec::Vec; @@ -73,6 +47,7 @@ use super::opt::ldm::HcRawSeq; pub(crate) mod gear_hash; pub(crate) mod params; pub(crate) mod search; +pub(crate) mod store; pub(crate) mod table; use gear_hash::{GearHashState, LDM_BATCH_SIZE}; @@ -84,6 +59,27 @@ use table::LdmHashTable; /// (`zstd_ldm.c:315`: `XXH64(split, minMatchLength, 0)`). const LDM_XXH64_SEED: u64 = 0; +/// How much of a long-distance match's body the strategy's own tables index: +/// the match is taken without being searched, and upstream catches its tables +/// up on no more than this much of it (`ZSTD_ldm_limitTableUpdate`, +/// `zstd_ldm.c:335`, brings `nextToUpdate` to within 512 of the match end). +pub(crate) const LONG_MATCH_INDEXED_TAIL: usize = 512; + +/// Bytes a frame's long-distance matching holds under `params`: its table and +/// the sequence vectors around it. Each vector keeps at most a block's matches +/// at `min_match_length` apiece (upstream's `ZSTD_ldm_getMaxNbSeq`), and five +/// are retained: what a search found, the store, which keeps the matches a +/// block took until they are half of it, the block's own, and the optimal +/// parser's copy of those. +pub(crate) fn estimated_workspace_bytes(params: &LdmParams) -> usize { + const RETAINED_SEQUENCE_VECTORS: usize = 5; + let block = crate::common::MAX_BLOCK_SIZE as usize; + let sequences = block / params.min_match_length.max(1) as usize + 1; + let vectors = sequences * core::mem::size_of::() * RETAINED_SEQUENCE_VECTORS; + LdmHashTable::estimated_workspace_bytes(params.hash_log, params.bucket_size_log) + .saturating_add(vectors) +} + /// LDM sequence producer — owns the rolling-hash state, bucket /// table, and scratch buffers needed to scan an input block and /// emit a stream of [`HcRawSeq`] candidates consumed by the @@ -95,7 +91,6 @@ const LDM_XXH64_SEED: u64 = 0; /// [`Self::clear`] only when starting a new frame (so the /// long-range history accumulated across blocks is preserved /// within a frame, mirroring upstream zstd's `ldmState_t` lifecycle). -#[derive(Clone)] pub(crate) struct LdmProducer { /// Parameter set this producer was built with. Used by the /// split walker (next commit) to honour `min_match_length` / @@ -132,12 +127,6 @@ impl LdmProducer { } } - /// Re-derive parameters from a `(window_log, strategy)` pair - /// using [`LdmParams::adjust_for`]. Convenience wrapper. - pub(crate) fn with_window_and_strategy(window_log: u32, strategy: u32) -> Self { - Self::new(LdmParams::adjust_for(window_log, strategy)) - } - /// Heap bytes this producer owns: the LDM hash table and the split-scratch. pub(crate) fn heap_size(&self) -> usize { self.hash_table.heap_size() + self.splits_scratch.capacity() * core::mem::size_of::() @@ -153,6 +142,51 @@ impl LdmProducer { ); } + /// Index `[start_abs, end_abs)` of `live_history` (whose first byte sits + /// at `history_abs_start`) without searching it: a dictionary loaded under + /// long-distance matching (upstream zstd `ZSTD_ldm_fillHashTable`, + /// `zstd_ldm.c:289-327`). Leaves the rolling hash for the next + /// [`Self::generate_into`], which re-seeds it anyway. + pub(crate) fn fill_table( + &mut self, + live_history: &[u8], + history_abs_start: usize, + start_abs: usize, + end_abs: usize, + ) { + debug_assert!(history_abs_start <= start_abs && start_abs <= end_abs); + debug_assert!(end_abs <= history_abs_start + live_history.len()); + let min_match = self.params.min_match_length as usize; + let hash_id_mask = self.hash_table.bucket_mask(); + let mut hash_state = GearHashState::new(min_match, self.params.hash_rate_log); + let mut ip_abs = start_abs; + while ip_abs < end_abs { + let chunk = &live_history[ip_abs - history_abs_start..end_abs - history_abs_start]; + let (hashed, num_splits) = + gear_hash::feed(&mut hash_state, chunk, &mut self.splits_scratch); + for &split in &self.splits_scratch[..num_splits] { + // A split closes a `min_match`-byte window; one that would open + // before the range is not a window of it. + if ip_abs + split < start_abs + min_match { + continue; + } + let split_abs = ip_abs + split - min_match; + let split_idx = split_abs - history_abs_start; + let mut hasher = XxHash64::with_seed(LDM_XXH64_SEED); + hasher.write(&live_history[split_idx..split_idx + min_match]); + let xxhash = hasher.finish(); + self.hash_table.ensure_room_for(split_abs); + self.hash_table.insert_absolute( + (xxhash as u32) & hash_id_mask, + split_abs, + (xxhash >> 32) as u32, + ); + } + // `feed` consumes at least one byte of a non-empty chunk. + ip_abs += hashed.max(1); + } + } + /// Read-only view of the parameter set for diagnostics / tests. pub(crate) fn params(&self) -> LdmParams { self.params diff --git a/zstd/src/encoding/ldm/params.rs b/zstd/src/encoding/ldm/params.rs index d2ae33ff4..67048d46d 100644 --- a/zstd/src/encoding/ldm/params.rs +++ b/zstd/src/encoding/ldm/params.rs @@ -145,11 +145,13 @@ impl LdmParams { } /// Hash-table size in entries: `1 << hash_log`. + #[cfg(test)] pub(crate) const fn hash_table_entries(&self) -> usize { 1usize << self.hash_log } /// Per-bucket slot count: `1 << bucket_size_log`. + #[cfg(test)] pub(crate) const fn bucket_slots(&self) -> usize { 1usize << self.bucket_size_log } diff --git a/zstd/src/encoding/ldm/store.rs b/zstd/src/encoding/ldm/store.rs new file mode 100644 index 000000000..c89d415f4 --- /dev/null +++ b/zstd/src/encoding/ldm/store.rs @@ -0,0 +1,132 @@ +//! Long-distance matches searched ahead of the blocks that consume them. +//! +//! The search runs over whatever input is known, which can reach past the +//! block about to be compressed: the block splitter asks where long matches +//! lie before it cuts. Each block then takes the matches covering it, the one +//! crossing its end cut there and carried into the next block, as upstream +//! carries a raw sequence store across blocks (`maybeSplitSequence` and +//! `ZSTD_ldm_skipSequences`, `zstd_ldm.c:607-666`). + +use alloc::vec::Vec; + +use crate::encoding::opt::ldm::HcRawSeq; + +/// One match found by the search, in stream positions. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +struct LongMatch { + start: usize, + end: usize, + offset: usize, +} + +#[derive(Default)] +pub(crate) struct LdmSequenceStore { + /// Found and not yet taken past, in stream order. + matches: Vec, + /// Index of the first match a block has not taken to its end. + pos: usize, + /// Stream position the next block starts at. + taken_end: usize, + /// Stream position the input is searched up to. + searched_end: usize, +} + +impl LdmSequenceStore { + /// Forget everything, with input from `at` on unsearched and untaken. + pub(crate) fn reset(&mut self, at: usize) { + self.matches.clear(); + self.pos = 0; + self.taken_end = at; + self.searched_end = at; + } + + /// Stream position the input is searched up to. + pub(crate) fn searched_end(&self) -> usize { + self.searched_end + } + + /// Stream position the next block starts at. + pub(crate) fn taken_end(&self) -> usize { + self.taken_end + } + + /// Heap bytes the matches hold. + pub(crate) fn heap_size(&self) -> usize { + self.matches.capacity() * core::mem::size_of::() + } + + /// Record the sequences a search of `[searched_end, to)` found, literal + /// lengths counted from the start of that range. + pub(crate) fn searched(&mut self, found: &[HcRawSeq], to: usize) { + let mut at = self.searched_end; + for seq in found { + let start = at + seq.lit_length; + let end = start + seq.match_length; + self.matches.push(LongMatch { + start, + end, + offset: seq.offset, + }); + at = end; + } + debug_assert!(at <= to, "a sequence reaches past the searched range"); + self.searched_end = to; + } + + /// `cut` bytes past the next block's start, moved out of any long match it + /// falls inside: to the match end, or to `window` when the match runs past + /// it. Cutting inside one buys a block whose first sequence repeats the + /// offset the previous one ended on. + pub(crate) fn cut_outside_matches(&self, window: usize, cut: usize) -> usize { + let cut_at = self.taken_end + cut; + for found in &self.matches[self.pos..] { + if found.start >= cut_at { + break; + } + if cut_at < found.end { + return (found.end - self.taken_end).min(window); + } + } + cut + } + + /// Take the matches of the next `len` bytes into `out` as sequences, + /// literal lengths counted from the block start. A match crossing the + /// block end is cut there, the rest carried into the next block; a piece + /// shorter than `min_match`, on either side of the cut, is left as + /// literals. + pub(crate) fn take(&mut self, len: usize, min_match: usize, out: &mut Vec) { + out.clear(); + let block_end = self.taken_end + len; + let mut at = self.taken_end; + while let Some(&found) = self.matches.get(self.pos) { + if found.start >= block_end { + break; + } + let start = found.start.max(self.taken_end); + let end = found.end.min(block_end); + if end - start >= min_match { + out.push(HcRawSeq { + lit_length: start - at, + offset: found.offset, + match_length: end - start, + }); + at = end; + } + if found.end > block_end { + break; + } + self.pos += 1; + } + self.taken_end = block_end; + // Taken matches go once they are the bulk of the vector, so a search + // far ahead of a long stream holds only what is still to come. + if self.pos > self.matches.len() / 2 { + self.matches.drain(..self.pos); + self.pos = 0; + } + } +} + +#[cfg(test)] +mod tests; diff --git a/zstd/src/encoding/ldm/store/tests.rs b/zstd/src/encoding/ldm/store/tests.rs new file mode 100644 index 000000000..87ab7a4ed --- /dev/null +++ b/zstd/src/encoding/ldm/store/tests.rs @@ -0,0 +1,104 @@ +use alloc::vec; +use alloc::vec::Vec; + +use super::LdmSequenceStore; +use crate::encoding::opt::ldm::HcRawSeq; + +fn seq(lit_length: usize, offset: usize, match_length: usize) -> HcRawSeq { + HcRawSeq { + lit_length, + offset, + match_length, + } +} + +fn triples(out: &[HcRawSeq]) -> Vec<(usize, usize, usize)> { + out.iter() + .map(|s| (s.lit_length, s.offset, s.match_length)) + .collect() +} + +/// A block takes its matches with literals counted from its own start, and a +/// later search's literal lengths count from where the previous one stopped. +#[test] +fn blocks_take_their_matches_relative_to_their_start() { + let mut store = LdmSequenceStore::default(); + store.reset(1000); + store.searched(&[seq(10, 500, 100), seq(20, 700, 64)], 1300); + store.searched(&[seq(50, 900, 80)], 1600); + let mut out = Vec::new(); + store.take(200, 4, &mut out); + // [1000, 1200): 10 literals, a 100-byte match, 20 literals, a 64-byte one. + assert_eq!(triples(&out), vec![(10, 500, 100), (20, 700, 64)]); + store.take(400, 4, &mut out); + // The third match starts at 1300 + 50, 150 bytes into this block. + assert_eq!(triples(&out), vec![(150, 900, 80)]); + assert_eq!(store.taken_end(), 1600); +} + +/// A match crossing a block end is cut there and its rest carried into the +/// next block at the same offset; a piece under `min_match` stays literals, +/// on either side. +#[test] +fn a_match_crossing_the_block_end_is_carried() { + let mut store = LdmSequenceStore::default(); + store.reset(0); + store.searched(&[seq(10, 300, 200)], 1000); + let mut out = Vec::new(); + store.take(110, 4, &mut out); + assert_eq!(triples(&out), vec![(10, 300, 100)]); + store.take(500, 4, &mut out); + assert_eq!(triples(&out), vec![(0, 300, 100)]); + + // Cut two bytes into the match: the head is under `min_match` and stays + // literals, the tail is taken. + let mut store = LdmSequenceStore::default(); + store.reset(0); + store.searched(&[seq(10, 300, 200)], 1000); + store.take(12, 4, &mut out); + assert!(out.is_empty()); + store.take(500, 4, &mut out); + assert_eq!(triples(&out), vec![(0, 300, 198)]); + + // And a short tail is literals of the next block. + let mut store = LdmSequenceStore::default(); + store.reset(0); + store.searched(&[seq(10, 300, 200), seq(30, 400, 50)], 1000); + store.take(208, 4, &mut out); + assert_eq!(triples(&out), vec![(10, 300, 198)]); + store.take(500, 4, &mut out); + assert_eq!(triples(&out), vec![(32, 400, 50)]); +} + +/// A cut inside a match moves to the match end, or to the window end when the +/// match runs past it; a cut between matches stays. +#[test] +fn a_cut_is_moved_out_of_a_match() { + let mut store = LdmSequenceStore::default(); + store.reset(100); + store.searched(&[seq(10, 300, 200), seq(100, 700, 5000)], 10_000); + // Inside the first match [110, 310): moved to its end, 210 from the start. + assert_eq!(store.cut_outside_matches(8192, 150), 210); + // Between the matches: kept. + assert_eq!(store.cut_outside_matches(8192, 250), 250); + // Inside the second, which runs past the window: the whole window. + assert_eq!(store.cut_outside_matches(4096, 1000), 4096); + // On a match's first byte is not inside it. + assert_eq!(store.cut_outside_matches(8192, 10), 10); +} + +/// Taken matches are dropped once they are the bulk of the store, without +/// moving what is still to come. +#[test] +fn taken_matches_are_dropped() { + let mut store = LdmSequenceStore::default(); + store.reset(0); + let found: Vec = (0..100).map(|_| seq(10, 64, 90)).collect(); + store.searched(&found, 10_000); + let mut out = Vec::new(); + store.take(6000, 4, &mut out); + assert_eq!(out.len(), 60); + store.take(4000, 4, &mut out); + assert_eq!(out.len(), 40); + assert_eq!(triples(&out)[0], (10, 64, 90)); +} diff --git a/zstd/src/encoding/ldm/table.rs b/zstd/src/encoding/ldm/table.rs index 0251e70a4..b66549257 100644 --- a/zstd/src/encoding/ldm/table.rs +++ b/zstd/src/encoding/ldm/table.rs @@ -91,7 +91,6 @@ const REBASE_GUARD_BAND: u32 = 1u32 << 30; /// [`LdmHashTable::insert_absolute`] / [`LdmHashTable::resolve`] /// helpers so absolute-position support extends past `u32::MAX` /// transparently — same pattern as `MatchTable`'s chain rebase. -#[derive(Clone)] pub(crate) struct LdmHashTable { entries: Vec, bucket_offsets: Vec, @@ -288,6 +287,7 @@ impl LdmHashTable { /// Absolute stream position that corresponds to the +1 /// relative-offset bias — i.e. an entry with `offset == 1` /// refers to absolute byte `position_base`. + #[cfg(test)] pub(crate) const fn position_base(&self) -> usize { self.position_base } diff --git a/zstd/src/encoding/levels/config.rs b/zstd/src/encoding/levels/config.rs index 3ea6451a2..eff1d6309 100644 --- a/zstd/src/encoding/levels/config.rs +++ b/zstd/src/encoding/levels/config.rs @@ -226,6 +226,52 @@ impl LevelParams { pub(crate) fn pre_split(&self) -> Option { Some(pre_split_for(self.strategy_tag, self.lazy_depth)) } + + /// `log2` of how much of a dictionary's end the match-finder tables index + /// when it is loaded: a longer dictionary is indexed only that far from its + /// end, since tables this size could not keep more of it anyway (upstream + /// `ZSTD_loadDictionaryContent`, `zstd_compress.c:4987`: + /// `MIN(MAX(hashLog + 3, chainLog + 1), 31)`). Every byte of it stays in + /// the window. + pub(crate) fn dictionary_reach_log(&self) -> u32 { + use crate::encoding::strategy::SearchMethod; + // The active backend's config: an override may leave the level row's + // own beside it. + let logs = match self.search { + SearchMethod::Fast => self.fast.map(|fast| (fast.hash_log as usize, 0)), + SearchMethod::DoubleFast => self + .dfast + .map(|dfast| (dfast.long_hash_log as usize, dfast.short_hash_log as usize)), + SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => { + self.row.map(|row| (row.hash_bits, row.chain_log)) + } + SearchMethod::HashChain | SearchMethod::BinaryTree => { + self.hc.map(|hc| (hc.hash_log, hc.chain_log)) + } + }; + // The reset synthesises the active config when a row lacks it, so a + // missing one only means a table-less level: nothing to index. + let Some((hash_log, chain_log)) = logs else { + return 0; + }; + (hash_log + 3).max(chain_log + 1).min(31) as u32 + } + + /// The level's upstream `cParams.minMatch`: the shortest match its search + /// accepts, and so the shortest piece of a long-distance match a block cut + /// leaves worth keeping (`maybeSplitSequence`, `zstd_ldm.c:644`). + #[cfg(feature = "ldm")] + pub(crate) fn min_match(&self) -> usize { + use crate::encoding::strategy::SearchMethod; + let configured = match self.search { + SearchMethod::Fast => self.fast.map(|fast| fast.mls as usize), + SearchMethod::DoubleFast => Some(crate::encoding::match_generator::DFAST_MIN_MATCH_LEN), + SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => self.row.map(|row| row.mls), + SearchMethod::HashChain => Some(crate::encoding::hc::HC_MIN_MATCH_LEN), + SearchMethod::BinaryTree => self.hc.map(|hc| hc.search_mls), + }; + configured.unwrap_or(crate::encoding::cost_model::HC_FORMAT_MINMATCH) + } } /// The pre-splitter level for an effective strategy (the tag plus, for the @@ -947,13 +993,11 @@ pub fn estimated_compression_workspace_bytes_for_run( #[cfg(feature = "ldm")] let ldm = if long_distance_matching { let strategy = ldm_strategy_ordinal(params.strategy_tag, params.lazy_depth); - let ldm_params = crate::encoding::ldm::params::LdmParams::adjust_for( - u32::from(params.window_log), - strategy, - ); - crate::encoding::ldm::table::LdmHashTable::estimated_workspace_bytes( - ldm_params.hash_log, - ldm_params.bucket_size_log, + crate::encoding::ldm::estimated_workspace_bytes( + &crate::encoding::ldm::params::LdmParams::adjust_for( + u32::from(params.window_log), + strategy, + ), ) } else { 0 @@ -1012,11 +1056,7 @@ pub fn estimated_compression_workspace_bytes_for_parameters( apply_frame_overrides(&mut params, &overrides, dictionary.is_some(), src_size_hint); #[cfg(feature = "ldm")] let ldm = overrides.ldm.map_or(0, |ldm| { - let ldm_params = frame_ldm_params(¶ms, &ldm); - crate::encoding::ldm::table::LdmHashTable::estimated_workspace_bytes( - ldm_params.hash_log, - ldm_params.bucket_size_log, - ) + crate::encoding::ldm::estimated_workspace_bytes(&frame_ldm_params(¶ms, &ldm)) }); #[cfg(not(feature = "ldm"))] let ldm = 0; diff --git a/zstd/src/encoding/levels/config/tests.rs b/zstd/src/encoding/levels/config/tests.rs index c73d955db..68e2ba504 100644 --- a/zstd/src/encoding/levels/config/tests.rs +++ b/zstd/src/encoding/levels/config/tests.rs @@ -296,6 +296,66 @@ fn a_strategy_override_counts_only_the_backend_it_selects() { assert_eq!(estimate(&as_fast), estimate(&level_one)); } +/// Long-distance matching holds its found sequences besides its table, a +/// block's worth at the matcher's minimum match apiece in each of the vectors +/// it keeps (upstream budgets one such store, `ZSTD_ldm_getMaxNbSeq`). The +/// shortest minimum match multiplies them, and both estimates count it. +#[cfg(feature = "ldm")] +#[test] +fn long_distance_estimates_count_the_sequence_storage() { + use crate::encoding::CompressionParameters; + let with_min_match = |min_match: u32| { + CompressionParameters::builder(CompressionLevel::Level(3)) + .enable_long_distance_matching(true) + .ldm_min_match(min_match) + .build() + .unwrap() + }; + let short = + super::estimated_compression_workspace_bytes_for_parameters(&with_min_match(4), None, None); + let long = super::estimated_compression_workspace_bytes_for_parameters( + &with_min_match(64), + None, + None, + ); + // One vector of a block at four bytes a match, the least the extra + // storage can be. + let one_store = (128 * 1024 / 4) * core::mem::size_of::(); + assert!( + short >= long + one_store, + "minMatch 4 budgets {short} bytes, minMatch 64 {long}" + ); + let off = super::estimated_compression_workspace_bytes_for_run( + CompressionLevel::Level(3), + None, + None, + false, + None, + ); + let on = super::estimated_compression_workspace_bytes_for_run( + CompressionLevel::Level(3), + None, + None, + true, + None, + ); + let table_only = { + let params = super::resolve_level_params(CompressionLevel::Level(3), None); + let ldm = crate::encoding::ldm::params::LdmParams::adjust_for( + u32::from(params.window_log), + super::ldm_strategy_ordinal(params.strategy_tag, params.lazy_depth), + ); + crate::encoding::ldm::table::LdmHashTable::estimated_workspace_bytes( + ldm.hash_log, + ldm.bucket_size_log, + ) + }; + assert!( + on > off + table_only, + "the run estimate counts the table alone: {on} with, {off} without" + ); +} + /// The parameters the encoder actually runs for a (level, source size) pair /// are upstream's `ZSTD_getCParams(level, size, 0)` for that pair: strategy, /// window / hash / chain widths, search depth, minMatch and targetLength, on diff --git a/zstd/src/encoding/levels/fastest.rs b/zstd/src/encoding/levels/fastest.rs index 3bc259d89..e6aa56280 100644 --- a/zstd/src/encoding/levels/fastest.rs +++ b/zstd/src/encoding/levels/fastest.rs @@ -88,7 +88,10 @@ pub(crate) fn compress_block_encoded( // the dictionary. The classifier's own sample is safe in a way this one is // not — it measures the block against itself, and a repeat is what the grid // then catches. - let dict_rejects_raw = dict_active; + // + // Long-distance matches are external to the block in the same way, and no + // sample sees them either: upstream searches every block for them. + let dict_rejects_raw = dict_active || state.matcher.matches_beyond_window_tables(); // The level and the window decide this for the whole frame, so it also says // whether the grid is worth keeping at all: nothing it records can be acted // on where no block may go out raw, and recording there would take its diff --git a/zstd/src/encoding/match_generator/dict_prime.rs b/zstd/src/encoding/match_generator/dict_prime.rs index b8de6a385..c863f31cc 100644 --- a/zstd/src/encoding/match_generator/dict_prime.rs +++ b/zstd/src/encoding/match_generator/dict_prime.rs @@ -60,6 +60,10 @@ impl MatchGeneratorDriver { } }; self.dictionary_retained_budget = inflated.saturating_sub(base); + // The reset kept the dictionary's bytes and index but cleared the + // long-distance table, which a freshly primed frame would hold filled. + #[cfg(feature = "ldm")] + self.index_dictionary_in_place(); } pub(super) fn prime_with_dictionary_impl( @@ -101,6 +105,17 @@ impl MatchGeneratorDriver { if dict_content.is_empty() { return; } + #[cfg(feature = "ldm")] + let ldm_dict_start = self.ldm_committed_end; + // The match-finder tables index only as much of the dictionary's end as + // they can hold (upstream `ZSTD_loadDictionaryContent`, + // `zstd_compress.c:4987`); the whole of it is committed all the same, + // so every byte stays in reach of an offset. + let reach_log = self + .reset_shape + .map_or(31, |(params, ..)| params.dictionary_reach_log()); + self.storage + .set_dictionary_index_floor(dict_content.len().saturating_sub(1usize << reach_log)); // Dictionary bytes should stay addressable until produced frame output // itself exceeds the live window size. We bump `max_window_size` @@ -215,6 +230,24 @@ impl MatchGeneratorDriver { committed_dict_budget += end - start; start = end; } + // The long-distance matcher indexes the dictionary as upstream does when + // it loads one (`ZSTD_ldm_fillHashTable` from + // `ZSTD_loadDictionaryContent`), so a frame can reach it from as far as + // the window allows. + #[cfg(feature = "ldm")] + { + self.ldm_dict_end = self.ldm_committed_end; + if let Some(producer) = self.ldm.as_mut() { + let window = self.storage.committed_window(); + let window_start = self.ldm_committed_end - window.len(); + producer.fill_table( + window, + window_start, + ldm_dict_start.max(window_start), + self.ldm_committed_end, + ); + } + } // Derive `granted_retained_budget` directly from the two real // bounds — bytes actually committed and bytes the cap allows @@ -311,6 +344,8 @@ impl MatchGeneratorDriver { super::super::strategy::BackendTag::Row => self.row_matcher_mut().mark_dict_primed(), _ => {} } + // The dictionary is indexed; later inserts are the input's own. + self.storage.set_dictionary_index_floor(0); } pub(super) fn restore_primed_dictionary_impl( @@ -327,7 +362,7 @@ impl MatchGeneratorDriver { // match this reset. Restoring it would let the encoder search past the // frame header's window (an undecodable match), so on a key mismatch we // refuse and the caller re-primes. - let Some((params, table_bits, fast_attach, fast_tagged, ldm)) = self.reset_shape else { + let Some((params, table_bits, fast_attach, fast_tagged)) = self.reset_shape else { return false; }; let key = PrimedKey { @@ -336,7 +371,6 @@ impl MatchGeneratorDriver { table_bits, fast_attach, fast_tagged, - ldm, }; let Some((snapshot, budget, captured_key)) = &mut self.primed else { return false; @@ -395,20 +429,12 @@ impl MatchGeneratorDriver { if tree_in_snapshot { tables.clone_from(&snapshot_tables); } - // The snapshot does not retain the LDM producer (it holds no - // dict state; see `capture_primed_dictionary`). Carry over the - // frame's freshly-reset producer, built this frame by `reset` - // with the same params the snapshot key pins. - #[cfg(feature = "ldm")] - let fresh_ldm = live.take_ldm_producer(); *live = snap.clone(); live.table.tables = tables; live.table.history = history; if !tree_in_snapshot { live.table.ensure_tables(); } - #[cfg(feature = "ldm")] - live.set_ldm_producer(fresh_ldm); snap.table.tables = snapshot_tables; snap.table.history = snapshot_history; } @@ -419,13 +445,32 @@ impl MatchGeneratorDriver { } } self.dictionary_retained_budget = budget; + #[cfg(feature = "ldm")] + self.index_dictionary_in_place(); true } + /// Index the dictionary the reset left in the window into the + /// long-distance matcher's cleared table, as priming it would. Upstream + /// re-indexes per frame as well: a CDict carries no long-distance table, + /// and a frame that loads the dictionary fills one + /// (`ZSTD_ldm_fillHashTable` from `ZSTD_loadDictionaryContent`). Keeping a + /// copy of the filled table instead would hold a second table, the + /// largest the frame has, and copying it costs more than the fill. + #[cfg(feature = "ldm")] + fn index_dictionary_in_place(&mut self) { + self.ldm_committed_end = self.storage.committed_window().len(); + self.ldm_dict_end = self.ldm_committed_end; + if let Some(producer) = self.ldm.as_mut() { + let window = self.storage.committed_window(); + producer.fill_table(window, 0, 0, window.len()); + } + } + pub(super) fn capture_primed_dictionary_impl(&mut self, level: super::super::CompressionLevel) { // No resolved shape means `reset` has not run for this frame — nothing // valid to key a snapshot on, so skip the capture. - let Some((params, table_bits, fast_attach, fast_tagged, ldm)) = self.reset_shape else { + let Some((params, table_bits, fast_attach, fast_tagged)) = self.reset_shape else { return; }; let key = PrimedKey { @@ -434,7 +479,6 @@ impl MatchGeneratorDriver { table_bits, fast_attach, fast_tagged, - ldm, }; // CDict-equivalent retained state. A binary-tree level in ATTACH mode // decouples the dictionary into `dms` (the upstream zstd `dictMatchState`); its @@ -468,24 +512,15 @@ impl MatchGeneratorDriver { let tables = core::mem::take(&mut hc.table.tables); let chain_off = core::mem::take(&mut hc.table.chain_off); let hash3_off = core::mem::take(&mut hc.table.hash3_off); - // The LDM producer carries no dictionary state (LDM is not - // dict-primed; its hash table is empty at capture), so it is not - // retained either — `restore` reinstates the frame's freshly-reset - // producer. Take it out so the clone does not duplicate its table. - #[cfg(feature = "ldm")] - let ldm_producer = hc.take_ldm_producer(); - // Clone the dict-state-only storage (live tables now empty Vecs, - // LDM producer detached). + // Clone the dict-state-only storage (live tables now empty Vecs). let snapshot = self.storage.clone(); - // Move the live tables (and LDM producer) back into the working storage. + // Move the live tables back into the working storage. let MatcherStorage::HashChain(hc) = &mut self.storage else { unreachable!("storage variant is stable across the take/put"); }; hc.table.tables = tables; hc.table.chain_off = chain_off; hc.table.hash3_off = hash3_off; - #[cfg(feature = "ldm")] - hc.set_ldm_producer(ldm_producer); self.primed = Some((snapshot, self.dictionary_retained_budget, key)); } else { self.primed = Some((self.storage.clone(), self.dictionary_retained_budget, key)); diff --git a/zstd/src/encoding/match_generator/long_distance.rs b/zstd/src/encoding/match_generator/long_distance.rs new file mode 100644 index 000000000..ece570cd0 --- /dev/null +++ b/zstd/src/encoding/match_generator/long_distance.rs @@ -0,0 +1,132 @@ +//! Long-distance matching in the [`MatchGeneratorDriver`]: the search ahead of +//! the blocks, the matches each block takes, and the block matched around them +//! below the optimal parser. +//! +//! A child module so it reaches the driver's private long-distance state and +//! storage directly. + +use super::{MatchGeneratorDriver, Sequence, owned_sink}; + +impl MatchGeneratorDriver { + /// Search the input from where the last search stopped up to `to`, a + /// stream position at most the end of the input read so far. + /// + /// What a match may reach is fixed against the END of the range, as + /// upstream enforces the distance once per chunk before generating + /// (`ZSTD_window_enforceMaxDist` on `chunkEnd`, `zstd_ldm.c:585`): one + /// window back from it, or the whole dictionary while the window has not + /// yet passed the dictionary's end. Every match then holds an offset valid + /// at its last byte, wherever in the range it lies. + fn search_long_distance_to(&mut self, to: usize) { + let Some(producer) = self.ldm.as_mut() else { + return; + }; + let from = self.ldm_store.searched_end(); + if from >= to { + return; + } + let window = self.reported_window_size; + let reach_floor = if self.ldm_dict_end != 0 && to <= window + self.ldm_dict_end { + 0 + } else { + to.saturating_sub(window) + }; + let live_start = self.ldm_committed_end - self.storage.committed_window().len(); + let live = self.storage.window_with_pending(); + let floor = reach_floor.max(live_start); + debug_assert!(floor <= from, "a search range wider than the window"); + self.ldm_found.clear(); + producer.generate_into( + &live[floor - live_start..], + floor, + from, + to, + &mut self.ldm_found, + ); + self.ldm_store.searched(&self.ldm_found, to); + } + + /// Line the matches up with the next block starting at `at`: anything + /// kept for another position (a prime, a reset, a skipped search) is gone. + fn align_long_distance_store(&mut self, at: usize) { + if self.ldm_store.taken_end() != at { + self.ldm_store.reset(at); + } + } + + /// The matches of the last committed block, into [`Self::ldm_block`], + /// searched first as far as the block reaches. Returns the block's length. + pub(super) fn take_long_distance_block(&mut self) -> usize { + let block_len = self.storage.last_block_len(); + let block_end = self.ldm_committed_end; + self.align_long_distance_store(block_end - block_len); + self.search_long_distance_to(block_end); + let min_match = self.reset_shape.map_or( + crate::encoding::cost_model::HC_FORMAT_MINMATCH, + |(params, ..)| params.min_match(), + ); + self.ldm_store + .take(block_len, min_match, &mut self.ldm_block); + block_len + } + + /// See [`crate::encoding::Matcher::settle_block_cut`]: the next `window` + /// uncommitted bytes are searched now, and the cut moved out of any long + /// match. + pub(super) fn settle_long_distance_cut(&mut self, window: usize, cut: usize) -> usize { + if self.ldm.is_none() { + return cut; + } + let start = self.ldm_committed_end; + self.align_long_distance_store(start); + self.search_long_distance_to(start + window); + self.ldm_store.cut_outside_matches(window, cut) + } + + /// Take the block's long-distance matches outright and match only the + /// literals between them with the frame's backend (upstream + /// `ZSTD_ldm_blockCompress` below btopt). The block is handed back to the + /// uncommitted input and committed again in pieces, so each gap is the + /// backend's whole block and each match body is indexed without being + /// searched. Returns `false` when the block has no long-distance match, + /// leaving it committed for the ordinary search. + pub(super) fn start_matching_long_distance( + &mut self, + handle_sequence: &mut impl FnMut(Sequence), + ) -> bool { + let block_len = self.take_long_distance_block(); + if self.ldm_block.is_empty() { + return false; + } + let sequences = core::mem::take(&mut self.ldm_block); + self.storage.uncommit_block(block_len); + let mut consumed = 0usize; + for seq in &sequences { + // The gap's trailing literals become the long match's own. + let mut gap = Some(0usize); + if seq.lit_length != 0 { + self.storage.commit_piece(seq.lit_length); + self.start_matching_owned(&mut owned_sink(handle_sequence, &mut gap)); + } + let gap_literals = gap.expect("the gap sink only ever writes Some"); + self.storage.commit_piece(seq.match_length); + self.storage.skip_long_match(); + handle_sequence(Sequence::Triple { + literal_len: gap_literals, + offset: seq.offset, + match_len: seq.match_length, + }); + self.storage.note_long_match(seq.offset, gap_literals); + consumed += seq.lit_length + seq.match_length; + } + let tail = block_len - consumed; + if tail != 0 { + self.storage.commit_piece(tail); + let mut forward = None; + self.start_matching_owned(&mut owned_sink(handle_sequence, &mut forward)); + } + self.ldm_block = sequences; + self.ldm_split_block = Some(block_len); + true + } +} diff --git a/zstd/src/encoding/match_generator/mod.rs b/zstd/src/encoding/match_generator/mod.rs index ec4a4169a..6c0be9155 100644 --- a/zstd/src/encoding/match_generator/mod.rs +++ b/zstd/src/encoding/match_generator/mod.rs @@ -5,7 +5,7 @@ //! //! The task here is to efficiently find matches in the already encoded data for the current suffix of the not yet encoded data. -#[cfg(feature = "bench-internals")] +#[cfg(any(feature = "bench-internals", feature = "ldm"))] use alloc::vec::Vec; // SIMD/CRC intrinsics now live in `crate::encoding::fastpath::*` where they // sit under per-CPU `#[target_feature]` umbrellas; no architecture-specific @@ -161,6 +161,10 @@ use super::hc::generator::HcMatchGenerator; // can reach the driver's private `primed` / `reset_shape` state directly; the // `Matcher` trait's dict entry points forward to its inherent `*_impl` helpers. mod dict_prime; +// Long-distance matching: the search ahead of the blocks and the blocks +// matched around its matches. A child module for the same private access. +#[cfg(feature = "ldm")] +mod long_distance; // `Strategy` and `StrategyTag` live in `crate::encoding::strategy`. // The driver carries a `StrategyTag` field set at `reset()` and @@ -239,6 +243,98 @@ impl MatcherStorage { } } + /// See the backends' `set_dictionary_index_floor`. + fn set_dictionary_index_floor(&mut self, floor: usize) { + match self { + Self::Simple(m) => m.set_dictionary_index_floor(floor), + Self::Dfast(m) => m.set_dictionary_index_floor(floor), + Self::Row(m) => m.set_dictionary_index_floor(floor), + Self::HashChain(m) => m.table.set_dictionary_index_floor(floor), + } + } + + /// The committed window of the owned history, oldest byte first. + #[cfg(feature = "ldm")] + fn committed_window(&self) -> &[u8] { + match self { + Self::Simple(m) => m.committed_window(), + Self::Dfast(m) => m.live_history(), + Self::Row(m) => m.live_history(), + Self::HashChain(m) => m.table.live_history(), + } + } + + /// The committed window followed by the input read but not committed. + #[cfg(feature = "ldm")] + fn window_with_pending(&self) -> &[u8] { + match self { + Self::Simple(m) => m.window_with_pending(), + Self::Dfast(m) => m.window_with_pending(), + Self::Row(m) => m.window_with_pending(), + Self::HashChain(m) => m.table.window_with_pending(), + } + } + + /// Length of the last committed block. + #[cfg(feature = "ldm")] + fn last_block_len(&self) -> usize { + match self { + Self::Simple(m) => m.last_committed_space().len(), + Self::Dfast(m) => m.get_last_space().len(), + Self::Row(m) => m.get_last_space().len(), + Self::HashChain(m) => m.table.get_last_space().len(), + } + } + + /// See the backends' `uncommit_block`. + #[cfg(feature = "ldm")] + fn uncommit_block(&mut self, len: usize) { + match self { + Self::Simple(m) => m.uncommit_block(len), + Self::Dfast(m) => m.uncommit_block(len), + Self::Row(m) => m.uncommit_block(len), + Self::HashChain(m) => m.table.uncommit_block(len), + } + } + + /// Commit `len` bytes of a block handed back by [`Self::uncommit_block`]. + /// The whole block fit the window when it was first committed, so a piece + /// of it evicts nothing. + #[cfg(feature = "ldm")] + fn commit_piece(&mut self, len: usize) { + match self { + Self::Simple(m) => { + let evicted = m.commit_block(len); + debug_assert_eq!(evicted, 0, "a recommitted piece evicted window bytes"); + } + Self::Dfast(m) => m.commit_block(len), + Self::Row(m) => m.commit_block(len), + Self::HashChain(m) => m.table.commit_block(len), + } + } + + /// See the backends' `skip_long_match`. + #[cfg(feature = "ldm")] + fn skip_long_match(&mut self) { + match self { + Self::Simple(m) => m.skip_long_match(), + Self::Dfast(m) => m.skip_long_match(), + Self::Row(m) => m.skip_long_match(), + Self::HashChain(m) => m.table.skip_long_match(), + } + } + + /// See the backends' `note_long_match`. + #[cfg(feature = "ldm")] + fn note_long_match(&mut self, offset: usize, literal_len: usize) { + match self { + Self::Simple(m) => m.note_long_match(offset, literal_len), + Self::Dfast(m) => m.note_long_match(offset, literal_len), + Self::Row(m) => m.note_long_match(offset, literal_len), + Self::HashChain(m) => m.table.note_long_match(offset, literal_len), + } + } + /// [`super::strategy::BackendTag`] family of the active variant. fn backend(&self) -> super::strategy::BackendTag { use super::strategy::BackendTag; @@ -327,17 +423,11 @@ pub struct MatchGeneratorDriver { reset_dict_attach_ok: bool, // Hint-resolved matcher shape from the last `reset`: the [`LevelParams`], the // active backend's applied Dfast/Row hash-table width (`0` for HC/Fast), the - // Fast attach-vs-copy mode, the Fast slot format, and the active LDM override - // (#27). Combined with the frame's level into the [`PrimedKey`] that keys - // the primed snapshot, so it is only restored into a reset that resolved the - // identical matcher AND LDM configuration. `None` before the first `reset`. - reset_shape: Option<( - LevelParams, - usize, - bool, - bool, - Option, - )>, + // Fast attach-vs-copy mode and the Fast slot format. Combined with the + // frame's level into the [`PrimedKey`] that keys the primed snapshot, so it + // is only restored into a reset that resolved the identical matcher. `None` + // before the first `reset`. + reset_shape: Option<(LevelParams, usize, bool, bool)>, // One-shot borrowed block range `[start, end)` staged by the borrowed // Fast frame path (`set_borrowed_block`) for the NEXT // `start_matching` / `skip_matching_with_hint`. `Some` routes that @@ -365,6 +455,33 @@ pub struct MatchGeneratorDriver { /// place rather than copied into the history. Decided at the reset, /// because it decides how large the history is laid out. frame_in_place: bool, + /// Long-distance matcher, `Some` while the frame runs with it on. It + /// belongs to the driver rather than to a backend, as upstream keeps + /// `ldmState` beside the match state: every strategy reads the same one. + #[cfg(feature = "ldm")] + ldm: Option, + /// The long-distance matches searched ahead of the blocks. + #[cfg(feature = "ldm")] + ldm_store: super::ldm::store::LdmSequenceStore, + /// What the last search found, before the store takes it. + #[cfg(feature = "ldm")] + ldm_found: Vec, + /// The long-distance sequences of the block being matched. + #[cfg(feature = "ldm")] + ldm_block: Vec, + /// Stream position of the committed end, in the coordinates the + /// long-distance matcher keeps its table in: the frame's first byte + /// (its dictionary's, when it has one) is `0`. + #[cfg(feature = "ldm")] + ldm_committed_end: usize, + /// Where the frame's dictionary ends in those coordinates, `0` without + /// one: what a match may still reach while the window has not passed it. + #[cfg(feature = "ldm")] + ldm_dict_end: usize, + /// Length of a block the long-distance matcher recommitted in pieces, so + /// [`Matcher::get_last_space`] reports the whole block, not its last piece. + #[cfg(feature = "ldm")] + ldm_split_block: Option, } /// Identity of the matcher configuration a primed snapshot was captured under: @@ -417,15 +534,6 @@ struct PrimedKey { /// frame from how full the input will leave the table, so two frames of one /// geometry can differ in it; a restore carries the snapshot's format in. fast_tagged: bool, - /// Fine-grained LDM override (#27) active at capture time. The - /// snapshot's cloned `storage` carries `BtMatcher::ldm_producer`, - /// which is configured from this override; restoring a snapshot - /// captured under a different LDM configuration (enable flip or - /// changed knobs) would reinstate a stale producer. `params` already - /// pins `window_log` / `strategy_tag` (the rest of the producer's - /// identity), so folding the override completes the LDM identity. - /// `None` = LDM off, matching `ParamOverrides::ldm`. - ldm: Option, } /// Whether a configured HashChain matcher attaches the dictionary for a frame @@ -636,6 +744,20 @@ impl MatchGeneratorDriver { primed: None, own_workspace: crate::encoding::workspace::Workspace::new(), frame_in_place: false, + #[cfg(feature = "ldm")] + ldm: None, + #[cfg(feature = "ldm")] + ldm_store: Default::default(), + #[cfg(feature = "ldm")] + ldm_found: Vec::new(), + #[cfg(feature = "ldm")] + ldm_block: Vec::new(), + #[cfg(feature = "ldm")] + ldm_committed_end: 0, + #[cfg(feature = "ldm")] + ldm_dict_end: 0, + #[cfg(feature = "ldm")] + ldm_split_block: None, } } @@ -1108,7 +1230,14 @@ impl Matcher for MatchGeneratorDriver { .primed .as_ref() .map_or(0, |(storage, _, _)| storage.heap_size()); - self.storage.heap_size() + snapshot + self.own_workspace.heap_bytes() + #[cfg(feature = "ldm")] + let ldm = self.ldm.as_ref().map_or(0, |p| p.heap_size()) + + self.ldm_store.heap_size() + + (self.ldm_found.capacity() + self.ldm_block.capacity()) + * core::mem::size_of::(); + #[cfg(not(feature = "ldm"))] + let ldm = 0; + self.storage.heap_size() + snapshot + self.own_workspace.heap_bytes() + ldm } fn clear_param_overrides(&mut self) { @@ -1247,6 +1376,12 @@ impl Matcher for MatchGeneratorDriver { hint, ); } + // The long-distance matcher's parameters for the frame, `None` without one. + #[cfg(feature = "ldm")] + let frame_ldm = self + .param_overrides + .and_then(|ov| ov.ldm) + .map(|ldm_ov| crate::encoding::levels::config::frame_ldm_params(¶ms, &ldm_ov)); // A dictionary frame's hash-chain / binary-tree widths are the CDict's // (`resolve_level_params_with_dict`): verbatim when the dictionary is // copied into the live tables (`ZSTD_resetCCtx_byCopyingCDict` builds @@ -1392,9 +1527,16 @@ impl Matcher for MatchGeneratorDriver { // taking it again. The kernels keep positions in `u32`, counting a // dictionary ahead of the slice, so a longer one is copied. let dict_len = dict_hint.map_or(0, |sizes| sizes.content); + // The long-distance matcher splits a block into pieces through the + // owned history, so its frames keep the input there. + #[cfg(feature = "ldm")] + let long_distance = self.param_overrides.is_some_and(|ov| ov.ldm.is_some()); + #[cfg(not(feature = "ldm"))] + let long_distance = false; self.frame_in_place = match workspace.ingest() { crate::encoding::workspace::IngestPlan::Slice(len) => { (dict_hint.is_none() || fast_attach) + && !long_distance && self.borrowed_supported() && len .checked_add(dict_len) @@ -1447,8 +1589,8 @@ impl Matcher for MatchGeneratorDriver { // the table contents and bias wholesale, so the reset's // full-table memset would be thrown away. The // key components mirror `reset_shape` below: Simple leaves - // `resolved_table_bits` 0, never carries an LDM override, - // and `fast_attach` is false in copy mode by construction. + // `resolved_table_bits` 0, and `fast_attach` is false in copy + // mode by construction. // The slot format is the one component this reset has yet to // choose, so it travels with the carry: the reset leaves the // table only if it chooses the snapshot's format, which is @@ -1469,7 +1611,6 @@ impl Matcher for MatchGeneratorDriver { table_bits: 0, fast_attach: false, fast_tagged: captured.fast_tagged, - ldm: None, } }) .map(|(_, _, captured)| captured.fast_tagged); @@ -1722,41 +1863,22 @@ impl Matcher for MatchGeneratorDriver { hc.reset(); } } - // LDM wiring (#27): attach (or clear) the long-distance-match - // producer on the optimal (BT) backend. LDM is the only - // back-reference path that crosses the regular window, so it - // only has a home on the `BtMatcher`; non-BT strategies drop the - // producer. Built AFTER `hc.reset()` because `BtMatcher::reset` - // clears an existing producer's table but does not null the - // slot — installing here gives the new frame a fresh producer. + // The long-distance matcher for the frame, whichever strategy runs it. + // The same parameters keep the table allocation and only clear it, as + // upstream reuses its `ldmState_t`. #[cfg(feature = "ldm")] { - // Resolve the derived LDM params first (immutable borrow of the - // overrides), then reuse the existing producer's allocation below. - let derived_ldm = self - .param_overrides - .as_ref() - .and_then(|ov| ov.ldm) - .map(|ldm_ov| crate::encoding::levels::config::frame_ldm_params(¶ms, &ldm_ov)); - if let MatcherStorage::HashChain(hc) = &mut self.storage { - // Reuse the existing producer's hash-table allocation when the - // derived params are unchanged: only `clear()` (re-zero the - // table + re-seed the rolling hash, no allocation) is needed for - // the new frame. A params change (or the first frame) forces a - // fresh `LdmProducer::new`. On the reused-encoder compress-dict - // path this avoids re-allocating the LDM hash table (large at - // btultra2) every frame — upstream zstd reuses its `ldmState_t` - // the same way. `clear()` is mandatory here for correctness - // regardless of what `BtMatcher::reset` did to the old table. - let producer = derived_ldm.map(|p| match hc.take_ldm_producer() { - Some(mut existing) if existing.params() == p => { - existing.clear(); - existing - } - _ => super::ldm::LdmProducer::new(p), - }); - hc.set_ldm_producer(producer); - } + self.ldm = frame_ldm.map(|p| match self.ldm.take() { + Some(mut existing) if existing.params() == p => { + existing.clear(); + existing + } + _ => super::ldm::LdmProducer::new(p), + }); + // A dictionary the reset kept in place is the frame's first bytes. + self.ldm_committed_end = self.storage.committed_window().len(); + self.ldm_dict_end = self.ldm_committed_end; + self.ldm_split_block = None; } // Record the resolved matcher shape for the primed-snapshot key. Captured // here (post-resolution, after the test-only param override) so the key @@ -1768,28 +1890,9 @@ impl Matcher for MatchGeneratorDriver { // resolved matcher geometry instead, so folding the Fast bit into their // key would over-key identical resolved shapes. `fast_attach` is the // decision `prime_with_dictionary` makes from the same `reset_size_log`. - // The LDM override is part of the snapshot identity ONLY on the - // optimal (BinaryTree) path: that is the only backend whose cloned - // `storage` carries a `BtMatcher::ldm_producer`. On Fast / Dfast / - // Row and lazy-HashChain resets the producer slot does not exist, - // so folding the override there would over-key the snapshot and - // force needless re-primes when LDM is toggled. Gated like - // `fast_attach` (a key bit only participates where it changes the - // cloned matcher shape). - let active_ldm = if matches!(params.search, super::strategy::SearchMethod::BinaryTree) { - self.param_overrides.and_then(|ov| ov.ldm) - } else { - None - }; // The Fast slot format this reset chose; only the Simple backend has one. let fast_tagged = matches!(&self.storage, MatcherStorage::Simple(m) if m.slots_tagged()); - self.reset_shape = Some(( - params, - resolved_table_bits, - fast_attach, - fast_tagged, - active_ldm, - )); + self.reset_shape = Some((params, resolved_table_bits, fast_attach, fast_tagged)); // Everything is laid out in `workspace` now. A driver reset on its own // before it joined a context still holds the workspace it used then, // which nothing points into any more; on its own the field is empty @@ -1864,7 +1967,30 @@ impl Matcher for MatchGeneratorDriver { self.reported_window_size as u64 } + fn matches_beyond_window_tables(&self) -> bool { + #[cfg(feature = "ldm")] + return self.ldm.is_some(); + #[cfg(not(feature = "ldm"))] + false + } + + fn settle_block_cut(&mut self, window: usize, cut: usize) -> usize { + #[cfg(feature = "ldm")] + return self.settle_long_distance_cut(window, cut); + #[cfg(not(feature = "ldm"))] + { + let _ = window; + cut + } + } + fn get_last_space(&mut self) -> &[u8] { + // A block matched in pieces is still one block to the encoder. + #[cfg(feature = "ldm")] + if let Some(len) = self.ldm_split_block { + let window = self.storage.committed_window(); + return &window[window.len() - len..]; + } match &self.storage { MatcherStorage::Simple(m) => m.last_committed_space(), MatcherStorage::Dfast(m) => m.get_last_space(), @@ -1909,6 +2035,11 @@ impl Matcher for MatchGeneratorDriver { /// matches older than the window the frame header reports, which encodes /// an offset no decoder can resolve. fn commit_filled(&mut self, len: usize) { + #[cfg(feature = "ldm")] + { + self.ldm_committed_end += len; + self.ldm_split_block = None; + } // The window backends evict whole blocks inside `commit_block`, // subtracting each evicted block from `window_size` and then adding // `len`, so `pre + len - post` is exactly what was evicted and never @@ -1941,13 +2072,16 @@ impl Matcher for MatchGeneratorDriver { } fn start_matching(&mut self, mut handle_sequence: impl FnMut(Sequence)) { - use super::strategy::{self, StrategyTag}; // Borrowed one-shot Fast path: if the frame driver staged a // block range via `set_borrowed_block`, scan it in place against // the borrowed window instead of the owned committed block. Only // the Simple backend is instrumented (the gate guarantees it), // and the stage is consumed so the next block re-stages. if let Some((block_start, block_end)) = self.borrowed_pending.take() { + // The borrowed scans share their kernels with the owned ones, so + // they report through the same sink type. + let mut forward = None; + let mut sink = owned_sink(&mut handle_sequence, &mut forward); match self.active_backend() { super::strategy::BackendTag::Simple => { let m = self.simple_mut(); @@ -1955,18 +2089,14 @@ impl Matcher for MatchGeneratorDriver { // Dict-attach borrowed scan: live matches read the // borrowed input in place, dict matches read the // committed dict prefix via the 2-segment counter. - m.start_matching_borrowed_dict( - block_start, - block_end, - &mut handle_sequence, - ); + m.start_matching_borrowed_dict(block_start, block_end, &mut sink); } else { - m.start_matching_borrowed(block_start, block_end, &mut handle_sequence); + m.start_matching_borrowed(block_start, block_end, &mut sink); } } super::strategy::BackendTag::Dfast => self .dfast_matcher_mut() - .start_matching_borrowed(block_start, block_end, &mut handle_sequence), + .start_matching_borrowed(block_start, block_end, &mut sink), super::strategy::BackendTag::Row => { // Same greedy/lazy parse split as the owned RowHash arm. let greedy = self.parse == super::strategy::ParseMode::Greedy; @@ -1974,13 +2104,13 @@ impl Matcher for MatchGeneratorDriver { block_start, block_end, greedy, - &mut handle_sequence, + &mut sink, ); } super::strategy::BackendTag::HashChain => match self.search { super::strategy::SearchMethod::HashChain => self .hc_matcher_mut() - .start_matching_lazy_borrowed(block_start, block_end, &mut handle_sequence), + .start_matching_lazy_borrowed(block_start, block_end, &mut sink), // `borrowed_supported()` keeps the optimal parsers on the // owned path and `set_borrowed_block` asserts it. other => { @@ -1990,50 +2120,23 @@ impl Matcher for MatchGeneratorDriver { } return; } - // Decoupled parse×search dispatch (fires once per block). The - // search axis (`self.search`) picks the candidate-finding backend; - // the parse axis (greedy vs lazy depth) is carried by the - // backend's runtime `lazy_depth`, set per level at `reset()`. - // The two are independent, so any parse can run on any search - // backend. The `BinaryTree` arm still selects the opt `Strategy` - // ZST off `strategy_tag` so `compress_block::` keeps its - // const-folded optimal-parser monomorphisation. - use super::strategy::SearchMethod; - match self.search { - SearchMethod::Fast => { - self.simple_mut().start_matching(&mut handle_sequence); - } - SearchMethod::DoubleFast => { - self.dfast_matcher_mut() - .start_matching(&mut handle_sequence); - } - SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => { - // One upstream `lazy_generic` body for greedy (depth 0: a - // repcode hit is stored without a search, no lookahead), lazy - // / lazy2 (`lazy_depth` 1 / 2) and btlazy2 (depth 2 over the - // binary-tree finder); the finder (rows / chain / tree) was - // resolved at `configure`. - self.row_matcher_mut().start_matching(&mut handle_sequence); - } - SearchMethod::HashChain => { - // Greedy/lazy/lazy2 all flow through the lazy parser; it - // reads `hc.lazy_depth` (0 = greedy commit). - self.hc_matcher_mut() - .start_matching_lazy(&mut handle_sequence); + #[cfg(feature = "ldm")] + if self.ldm.is_some() { + // The optimal parser weighs long-distance matches as candidates; + // below it they are taken outright (upstream + // `ZSTD_ldm_blockCompress`, `zstd_ldm.c:685-749`). + if matches!(self.search, super::strategy::SearchMethod::BinaryTree) { + self.take_long_distance_block(); + let MatcherStorage::HashChain(hc) = &mut self.storage else { + unreachable!("the optimal parser runs on the hash-chain backend"); + }; + core::mem::swap(&mut self.ldm_block, &mut hc.backend.bt_mut().ldm_sequences); + } else if self.start_matching_long_distance(&mut handle_sequence) { + return; } - SearchMethod::BinaryTree => match self.strategy_tag { - StrategyTag::BtOpt => self.compress_block::(&mut handle_sequence), - StrategyTag::BtUltra => { - self.compress_block::(&mut handle_sequence) - } - StrategyTag::BtUltra2 => { - self.compress_block::(&mut handle_sequence) - } - _ => unreachable!( - "SearchMethod::BinaryTree requires an optimal strategy tag (BtOpt/BtUltra/BtUltra2)" - ), - }, } + let mut gap_literals = None; + self.start_matching_owned(&mut owned_sink(&mut handle_sequence, &mut gap_literals)); } fn skip_matching(&mut self) { @@ -2066,6 +2169,13 @@ impl Matcher for MatchGeneratorDriver { } return; } + // A block written out unsearched is still searched for long matches, + // as upstream searches every block it compresses, and its matches are + // taken past. + #[cfg(feature = "ldm")] + if self.ldm.is_some() { + self.take_long_distance_block(); + } match self.active_backend() { super::strategy::BackendTag::Simple => { self.simple_mut() @@ -2084,6 +2194,82 @@ impl Matcher for MatchGeneratorDriver { } } +/// The sink every owned match runs against: `handle_sequence`, except that a +/// block's trailing literals go to `gap_literals` when it is `Some`, as the +/// gap before a long-distance match needs them. One sink type for the plain +/// block and for those gaps compiles each backend's scan once per caller sink +/// instead of twice. +fn owned_sink<'a, F: FnMut(Sequence)>( + handle_sequence: &'a mut F, + gap_literals: &'a mut Option, +) -> impl FnMut(Sequence) + 'a { + // The guard is read only for the block's last report, never per triple. + // Each arm rebuilds the report from its fields: forwarding the matched + // value whole copies it as one 32-byte block out of the separate stores + // that wrote it, and that copy stalls on every triple. + move |sequence| match sequence { + Sequence::Triple { + literal_len, + offset, + match_len, + } => handle_sequence(Sequence::Triple { + literal_len, + offset, + match_len, + }), + Sequence::Literals { len } => match gap_literals { + Some(gap) => *gap = len, + None => handle_sequence(Sequence::Literals { len }), + }, + } +} + +impl MatchGeneratorDriver { + /// Match the last committed block of the owned history with the frame's + /// backend. + fn start_matching_owned(&mut self, handle_sequence: &mut impl FnMut(Sequence)) { + use super::strategy::{self, StrategyTag}; + // Decoupled parse×search dispatch (fires once per block). The + // search axis (`self.search`) picks the candidate-finding backend; + // the parse axis (greedy vs lazy depth) is carried by the + // backend's runtime `lazy_depth`, set per level at `reset()`. + // The two are independent, so any parse can run on any search + // backend. The `BinaryTree` arm still selects the opt `Strategy` + // ZST off `strategy_tag` so `compress_block::` keeps its + // const-folded optimal-parser monomorphisation. + use super::strategy::SearchMethod; + match self.search { + SearchMethod::Fast => { + self.simple_mut().start_matching(handle_sequence); + } + SearchMethod::DoubleFast => { + self.dfast_matcher_mut().start_matching(handle_sequence); + } + SearchMethod::RowHash | SearchMethod::BinaryTreeLazy => { + // One upstream `lazy_generic` body for greedy (depth 0: a + // repcode hit is stored without a search, no lookahead), lazy + // / lazy2 (`lazy_depth` 1 / 2) and btlazy2 (depth 2 over the + // binary-tree finder); the finder (rows / chain / tree) was + // resolved at `configure`. + self.row_matcher_mut().start_matching(handle_sequence); + } + SearchMethod::HashChain => { + // Greedy/lazy/lazy2 all flow through the lazy parser; it + // reads `hc.lazy_depth` (0 = greedy commit). + self.hc_matcher_mut().start_matching_lazy(handle_sequence); + } + SearchMethod::BinaryTree => match self.strategy_tag { + StrategyTag::BtOpt => self.compress_block::(handle_sequence), + StrategyTag::BtUltra => self.compress_block::(handle_sequence), + StrategyTag::BtUltra2 => self.compress_block::(handle_sequence), + _ => unreachable!( + "SearchMethod::BinaryTree requires an optimal strategy tag (BtOpt/BtUltra/BtUltra2)" + ), + }, + } + } +} + impl MatchGeneratorDriver { /// Monomorphised optimal-parser entry point. Only the `BinaryTree` /// search arm of [`Matcher::start_matching`] routes here, selecting diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs index 9878c7b68..4ca19c42e 100644 --- a/zstd/src/encoding/match_generator/tests.rs +++ b/zstd/src/encoding/match_generator/tests.rs @@ -131,6 +131,81 @@ fn parse_search_matrix_decoupled_roundtrips() { assert_eq!(got, data, "lazy-on-rowhash diverged"); } +/// Long-distance matches are taken outright on the lazy hash-chain search as +/// on every other non-optimal one: a repeat beyond the chain's reach is +/// matched, the stream rebuilds, and the chain carries on around the matches. +#[cfg(feature = "ldm")] +#[test] +fn hash_chain_lazy_takes_long_distance_matches() { + use super::super::parameters::CompressionParameters; + use super::super::strategy::{ParseMode, SearchMethod}; + + // Six-bit symbols from an LCG's high bits: no repeat the chain finds. + let mut state = 0x0569_C4A1u32; + let half: Vec = (0..256 * 1024) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state >> 26) as u8 + }) + .collect(); + // The copy in shuffled pieces, so one hit cannot carry the rest. + const PIECE: usize = 4096; + let pieces = half.len() / PIECE; + let mut data = half.clone(); + for i in 0..pieces { + let piece = (i * 37) % pieces; + data.extend_from_slice(&half[piece * PIECE..(piece + 1) * PIECE]); + } + // Literal bytes the parse leaves, after checking the stream rebuilds. + let literals = |long: bool| { + let overrides = CompressionParameters::builder(CompressionLevel::Level(5)) + .window_log(22) + .hash_log(12) + .chain_log(12) + .enable_long_distance_matching(long) + .build() + .unwrap() + .overrides(); + let mut driver = MatchGeneratorDriver::new(1 << 17, 8); + driver.set_param_overrides(Some(overrides)); + driver.set_config_override(SearchMethod::HashChain, ParseMode::Lazy2); + driver.reset(CompressionLevel::Level(5)); + assert_eq!(driver.search, SearchMethod::HashChain); + let mut out = Vec::with_capacity(data.len()); + let mut literals = 0usize; + let mut consumed = 0usize; + while consumed < data.len() { + let take = (data.len() - consumed).min(driver.slice_size); + let block = &data[consumed..consumed + take]; + driver.commit_input(block); + consumed += take; + let mut replay = BlockReplay::new(block); + driver.start_matching(|seq| { + literals += match seq { + Sequence::Triple { literal_len, .. } => literal_len, + Sequence::Literals { len } => len, + }; + replay.apply(&mut out, seq); + }); + } + assert_eq!( + out, data, + "long-distance matching {long}: the stream does not rebuild" + ); + literals + }; + let without = literals(false); + let with = literals(true); + assert!( + without > half.len() * 3 / 2, + "the chain reached the repeat by itself: {without} literal bytes" + ); + assert!( + with < half.len() + half.len() / 8, + "the repeat was not matched: {with} literal bytes, {without} without" + ); +} + /// The row `mls` knob (C-like `minMatch`) is respected: every accepted /// match (regular row + repcode, on the lazy parse) is at least `mls` /// bytes, and the stream still round-trips for the whole 4..=7 range. The @@ -2834,55 +2909,6 @@ fn hc_prime_with_empty_dictionary_disables_btultra2_seed_pass() { ); } -#[test] -fn primed_snapshot_not_restored_across_ldm_config_change() { - // The CDict-equivalent primed snapshot clones `storage`, which on the - // BT backend carries `BtMatcher::ldm_producer`. A snapshot captured - // under one LDM configuration must NOT be restored into a reset that - // resolved a different LDM configuration (else the restored producer - // is stale). `PrimedKey` must fold the LDM override into the key so - // such a restore is refused and the caller re-primes. - use super::super::parameters::CompressionParameters; - - let dict = b"abcdefghabcdefghabcdefgh"; - let ldm_on = CompressionParameters::builder(CompressionLevel::Level(19)) - .enable_long_distance_matching(true) - .build() - .unwrap() - .overrides(); - let ldm_off = CompressionParameters::builder(CompressionLevel::Level(19)) - .build() - .unwrap() - .overrides(); - - let mut driver = MatchGeneratorDriver::new(1024, 1); - - // Capture a snapshot primed under LDM-on at level 19. - driver.set_param_overrides(Some(ldm_on)); - driver.reset(CompressionLevel::Level(19)); - driver.prime_with_dictionary(dict, [1, 4, 8]); - driver.capture_primed_dictionary(CompressionLevel::Level(19)); - - // Same dictionary + level, but LDM now OFF: the snapshot's LDM state - // is stale, so restore must be refused. - driver.set_param_overrides(Some(ldm_off)); - driver.reset(CompressionLevel::Level(19)); - assert!( - !driver.restore_primed_dictionary(CompressionLevel::Level(19)), - "primed snapshot restored across an LDM config change (stale producer)", - ); - - // Sanity: re-priming + capturing under LDM-off, then restoring under - // the IDENTICAL LDM-off config DOES match (the key is not over-tight). - driver.prime_with_dictionary(dict, [1, 4, 8]); - driver.capture_primed_dictionary(CompressionLevel::Level(19)); - driver.reset(CompressionLevel::Level(19)); - assert!( - driver.restore_primed_dictionary(CompressionLevel::Level(19)), - "primed snapshot not restored under identical LDM config", - ); -} - /// btultra2 parses a frame's first block twice and hides the first pass from /// the second by re-encoding the stored positions. A reused matcher keeps the /// previous frame's entries below the floor, and the re-encoding has to leave diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs index 80e266b86..30685c6ae 100644 --- a/zstd/src/encoding/match_table/storage.rs +++ b/zstd/src/encoding/match_table/storage.rs @@ -261,6 +261,9 @@ pub(crate) struct MatchTable { pub(crate) hash3_log: usize, pub(crate) next_to_update3: usize, pub(crate) skip_insert_until_abs: usize, + /// While a dictionary is primed: how far into it the tables start + /// indexing, `0` for all of it (see [`Self::set_dictionary_index_floor`]). + pub(crate) dict_index_floor: usize, pub(crate) dictionary_limit_abs: Option, pub(crate) dictionary_primed_for_frame: bool, /// Sticky across resets: set once a dictionary is primed into this table @@ -360,6 +363,7 @@ impl Clone for MatchTable { hash3_log: self.hash3_log, next_to_update3: self.next_to_update3, skip_insert_until_abs: self.skip_insert_until_abs, + dict_index_floor: self.dict_index_floor, dictionary_limit_abs: self.dictionary_limit_abs, dictionary_primed_for_frame: self.dictionary_primed_for_frame, dictionary_active: self.dictionary_active, @@ -407,6 +411,7 @@ impl Clone for MatchTable { self.hash3_log = source.hash3_log; self.next_to_update3 = source.next_to_update3; self.skip_insert_until_abs = source.skip_insert_until_abs; + self.dict_index_floor = source.dict_index_floor; self.dictionary_limit_abs = source.dictionary_limit_abs; self.dictionary_primed_for_frame = source.dictionary_primed_for_frame; self.dictionary_active = source.dictionary_active; @@ -541,6 +546,7 @@ impl MatchTable { hash3_log: HC3_HASH_LOG, next_to_update3: 0, skip_insert_until_abs: 0, + dict_index_floor: 0, dictionary_limit_abs: None, dictionary_primed_for_frame: false, dictionary_active: false, @@ -984,6 +990,7 @@ impl MatchTable { // generous fixed depth keeps the tree well-ordered without the live // searchLog cap. Mirrors upstream zstd `ZSTD_insertBt1` nbCompares. const DMS_BUILD_DEPTH: usize = 1 << 9; + let floor = self.dict_index_floor; // 2 children per dict position: [smaller, larger]. build_dms!( self, @@ -993,7 +1000,7 @@ impl MatchTable { read, 2, |concat, hash_table, chain_table, dms_hash_log, region| { - let mut current = 0usize; + let mut current = floor; while current + read <= region { let h = Self::hash_position_at(concat, current, dms_hash_log, mls); // Insert `current` as the new root; splay the old tree into @@ -1061,6 +1068,7 @@ impl MatchTable { /// chain. Cached across frames via `DictAttach::is_primed`. pub(crate) fn prime_dms_hc(&mut self, region_len: usize) { let mls = HC_MIN_MATCH_LEN; + let floor = self.dict_index_floor; // Single `next` link per dict position (HC4 chain, not the BT's 2/pos). build_dms!( self, @@ -1070,7 +1078,7 @@ impl MatchTable { mls, 1, |concat, hash_table, chain_table, dms_hash_log, region| { - let mut current = 0usize; + let mut current = floor; while current + mls <= region { let h = Self::hash_position_at(concat, current, dms_hash_log, mls); // Prepend `current` to its bucket chain (upstream zstd @@ -2401,10 +2409,75 @@ impl MatchTable { self.next_to_update3 = self.next_to_update3.max(committed_end); } + /// Index a dictionary being primed only from `floor` bytes into it, `0` + /// for all of it; the driver sets it for the prime and clears it after. + /// The tree's insertion cursors start from the floor too, since a copied + /// dictionary is threaded through them. + pub(crate) fn set_dictionary_index_floor(&mut self, floor: usize) { + self.dict_index_floor = floor; + if floor != 0 { + let floor_abs = self.history_abs_start + floor; + self.skip_insert_until_abs = self.skip_insert_until_abs.max(floor_abs); + self.next_to_update3 = self.next_to_update3.max(floor_abs); + } + } + + /// The committed window followed by the input read but not yet committed. + #[cfg(feature = "ldm")] + pub(crate) fn window_with_pending(&self) -> &[u8] { + &self.history[self.history_start..] + } + + /// Hand the last committed block of `len` bytes back to the uncommitted + /// input, undoing only the bookkeeping of [`Self::commit_block`]: what it + /// evicted stays evicted, so committing the same bytes again in pieces + /// evicts nothing more. + #[cfg(feature = "ldm")] + pub(crate) fn uncommit_block(&mut self, len: usize) { + assert_eq!(self.chunk_lens.pop_back(), Some(len)); + self.window_size -= len; + self.uncommitted_len += len; + } + + /// Index the last committed block as the body of a long-distance match on + /// the lazy chain: only its tail, which is what upstream's lazy search + /// catches up on after one (`ZSTD_ldm_limitTableUpdate` moves + /// `nextToUpdate` to it), since the match itself is never searched. + #[cfg(feature = "ldm")] + pub(crate) fn skip_long_match(&mut self) { + debug_assert!( + !self.uses_bt, + "the optimal parser takes LDM matches as candidates" + ); + self.ensure_tables(); + let (start, len) = self.current_block_range(); + let end = start + len; + self.backfill_boundary_positions(start, end); + self.insert_positions( + end - len.min(crate::encoding::ldm::LONG_MATCH_INDEXED_TAIL), + end, + ); + } + + /// Carry a long-distance match the driver emitted into the repeat + /// offsets, as the encoder will code it. + #[cfg(feature = "ldm")] + pub(crate) fn note_long_match(&mut self, offset: usize, literal_len: usize) { + let _ = crate::encoding::blocks::encode_offset_with_history( + offset as u32, + literal_len as u32, + &mut self.offset_hist, + ); + } + pub(crate) fn skip_matching(&mut self, incompressible_hint: Option) { self.ensure_tables(); let (current_abs_start, current_len) = self.current_block_range(); let current_abs_end = current_abs_start + current_len; + // A dictionary being primed sits at the front of the history and is + // indexed only from its floor on. + let floor = (self.history_abs_start + self.dict_index_floor).min(current_abs_end); + let current_abs_start = current_abs_start.max(floor); self.backfill_boundary_positions(current_abs_start, current_abs_end); if self.uses_bt { if incompressible_hint == Some(true) { diff --git a/zstd/src/encoding/mod.rs b/zstd/src/encoding/mod.rs index add752c44..4fe52caff 100644 --- a/zstd/src/encoding/mod.rs +++ b/zstd/src/encoding/mod.rs @@ -613,6 +613,23 @@ pub trait Matcher { fn block_samples_match_dict(&self, _block: &[u8]) -> bool { false } + /// Whether this matcher finds matches beyond what its regular tables + /// reach (long-distance matching). A block that looks incompressible can + /// then still be a repeat from far back, so the encoder searches every + /// block instead of writing one off on a sample. Defaults to `false`. + fn matches_beyond_window_tables(&self) -> bool { + false + } + /// Where to cut the next block, given the cut the block splitter + /// proposes `cut` bytes into the `window` bytes of + /// [`uncommitted_input`](Self::uncommitted_input) it looked at. A matcher + /// that already knows a match crossing the cut moves it out of the match, + /// at most to `window`; cutting inside one only makes the next block + /// start by repeating it. Defaults to `cut`. + fn settle_block_cut(&mut self, window: usize, cut: usize) -> usize { + let _ = window; + cut + } /// Heap bytes this matcher's allocations hold (tables, history, scratch), /// excluding the inline struct itself. Lets a context report its true /// footprint via `ZSTD_sizeof_CCtx`. Defaults to `0` for custom matchers. diff --git a/zstd/src/encoding/parameters.rs b/zstd/src/encoding/parameters.rs index cc4f7616c..ef7a68fcd 100644 --- a/zstd/src/encoding/parameters.rs +++ b/zstd/src/encoding/parameters.rs @@ -40,9 +40,9 @@ //! never enables LDM — even at level 22. It is activated either by //! [`CompressionParametersBuilder::enable_long_distance_matching`] or by any of //! the `ldm_*` setters, which each imply `enable_long_distance_matching(true)`. -//! When enabled, the LDM producer attaches to the optimal (`btopt` / `btultra` -//! / `btultra2`) match-finder; pair it with an optimal [`Strategy`] (or a level -//! ≥ 16) for it to take effect. +//! It works under every [`Strategy`]: the optimal parsers weigh its matches as +//! candidates, the others take them outright and search only the bytes between +//! them, as upstream does. A dictionary loaded under it is indexed for it too. use crate::encoding::CompressionLevel; diff --git a/zstd/src/encoding/row/mod.rs b/zstd/src/encoding/row/mod.rs index 12665f9bb..eee8a2c39 100644 --- a/zstd/src/encoding/row/mod.rs +++ b/zstd/src/encoding/row/mod.rs @@ -2419,6 +2419,9 @@ pub(crate) struct RowMatchGenerator { /// stores it raw and rotates, so the encoder-side history and /// upstream's parse-side reps legitimately differ. lazy_reps: Option<[usize; 2]>, + /// While a dictionary is primed: how far into it the tables start + /// indexing, `0` for all of it (see [`Self::set_dictionary_index_floor`]). + dict_index_floor: usize, pub(crate) lazy_depth: u8, /// Cached fastpath kernel for the per-candidate `common_prefix_len` /// compare, resolved once per matcher so the rep probe skips the @@ -2547,6 +2550,7 @@ impl RowMatchGenerator { row_hash_mls: ROW_MIN_MATCH_LEN.clamp(4, 6) as u32, lazy_next_to_update: 0, lazy_reps: None, + dict_index_floor: 0, lazy_depth: 1, cpl_kernel: crate::encoding::fastpath::select_kernel(), rows_len: 0, @@ -4012,6 +4016,54 @@ impl RowMatchGenerator { crate::encoding::fastpath::scalar::common_prefix_len_ptr ); + /// Index a dictionary being primed only from `floor` bytes into it, `0` + /// for all of it; the driver sets it for the prime and clears it after. + pub(crate) fn set_dictionary_index_floor(&mut self, floor: usize) { + self.dict_index_floor = floor; + } + + /// The committed window followed by the input read but not yet committed. + #[cfg(feature = "ldm")] + pub(crate) fn window_with_pending(&self) -> &[u8] { + &self.history[self.history_start..] + } + + /// Hand the last committed block of `len` bytes back to the uncommitted + /// input, undoing only the bookkeeping of [`Self::commit_block`]: what it + /// evicted stays evicted, so committing the same bytes again in pieces + /// evicts nothing more. + #[cfg(feature = "ldm")] + pub(crate) fn uncommit_block(&mut self, len: usize) { + assert_eq!(self.chunk_lens.pop_back(), Some(len)); + self.window_size -= len; + self.uncommitted_len += len; + } + + /// Take the last committed block as the body of a long-distance match: + /// nothing in it is indexed now, and the next block resumes indexing its + /// tail, as upstream's `ZSTD_ldm_limitTableUpdate` leaves `nextToUpdate`. + #[cfg(feature = "ldm")] + pub(crate) fn skip_long_match(&mut self) { + let (start, len) = self.current_block_range(); + self.skip_matching_with_hint(None); + let end = start + len; + self.lazy_next_to_update = end - len.min(crate::encoding::ldm::LONG_MATCH_INDEXED_TAIL); + } + + /// Carry a long-distance match the driver emitted into the repeat offsets: + /// the lazy parse's carried pair takes it as the newest (upstream + /// `ZSTD_ldm_blockCompress` shifts `rep`), the encoder-side history as the + /// encoder will code it. + #[cfg(feature = "ldm")] + pub(crate) fn note_long_match(&mut self, offset: usize, literal_len: usize) { + let newest = self + .lazy_reps + .map_or(self.offset_hist[0] as usize, |reps| reps[0]); + self.lazy_reps = Some([offset, newest]); + let _ = + encode_offset_with_history(offset as u32, literal_len as u32, &mut self.offset_hist); + } + pub(crate) fn skip_matching_with_hint(&mut self, incompressible_hint: Option) { match self.row_log { 4 => self.skip_matching_with_hint_rl::<4>(incompressible_hint), @@ -4438,6 +4490,7 @@ impl RowMatchGenerator { bt: &mut [u32], content: *const u8, concat_len: usize, + start: usize, index_base: usize, hash_log: usize, chain_log: usize, @@ -4445,7 +4498,7 @@ impl RowMatchGenerator { mls: u32, ) { let target = concat_len.saturating_sub(8); - let mut idx = 0usize; + let mut idx = start; while idx < target { // SAFETY: `idx + 8 <= concat_len`; the tree holds only earlier // dictionary positions (this loop's own inserts). @@ -4490,6 +4543,7 @@ impl RowMatchGenerator { hc_chain, base.add(history_start), concat_len, + self.dict_index_floor, index_base, hash_log, chain_log, @@ -4521,6 +4575,7 @@ impl RowMatchGenerator { } let base = self.history.as_ptr(); let history_start = self.history_start; + let floor = self.dict_index_floor; let dict = self.dict.table_mut_or_init(|| RowDictTables { heads: Vec::new(), positions: Vec::new(), @@ -4545,6 +4600,7 @@ impl RowMatchGenerator { &mut dict.hc_chain, base.add(history_start), concat_len, + floor, BT_IDX_BASE, hash_log, chain_log, @@ -4608,7 +4664,8 @@ impl RowMatchGenerator { // tables. The index cursor carries across the dictionary's // slices; `mark_dict_primed` moves it to the dictionary end. self.dict.set_region_len(concat_len); - let from = self.lazy_next_to_update.max(hist_start) - hist_start; + let from = + (self.lazy_next_to_update.max(hist_start) - hist_start).max(self.dict_index_floor); if from < indexable_end { let scan = self.scan_ctx(); if self.finder == LazyFinder::Chain { @@ -4673,7 +4730,7 @@ impl RowMatchGenerator { if self.dict.is_primed() { return; } - let from = self.dict.next_to_update(); + let from = self.dict.next_to_update().max(self.dict_index_floor); if from >= indexable_end { return; } diff --git a/zstd/src/encoding/simple/fast_matcher.rs b/zstd/src/encoding/simple/fast_matcher.rs index 1efb1705c..84b194a39 100644 --- a/zstd/src/encoding/simple/fast_matcher.rs +++ b/zstd/src/encoding/simple/fast_matcher.rs @@ -341,6 +341,9 @@ pub(crate) struct FastKernelMatcher { /// the slices one dictionary is committed in so the phase stays continuous. /// See [`Self::prime_hash_table_for_dict_copy`]. Reset with the table. dict_copy_fill_next: usize, + /// While a dictionary is primed: how far into it the tables start + /// indexing, `0` for all of it (see [`Self::set_dictionary_index_floor`]). + dict_index_floor: usize, } impl Clone for FastKernelMatcher { @@ -367,6 +370,7 @@ impl Clone for FastKernelMatcher { dict_table_hash_log: self.dict_table_hash_log, loaded_dict_end: self.loaded_dict_end, dict_copy_fill_next: self.dict_copy_fill_next, + dict_index_floor: self.dict_index_floor, } } @@ -395,6 +399,7 @@ impl Clone for FastKernelMatcher { self.dict_resident = source.dict_resident; self.loaded_dict_end = source.loaded_dict_end; self.dict_copy_fill_next = source.dict_copy_fill_next; + self.dict_index_floor = source.dict_index_floor; } } @@ -632,6 +637,7 @@ impl FastKernelMatcher { dict_table_hash_log: None, loaded_dict_end: 0, dict_copy_fill_next: 0, + dict_index_floor: 0, } } @@ -1653,6 +1659,61 @@ impl FastKernelMatcher { } } + /// Index a dictionary being primed only from `floor` bytes into it, `0` + /// for all of it; the driver sets it for the prime and clears it after. + pub(crate) fn set_dictionary_index_floor(&mut self, floor: usize) { + self.dict_index_floor = floor; + } + + /// The committed window, oldest byte first. + #[cfg(feature = "ldm")] + pub(crate) fn committed_window(&self) -> &[u8] { + self.history_bytes() + } + + /// The committed window followed by the input read but not yet committed. + #[cfg(feature = "ldm")] + pub(crate) fn window_with_pending(&self) -> &[u8] { + &self.history[..] + } + + /// Hand the staged block of `len` bytes back to the uncommitted input, + /// undoing only the bookkeeping of [`Self::commit_block`]: what it evicted + /// stays evicted, so committing the same bytes again in pieces evicts + /// nothing more. + #[cfg(feature = "ldm")] + pub(crate) fn uncommit_block(&mut self, len: usize) { + assert_eq!(self.staged_block.take(), Some(len)); + self.uncommitted_len += len; + } + + /// Index the staged block as the body of a long-distance match: only its + /// tail, the way upstream fills the fast table after one + /// (`ZSTD_ldm_limitTableUpdate` + `ZSTD_fillHashTable`, every third + /// position), since the match itself is never searched. + #[cfg(feature = "ldm")] + pub(crate) fn skip_long_match(&mut self) { + const FILL_STEP: usize = 3; + let block_start = self.take_staged_block(); + let end = self.committed_len(); + let from = end - (end - block_start).min(crate::encoding::ldm::LONG_MATCH_INDEXED_TAIL); + self.prime_hash_table_for_range_stepped(from, FILL_STEP); + } + + /// Carry a long-distance match the driver emitted into the repeat + /// offsets: the kernel's pair shifts it in as the newest (upstream + /// `ZSTD_ldm_blockCompress`), the encoder-side history takes it as the + /// encoder will code it. + #[cfg(feature = "ldm")] + pub(crate) fn note_long_match(&mut self, offset: usize, literal_len: usize) { + self.rep = [offset as u32, self.rep[0]]; + let _ = crate::encoding::blocks::encode_offset_with_history( + offset as u32, + literal_len as u32, + &mut self.offset_hist, + ); + } + /// Commit one slice of a COPY-mode dictionary: append it to history, index /// it into the LIVE table with upstream's dictionary fill, and carry the /// dictionary boundary. @@ -1892,7 +1953,7 @@ impl FastKernelMatcher { return; } let last_hashable = history_len - HASH_READ_SIZE; - let fill_start = self.dict_copy_fill_next; + let fill_start = self.dict_copy_fill_next.max(self.dict_index_floor); if fill_start > last_hashable { return; } @@ -2218,7 +2279,8 @@ impl FastKernelMatcher { "dict fill origin {} ran ahead of slice start {range_start}", self.dict.next_to_update(), ); - if self.dict.next_to_update() > last_hashable { + // Nothing below the index floor is indexed. + if self.dict.next_to_update().max(self.dict_index_floor) > last_hashable { return; } // Size the dict table to the CDict geometry of the WHOLE dictionary @@ -2252,7 +2314,7 @@ impl FastKernelMatcher { self.dict.invalidate(); self.dict.set_region_len(history_len); } - let fill_start = self.dict.next_to_update(); + let fill_start = self.dict.next_to_update().max(self.dict_index_floor); self.dict .table_mut_or_init(|| FastHashTable::new(hash_log, mls)); let base = self.history.as_ptr(); diff --git a/zstd/src/encoding/streaming_encoder.rs b/zstd/src/encoding/streaming_encoder.rs index cb4cbabeb..3ab885e4b 100644 --- a/zstd/src/encoding/streaming_encoder.rs +++ b/zstd/src/encoding/streaming_encoder.rs @@ -1103,19 +1103,27 @@ impl CompressionContext { /// `remaining` is the input still to come as far as the splitter knows: /// a full block again while writes continue, the buffered bytes at the /// end of the frame. - fn pre_split_len(&self, block_capacity: usize, remaining: usize) -> usize { + fn pre_split_len(&mut self, block_capacity: usize, remaining: usize) -> usize { if matches!(self.compression_level, CompressionLevel::Uncompressed) { return self.pending_len(); } let pending = self.state.matcher.uncommitted_input(); - crate::encoding::frame_compressor::optimal_block_size_with( + let window = pending.len(); + let cut = crate::encoding::frame_compressor::optimal_block_size_with( self.state.pre_split.map(usize::from), pending, remaining, block_capacity, self.savings, ) - .min(pending.len()) + .min(window); + // The splitter sees byte statistics only; a cut it proposes inside a + // match the matcher already knows of is moved out. + if cut < window { + self.state.matcher.settle_block_cut(window, cut) + } else { + cut + } } /// Emit the pending block once it is full, cut where the pre-splitter diff --git a/zstd/src/tests/parameters_test.rs b/zstd/src/tests/parameters_test.rs index c90fff909..1831e009e 100644 --- a/zstd/src/tests/parameters_test.rs +++ b/zstd/src/tests/parameters_test.rs @@ -251,6 +251,329 @@ fn ldm_custom_knobs_round_trip() { assert_eq!(decode(&compress_with_parameters(&data, ¶ms)), data); } +/// Long-distance matching runs under every strategy, not only the optimal +/// ones: a random half repeated at a distance the strategy's own tables +/// cannot reach is found by LDM alone, so the second copy costs next to +/// nothing. Upstream applies LDM below btopt through `ZSTD_ldm_blockCompress`. +/// Six-bit symbols keep every block searched (a full-entropy block would be +/// written off by the raw skip, whose repeat grid could find the copy instead) +/// while leaving short local matches rare. The copy is laid out in shuffled +/// 4 KiB pieces, so each piece sits at an offset of its own and one lucky hit +/// cannot carry the rest through the repeat code. +#[cfg(feature = "ldm")] +#[test] +fn ldm_finds_far_repeat_under_every_strategy() { + const PIECE: usize = 4096; + let half: Vec = generate_data(0x5EED_0569, 512 * 1024) + .into_iter() + .map(|byte| byte & 63) + .collect(); + let pieces = half.len() / PIECE; + let mut data = half.clone(); + for i in 0..pieces { + // 37 is coprime with the piece count, so this visits every piece once. + let piece = (i * 37) % pieces; + data.extend_from_slice(&half[piece * PIECE..(piece + 1) * PIECE]); + } + for strategy in [ + Strategy::Fast, + Strategy::Dfast, + Strategy::Greedy, + Strategy::Lazy, + Strategy::Lazy2, + Strategy::Btlazy2, + Strategy::Btopt, + ] { + // Tables far smaller than the distance, so nothing but LDM reaches the + // first copy. + let builder = || { + CompressionParameters::builder(CompressionLevel::Level(3)) + .strategy(strategy) + .window_log(22) + .hash_log(12) + .chain_log(12) + }; + let off = compress_with_parameters(&data, &builder().build().unwrap()); + let on = compress_with_parameters( + &data, + &builder() + .enable_long_distance_matching(true) + .build() + .unwrap(), + ); + // Each copy costs about three quarters of its size in literals; the + // second one costs nearly that again without LDM (a few pieces are + // still found by chance) and almost nothing with it. + assert!( + off.len() > data.len() * 13 / 20, + "{strategy:?}: the regular finder already reached the repeat ({} of {})", + off.len(), + data.len(), + ); + assert!( + on.len() < half.len() * 8 / 10, + "{strategy:?}: LDM did not find the repeat ({} of {}, {} without LDM)", + on.len(), + data.len(), + off.len(), + ); + assert_eq!(decode(&on), data, "{strategy:?}: LDM frame round-trip"); + } +} + +/// The block splitter does not cut inside a long-distance match: a repeat of +/// random data is one match per block, and a cut through it would only buy a +/// block whose single sequence carries the same match on. Both frame loops +/// ask, the one-shot one and the stream's. +#[cfg(feature = "ldm")] +#[test] +fn ldm_matches_are_not_cut_by_the_block_splitter() { + use crate::encoding::StreamingEncoder; + use crate::io::Write; + + let half = generate_data(0xC07_0569, 1 << 20); + let mut data = half.clone(); + data.extend_from_slice(&half); + let params = CompressionParameters::builder(CompressionLevel::Level(3)) + .window_log(22) + .enable_long_distance_matching(true) + .build() + .unwrap(); + let one_shot = compress_with_parameters(&data, ¶ms); + let mut encoder = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(3)); + encoder.set_parameters(¶ms).unwrap(); + encoder.write_all(&data).unwrap(); + let streamed = encoder.finish().unwrap(); + for (path, frame) in [("one-shot", &one_shot), ("stream", &streamed)] { + // The first copy goes out raw; the second is eight blocks of one + // sequence each, where a cut at every splitter boundary would make it + // about a hundred. + assert!( + frame.len() < half.len() + 512, + "{path}: the repeat cost {} bytes", + frame.len() - half.len() + ); + assert_eq!(decode(frame), data, "{path}: round-trip"); + } +} + +/// Long-distance matching reaches into a dictionary, under every strategy: +/// the dictionary is indexed into its table when it is loaded (upstream +/// `ZSTD_ldm_fillHashTable` from `ZSTD_loadDictionaryContent`), so a source +/// made of shuffled pieces of a large dictionary costs next to nothing. A +/// second frame on the same compressor restores the primed dictionary and must +/// write what a freshly primed compressor writes. +#[cfg(feature = "ldm")] +#[test] +fn ldm_reaches_into_the_dictionary_under_every_strategy() { + use crate::decoding::{Dictionary, FrameDecoder}; + + const PIECE: usize = 4096; + const DICT_ID: u32 = 0x0569_D1C7; + let dict: Vec = generate_data(0xD1C7_0569, 256 * 1024) + .into_iter() + .map(|byte| byte & 63) + .collect(); + let pieces = dict.len() / PIECE; + let mut source = Vec::new(); + for i in 0..pieces { + let piece = (i * 37) % pieces; + source.extend_from_slice(&dict[piece * PIECE..(piece + 1) * PIECE]); + } + for strategy in [ + Strategy::Fast, + Strategy::Dfast, + Strategy::Greedy, + Strategy::Lazy2, + Strategy::Btlazy2, + Strategy::Btopt, + ] { + let params = CompressionParameters::builder(CompressionLevel::Level(3)) + .strategy(strategy) + .window_log(22) + .hash_log(12) + .chain_log(12) + .enable_long_distance_matching(true) + .build() + .unwrap(); + let compressor = || { + let mut compressor: FrameCompressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_parameters(¶ms); + compressor + .set_dictionary(Dictionary::from_raw_content(DICT_ID, dict.clone()).unwrap()) + .unwrap(); + compressor + }; + let mut warm = compressor(); + let first = warm.compress_independent_frame(source.as_slice()); + let restored = warm.compress_independent_frame(source.as_slice()); + let fresh = compressor().compress_independent_frame(source.as_slice()); + assert!( + first.len() < source.len() / 10, + "{strategy:?}: LDM did not reach into the dictionary ({} of {})", + first.len(), + source.len(), + ); + assert_eq!( + first, fresh, + "{strategy:?}: a fresh compressor wrote another frame" + ); + assert_eq!( + restored, fresh, + "{strategy:?}: the restored dictionary wrote another frame" + ); + let mut decoder = FrameDecoder::new(); + decoder + .add_dict(Dictionary::from_raw_content(DICT_ID, dict.clone()).unwrap()) + .unwrap(); + let mut decoded = Vec::with_capacity(source.len()); + decoder.decode_all_to_vec(&restored, &mut decoded).unwrap(); + assert_eq!(decoded, source, "{strategy:?}: round-trip"); + } +} + +/// The primed dictionary snapshot carries no long-distance state, so it is +/// restored whether the next frame turns long-distance matching on or off, +/// and the frame matches what a freshly primed compressor writes either way. +#[cfg(feature = "ldm")] +#[test] +fn a_primed_dictionary_restores_across_a_long_distance_switch() { + use crate::decoding::Dictionary; + + const PIECE: usize = 4096; + const DICT_ID: u32 = 0x0569_D1C8; + let dict: Vec = generate_data(0xD1C7_0571, 256 * 1024) + .into_iter() + .map(|byte| byte & 63) + .collect(); + let pieces = dict.len() / PIECE; + let mut source = Vec::new(); + for i in 0..pieces { + let piece = (i * 37) % pieces; + source.extend_from_slice(&dict[piece * PIECE..(piece + 1) * PIECE]); + } + for strategy in [Strategy::Fast, Strategy::Lazy2, Strategy::Btopt] { + let params = |long: bool| { + CompressionParameters::builder(CompressionLevel::Level(3)) + .strategy(strategy) + .window_log(22) + .hash_log(12) + .chain_log(12) + .enable_long_distance_matching(long) + .build() + .unwrap() + }; + let compressor = |long: bool| { + let mut compressor: FrameCompressor = FrameCompressor::new(CompressionLevel::Default); + compressor.set_parameters(¶ms(long)); + compressor + .set_dictionary(Dictionary::from_raw_content(DICT_ID, dict.clone()).unwrap()) + .unwrap(); + compressor + }; + for first in [false, true] { + let mut warm = compressor(first); + warm.compress_independent_frame(source.as_slice()); + warm.set_parameters(¶ms(!first)); + let restored = warm.compress_independent_frame(source.as_slice()); + let fresh = compressor(!first).compress_independent_frame(source.as_slice()); + assert!( + restored == fresh, + "{strategy:?}: long-distance matching {} after {}: {} bytes, fresh {}", + !first, + first, + restored.len(), + fresh.len(), + ); + } + } +} + +/// A dictionary a reused context keeps resident between unsized frames stays +/// in reach of long-distance matching: every frame on the context writes what +/// a fresh encoder writes, the first one included. The pieces come from the +/// dictionary's start, which the regular tables leave unindexed. +#[cfg(feature = "ldm")] +#[test] +fn ldm_reaches_into_a_resident_dictionary_frame_after_frame() { + use crate::encoding::{CompressionContext, EncoderDictionary, StreamingEncoder}; + use crate::io::Write; + + const PIECE: usize = 4096; + let dict: Vec = generate_data(0xD1C7_0570, 256 * 1024) + .into_iter() + .map(|byte| byte & 63) + .collect(); + let mut source = Vec::new(); + for piece in [3, 17, 9] { + source.extend_from_slice(&dict[piece * PIECE..(piece + 1) * PIECE]); + } + let dictionary = EncoderDictionary::from_serialized_or_raw_content(&dict).unwrap(); + for strategy in [ + Strategy::Fast, + Strategy::Dfast, + Strategy::Greedy, + Strategy::Lazy2, + Strategy::Btlazy2, + Strategy::Btopt, + ] { + let params = CompressionParameters::builder(CompressionLevel::Level(3)) + .strategy(strategy) + .window_log(22) + .hash_log(12) + .chain_log(12) + .enable_long_distance_matching(true) + .build() + .unwrap(); + let mut fresh = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(3)); + fresh.set_parameters(¶ms).unwrap(); + fresh.set_encoder_dictionary(dictionary.clone()).unwrap(); + fresh.write_all(&source).unwrap(); + let expected = fresh.finish().unwrap(); + let mut without = StreamingEncoder::new(Vec::new(), CompressionLevel::Level(3)); + without + .set_parameters( + &CompressionParameters::builder(CompressionLevel::Level(3)) + .strategy(strategy) + .window_log(22) + .hash_log(12) + .chain_log(12) + .build() + .unwrap(), + ) + .unwrap(); + without.set_encoder_dictionary(dictionary.clone()).unwrap(); + without.write_all(&source).unwrap(); + let without = without.finish().unwrap(); + // The optimal parser weighs the long matches as candidates rather than + // taking them outright, so it is held to a gain, not to a size. + assert!( + expected.len() < without.len() / 2, + "{strategy:?}: LDM did not reach into the dictionary ({} bytes, {} without)", + expected.len(), + without.len(), + ); + let mut context = CompressionContext::new(CompressionLevel::Level(3)); + context.set_parameters(¶ms).unwrap(); + context.set_encoder_dictionary(dictionary.clone()).unwrap(); + for frame_index in 0..3 { + let mut frame = Vec::new(); + let mut rest = source.as_slice(); + while !rest.is_empty() { + let taken = context.write(&mut frame, rest).unwrap(); + rest = &rest[taken..]; + } + context.finish_frame(&mut frame).unwrap(); + assert!( + frame == expected, + "{strategy:?}: frame {frame_index} on the reused context wrote {} bytes, a fresh encoder {}", + frame.len(), + expected.len(), + ); + } + } +} + /// `FrameCompressor::set_parameters` on the streaming path reuses one /// compressor across frames and produces decodable output. #[test]