From 625090d4b29ecec94d6e9edc29a67d76b2ad4a59 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 12:38:25 +0000 Subject: [PATCH 1/2] benches: Lance 13 source-pruning probe (D-TELEPORT-2) Measures whether Lance 13's existing scanner and index paths prune source reads before the Quack/mask-risc fold, with identical answers. Arms: full scan + mask-risc fold, row-range prefilter (sorted only), zone-map filter, B-tree filter, and B-tree COUNT with an empty projection. Layouts sorted, partial, periodic and shuffled, with and without NULLs; a deletion pass checks COUNT stays exact. Every arm is asserted against the oracle. Findings: on ordered data the B-tree reads bytes proportional to selectivity; the zone map prunes only what clustering allows; on shuffled data at 50% the B-tree reads a third of the bytes and is ~9x slower than the full scan; index-only COUNT reads no data columns (index I/O is not in Lance's bytes_read counter). No new primitive is needed; the open item is a scan-vs-take cost choice. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015JH7YkXs5us6g3tG8NiFfP --- .../entries/2026-10-11-lance-source-probe.md | 79 +++ crates/lance-graph-benches/Cargo.toml | 2 + .../examples/lance_source_probe.rs | 557 ++++++++++++++++++ 3 files changed, 638 insertions(+) create mode 100644 .claude/board/entries/2026-10-11-lance-source-probe.md create mode 100644 crates/lance-graph-benches/examples/lance_source_probe.rs diff --git a/.claude/board/entries/2026-10-11-lance-source-probe.md b/.claude/board/entries/2026-10-11-lance-source-probe.md new file mode 100644 index 000000000..2a4e649d4 --- /dev/null +++ b/.claude/board/entries/2026-10-11-lance-source-probe.md @@ -0,0 +1,79 @@ +# 2026-10-11 — Lance 13 source pruning before the fold (D-TELEPORT-2) + +**Status:** MEASURED. Probe only, no production change. +`crates/lance-graph-benches/examples/lance_source_probe.rs`. + +```text +CARGO_PROFILE_RELEASE_DEBUG=0 cargo run --release -p lance-graph-benches --example lance_source_probe +``` + +The question: can Lance 13's existing scanner and index paths prune source +reads before the Quack/mask-risc fold, without a row-sized intermediate +population, and with identical answers? Every arm is asserted against the +oracle COUNT and SUM. No new executor, opcode, IR or OGAR code. + +Setup: 1,048,576 rows, fragments of 131,072, columns `x`, `xz`, `xb` +(identical i32 values) and `v`; zone map on `xz`, B-tree on `xb`. Cold = +fresh `Dataset::open` after `drop_caches` (works in this container); warm = +best of 3. Bytes and iops come from Lance's `ExecutionSummaryCounts`. +Host: 4 shared vCPUs; only order-of-magnitude differences are read. + +## Arms + +- **A** full scan of `x`, `v`; mask-risc `x < t` under the validity plane, then COUNT and masked SUM over the borrowed Arrow buffers. +- **C** sorted, no NULLs only: `with_row_addr_prefilter` over the prefix `[0, k)`, then a Range-mask SUM. Uses the fixture's known order. +- **Dz** `filter("xz < t")`, project `v`; no recheck in mask-risc. +- **Db** `filter("xb < t")`, project `v`. +- **N** `filter("xb < t")`, empty projection, COUNT only. + +## Results (bytes read / warm ms) + +| layout | sel | A | C | Dz | Db | N | +|---|---|---|---|---|---|---| +| sorted | 0.5 | 3.87 MB / 7.5 | 664 KB / 2.1 | 1.85 MB / 5.5 | 664 KB / 8.6 | 0 / 5.7 | +| sorted | 0.01 | 3.83 MB / 5.9 | 14 KB / 0.8 | 48 KB / 2.1 | 14 KB / 1.1 | 0 / 0.7 | +| sorted | 0.0001 | 3.83 MB / 6.5 | 1.3 KB / 0.7 | 23 KB / 1.7 | 1.3 KB / 1.1 | 0 / 0.6 | +| partial (shuffled within 64 Ki) | 0.5 | 3.84 MB / 8.1 | — | 1.86 MB / 4.1 | 664 KB / 22.6 | 0 / 14.2 | +| periodic (16 ramps) | 0.01 | 3.83 MB / 5.6 | — | 432 KB / 3.4 | 21 KB / 3.0 | 0 / 1.8 | +| shuffled | 0.5 | 3.96 MB / 9.1 | — | 3.96 MB / 9.8 | 1.33 MB / 80.4 | 0 / 35.4 | +| shuffled | 0.0001 | 3.96 MB / 5.3 | — | 2.17 MB / 5.9 | 214 KB / 2.6 | 0 / 0.9 | + +NULLs (every 17th row) change none of the conclusions. After deleting +`x % 10 = 0`, N's COUNT stays exact (471,856 / 9,439 / 93) with 0 data bytes. + +## Readings + +1. **Pruning happens before the fold, and it is selectivity-proportional on + ordered data.** Sorted: Db and C read the same bytes, 664 KB at 50% down + to 1.3 KB at 0.01%, against a constant ~3.8 MB full scan. Nothing + row-sized is materialised on the Java/Quack side: the scanner's survivors + ARE the fold input. +2. **The zone map prunes only what clustering lets it.** Sorted it reads + ~2–20× the B-tree's bytes (zone granularity 8,192 rows); fully shuffled + it reads the whole column at every selectivity above 0.01%. Shuffled + layouts are the control for lost pruning, not a timing target. +3. **Bytes and latency diverge.** On shuffled data at 50%, Db reads a third + of A's bytes and is ~9× slower warm (80 ms vs 9 ms): a scattered + row-address take is worse than a sequential scan. The B-tree pays off + only at low selectivity or on ordered data. +4. **COUNT needs no data column.** N reads 0 data bytes and 0 data iops, + stays exact after deletions, and is the fastest arm below ~1% on every + layout. Caveat: `ExecutionSummaryCounts.bytes_read` is recorded by + `FilteredReadExec`, `TakeExec` and KNN; no scalar-index exec records it, + so B-tree page I/O is NOT in the 0. Cold N at 50% sorted (21.8 ms vs + 5.7 warm) shows that index I/O exists. "0 bytes" means "no data + columns", never "no I/O". +5. **Index build cost:** zone map 22–117 ms, B-tree 136–229 ms per million + rows here; amortised over queries, not per query. + +## Smallest remaining seam + +None is required for this to work: Lance's own `filter()` + index paths do +the pruning, and the fold consumes the scanner's output directly. The one +missing piece is a **cost choice between full scan and index take**, driven +by selectivity and clustering (reading 3 above). That is a planner +decision over existing scanner calls, not a new primitive; it is OPEN until +a real workload measures where the crossover sits. `with_row_addr_prefilter` +(arm C) is only correct when the order is witnessed; nothing in lance-graph +carries that witness yet (`ISS-WITNESSED-RANGE-DOES-NOT-ATTEST-PLANE-ORDER`), +so C stays a probe arm. diff --git a/crates/lance-graph-benches/Cargo.toml b/crates/lance-graph-benches/Cargo.toml index dac084d41..35c1ecf2f 100644 --- a/crates/lance-graph-benches/Cargo.toml +++ b/crates/lance-graph-benches/Cargo.toml @@ -8,6 +8,8 @@ description = "Benchmarks for lance-graph (not published)" [dependencies] lance-graph = { path = "../lance-graph", version = "0.5.3" } lance = { workspace = true } +# `examples/lance_source_probe.rs` builds zone-map and B-tree indices. +lance-index = { workspace = true } arrow = { workspace = true } arrow-array = { workspace = true } arrow-schema = { workspace = true } diff --git a/crates/lance-graph-benches/examples/lance_source_probe.rs b/crates/lance-graph-benches/examples/lance_source_probe.rs new file mode 100644 index 000000000..af6f93404 --- /dev/null +++ b/crates/lance-graph-benches/examples/lance_source_probe.rs @@ -0,0 +1,557 @@ +//! D-TELEPORT-2: can Lance prune source reads before mask-risc folds, without +//! a row-sized intermediate population? +//! +//! Every arm answers the same query over a real Lance 13 dataset: +//! +//! ```text +//! SELECT COUNT(*), SUM(v) WHERE x < t +//! ``` +//! +//! and is checked against a row-by-row oracle computed from the generator. +//! The probe adds no op, no lowering and no Lance-side code: it uses the +//! public scanner (`filter`, `project`, `with_row_addr_prefilter`, +//! `scan_stats_callback`) and folds the returned Arrow columns in place +//! through mask-risc. +//! +//! | arm | what Lance does | what mask-risc does | +//! |---|---|---| +//! | A | full scan of `x`, `v` | `x < t` per batch, then COUNT and SUM(v) | +//! | C | `with_row_addr_prefilter` of the row range the sorted fixture puts `x < t` in | SUM(v) only; the predicate is already the range | +//! | Dz | `filter("xz < t")` through a zone-map index (approximate; Lance rechecks) | SUM(v) only | +//! | Db | `filter("xb < t")` through a B-tree index (exact) | SUM(v) only | +//! | N | `Db` with an empty projection: COUNT only | nothing | +//! +//! `xz` and `xb` hold the same values as `x`, so each index sits on its own +//! column and the planner cannot pick the other one. +//! +//! Neither D arm rechecks `x < t` in mask-risc: Lance's scan has evaluated it +//! (exactly for the B-tree, by its own recheck for the zone map), so the fold +//! reads only `v`. The folds borrow the Arrow buffers; the only per-batch +//! allocation is the validity bitmap for the nullable fixtures (one word per +//! 64 rows, into a buffer reused across batches). +//! +//! I/O is read from Lance's own `ExecutionSummaryCounts` (`bytes_read`, +//! `iops`), so physical reduction is reported apart from latency. Cold runs +//! open a fresh `Dataset` (empty index cache) and try to drop the OS page +//! cache; whether that succeeded is printed, never assumed. +//! +//! ```text +//! cargo run --release -p lance-graph-benches --example lance_source_probe +//! PROBE_N=16777216 cargo run --release -p lance-graph-benches --example lance_source_probe +//! ``` + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Instant; + +use arrow::array::AsArray; +use arrow::datatypes::Int32Type; +use arrow_array::{Array, Int32Array, RecordBatch, RecordBatchIterator}; +use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; +use lance::dataset::scanner::{RowAddrMask, RowAddrTreeMap}; +use lance::dataset::{Dataset, WriteMode, WriteParams}; +use lance::index::DatasetIndexExt; +use lance_graph_mask_risc::exec::{execute_into, Scratch}; +use lance_graph_mask_risc::{ + scratch_words_for, tile_words_for, Foreign, LaneRef, MaskOp, Operand, Out, Planes, Pred, + Program, Terminal, Value, +}; +use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; +use lance_index::IndexType; + +const SCALE: i32 = 1_000_000; +const FRAG_ROWS: usize = 131_072; +const BATCH: usize = 65_536; + +// ───────────────────────── data ───────────────────────── + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Layout { + /// `x` ascending with the row: `x < t` is one leading run. + Sorted, + /// The same values, shuffled: every zone spans the whole domain. + Shuffled, + /// Sorted, then shuffled inside windows of 65,536 rows. + Partial, + /// 16 ascending ramps: `x < t` is 16 disjoint runs. + Periodic, +} + +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn shuffle(&mut self, s: &mut [T]) { + for i in (1..s.len()).rev() { + s.swap(i, (self.next() % (i as u64 + 1)) as usize); + } + } +} + +struct Data { + x: Vec, + v: Vec, + /// `false` where `x` is NULL. + valid: Vec, +} + +fn data(n: usize, layout: Layout, nulls: bool) -> Data { + let mut x: Vec = match layout { + Layout::Periodic => { + let p = n / 16; + (0..n) + .map(|i| ((i % p) as i64 * i64::from(SCALE) / p as i64) as i32) + .collect() + } + _ => (0..n) + .map(|i| (i as i64 * i64::from(SCALE) / n as i64) as i32) + .collect(), + }; + let mut r = Rng(0x05EE_D0F1_A9CE); + match layout { + Layout::Shuffled => r.shuffle(&mut x), + Layout::Partial => x.chunks_mut(65_536).for_each(|c| r.shuffle(c)), + _ => {} + } + let v = (0..n).map(|_| (r.next() % 1_000) as i32).collect(); + let valid = (0..n).map(|i| !(nulls && i % 17 == 0)).collect(); + Data { x, v, valid } +} + +/// The oracle: per row, no Lance, no mask. +fn oracle(d: &Data, t: i32) -> (u64, i64) { + let (mut c, mut s) = (0u64, 0i64); + for i in 0..d.x.len() { + if d.valid[i] && d.x[i] < t { + c += 1; + s += i64::from(d.v[i]); + } + } + (c, s) +} + +fn schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("x", DataType::Int32, true), + Field::new("xz", DataType::Int32, true), + Field::new("xb", DataType::Int32, true), + Field::new("v", DataType::Int32, false), + ])) +} + +/// Write `d` as fragments of `FRAG_ROWS`, then build a zone map on `xz` and a +/// B-tree on `xb`. Returns the dataset and the two build times. +async fn write(d: &Data, path: &str) -> (Dataset, f64, f64) { + let opt = |i: usize| d.valid[i].then_some(d.x[i]); + let col = || Arc::new(Int32Array::from_iter((0..d.x.len()).map(opt))); + let batch = RecordBatch::try_new( + schema(), + vec![col(), col(), col(), Arc::new(Int32Array::from(d.v.clone()))], + ) + .expect("batch"); + let params = WriteParams { + mode: WriteMode::Create, + max_rows_per_file: FRAG_ROWS, + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema()); + let mut ds = Dataset::write(reader, path, Some(params)) + .await + .expect("write"); + + let t = Instant::now(); + ds.create_index_builder( + &["xz"], + IndexType::ZoneMap, + &ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap), + ) + .name("xz_zone".into()) + .await + .expect("zone map"); + let zone_ms = t.elapsed().as_secs_f64() * 1e3; + let t = Instant::now(); + ds.create_index_builder(&["xb"], IndexType::BTree, &ScalarIndexParams::default()) + .name("xb_btree".into()) + .await + .expect("btree"); + let btree_ms = t.elapsed().as_secs_f64() * 1e3; + (ds, zone_ms, btree_ms) +} + +// ───────────────────────── folds ───────────────────────── + +/// Reused fold state: one scratch arena and one validity buffer, sized for +/// the largest batch, allocated once per probe. +struct Folder { + scratch: Vec, + validity: Vec, +} + +impl Folder { + fn new() -> Self { + Folder { + scratch: vec![0; scratch_words_for(tile_words_for(BATCH), 2).expect("small")], + validity: vec![0; BATCH.div_ceil(64)], + } + } + + fn run(&mut self, p: &Program, planes: &Planes<'_>) -> Value { + let mut s = Scratch::over( + &mut self.scratch, + tile_words_for(planes.n_rows), + p.scratch_slots as usize, + ) + .expect("scratch"); + execute_into(p, planes, &Foreign::NONE, &mut s, Out::None).expect("runs") + } + + /// Arm A: `x < t` (gated by validity when `x` has NULLs), then COUNT and + /// SUM(v), over the batch's own buffers. + fn predicate(&mut self, b: &RecordBatch, t: i32) -> (u64, i64) { + let n = b.num_rows(); + let x = b + .column_by_name("x") + .expect("x") + .as_primitive::(); + let v = b + .column_by_name("v") + .expect("v") + .as_primitive::(); + let lanes = [LaneRef::I32(x.values()), LaneRef::I32(v.values())]; + let gated = x.null_count() > 0; + if gated { + let words = n.div_ceil(64); + self.validity[..words].iter_mut().for_each(|w| *w = 0); + let nb = x.nulls().expect("nulls"); + for i in 0..n { + if nb.is_valid(i) { + self.validity[i / 64] |= 1 << (i % 64); + } + } + } + let validity = std::mem::take(&mut self.validity); + let masks: [&[u64]; 1] = [&validity[..n.div_ceil(64)]]; + let planes = Planes { + n_rows: n, + masks: if gated { &masks } else { &[] }, + lanes: &lanes, + }; + let pred = MaskOp::Pred { + pred: Pred::LtI32 { lane: 0, t }, + under: gated.then_some(Operand::Plane(0)), + dst: 0, + }; + let m = Operand::Scratch(0); + let c = match self.run( + &Program::new(vec![pred], Terminal::Count { mask: m }), + &planes, + ) { + Value::Count(c) => c as u64, + other => panic!("{other:?}"), + }; + let s = match self.run( + &Program::new(vec![pred], Terminal::MaskedSumI32 { mask: m, lane: 1 }), + &planes, + ) { + Value::SumI64(s) => s, + other => panic!("{other:?}"), + }; + self.validity = validity; + (c, s) + } + + /// Arms C and D: every returned row is a survivor, so COUNT is the row + /// count and SUM(v) folds `v` under an all-rows range (no lane read for + /// the mask). + fn survivors(&mut self, b: &RecordBatch) -> (u64, i64) { + let n = b.num_rows(); + let v = b + .column_by_name("v") + .expect("v") + .as_primitive::(); + let lanes = [LaneRef::I32(v.values())]; + let planes = Planes { + n_rows: n, + masks: &[], + lanes: &lanes, + }; + let p = Program::new( + vec![MaskOp::Pred { + pred: Pred::Range { + lo: 0, + hi: n as u32, + }, + under: None, + dst: 0, + }], + Terminal::MaskedSumI32 { + mask: Operand::Scratch(0), + lane: 0, + }, + ); + match self.run(&p, &planes) { + Value::SumI64(s) => (n as u64, s), + other => panic!("{other:?}"), + } + } +} + +// ───────────────────────── arms ───────────────────────── + +#[derive(Default, Clone, Copy)] +struct Io { + bytes: usize, + iops: usize, +} + +#[derive(Clone, Copy)] +enum Arm { + A, + C, + Dz, + Db, + N, +} + +impl Arm { + fn name(self) -> &'static str { + match self { + Arm::A => "A full scan + fold", + Arm::C => "C row-range prefilter", + Arm::Dz => "Dz zone map filter", + Arm::Db => "Db btree filter", + Arm::N => "N btree COUNT, no columns", + } + } +} + +struct Run { + count: u64, + sum: Option, + rows_read: u64, + io: Io, + ms: f64, + /// Time to build the row-address mask (arm C only). + mask_ms: f64, +} + +/// The row addresses holding `x < t` on the sorted fixture: rows `0..k`, +/// split across fragments of `FRAG_ROWS` (fragment ids 0, 1, … in write +/// order, checked at setup). +fn sorted_prefix_mask(k: usize) -> RowAddrMask { + let mut m = RowAddrTreeMap::new(); + let mut row = 0usize; + let mut frag = 0u64; + while row < k { + let take = (k - row).min(FRAG_ROWS); + m.insert_range((frag << 32)..((frag << 32) + take as u64)); + row += take; + frag += 1; + } + RowAddrMask::from_allowed(m) +} + +async fn run(ds: &Dataset, arm: Arm, t: i32, k_sorted: Option, f: &mut Folder) -> Run { + let io = Arc::new((AtomicUsize::new(0), AtomicUsize::new(0))); + let sink = Arc::clone(&io); + let start = Instant::now(); + let mut sc = ds.scan(); + sc.batch_size(BATCH); + sc.scan_stats_callback(Arc::new(move |c| { + sink.0.fetch_add(c.bytes_read, Ordering::Relaxed); + sink.1.fetch_add(c.iops, Ordering::Relaxed); + })); + let mut mask_ms = 0.0; + match arm { + Arm::A => { + sc.project(&["x", "v"]).expect("project"); + } + Arm::C => { + let tm = Instant::now(); + let mask = sorted_prefix_mask(k_sorted.expect("sorted only")); + mask_ms = tm.elapsed().as_secs_f64() * 1e3; + sc.with_row_addr_prefilter(mask); + sc.project(&["v"]).expect("project"); + } + Arm::Dz => { + sc.filter(&format!("xz < {t}")).expect("filter"); + sc.project(&["v"]).expect("project"); + } + Arm::Db => { + sc.filter(&format!("xb < {t}")).expect("filter"); + sc.project(&["v"]).expect("project"); + } + Arm::N => { + sc.filter(&format!("xb < {t}")).expect("filter"); + sc.project::<&str>(&[]).expect("project"); + } + } + let mut stream = sc.try_into_stream().await.expect("stream"); + let (mut count, mut sum, mut rows) = (0u64, 0i64, 0u64); + while let Some(b) = stream.try_next().await.expect("batch") { + rows += b.num_rows() as u64; + let (c, s) = match arm { + Arm::A => f.predicate(&b, t), + Arm::N => (b.num_rows() as u64, 0), + _ => f.survivors(&b), + }; + count += c; + sum += s; + } + drop(stream); + let ms = start.elapsed().as_secs_f64() * 1e3; + Run { + count, + sum: (!matches!(arm, Arm::N)).then_some(sum), + rows_read: rows, + io: Io { + bytes: io.0.load(Ordering::Relaxed), + iops: io.1.load(Ordering::Relaxed), + }, + ms, + mask_ms, + } +} + +fn drop_page_cache() -> bool { + std::fs::write("/proc/sys/vm/drop_caches", "3").is_ok() +} + +// ───────────────────────── main ───────────────────────── + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + let n: usize = std::env::var("PROBE_N") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(1 << 20); + let sels = [0.5, 0.1, 0.01, 0.001, 0.0001]; + let dir = tempfile::tempdir().expect("tempdir"); + let can_drop = drop_page_cache(); + println!( + "n = {n}, fragments of {FRAG_ROWS}, batch {BATCH}; OS page-cache drop {}", + if can_drop { + "WORKS (cold = dropped)" + } else { + "NOT PERMITTED (cold = fresh Dataset only)" + } + ); + + let cases = [ + (Layout::Sorted, false), + (Layout::Sorted, true), + (Layout::Partial, false), + (Layout::Periodic, false), + (Layout::Shuffled, false), + (Layout::Shuffled, true), + ]; + let mut f = Folder::new(); + for (layout, nulls) in cases { + let d = data(n, layout, nulls); + let path = dir + .path() + .join(format!("{layout:?}-{nulls}")) + .to_string_lossy() + .to_string(); + let tw = Instant::now(); + let (_, zone_ms, btree_ms) = write(&d, &path).await; + let write_ms = tw.elapsed().as_secs_f64() * 1e3 - zone_ms - btree_ms; + let ds = Dataset::open(&path).await.expect("open"); + + // The fragment ids `sorted_prefix_mask` assumes. + let ids: Vec = ds.get_fragments().iter().map(|fr| fr.id() as u64).collect(); + assert_eq!( + ids, + (0..ids.len() as u64).collect::>(), + "fragment ids" + ); + + println!( + "\n== {layout:?}, nulls {nulls}: write {write_ms:.0} ms, zone-map build {zone_ms:.0} ms, btree build {btree_ms:.0} ms ==" + ); + println!( + " {:<7} {:<28} {:>9} {:>11} {:>7} {:>10} {:>10} {:>9}", + "sel", "arm", "rows", "bytes", "iops", "cold ms", "warm ms", "mask ms" + ); + for sel in sels { + let t = (sel * f64::from(SCALE)) as i32; + let (want_c, want_s) = oracle(&d, t); + let k_sorted = + (layout == Layout::Sorted && !nulls).then(|| d.x.partition_point(|&x| x < t)); + for arm in [Arm::A, Arm::C, Arm::Dz, Arm::Db, Arm::N] { + if matches!(arm, Arm::C) && k_sorted.is_none() { + continue; + } + // Cold: a fresh Dataset (empty index cache), page cache dropped + // when permitted. + drop_page_cache(); + let cold_ds = Dataset::open(&path).await.expect("open"); + let cold = run(&cold_ds, arm, t, k_sorted, &mut f).await; + // Warm: best of three on one handle. + let mut warm = run(&ds, arm, t, k_sorted, &mut f).await; + for _ in 0..2 { + let w = run(&ds, arm, t, k_sorted, &mut f).await; + if w.ms < warm.ms { + warm = w; + } + } + for r in [&cold, &warm] { + assert_eq!( + r.count, + want_c, + "{layout:?} {nulls} {sel} {}: count", + arm.name() + ); + if let Some(s) = r.sum { + assert_eq!(s, want_s, "{layout:?} {nulls} {sel} {}: sum", arm.name()); + } + } + println!( + " {:<7} {:<28} {:>9} {:>11} {:>7} {:>10.2} {:>10.2} {:>9.3}", + sel, + arm.name(), + warm.rows_read, + warm.io.bytes, + warm.io.iops, + cold.ms, + warm.ms, + warm.mask_ms + ); + } + } + } + + deletion_aware_count(n, dir.path()).await; +} + +/// The COUNT-only arm against deletions: does an exact index selection give a +/// deletion-aware cardinality, and what does it read? +async fn deletion_aware_count(n: usize, root: &std::path::Path) { + let d = data(n, Layout::Sorted, false); + let path = root.join("deleted").to_string_lossy().to_string(); + let (mut ds, _, _) = write(&d, &path).await; + // Delete every 10th row by `v`-independent position: rows whose x is a + // multiple of 10 in the sorted fixture. + ds.delete("x % 10 = 0").await.expect("delete"); + let ds = Dataset::open(&path).await.expect("open"); + println!("\n== COUNT after deleting rows with x % 10 = 0 (sorted, btree) =="); + let mut f = Folder::new(); + for sel in [0.5, 0.01, 0.0001] { + let t = (sel * f64::from(SCALE)) as i32; + let want = d.x.iter().filter(|&&x| x < t && x % 10 != 0).count() as u64; + let r = run(&ds, Arm::N, t, None, &mut f).await; + assert_eq!(r.count, want, "deletion-aware count at {sel}"); + let full = run(&ds, Arm::Db, t, None, &mut f).await; + assert_eq!(full.count, want); + println!( + " sel {sel:<7} COUNT {:>9} N bytes {:>9} iops {:>4} {:>8.2} ms | Db bytes {:>9} {:>8.2} ms", + r.count, r.io.bytes, r.io.iops, r.ms, full.io.bytes, full.ms + ); + } +} From d13dc8a06706b7b285b90557e3365fdcfcd753d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 12:38:25 +0000 Subject: [PATCH 2/2] quack: D-PHOTO-0 probe over 64 resident populations 64 populations x 1,048,576 addresses in four regimes (sparse, dense, clustered, 256 archetypes). Compares quack-fused and in-place plane lowering, single-range extents, per-address u64 signatures and signature-equivalence classes on COUNT, Any, SUM and Rows; the canonical expression plus 20 random ones equal the oracle in every arm. Also measures transpose, class build, 1,000-address updates and spill/reload. Bit-sliced planes answer in ~0.03-0.05 ms per query regardless of data; signatures lose by 250-600x; classes win only at a few hundred classes and cost 30-100x more per update. Board: entries for D-TELEPORT-2 and D-PHOTO-0, D-TELEPORT-1 marked shipped (#1487). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015JH7YkXs5us6g3tG8NiFfP --- .claude/board/STATUS_BOARD.md | 4 +- .../2026-10-11-photo-probe-64-populations.md | 72 +++ .claude/board/entries/README.md | 4 +- .../lance-graph-quack/examples/photo_probe.rs | 545 ++++++++++++++++++ 4 files changed, 623 insertions(+), 2 deletions(-) create mode 100644 .claude/board/entries/2026-10-11-photo-probe-64-populations.md create mode 100644 crates/lance-graph-quack/examples/photo_probe.rs diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index efc49c557..202432ffb 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -7,7 +7,9 @@ Plan: `.claude/plans/sql-polyfill-r2il-v1.md`. | **D-SQLP-0** | Plan + board rows | Shipped (#1476, #1478) | — | | **D-SQLP-1** | `FoldDialect` moved into a `lance-graph-r2il` crate | Shipped (#1479; workspace-excluded, own CI step; probe 40/40 unchanged, crate 7 tests) | existing `row_bridge.rs` tests unchanged and green; one arity table | | **D-TELEPORT-0** | Source-restricted fold probe: scan/join vs gated gather vs extent; range intersection; sorted-lane crossing point | Shipped (#1484, probe) | every arm equals the oracle on clustered and scattered layouts | -| **D-TELEPORT-1** | quack `fold_ranges`: `Range ∧ Range` on one column lowers to the intersected range (empty stays a valid empty range; different columns, `OR`/`NOT` and `lo > hi` left alone) | In PR | one op and `Lowering::Range` for COUNT; 400 seeded conjunctions equal the oracle under both lowerings; disable runs red | +| **D-TELEPORT-1** | quack `fold_ranges`: `Range ∧ Range` on one column lowers to the intersected range (empty stays a valid empty range; different columns, `OR`/`NOT` and `lo > hi` left alone) | Shipped (#1487) | one op and `Lowering::Range` for COUNT; 400 seeded conjunctions equal the oracle under both lowerings; disable runs red | +| **D-TELEPORT-2** | Lance 13 source-pruning probe: full scan vs row-range prefilter vs zone map vs B-tree vs index-only COUNT, sorted/partial/periodic/shuffled, with and without NULLs | In PR (probe) | every arm equals the oracle; index COUNT reads no data columns and stays exact after deletes | +| **D-PHOTO-0** | 64 resident populations × 1 Mi addresses: bit-sliced planes vs extents vs u64 signatures vs signature classes; update and spill costs | In PR (probe) | canonical + 20 random expressions equal the oracle under COUNT, Any, SUM and Rows in four regimes | | **D-SQLP-2** | First procedures (`filter_lt`, `sum`, `group_sum`, `margin`) as inlined R2IL call sequences (`lance-graph-r2il::procs`); `FoldDialect` wires `IntOr` and adds a fused mode (mask-risc `fuse`) beside the in-place mode | Shipped (#1484; 8 tests, six disable runs red; probe suite unchanged) | scalar-reference parity; disable runs red; both modes keep the same rows, fused holds ternlogs | | **D-SQLP-3** | Benchmark: direct ndarray vs hand-built `Program` vs interpreted procedure | Queued | interpreted ≤ 1.05× hand-built at 65,536 and 1M rows (pre-registered) | | **D-SQLP-4** | Build-time generator: declared procedure → Rust with const-generic opcodes | Queued, gated on D-SQLP-3 | ≤ 1.05× direct ndarray; no interpreter dispatch in the hot loop | diff --git a/.claude/board/entries/2026-10-11-photo-probe-64-populations.md b/.claude/board/entries/2026-10-11-photo-probe-64-populations.md new file mode 100644 index 000000000..1513c5cb4 --- /dev/null +++ b/.claude/board/entries/2026-10-11-photo-probe-64-populations.md @@ -0,0 +1,72 @@ +# 2026-10-11 — 64 resident populations: bit-sliced planes vs signatures vs classes (D-PHOTO-0) + +**Status:** MEASURED. Probe only, no production change. +`crates/lance-graph-quack/examples/photo_probe.rs`. + +```text +CARGO_PROFILE_RELEASE_DEBUG=0 cargo run --release -p lance-graph-quack --example photo_probe +``` + +The question: with 64 already-derived populations over 1,048,576 addresses +resident, which representation answers boolean questions over them fastest, +and what does keeping them cost? Every arm answers COUNT, Any, SUM and Rows +for the canonical `(P0|P1|P2|P3) & P17 & !P29` plus 20 random expressions, +asserted equal to an oracle. Population creation is a generator stand-in; +no real reasoning baseline exists yet. + +## Arms + +- **B** quack `lower_fused` / **Bi** `lower` (in-place) over the 64 mask-risc planes. +- **D** `execute_extent` over each expression's support range, from one occupied range per population (AND intersects, OR takes the hull, NOT gives up). +- **C** probe-local transpose to one u64 signature per address; the expression is tested per address. +- **E** HashMap signature → (count, sum); the expression is tested per class. + +## Results (per query, COUNT, steady state; 1,000 queries for C/E) + +| regime | B | Bi | D | C | E | classes | +|---|---|---|---|---|---|---| +| sparse (1%) | 0.11 ms | 0.046 ms | 0.037 ms | 12.2 ms | 0.39 ms | 24,180 | +| dense (50%) | 0.11 ms | 0.048 ms | 0.036 ms | 22.8 ms | 29.9 ms | 1,048,576 | +| clustered | 0.11 ms | 0.047 ms | 0.028 ms | 12.3 ms | 0.006 ms | 274 | +| archetypes | 0.13 ms | 0.048 ms | 0.036 ms | 22.5 ms | 0.009 ms | 256 | + +Costs: transpose 4–51 ms; class build 14–176 ms; updating 1,000 addresses +of one population: planes 0.005–0.012 ms, signatures + classes 0.30–0.95 ms. +Spill of 8 MiB of planes 19–56 ms, reload 1.7–7.3 ms (page cache, not +cold). Nonzero words: sparse 47.5%, dense 100%, clustered 1.6%. + +## Readings + +1. **Bit-sliced planes win on every regime but one.** They touch only the + planes the expression names (~5 of 64, ~82k words) and cost the same + whatever the data looks like. Signatures (C) always test all 1M + addresses and lose by 250–600×. +2. **Classes win only when there are few of them.** At 256–274 classes E is + ~5× faster than planes; at 24k classes it is ~10× slower; at 1M classes + (dense) it is slower than C. Class count is data-dependent and unknown + in advance, and every update rebuilds them; planes update 30–100× cheaper. +3. **One occupied range per population is a weak descriptor.** On clustered + data the populations occupy 1.6% of the words, yet D cuts per-query + words only from ~82k to ~56k, because OR takes the hull of four runs. + Multiple ranges per population would be needed to use the clustering. +4. **Spill/replay is cheap at this size** (8 MiB in tens of ms), and a + sparse spill is worth it only for clustered populations. + +## Against the real alpha channel + +`AlphaMask` must not hold these: `alpha.rs` keeps motion and reference +frames as separate types that "may share a bitmap layout; they must not +share a type, a name or a conversion" (`ISS-LXA-ALPHA-FIT`). A cached +population therefore needs its own type over the same word layout, scoped +to (universe, snapshot version, expression identity, principal and policy +version for authorization-derived populations). Alpha persistence exists +only in MedCare-rs test writers, so there is no shipped spill path to reuse. + +## Smallest missing seam + +None for evaluation: quack + mask-risc already answer every query shape +here, fused, at ~0.03–0.05 ms. What is missing is a **population-cache +descriptor** (the scope tuple above plus exact/approximate and dependency +state) so a cached plane can be reused safely; and, if clustered data +matters, multi-range extents. Both are OPEN until a real reasoning +workload produces populations to measure. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 897b79d3c..986f450da 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,12 +25,14 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -303 entries, 2026-08-06 .. 2026-10-11. +305 entries, 2026-08-06 .. 2026-10-11. | date | entry id | finding | file | |---|---|---|---| | 2026-10-11 | `D-TELEPORT-0` | | [2026-10-11-teleport-probe-source-restriction.md](2026-10-11-teleport-probe-source-restriction.md) | | 2026-10-11 | `synergy-ledger-classid-hotplug` | | [2026-10-11-synergy-ledger-classid-hotplug.md](2026-10-11-synergy-ledger-classid-hotplug.md) | +| 2026-10-11 | `D-PHOTO-0` | | [2026-10-11-photo-probe-64-populations.md](2026-10-11-photo-probe-64-populations.md) | +| 2026-10-11 | `D-TELEPORT-2` | | [2026-10-11-lance-source-probe.md](2026-10-11-lance-source-probe.md) | | 2026-10-11 | `coresearch-labels-address-lifetime` | | [2026-10-11-coresearch-labels-address-lifetime.md](2026-10-11-coresearch-labels-address-lifetime.md) | | 2026-10-11 | `coresearch-instance-authz` | | [2026-10-11-coresearch-instance-authz.md](2026-10-11-coresearch-instance-authz.md) | | 2026-10-10 | `D-XGP-8` | | [2026-10-10-woa-anchors-timesheet.md](2026-10-10-woa-anchors-timesheet.md) | diff --git a/crates/lance-graph-quack/examples/photo_probe.rs b/crates/lance-graph-quack/examples/photo_probe.rs new file mode 100644 index 000000000..752f0c40a --- /dev/null +++ b/crates/lance-graph-quack/examples/photo_probe.rs @@ -0,0 +1,545 @@ +//! D-PHOTO-0: exposing 64 resident populations instead of recomputing them. +//! +//! 64 membership bitplanes `P0..P63` over one universe of 2^20 addresses are +//! built once. Each query is a Boolean expression over them ("an exposure"), +//! answered as COUNT, EXISTS, KEEP or SUM. The question is what a NEW +//! expression costs once the populations exist, and what each cached +//! representation costs to build, update, spill and reload. +//! +//! | arm | representation | per-query work | +//! |---|---|---| +//! | B | 64 bitplanes, Quack `lower_fused` → mask-risc | the words of the planes the expression names | +//! | Bi | same, Quack `lower` (in place) | same words, more passes | +//! | D | B over the expression's support range only (`execute_extent`) | the words inside the range | +//! | C | one `u64` signature per address (probe-local transpose) | every address | +//! | E | signature → count classes (probe-local) | one test per distinct signature | +//! +//! There is no executable "reasoning" producing these populations in the +//! tree, so arm A (recompute the reasoning per query) has no honest +//! baseline; the generator's cost is printed as population-creation cost +//! and labelled as such. C and E are probe-local because nothing in +//! lance-graph or ndarray transposes bitplanes to per-address signatures. +//! +//! Every answer is checked against a per-address oracle that reads the +//! planes bit by bit (on the first queries of each batch; the repeated-query +//! timings reuse checked expressions). +//! +//! ```text +//! CARGO_PROFILE_RELEASE_DEBUG=0 cargo run --release -p lance-graph-quack --example photo_probe +//! ``` + +use std::collections::HashMap; +use std::time::Instant; + +use lance_graph_mask_risc::exec::{execute_extent, execute_into, Scratch}; +use lance_graph_mask_risc::{words_for, Foreign, LaneRef, Out, Planes, Program, Value}; +use lance_graph_quack::{lower, lower_fused, Agg, Col, Filter, Mask, Query}; + +const N: usize = 1 << 20; +const W: usize = N / 64; +const P: usize = 64; + +// ───────────────────────── populations ───────────────────────── + +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn below(&mut self, m: u64) -> u64 { + self.next() % m + } +} + +#[derive(Clone, Copy, Debug)] +enum Regime { + /// Independent populations, each ~1% of addresses. + Sparse, + /// Independent populations, each ~50%. + Dense, + /// Each population is 4 runs of ~4,096 addresses: mostly empty words. + Clustered, + /// Every address copies one of 256 archetype signatures: low diversity, + /// heavily overlapping populations. + Archetypes, +} + +/// Build the 64 planes; the time this takes stands in for "the reasoning". +fn populations(regime: Regime) -> Vec> { + let mut r = Rng(0xF0_70 + regime as u64); + let mut planes = vec![vec![0u64; W]; P]; + match regime { + Regime::Sparse | Regime::Dense => { + let d = if matches!(regime, Regime::Sparse) { + 1 + } else { + 50 + }; + for p in planes.iter_mut() { + for a in 0..N { + if r.below(100) < d { + p[a / 64] |= 1 << (a % 64); + } + } + } + } + Regime::Clustered => { + for p in planes.iter_mut() { + for _ in 0..4 { + let lo = r.below((N - 4096) as u64) as usize; + for a in lo..lo + 4096 { + p[a / 64] |= 1 << (a % 64); + } + } + } + } + Regime::Archetypes => { + let arche: Vec = (0..256).map(|_| r.next()).collect(); + for a in 0..N { + let s = arche[r.below(256) as usize]; + for (i, p) in planes.iter_mut().enumerate() { + if s >> i & 1 == 1 { + p[a / 64] |= 1 << (a % 64); + } + } + } + } + } + planes +} + +fn bit(p: &[u64], a: usize) -> bool { + p[a / 64] >> (a % 64) & 1 == 1 +} + +// ───────────────────────── expressions ───────────────────────── + +#[derive(Clone, Debug)] +enum Ex { + Leaf(u16), + And(Vec), + Or(Vec), + Not(Box), +} + +impl Ex { + fn filter(&self) -> Filter { + match self { + Ex::Leaf(i) => Filter::Plane(Mask(*i)), + Ex::And(v) => Filter::And(v.iter().map(Ex::filter).collect()), + Ex::Or(v) => Filter::Or(v.iter().map(Ex::filter).collect()), + Ex::Not(e) => Filter::Not(Box::new(e.filter())), + } + } + /// The oracle reads the planes, one bit per leaf per address. + fn on_planes(&self, planes: &[Vec], a: usize) -> bool { + match self { + Ex::Leaf(i) => bit(&planes[*i as usize], a), + Ex::And(v) => v.iter().all(|e| e.on_planes(planes, a)), + Ex::Or(v) => v.iter().any(|e| e.on_planes(planes, a)), + Ex::Not(e) => !e.on_planes(planes, a), + } + } + fn on_sig(&self, s: u64) -> bool { + match self { + Ex::Leaf(i) => s >> i & 1 == 1, + Ex::And(v) => v.iter().all(|e| e.on_sig(s)), + Ex::Or(v) => v.iter().any(|e| e.on_sig(s)), + Ex::Not(e) => !e.on_sig(s), + } + } + fn leaves(&self, out: &mut Vec) { + match self { + Ex::Leaf(i) => { + if !out.contains(i) { + out.push(*i) + } + } + Ex::And(v) | Ex::Or(v) => v.iter().for_each(|e| e.leaves(out)), + Ex::Not(e) => e.leaves(out), + } + } + /// A row range outside which the expression is certainly false, from + /// each population's own occupied range: AND intersects, OR takes the + /// hull, NOT gives up. The same algebra as quack's `fold_ranges`, at the + /// population level. + fn support(&self, occ: &[(usize, usize)]) -> (usize, usize) { + match self { + Ex::Leaf(i) => occ[*i as usize], + Ex::And(v) => v + .iter() + .map(|e| e.support(occ)) + .fold((0, N), |(a, b), (c, d)| { + let lo = a.max(c); + (lo, b.min(d).max(lo)) + }), + Ex::Or(v) => v + .iter() + .map(|e| e.support(occ)) + .filter(|(a, b)| a < b) + .fold((N, 0), |(a, b), (c, d)| (a.min(c), b.max(d))) + .pipe(|(a, b)| if a > b { (0, 0) } else { (a, b) }), + Ex::Not(_) => (0, N), + } + } +} + +trait Pipe: Sized { + fn pipe(self, f: impl FnOnce(Self) -> R) -> R { + f(self) + } +} +impl Pipe for T {} + +/// The canonical example: (P0 | P1 | P2 | P3) & P17 & !P29. +fn canonical() -> Ex { + Ex::And(vec![ + Ex::Or((0..4).map(Ex::Leaf).collect()), + Ex::Leaf(17), + Ex::Not(Box::new(Ex::Leaf(29))), + ]) +} + +/// A random expression of 3-7 distinct leaves, mixing AND / OR / NOT. +fn random_ex(r: &mut Rng) -> Ex { + let k = 3 + r.below(5) as usize; + let mut leaves: Vec = Vec::new(); + while leaves.len() < k { + let l = r.below(P as u64) as u16; + if !leaves.contains(&l) { + leaves.push(l); + } + } + let mut terms: Vec = leaves + .into_iter() + .map(|l| { + if r.below(4) == 0 { + Ex::Not(Box::new(Ex::Leaf(l))) + } else { + Ex::Leaf(l) + } + }) + .collect(); + while terms.len() > 1 { + let b = terms.pop().unwrap(); + let a = terms.pop().unwrap(); + terms.push(if r.below(2) == 0 { + Ex::And(vec![a, b]) + } else { + Ex::Or(vec![a, b]) + }); + } + terms.pop().unwrap() +} + +// ───────────────────────── execution ───────────────────────── + +fn run( + p: &Program, + planes: &Planes<'_>, + extent: (usize, usize), + keep: Option<&mut [u64]>, +) -> Value { + let mut s = Scratch::for_program(p, N).expect("scratch"); + let out = match keep { + Some(k) => Out::Mask(k), + None => Out::None, + }; + if extent == (0, N) { + execute_into(p, planes, &Foreign::NONE, &mut s, out).expect("runs") + } else { + execute_extent(p, planes, &Foreign::NONE, &mut s, out, extent.0..extent.1).expect("runs") + } +} + +fn count_of(v: Value) -> u64 { + match v { + Value::Count(c) => c as u64, + other => panic!("{other:?}"), + } +} + +fn ms(t: Instant) -> f64 { + t.elapsed().as_secs_f64() * 1e3 +} + +// ───────────────────────── main ───────────────────────── + +fn main() { + let vals: Vec = (0..N).map(|i| (i % 1000) as i32).collect(); + for regime in [ + Regime::Sparse, + Regime::Dense, + Regime::Clustered, + Regime::Archetypes, + ] { + section(regime, &vals); + } +} + +fn section(regime: Regime, vals: &[i32]) { + let t = Instant::now(); + let pl = populations(regime); + let create_ms = ms(t); + let masks: Vec<&[u64]> = pl.iter().map(Vec::as_slice).collect(); + let lanes = [LaneRef::I32(vals)]; + let planes = Planes { + n_rows: N, + masks: &masks, + lanes: &lanes, + }; + + // Population descriptors: occupied word range per plane (for D). + let t = Instant::now(); + let occ: Vec<(usize, usize)> = pl + .iter() + .map(|p| { + let lo = p.iter().position(|w| *w != 0).map_or(N, |w| w * 64); + let hi = p + .iter() + .rposition(|w| *w != 0) + .map_or(0, |w| ((w + 1) * 64).min(N)); + if lo >= hi { + (0, 0) + } else { + (lo, hi) + } + }) + .collect(); + let occ_ms = ms(t); + + // C: transpose to one signature per address. + let t = Instant::now(); + let mut sig = vec![0u64; N]; + for (i, p) in pl.iter().enumerate() { + for (w, &word) in p.iter().enumerate() { + let mut x = word; + while x != 0 { + let b = x.trailing_zeros() as usize; + sig[w * 64 + b] |= 1 << i; + x &= x - 1; + } + } + } + let transpose_ms = ms(t); + + // E: signature → (count, sum of vals). + let t = Instant::now(); + let mut classes: HashMap = HashMap::new(); + for (a, &s) in sig.iter().enumerate() { + let e = classes.entry(s).or_insert((0, 0)); + e.0 += 1; + e.1 += i64::from(vals[a]); + } + let class_ms = ms(t); + + println!( + "\n== {regime:?}: create {create_ms:.0} ms (stand-in for reasoning) | occupied ranges {occ_ms:.2} ms | transpose {transpose_ms:.0} ms | {} signature classes built in {class_ms:.0} ms ==", + classes.len() + ); + println!( + " resident: planes {} KiB, signatures {} KiB, classes ~{} KiB", + P * W * 8 / 1024, + N * 8 / 1024, + classes.len() * 24 / 1024 + ); + + // Correctness: the canonical expression plus 20 random ones, every + // representation and every terminal, against the per-address oracle. + let mut r = Rng(0xE7_905E); + let mut checked: Vec = vec![canonical()]; + checked.extend((0..20).map(|_| random_ex(&mut r))); + for ex in &checked { + let want: Vec = (0..N).filter(|&a| ex.on_planes(&pl, a)).collect(); + let want_sum: i64 = want.iter().map(|&a| i64::from(vals[a])).sum(); + let f = ex.filter(); + let sup = ex.support(&occ); + for (fused, ext) in [(true, (0, N)), (false, (0, N)), (true, sup)] { + let lo = |agg| { + let q = Query { + filter: f.clone(), + agg, + }; + if fused { lower_fused(&q) } else { lower(&q) }.expect("lowers") + }; + let c = if ext.0 >= ext.1 { + 0 + } else { + count_of(run(&lo(Agg::Count), &planes, ext, None)) + }; + assert_eq!( + c, + want.len() as u64, + "{regime:?} count {ex:?} fused={fused} ext={ext:?}" + ); + if ext.0 < ext.1 { + let any = matches!(run(&lo(Agg::Any), &planes, ext, None), Value::Bool(true)); + assert_eq!(any, !want.is_empty()); + let s = match run(&lo(Agg::SumI32(Col(0))), &planes, ext, None) { + Value::SumI64(s) => s, + other => panic!("{other:?}"), + }; + assert_eq!(s, want_sum); + let mut kept = vec![0u64; words_for(N)]; + run(&lo(Agg::Rows), &planes, ext, Some(&mut kept)); + let got: Vec = (0..N).filter(|&a| bit(&kept, a)).collect(); + assert_eq!(got, want); + } + } + let c_sig = sig.iter().filter(|&&s| ex.on_sig(s)).count(); + assert_eq!(c_sig, want.len()); + let (c_cls, s_cls) = classes + .iter() + .filter(|(s, _)| ex.on_sig(**s)) + .fold((0u64, 0i64), |(c, s), (_, v)| (c + v.0, s + v.1)); + assert_eq!((c_cls, s_cls), (want.len() as u64, want_sum)); + } + println!(" correctness: canonical + 20 random expressions, all arms and terminals equal the oracle"); + + // Work and time per new exposure: 1..10,000 fresh expressions, COUNT. + println!( + " {:>6} {:>12} {:>12} {:>12} {:>12} {:>12} | words/query B {:>7} D {:>7}", + "q", "B fused", "B in-place", "D extent", "C sigs", "E classes", "", "" + ); + for nq in [1usize, 10, 100, 1_000, 10_000] { + let mut r = Rng(0x0901 + nq as u64); + let exs: Vec = (0..nq).map(|_| random_ex(&mut r)).collect(); + let mut words_b = 0usize; + let mut words_d = 0usize; + let mut times = [0f64; 5]; + let mut answers = [0u64; 5]; + for ex in &exs { + let f = ex.filter(); + let mut leaves = Vec::new(); + ex.leaves(&mut leaves); + let sup = ex.support(&occ); + words_b += leaves.len() * W; + words_d += leaves.len() * (sup.1.saturating_sub(sup.0)).div_ceil(64); + let q = Query { + filter: f, + agg: Agg::Count, + }; + let t = Instant::now(); + answers[0] += count_of(run(&lower_fused(&q).unwrap(), &planes, (0, N), None)); + times[0] += ms(t); + let t = Instant::now(); + answers[1] += count_of(run(&lower(&q).unwrap(), &planes, (0, N), None)); + times[1] += ms(t); + let t = Instant::now(); + if sup.0 < sup.1 { + answers[2] += count_of(run(&lower_fused(&q).unwrap(), &planes, sup, None)); + } + times[2] += ms(t); + // C and E cost the same per query at every repetition count, so + // they stop at 1,000 queries; their per-query time is still the + // mean over the queries they ran. + if nq <= 1_000 { + let t = Instant::now(); + answers[3] += sig.iter().filter(|&&s| ex.on_sig(s)).count() as u64; + times[3] += ms(t); + let t = Instant::now(); + answers[4] += classes + .iter() + .filter(|(s, _)| ex.on_sig(**s)) + .map(|(_, v)| v.0) + .sum::(); + times[4] += ms(t); + } + } + assert!( + answers[..3].iter().all(|&a| a == answers[0]), + "{regime:?} {answers:?}" + ); + if nq <= 1_000 { + assert!( + answers[3] == answers[0] && answers[4] == answers[0], + "{regime:?} {answers:?}" + ); + } + if nq > 1_000 { + // Not run: printed as NaN, never as a time. + times[3] = f64::NAN; + times[4] = f64::NAN; + } + println!( + " {:>6} {:>10.3}ms {:>10.3}ms {:>10.3}ms {:>10.3}ms {:>10.3}ms | per query {:>7} {:>7}", + nq, + times[0] / nq as f64, + times[1] / nq as f64, + times[2] / nq as f64, + times[3] / nq as f64, + times[4] / nq as f64, + words_b / nq, + words_d / nq + ); + } + + // Incremental update: flip 1,000 addresses of P5. + let mut r = Rng(0x0DE1); + let flips: Vec = (0..1000).map(|_| r.below(N as u64) as usize).collect(); + let mut pl2 = pl.clone(); + let mut sig2 = sig.clone(); + let mut classes2 = classes.clone(); + let t = Instant::now(); + for &a in &flips { + pl2[5][a / 64] ^= 1 << (a % 64); + } + let upd_b = ms(t); + let t = Instant::now(); + for &a in &flips { + let old = sig2[a]; + let new = old ^ (1 << 5); + sig2[a] = new; + let e = classes2.get_mut(&old).unwrap(); + e.0 -= 1; + e.1 -= i64::from(vals[a]); + if e.0 == 0 { + classes2.remove(&old); + } + let e = classes2.entry(new).or_insert((0, 0)); + e.0 += 1; + e.1 += i64::from(vals[a]); + } + let upd_ce = ms(t); + let t = Instant::now(); + let p = &pl2[5]; + let _ = ( + p.iter().position(|w| *w != 0), + p.iter().rposition(|w| *w != 0), + ); + let upd_d = ms(t); + println!(" update 1,000 addresses of one population: planes {upd_b:.3} ms, signatures+classes {upd_ce:.3} ms, recompute its occupied range {upd_d:.3} ms"); + + // Spill and reload: planes vs signatures, to a local file. + let path = + std::env::temp_dir().join(format!("photo_probe_{}_{regime:?}.bin", std::process::id())); + let bytes: Vec = pl + .iter() + .flat_map(|p| p.iter().flat_map(|w| w.to_le_bytes())) + .collect(); + let t = Instant::now(); + std::fs::write(&path, &bytes).unwrap(); + let spill = ms(t); + let t = Instant::now(); + let back = std::fs::read(&path).unwrap(); + let reload = ms(t); + assert_eq!(back.len(), bytes.len()); + let _ = std::fs::remove_file(&path); + // Sparse spill: only the nonzero words, as (word index, word). + let nz: usize = pl + .iter() + .map(|p| p.iter().filter(|w| **w != 0).count()) + .sum(); + println!( + " spill planes {} KiB in {spill:.1} ms, reload {reload:.1} ms (page cache, not cold) | nonzero words {} of {} ({:.1}% → sparse spill ~{} KiB)", + bytes.len() / 1024, + nz, + P * W, + 100.0 * nz as f64 / (P * W) as f64, + nz * 12 / 1024 + ); +}