From 1c0052a0e1795239305f7edabb38a04523659e71 Mon Sep 17 00:00:00 2001 From: tamirms Date: Thu, 8 Oct 2026 20:32:57 +0100 Subject: [PATCH 1/3] runcontainer: count or test before allocating in run-bitmap intersections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Intersecting a run container with a bitmap built an 8 KiB bitmap container before knowing whether the result was empty, a small array or a bitmap, and runContainer16.intersects built a whole container to answer a boolean. andBitmapContainer and iandRun16 now count the result first, with the existing andBitmapContainerCardinality, and build only what is returned; in place, the gaps between intervals are cleared directly. A run with more than 64 intervals goes through a stack scratch bitmap instead when it covers more than 4096 values or the CPU has the VBMI2 vector fill. intersects returns at the first shared value. An in-place And with a full range returns the container unchanged, and resetBitmapRange clears whole words with clear. Xeon 8375C, BenchmarkRunAndBitmap (added): And, empty result 2.8 µs -> 594 ns (4.7x) And, bitmap result 3.7 µs -> 2.0 µs (1.9x) in-place And, bitmap result 2.2 µs -> 475 ns (4.6x), no allocation Intersects, disjoint 2.8 µs -> 118 ns (23x), no allocation Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0198cmGsvonq19wTyzKBsw2Z --- bitmapcontainer.go | 43 ++++++- runbitmap_and_test.go | 274 ++++++++++++++++++++++++++++++++++++++++++ runcontainer.go | 115 +++++++++++++++++- util.go | 74 +++++++++++- 4 files changed, 494 insertions(+), 12 deletions(-) create mode 100644 runbitmap_and_test.go diff --git a/bitmapcontainer.go b/bitmapcontainer.go index 6ca8f679..c776caab 100644 --- a/bitmapcontainer.go +++ b/bitmapcontainer.go @@ -318,6 +318,11 @@ func bitmapEquals(a, b []uint64) bool { // is the array-to-bitmap conversion point and leaves a comfortable margin. const bitmapContainerVectorFillMinCardinality = 4096 +// arrayVectorFillMinCardinality is where decoding a whole key with the vector +// fill beats decoding only the words that hold the values. Measured crossover +// is near 1000 values. +const arrayVectorFillMinCardinality = 1024 + func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int { if useVectorFill && bc.cardinality >= bitmapContainerVectorFillMinCardinality { return fillLeastSignificant16bitsVector(bc.bitmap, x, i, mask) @@ -809,7 +814,7 @@ func (bc *bitmapContainer) iand(a container) container { return bc.iandBitmap(x) case *runContainer16: if x.isFull() { - return bc.clone() + return bc } return bc.iandRun16(x) } @@ -817,8 +822,20 @@ func (bc *bitmapContainer) iand(a container) container { } func (bc *bitmapContainer) iandRun16(rc *runContainer16) container { - rcb := newBitmapContainerFromRun(rc) - return bc.iandBitmap(rcb) + if rc.andInScratch() { + var scratch [bitmapContainerSize]uint64 + for i := range rc.iv { + setBitmapRange(scratch[:], int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + return bc.iandBitmap(&bitmapContainer{bitmap: scratch[:]}) + } + card := rc.andBitmapContainerCardinality(bc) + if card <= arrayDefaultMaxSize { + return rc.andBitmapArray(bc, card) + } + clearBitmapGaps(bc.bitmap, rc.iv, 0, maxCapacity) + bc.cardinality = card + return bc } func (bc *bitmapContainer) iandArray(ac *arrayContainer) container { @@ -850,6 +867,26 @@ func (bc *bitmapContainer) andArrayCardinality(value2 *arrayContainer) int { return pos } +func (bc *bitmapContainer) intersectsRange(start, end uint) bool { + if start >= end { + return false + } + firstword, endword := start/64, (end-1)/64 + lo, hi := ^uint64(0)<<(start%64), ^uint64(0)>>((64-end)&63) + if firstword == endword { + return bc.bitmap[firstword]&lo&hi != 0 + } + if bc.bitmap[firstword]&lo != 0 || bc.bitmap[endword]&hi != 0 { + return true + } + for _, w := range bc.bitmap[firstword+1 : endword] { + if w != 0 { + return true + } + } + return false +} + func (bc *bitmapContainer) getCardinalityInRange(start, end uint) int { if start >= end { return 0 diff --git a/runbitmap_and_test.go b/runbitmap_and_test.go new file mode 100644 index 00000000..bb3f5abf --- /dev/null +++ b/runbitmap_and_test.go @@ -0,0 +1,274 @@ +package roaring + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +// randomRun builds a canonical run container: up to n sorted intervals of +// at most maxLen values separated by gaps of at most maxGap, optionally +// touching 0 and 65535. +func randomRun(rng *rand.Rand, n, maxLen, maxGap int, touchEdges bool) *runContainer16 { + rc := &runContainer16{} + pos := 0 + if !touchEdges { + pos = rng.Intn(100) + } + for i := 0; i < n && pos < 65536; i++ { + length := min(1+rng.Intn(1+rng.Intn(maxLen)), 65536-pos) + rc.iv = append(rc.iv, interval16{start: uint16(pos), length: uint16(length - 1)}) + pos += length + 1 + rng.Intn(maxGap) + } + if last := int(rc.maximum()); touchEdges && last != 65535 { + if start := 65535 - rng.Intn(500); last+2 <= start { + rc.iv = append(rc.iv, interval16{start: uint16(start), length: uint16(65535 - start)}) + } + } + return rc +} + +// randomBitmapContainer ANDs ands random words per position, so the density +// is 1/2^ands. +func randomBitmapContainer(rng *rand.Rand, ands int) *bitmapContainer { + bc := newBitmapContainer() + for i := range bc.bitmap { + w := rng.Uint64() + for k := 1; k < ands; k++ { + w &= rng.Uint64() + } + bc.bitmap[i] = w + } + bc.computeCardinality() + return bc +} + +func requireAndResult(t *testing.T, want, got container) { + t.Helper() + require.True(t, want.equals(got), "want %d values, got %d", want.getCardinality(), got.getCardinality()) + _, isArray := got.(*arrayContainer) + require.Equal(t, got.getCardinality() <= arrayDefaultMaxSize, isArray, "container kind must follow its cardinality") +} + +// The reference is the old path: materialize the run, then the unchanged +// bitmap-bitmap intersection. Runs of one to hundreds of intervals check +// both the per-interval and the scratch paths, against every density. +func TestRunAndBitmapContainer(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + regular := func(stride, length int) *runContainer16 { + rc := &runContainer16{} + for start := 0; start+length <= 65536; start += stride { + rc.iv = append(rc.iv, interval16{start: uint16(start), length: uint16(length - 1)}) + } + return rc + } + runs := []*runContainer16{regular(32, 8), regular(64, 1), regular(3, 2), regular(200, 100)} + for i := 0; i < 120; i++ { + runs = append(runs, randomRun(rng, 1+rng.Intn(40), 3000, 2000, i%5 == 0), randomRun(rng, 65+rng.Intn(400), 60, 120, false)) + } + // Only values in the gaps of regular(32, 8), so that run's result is empty. + gaps := newBitmapContainer() + for v := 0; v < 65536; v++ { + if v%32 >= 8 { + gaps.iadd(uint16(v)) + } + } + for i, rc := range runs { + for _, bc := range []*bitmapContainer{randomBitmapContainer(rng, 6), randomBitmapContainer(rng, 2), randomBitmapContainer(rng, 1), newBitmapContainerwithRange(0, 65535), gaps} { + want := newBitmapContainerFromRun(rc).andBitmap(bc) + requireAndResult(t, want, rc.and(bc)) + requireAndResult(t, want, bc.and(rc)) + requireAndResult(t, want, bc.clone().iand(rc)) + require.Equal(t, !want.isEmpty(), rc.intersects(bc), "run %d", i) + } + } +} + +func TestRunAndBitmapContainerEdges(t *testing.T) { + empty := newBitmapContainer() + full := newBitmapContainerwithRange(0, 65535) + single := &runContainer16{iv: []interval16{{start: 100, length: 9}}} + edges := &runContainer16{iv: []interval16{{start: 0, length: 0}, {start: 65535, length: 0}}} + atThreshold := &runContainer16{iv: []interval16{{start: 0, length: arrayDefaultMaxSize - 1}}} + pastThreshold := &runContainer16{iv: []interval16{{start: 0, length: arrayDefaultMaxSize}}} + for _, rc := range []*runContainer16{single, edges, atThreshold, pastThreshold} { + for _, bc := range []*bitmapContainer{empty, full} { + want := newBitmapContainerFromRun(rc).andBitmap(bc) + requireAndResult(t, want, rc.and(bc)) + requireAndResult(t, want, bc.clone().iand(rc)) + } + } + // Exactly 4096 values through the scratch path: every even value below + // 16384 against the values below 8192. + evens := &runContainer16{} + for v := 0; v < 4*arrayDefaultMaxSize; v += 2 { + evens.iv = append(evens.iv, interval16{start: uint16(v)}) + } + half := newBitmapContainerwithRange(0, 2*arrayDefaultMaxSize-1) + requireAndResult(t, newBitmapContainerFromRun(evens).andBitmap(half), evens.and(half)) +} + +// An empty intersection allocates only its empty array container, and an +// in-place And with a full run and Intersects allocate nothing; the old paths +// allocated an 8 KiB bitmap first. +func TestRunAndBitmapContainerAllocations(t *testing.T) { + rc := &runContainer16{iv: []interval16{{start: 0, length: 999}, {start: 30000, length: 999}}} + bc := newBitmapContainerwithRange(2000, 20000) + require.LessOrEqual(t, testing.AllocsPerRun(100, func() { + if !rc.and(bc).isEmpty() { + t.Fatal("expected empty") + } + }), 1.0) + // The clone costs two allocations, the empty result one. + require.LessOrEqual(t, testing.AllocsPerRun(100, func() { + _ = bc.clone().iand(rc) + }), 3.0) + full := newRunContainer16Range(0, MaxUint16) + require.Zero(t, testing.AllocsPerRun(100, func() { _ = bc.iand(full) })) + require.Zero(t, testing.AllocsPerRun(100, func() { _ = rc.iand(full) })) + for _, c := range []container{bc, newArrayContainerRange(2000, 3000), &runContainer16{iv: []interval16{{start: 5000, length: 100}}}} { + require.Zero(t, testing.AllocsPerRun(100, func() { _ = rc.intersects(c) }), "%T", c) + } +} + +// The reference is the old definition, !rc.and(c).isEmpty(). +func TestRunContainerIntersects(t *testing.T) { + rng := rand.New(rand.NewSource(11)) + for trial := 0; trial < 400; trial++ { + rc := randomRun(rng, 1+rng.Intn(30), 3000, 2000, trial%7 == 0) + ac := newArrayContainer() + for range rng.Intn(300) { + ac.iadd(uint16(rng.Intn(65536))) + } + others := []container{randomBitmapContainer(rng, 1+trial%3*3), ac, randomRun(rng, 1+rng.Intn(30), 3000, 2000, trial%5 == 0)} + for _, c := range others { + want := !rc.and(c).isEmpty() + require.Equal(t, want, rc.intersects(c), "trial %d run vs %T", trial, c) + require.Equal(t, want, c.intersects(rc), "trial %d %T vs run", trial, c) + } + } + // A few values against many intervals take the search branch; the + // reference is the same definition. + many := randomRun(rng, 300, 60, 120, false) + require.Greater(t, len(many.iv), 200) + for trial := 0; trial < 200; trial++ { + few := newArrayContainer() + for range 1 + rng.Intn(6) { + few.iadd(uint16(rng.Intn(65536))) + } + want := !many.and(few).isEmpty() + require.Equal(t, want, many.intersects(few), "trial %d few vs many", trial) + require.Equal(t, want, few.intersects(many), "trial %d many vs few", trial) + } + // Edges: empty array, touching intervals, single values at both ends. + empty := newArrayContainer() + edges := &runContainer16{iv: []interval16{{start: 0, length: 0}, {start: 65535, length: 0}}} + require.False(t, edges.intersects(empty)) + one := newArrayContainer() + one.iadd(65535) + require.True(t, edges.intersects(one)) + touching := &runContainer16{iv: []interval16{{start: 1, length: 65533}}} + require.False(t, edges.intersects(touching)) + require.True(t, edges.intersects(newBitmapContainerwithRange(0, 0))) +} + +// runBitmap holds one run container at key 0 covering n intervals of the +// given length, spaced stride apart. +func runBitmap(n, length, stride int) *Bitmap { + bm := New() + for i := 0; i < n; i++ { + bm.AddRange(uint64(i*stride), uint64(i*stride+length)) + } + bm.RunOptimize() + return bm +} + +// BenchmarkRunAndBitmap intersects a run-optimized bitmap with a dense one +// over a single key through the public API, one case per result kind, plus +// the in-place form, alone and with the copy it needs, and the Intersects test. +func BenchmarkRunAndBitmap(b *testing.B) { + rng := rand.New(rand.NewSource(7)) + short := runBitmap(20, 300, 3000) // 6000 values + long := runBitmap(4, 10000, 16000) // 40000 values + many := runBitmap(1024, 8, 64) // 8192 values in 1024 intervals + gaps, quarter, half, dense := New(), New(), New(), New() + for v := uint32(0); v < 65536; v++ { + if !short.Contains(v) && rng.Intn(2) == 0 { + gaps.Add(v) + } + if rng.Intn(4) == 0 { + quarter.Add(v) + } + if rng.Intn(2) == 0 { + half.Add(v) + } + if rng.Intn(10) != 0 { + dense.Add(v) + } + } + for _, tc := range []struct { + name string + x, y *Bitmap + }{ + {"and/empty", short, gaps}, + {"and/array", short, half}, + {"and/bitmap", long, dense}, + {"and/many-intervals/array", many, quarter}, + {"and/many-intervals/bitmap", many, dense}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = And(tc.x, tc.y) + } + }) + } + for _, tc := range []struct { + name string + x *Bitmap + }{ + {"iand/bitmap", long}, + {"iand/many-intervals", many}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + var xs []*Bitmap + for b.Loop() { + if len(xs) == 0 { + // Fresh copies, made outside the timer: one And turns a copy into its result. + b.StopTimer() + for range 64 { + xs = append(xs, dense.Clone()) + } + b.StartTimer() + } + xs[len(xs)-1].And(tc.x) + xs = xs[:len(xs)-1] + } + }) + b.Run("clone+"+tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + dense.Clone().And(tc.x) + } + }) + } + few := New() + few.AddMany([]uint32{100, 30000, 60000}) + for _, tc := range []struct { + name string + x, y *Bitmap + }{ + {"intersects/disjoint", short, gaps}, + {"intersects/overlapping", short, half}, + {"intersects/few-values-many-intervals", many, few}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = tc.x.Intersects(tc.y) + } + }) + } +} diff --git a/runcontainer.go b/runcontainer.go index 7c369b75..f83c8021 100644 --- a/runcontainer.go +++ b/runcontainer.go @@ -1855,10 +1855,63 @@ func (rc *runContainer16) andCardinality(a container) int { panic("unsupported container type") } -// andBitmapContainer finds the intersection of rc and b. +const runAndScratchIntervals = 64 + +// andInScratch reports whether ANDing rc with a bitmap is cheaper in a +// scratch bitmap than interval by interval: past runAndScratchIntervals +// intervals, the measured crossover, if the run covers more than 4096 values +// or the vector fill makes reading the scratch back cheap. +func (rc *runContainer16) andInScratch() bool { + return len(rc.iv) > runAndScratchIntervals && (useVectorFill || rc.getCardinality() > arrayDefaultMaxSize) +} + +// andBitmapContainer intersects rc with bc, counting before allocating. func (rc *runContainer16) andBitmapContainer(bc *bitmapContainer) container { - bc2 := newBitmapContainerFromRun(rc) - return bc2.andBitmap(bc) + if rc.andInScratch() { + return rc.andBitmapContainerScratch(bc) + } + card := rc.andBitmapContainerCardinality(bc) + if card > arrayDefaultMaxSize { + answer := newBitmapContainer() + for i := range rc.iv { + copyBitmapRange(answer.bitmap, bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + answer.cardinality = card + return answer + } + return rc.andBitmapArray(bc, card) +} + +// andBitmapArray returns the card values of bc inside rc's intervals as an +// array container. +func (rc *runContainer16) andBitmapArray(bc *bitmapContainer, card int) *arrayContainer { + if useVectorFill && card >= arrayVectorFillMinCardinality { + var runBits [bitmapContainerSize]uint64 + for i := range rc.iv { + setBitmapRange(runBits[:], int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + ac := newArrayContainerSize(card) + fillArrayAND(ac.content, runBits[:], bc.bitmap) + return ac + } + ac := newArrayContainerCapacity(card) + for i := range rc.iv { + ac.content = appendBitmapRange(ac.content, bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + return ac +} + +func (rc *runContainer16) andBitmapContainerScratch(bc *bitmapContainer) container { + first, last := int(rc.iv[0].start)/64, int(rc.maximum())/64 + var scratch [bitmapContainerSize]uint64 + for i := range rc.iv { + copyBitmapRange(scratch[:], bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + card := int(popcntSlice(scratch[first : last+1])) + if card == 0 { + return newArrayContainerCapacity(0) + } + return containerFromWords(scratch[:], first, last, card) } func (rc *runContainer16) andArrayCardinality(ac *arrayContainer) int { @@ -1894,6 +1947,9 @@ func (rc *runContainer16) iand(a container) container { if rc.isFull() { return a.clone() } + if a.isFull() { + return rc + } switch c := a.(type) { case *runContainer16: // Important: there is no reason to believe that the @@ -2418,9 +2474,56 @@ func (rc *runContainer16) lazyOR(a container) container { } func (rc *runContainer16) intersects(a container) bool { - // TODO: optimize by doing inplace/less allocation - isect := rc.and(a) - return !isect.isEmpty() + switch c := a.(type) { + case *bitmapContainer: + for i := range rc.iv { + if c.intersectsRange(uint(rc.iv[i].start), uint(rc.iv[i].last())+1) { + return true + } + } + return false + case *arrayContainer: + if 2*len(c.content)*bits.Len(uint(len(rc.iv))) < len(rc.iv) { + // Few values against many intervals: a search each beats + // walking every interval. + for _, v := range c.content { + if rc.contains(v) { + return true + } + } + return false + } + // Otherwise andArray's merge, stopping at the first shared value. + for i, pos := 0, 0; i < len(rc.iv) && pos < len(c.content); { + v := c.content[pos] + for rc.iv[i].last() < v { + i++ + if i == len(rc.iv) { + return false + } + } + if rc.iv[i].start <= v { + return true + } + pos = advanceUntil(c.content, pos, len(c.content), rc.iv[i].start) + } + return false + case *runContainer16: + i, j := 0, 0 + for i < len(rc.iv) && j < len(c.iv) { + x, y := rc.iv[i], c.iv[j] + if haveOverlap16(x, y) { + return true + } + if x.last() < y.last() { + i++ + } else { + j++ + } + } + return false + } + panic("unsupported container type") } func (rc *runContainer16) xor(a container) container { diff --git a/util.go b/util.go index e49ad29c..ac133947 100644 --- a/util.go +++ b/util.go @@ -152,6 +152,76 @@ func flipBitmapRange(bitmap []uint64, start int, end int) { bitmap[endword] ^= ^uint64(0) >> (uint(-end) % 64) } +// copyBitmapRange copies the bits of src within [start, end) into dst, which +// must be zero there. +func copyBitmapRange(dst, src []uint64, start int, end int) { + if start >= end { + return + } + firstword := start / 64 + endword := (end - 1) / 64 + if firstword == endword { + dst[firstword] |= src[firstword] & (^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64)) + return + } + dst[firstword] |= src[firstword] & (^uint64(0) << uint(start%64)) + copy(dst[firstword+1:endword], src[firstword+1:endword]) + dst[endword] |= src[endword] & (^uint64(0) >> (uint(-end) % 64)) +} + +// appendBitmapRange appends the positions of the bits set in [start, end). +func appendBitmapRange(out []uint16, bitmap []uint64, start int, end int) []uint16 { + if start >= end { + return out + } + firstword := start / 64 + endword := (end - 1) / 64 + for i := firstword; i <= endword; i++ { + w := bitmap[i] + if i == firstword { + w &= ^uint64(0) << uint(start%64) + } + if i == endword { + w &= ^uint64(0) >> (uint(-end) % 64) + } + for w != 0 { + out = append(out, uint16(i*64+bits.TrailingZeros64(w))) + w &= w - 1 + } + } + return out +} + +// clearBitmapGaps clears the bits in [from, to) that no interval of iv covers. +func clearBitmapGaps(bitmap []uint64, iv []interval16, from, to int) { + for _, x := range iv { + if int(x.start) >= to { + break + } + resetBitmapRange(bitmap, from, int(x.start)) + from = max(from, int(x.last())+1) + } + resetBitmapRange(bitmap, from, to) +} + +// containerFromWords builds an array or bitmap container from the card bits +// set in words [first, last] of w, which spans a whole key and is zero +// outside them. +func containerFromWords(w []uint64, first, last, card int) container { + if card <= arrayDefaultMaxSize { + if useVectorFill && card >= arrayVectorFillMinCardinality { + return newArrayContainerFromBitmap(&bitmapContainer{cardinality: card, bitmap: w}) + } + ac := newArrayContainerCapacity(card) + ac.content = appendBitmapRange(ac.content, w, first*64, (last+1)*64) + return ac + } + bc := newBitmapContainer() + copy(bc.bitmap[first:last+1], w[first:last+1]) + bc.cardinality = card + return bc +} + func resetBitmapRange(bitmap []uint64, start int, end int) { if start >= end { return @@ -163,9 +233,7 @@ func resetBitmapRange(bitmap []uint64, start int, end int) { return } bitmap[firstword] &= ^(^uint64(0) << uint(start%64)) - for i := firstword + 1; i < endword; i++ { - bitmap[i] = 0 - } + clear(bitmap[firstword+1 : endword]) bitmap[endword] &= ^(^uint64(0) >> (uint(-end) % 64)) } From fccdc6306082591981eefce5942434df3efd65d2 Mon Sep 17 00:00:00 2001 From: tamirms Date: Thu, 8 Oct 2026 20:32:57 +0100 Subject: [PATCH 2/3] fastaggregation: count before allocating in FastAnd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FastAnd with three or more inputs intersected them pairwise, allocating an intermediate container per input and key even when the result was empty. The input with the fewest keys now drives: each of its keys is looked up in the first four inputs with stack cursors, and an array key is also checked for a shared value, before anything is allocated. A key that survives is intersected count-first: bitmaps in a stack scratch bitmap, with runs clearing what they do not cover, and arrays through the existing and and iand. Any further inputs are intersected into the result one at a time, as before; walking every input for every key was up to 1.9x slower than pairwise And when many inputs keep their keys. bitmapContainer.andArray hoists its slice headers out of the loop, a quarter faster on amd64 and arm64, and ParAnd's worker now calls the same andKChain. Xeon 8375C, against pairwise And in the same binary: jointly-empty 14.0 µs -> 1.9 µs (7.3x), 1 alloc instead of 18 singletons-10000 25.9 µs -> 12.4 µs (2.1x) array30-runs 486 ns -> 286 ns (1.7x) BenchmarkRealDataFastAnd (added) is 1.9x to 97x faster than master on all twelve datasets. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0198cmGsvonq19wTyzKBsw2Z --- bitmapcontainer.go | 12 +- fastaggregation.go | 184 ++++++++++++++++++++++-- fastand_kway_test.go | 280 ++++++++++++++++++++++++++++++++++++ parallel.go | 14 +- real_data_benchmark_test.go | 6 + 5 files changed, 466 insertions(+), 30 deletions(-) create mode 100644 fastand_kway_test.go diff --git a/bitmapcontainer.go b/bitmapcontainer.go index c776caab..dcc8d33c 100644 --- a/bitmapcontainer.go +++ b/bitmapcontainer.go @@ -845,15 +845,13 @@ func (bc *bitmapContainer) iandArray(ac *arrayContainer) container { func (bc *bitmapContainer) andArray(value2 *arrayContainer) *arrayContainer { answer := newArrayContainerCapacity(len(value2.content)) - answer.content = answer.content[:cap(answer.content)] - c := value2.getCardinality() + out, bitmap := answer.content[:cap(answer.content)], bc.bitmap pos := 0 - for k := 0; k < c; k++ { - v := value2.content[k] - answer.content[pos] = v - pos += int(bc.bitValue(v)) + for _, v := range value2.content { + out[pos] = v + pos += int((bitmap[v>>6] >> (v & 63)) & 1) } - answer.content = answer.content[:pos] + answer.content = out[:pos] return answer } diff --git a/fastaggregation.go b/fastaggregation.go index 3881b88a..a344f739 100644 --- a/fastaggregation.go +++ b/fastaggregation.go @@ -125,26 +125,188 @@ func (x1 *Bitmap) repairAfterLazy() { } } -// FastAnd computes the intersection between many bitmaps quickly -// Compared to the And function, it can take many bitmaps as input, thus saving the trouble -// of manually calling "And" many times. +// FastAnd computes the intersection between many bitmaps quickly. +// Compared to the And function, it can take many bitmaps as input. // -// Performance hints: if you have very large and tiny bitmaps, -// it may be beneficial performance-wise to put a tiny bitmap -// in first position. +// With three or more inputs, FastAnd walks the keys of the input with the +// fewest keys and looks each one up in the first four inputs, also checking +// an array container for a shared value, before allocating anything for it. +// The remaining inputs are then intersected into the result one at a time. func FastAnd(bitmaps ...*Bitmap) *Bitmap { - if len(bitmaps) == 0 { + switch len(bitmaps) { + case 0: return NewBitmap() - } else if len(bitmaps) == 1 { + case 1: return bitmaps[0].Clone() + case 2: + return And(bitmaps[0], bitmaps[1]) } - answer := And(bitmaps[0], bitmaps[1]) - for _, bm := range bitmaps[2:] { - answer.And(bm) + driver := 0 + for i := 1; i < len(bitmaps); i++ { + if bitmaps[i].highlowcontainer.size() < bitmaps[driver].highlowcontainer.size() { + driver = i + } + } + dra := &bitmaps[driver].highlowcontainer + // The walk covers the first width inputs, each with a cursor that resumes + // where the previous key stopped. Past those, intersecting an input into + // the result costs less per key than walking it. + const width = 4 + n := min(len(bitmaps), width) + var pos [width]int + var cs [width + 1]container // the window and a driver past it + answer := NewBitmap() +keys: + for j := 0; j < dra.size(); j++ { + key := dra.getKeyAtIndex(j) + dc := dra.getContainerAtIndex(j) + // Only an array is checked for a value it shares with each input: a + // lookup per value that usually stops at the first. For a run or a + // bitmap the check costs about as much as the intersection. + _, check := dc.(*arrayContainer) + m := 0 + if !dc.isFull() { // a full container changes nothing + cs[0] = dc + m = 1 + } + for i, bm := range bitmaps[:n] { + if i == driver { + continue + } + ra := &bm.highlowcontainer + p := pos[i] + if p < ra.size() && ra.getKeyAtIndex(p) < key { + p = ra.advanceUntil(key, p) + } + pos[i] = p + if p == ra.size() { + break keys + } + if ra.getKeyAtIndex(p) != key { + continue keys + } + pos[i]++ + c := ra.getContainerAtIndex(p) + if c.isFull() { + continue + } + if check && !dc.intersects(c) { + continue keys + } + cs[m] = c + m++ + } + if c := andK(cs[:m]); c != nil { + if answer.IsEmpty() { + // Room for up to 64 of the driver's remaining keys at once. + k := min(dra.size()-j, 64) + answer.highlowcontainer = roaringArray{ + keys: make([]uint16, 0, k), + containers: make([]container, 0, k), + needCopyOnWrite: make([]bool, 0, k), + } + } + answer.highlowcontainer.appendContainer(key, c, false) + } + } + for i := n; i < len(bitmaps) && !answer.IsEmpty(); i++ { + if i != driver { + answer.And(bitmaps[i]) + } } return answer } +// andK intersects the containers of one key, none of them full, and returns +// nil when the intersection is empty. It may reorder cs. +func andK(cs []container) container { + switch len(cs) { + case 0: + return newRunContainer16Range(0, maxCapacity-1) + case 1: + return cs[0].clone() + } + // The smallest array goes first; the rest keep their order. + smallest := -1 + for i, c := range cs { + if x, ok := c.(*arrayContainer); ok && (smallest < 0 || x.getCardinality() < cs[smallest].getCardinality()) { + smallest = i + } + } + if smallest >= 0 { + a := cs[smallest] + copy(cs[1:smallest+1], cs[:smallest]) + cs[0] = a + return andKChain(cs) + } + n := 0 + for i, c := range cs { + if _, ok := c.(*runContainer16); ok { + cs[n], cs[i] = cs[i], cs[n] + n++ + } + } + if n == len(cs) { + return andKChain(cs) + } + return andKBitmaps(cs[n:], cs[:n]) +} + +// andKChain intersects cs in order: the first two into a new container, then +// the rest into it in place, so nothing bigger than the first result is built. +func andKChain(cs []container) container { + c := cs[0].and(cs[1]) + for i := 2; i < len(cs) && !c.isEmpty(); i++ { + c = c.iand(cs[i]) + } + if c.isEmpty() { + return nil + } + return c +} + +// andKBitmaps ANDs the bitmaps in a scratch bitmap on the stack, stopping as +// soon as the result is empty. Runs act as masks: only the words between +// their common start and end are ANDed, and the gaps inside a run are cleared +// afterwards. +func andKBitmaps(bms, runs []container) container { + lo, hi := 0, maxCapacity-1 + gaps := false + for _, c := range runs { + rc := c.(*runContainer16) + lo, hi = max(lo, int(rc.minimum())), min(hi, int(rc.maximum())) + gaps = gaps || len(rc.iv) > 1 + } + if lo > hi { + return nil + } + first, last := lo/64, hi/64 + var scratch [bitmapContainerSize]uint64 + w := scratch[first : last+1] + src := bms[0].(*bitmapContainer).bitmap[first : last+1] + card := uint64(0) + if len(bms) == 1 { + copy(w, src) + } + for _, c := range bms[1:] { + if card = andCardSlice(w, src, c.(*bitmapContainer).bitmap[first:last+1]); card == 0 { + return nil + } + src = w + } + if len(bms) == 1 || gaps || lo%64 != 0 || (hi+1)%64 != 0 { + resetBitmapRange(scratch[:], first*64, lo) + resetBitmapRange(scratch[:], hi+1, (last+1)*64) + for _, c := range runs { + clearBitmapGaps(scratch[:], c.(*runContainer16).iv, lo, hi+1) + } + if card = popcntSlice(w); card == 0 { + return nil + } + } + return containerFromWords(scratch[:], first, last, int(card)) +} + // FastOr computes the union between many bitmaps quickly, as opposed to having to call Or repeatedly. // It might also be faster than calling Or repeatedly. func FastOr(bitmaps ...*Bitmap) *Bitmap { diff --git a/fastand_kway_test.go b/fastand_kway_test.go new file mode 100644 index 00000000..d70a7fbb --- /dev/null +++ b/fastand_kway_test.go @@ -0,0 +1,280 @@ +package roaring + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +// kwayKey picks the container key for the k-th key of an input: inputs share +// their keys, but each leaves some out, the way real inputs do. +func kwayKey(rng *rand.Rand, k int) (uint64, bool) { + return uint64(k) << 16, rng.Intn(8) != 0 +} + +// kwayBitmap builds a bitmap over up to keys keys holding random values at +// the given density: arrays when sparse, bitmaps when dense. +func kwayBitmap(rng *rand.Rand, keys int, density float64) *Bitmap { + bm := New() + for k := 0; k < keys; k++ { + base, present := kwayKey(rng, k) + if !present { + continue + } + for i := 0; i < int(density*65536); i++ { + bm.Add(uint32(base) + uint32(rng.Intn(65536))) + } + } + return bm +} + +// kwayRuns builds a run-optimized bitmap of random ranges over up to keys keys. +func kwayRuns(rng *rand.Rand, keys int) *Bitmap { + bm := New() + for k := 0; k < keys; k++ { + base, present := kwayKey(rng, k) + if !present { + continue + } + for start := 0; start < 65536; start += 1000 + rng.Intn(3000) { + bm.AddRange(base+uint64(start), base+uint64(min(start+1+rng.Intn(800), 65536))) + } + } + bm.RunOptimize() + return bm +} + +// intervalBitmap holds one run container at key 0 made of the given +// half-open intervals. +func intervalBitmap(t *testing.T, intervals [][2]uint64) *Bitmap { + t.Helper() + bm := New() + for _, iv := range intervals { + bm.AddRange(iv[0], iv[1]) + } + bm.RunOptimize() + require.Equal(t, uint64(1), bm.Stats().RunContainers, "%v", intervals) + return bm +} + +func randomIntervals(rng *rand.Rand) [][2]uint64 { + var out [][2]uint64 + pos := uint64(rng.Intn(64)) + for n := 1 + rng.Intn(40); n > 0 && pos < 65536; n-- { + length := min(uint64(4+rng.Intn(3000)), 65536-pos) + out = append(out, [2]uint64{pos, pos + length}) + pos += length + uint64(rng.Intn(200)) + } + return out +} + +// requireFastAnd checks FastAnd against the pairwise definition, that the +// result is valid, and that it shares no storage with its inputs, and +// reports whether the result is empty. +func requireFastAnd(t *testing.T, inputs ...*Bitmap) bool { + t.Helper() + before := make([]*Bitmap, len(inputs)) + for i, in := range inputs { + before[i] = in.Clone() + } + want := pairwiseAnd(inputs) + got := FastAnd(inputs...) + require.NoError(t, got.Validate()) + require.True(t, got.Equals(want), "%d values, want %d", got.GetCardinality(), want.GetCardinality()) + empty := got.IsEmpty() + got.Flip(0, 1<<21) // rewrites every container the inputs could share + for i, in := range inputs { + require.True(t, in.Equals(before[i]), "input %d was modified through the result", i) + } + return empty +} + +func TestFastAndMatchesPairwise(t *testing.T) { + rng := rand.New(rand.NewSource(3)) + densities := []float64{0.0005, 0.01, 0.05, 0.3, 0.8, 0.98} + whole := New() + whole.AddRange(0, 4<<16) + nonEmpty := 0 + for trial := 0; trial < 400; trial++ { + inputs := make([]*Bitmap, 3+rng.Intn(10)) + for i := range inputs { + switch rng.Intn(6) { + case 0: + inputs[i] = kwayRuns(rng, 1+rng.Intn(4)) + case 1: + inputs[i] = intervalBitmap(t, randomIntervals(rng)) + case 2: + inputs[i] = whole + default: + inputs[i] = kwayBitmap(rng, 1+rng.Intn(4), densities[rng.Intn(len(densities))]) + } + } + if !requireFastAnd(t, inputs...) { + nonEmpty++ + } + } + require.Greater(t, nonEmpty, 100, "most trials must reach the intersection, not stop at a missing key") +} + +func TestFastAndEdges(t *testing.T) { + rng := rand.New(rand.NewSource(12)) + dense := func() *Bitmap { + bm := New() + for range 32768 { + bm.Add(uint32(rng.Intn(65536))) + } + bm.AddRange(0, 256) // so a mask's edges always have values to drop + return bm + } + full := intervalBitmap(t, [][2]uint64{{0, 65536}}) + // A range ending on a word boundary on one side only is masked on the other. + requireFastAnd(t, dense(), intervalBitmap(t, [][2]uint64{{64, 100}}), dense()) + requireFastAnd(t, dense(), intervalBitmap(t, [][2]uint64{{70, 128}}), dense()) + // Two ranges spanning whole words still mask the gap between them. + requireFastAnd(t, dense(), intervalBitmap(t, [][2]uint64{{0, 64}, {128, 192}}), dense()) + // A range narrower than a many-interval run's span clips it. + var manyShort [][2]uint64 + for p := uint64(0); p < 65536; p += 64 { + manyShort = append(manyShort, [2]uint64{p, p + 8}) + } + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{100, 40000}}), intervalBitmap(t, manyShort), dense(), dense()) + // Two ranges that each meet the bitmap but overlap only where it is + // empty, so the masked result is empty. + gap := New() + gap.Add(0) + gap.AddRange(128, 5300) + requireFastAnd(t, gap, intervalBitmap(t, [][2]uint64{{0, 128}}), intervalBitmap(t, [][2]uint64{{64, 192}})) + // Ranges that each meet the first one but have no common value end the + // key before a bitmap is read, as do disjoint runs. + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{0, 40000}}), intervalBitmap(t, [][2]uint64{{30000, 65536}}), intervalBitmap(t, [][2]uint64{{0, 20000}}), dense()) + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{0, 1000}}), intervalBitmap(t, [][2]uint64{{2000, 3000}}), dense()) + // Whole-key runs change nothing. + y := New() + y.AddMany([]uint32{1, 5, 9, 70000}) + require.True(t, FastAnd(y, full, y).Equals(And(y, full))) + require.True(t, FastAnd(full, full, full).Equals(full)) +} + +// singletons returns n inputs holding the values 0 to n-1, one each: the +// second input already fails the check. +func singletons(n int) []*Bitmap { + inputs := make([]*Bitmap, n) + for i := range inputs { + inputs[i] = New() + inputs[i].Add(uint32(i)) + } + return inputs +} + +// kwayTriple returns three dense inputs whose pairs all overlap but whose +// intersection is empty. +func kwayTriple() (a, b, c *Bitmap) { + a, b, c = New(), New(), New() + for id := uint32(0); id < 4<<16; id++ { + switch id % 3 { + case 0: + a.Add(id) + b.Add(id) + case 1: + b.Add(id) + c.Add(id) + case 2: + a.Add(id) + c.Add(id) + } + } + return a, b, c +} + +func TestFastAndEmptyAllocatesNoContainer(t *testing.T) { + a, b, c := kwayTriple() + require.True(t, a.Intersects(b) && b.Intersects(c) && a.Intersects(c)) + require.True(t, FastAnd(a, b, c).IsEmpty()) + // Only the answer bitmap; never a container. + require.LessOrEqual(t, testing.AllocsPerRun(50, func() { FastAnd(a, b, c) }), 1.0) + + mask := intervalBitmap(t, [][2]uint64{{0, 65536}}) + evens, odds := New(), New() + for v := uint32(0); v < 65536; v += 2 { + evens.Add(v) + odds.Add(v + 1) + } + require.True(t, FastAnd(mask, evens, odds).IsEmpty()) + require.LessOrEqual(t, testing.AllocsPerRun(50, func() { FastAnd(mask, evens, odds) }), 1.0) + + // Nothing is allocated for the inputs past an empty result. + inputs := singletons(1000) + require.True(t, FastAnd(inputs...).IsEmpty()) + require.LessOrEqual(t, testing.AllocsPerRun(50, func() { FastAnd(inputs...) }), 1.0) +} + +// randomKey holds card random values on key. +func randomKey(rng *rand.Rand, card int, key uint32) *Bitmap { + bm := New() + for bm.GetCardinality() < uint64(card) { + bm.Add(key<<16 | uint32(rng.Intn(65536))) + } + return bm +} + +func pairwiseAnd(bms []*Bitmap) *Bitmap { + r := And(bms[0], bms[1]) + for _, bm := range bms[2:] { + r.And(bm) + } + return r +} + +// BenchmarkFastAndShapes runs FastAnd against pairwise And on the same +// inputs, one shape per container pairing plus many-input shapes; the first +// seven add a whole-key range like a query window. +func BenchmarkFastAndShapes(b *testing.B) { + rng := rand.New(rand.NewSource(9)) + window := New() + window.AddRange(0, 65536) + wide := New() + wide.AddRange(0, 8<<16) + spread := func(card int) *Bitmap { + bm := New() + for k := uint32(0); k < 8; k++ { + bm.Or(randomKey(rng, card, k)) + } + return bm + } + x, y, z := kwayTriple() + same := make([]*Bitmap, 128) + base := spread(30) + for i := range same { + same[i] = base.Clone() + } + for _, tc := range []struct { + name string + inputs []*Bitmap + }{ + {"array30-bitmap", []*Bitmap{randomKey(rng, 30, 0), randomKey(rng, 33000, 0), window}}, + {"array4000-bitmap", []*Bitmap{randomKey(rng, 4000, 0), randomKey(rng, 33000, 0), window}}, + {"array30-runs", []*Bitmap{randomKey(rng, 30, 0), runBitmap(250, 16, 32), window}}, + {"array4000-runs", []*Bitmap{randomKey(rng, 4000, 0), runBitmap(250, 16, 32), window}}, + {"bitmap-bitmap", []*Bitmap{randomKey(rng, 33000, 0), randomKey(rng, 33000, 0), window}}, + {"runs-bitmap", []*Bitmap{runBitmap(250, 16, 32), randomKey(rng, 33000, 0), window}}, + {"runs-runs", []*Bitmap{runBitmap(250, 16, 32), runBitmap(1875, 16, 32), window}}, + {"jointly-empty", []*Bitmap{x, y, z}}, + {"eight-keys", []*Bitmap{spread(30), spread(33000), wide}}, + {"singletons-10000", singletons(10000)}, + {"equal-x128", same}, + } { + b.Run(tc.name+"/fastand", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = FastAnd(tc.inputs...) + } + }) + b.Run(tc.name+"/pairwise", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = pairwiseAnd(tc.inputs) + } + }) + } +} diff --git a/parallel.go b/parallel.go index 01ba856f..9703912f 100644 --- a/parallel.go +++ b/parallel.go @@ -283,18 +283,8 @@ func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap { andFunc := func() { // Assumes only structs with >=2 containers are passed for input := range inputChan { - c := input.containers[0].and(input.containers[1]) - for _, next := range input.containers[2:] { - if c.isEmpty() { - break - } - c = c.iand(next) - } - - // Send a nil explicitly if the result of the intersection is an empty container - if c.isEmpty() { - c = nil - } + // A nil result tells the receiver the intersection is empty. + c := andKChain(input.containers) kx := keyedContainer{ input.key, diff --git a/real_data_benchmark_test.go b/real_data_benchmark_test.go index 7e47a323..f40a12a4 100644 --- a/real_data_benchmark_test.go +++ b/real_data_benchmark_test.go @@ -186,3 +186,9 @@ func BenchmarkRealDataFastOr(b *testing.B) { return FastOr(bitmaps...).GetCardinality() }) } + +func BenchmarkRealDataFastAnd(b *testing.B) { + benchmarkRealDataAggregate(b, func(bitmaps []*Bitmap) uint64 { + return FastAnd(bitmaps...).GetCardinality() + }) +} From 64285f3c1b67e88ada0c75fe8a7da54575473ec0 Mon Sep 17 00:00:00 2001 From: tamirms Date: Fri, 9 Oct 2026 12:50:41 +0100 Subject: [PATCH 3/3] fastaggregation: add a surviving-key row to BenchmarkFastAndShapes 1,024 identical inputs of 256 keys of 32 values each, so every key survives every input: the widest walk, kept as a benchmark as asked in review. Xeon 8375C: 13.3 ms against 13.1 ms for pairwise And, with 522 allocations against 535. Co-Authored-By: Claude Opus 5.5 (1M context) --- fastand_kway_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/fastand_kway_test.go b/fastand_kway_test.go index d70a7fbb..4c1ce4e0 100644 --- a/fastand_kway_test.go +++ b/fastand_kway_test.go @@ -248,6 +248,17 @@ func BenchmarkFastAndShapes(b *testing.B) { for i := range same { same[i] = base.Clone() } + // 1,024 inputs of 256 keys that all survive: the widest walk. + keys256 := New() + for k := uint32(0); k < 256; k++ { + for v := uint32(0); v < 64; v += 2 { + keys256.Add(k<<16 | v) + } + } + surviving := make([]*Bitmap, 1024) + for i := range surviving { + surviving[i] = keys256.Clone() + } for _, tc := range []struct { name string inputs []*Bitmap @@ -263,6 +274,7 @@ func BenchmarkFastAndShapes(b *testing.B) { {"eight-keys", []*Bitmap{spread(30), spread(33000), wide}}, {"singletons-10000", singletons(10000)}, {"equal-x128", same}, + {"surviving-1024x256", surviving}, } { b.Run(tc.name+"/fastand", func(b *testing.B) { b.ReportAllocs()