diff --git a/bitmapcontainer.go b/bitmapcontainer.go index 6ca8f679..dcc8d33c 100644 --- a/bitmapcontainer.go +++ b/bitmapcontainer.go @@ -318,6 +318,11 @@ func bitmapEquals(a, b []uint64) bool { // is the array-to-bitmap conversion point and leaves a comfortable margin. const bitmapContainerVectorFillMinCardinality = 4096 +// arrayVectorFillMinCardinality is where decoding a whole key with the vector +// fill beats decoding only the words that hold the values. Measured crossover +// is near 1000 values. +const arrayVectorFillMinCardinality = 1024 + func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int { if useVectorFill && bc.cardinality >= bitmapContainerVectorFillMinCardinality { return fillLeastSignificant16bitsVector(bc.bitmap, x, i, mask) @@ -809,7 +814,7 @@ func (bc *bitmapContainer) iand(a container) container { return bc.iandBitmap(x) case *runContainer16: if x.isFull() { - return bc.clone() + return bc } return bc.iandRun16(x) } @@ -817,8 +822,20 @@ func (bc *bitmapContainer) iand(a container) container { } func (bc *bitmapContainer) iandRun16(rc *runContainer16) container { - rcb := newBitmapContainerFromRun(rc) - return bc.iandBitmap(rcb) + if rc.andInScratch() { + var scratch [bitmapContainerSize]uint64 + for i := range rc.iv { + setBitmapRange(scratch[:], int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + return bc.iandBitmap(&bitmapContainer{bitmap: scratch[:]}) + } + card := rc.andBitmapContainerCardinality(bc) + if card <= arrayDefaultMaxSize { + return rc.andBitmapArray(bc, card) + } + clearBitmapGaps(bc.bitmap, rc.iv, 0, maxCapacity) + bc.cardinality = card + return bc } func (bc *bitmapContainer) iandArray(ac *arrayContainer) container { @@ -828,15 +845,13 @@ func (bc *bitmapContainer) iandArray(ac *arrayContainer) container { func (bc *bitmapContainer) andArray(value2 *arrayContainer) *arrayContainer { answer := newArrayContainerCapacity(len(value2.content)) - answer.content = answer.content[:cap(answer.content)] - c := value2.getCardinality() + out, bitmap := answer.content[:cap(answer.content)], bc.bitmap pos := 0 - for k := 0; k < c; k++ { - v := value2.content[k] - answer.content[pos] = v - pos += int(bc.bitValue(v)) + for _, v := range value2.content { + out[pos] = v + pos += int((bitmap[v>>6] >> (v & 63)) & 1) } - answer.content = answer.content[:pos] + answer.content = out[:pos] return answer } @@ -850,6 +865,26 @@ func (bc *bitmapContainer) andArrayCardinality(value2 *arrayContainer) int { return pos } +func (bc *bitmapContainer) intersectsRange(start, end uint) bool { + if start >= end { + return false + } + firstword, endword := start/64, (end-1)/64 + lo, hi := ^uint64(0)<<(start%64), ^uint64(0)>>((64-end)&63) + if firstword == endword { + return bc.bitmap[firstword]&lo&hi != 0 + } + if bc.bitmap[firstword]&lo != 0 || bc.bitmap[endword]&hi != 0 { + return true + } + for _, w := range bc.bitmap[firstword+1 : endword] { + if w != 0 { + return true + } + } + return false +} + func (bc *bitmapContainer) getCardinalityInRange(start, end uint) int { if start >= end { return 0 diff --git a/fastaggregation.go b/fastaggregation.go index 3881b88a..a344f739 100644 --- a/fastaggregation.go +++ b/fastaggregation.go @@ -125,26 +125,188 @@ func (x1 *Bitmap) repairAfterLazy() { } } -// FastAnd computes the intersection between many bitmaps quickly -// Compared to the And function, it can take many bitmaps as input, thus saving the trouble -// of manually calling "And" many times. +// FastAnd computes the intersection between many bitmaps quickly. +// Compared to the And function, it can take many bitmaps as input. // -// Performance hints: if you have very large and tiny bitmaps, -// it may be beneficial performance-wise to put a tiny bitmap -// in first position. +// With three or more inputs, FastAnd walks the keys of the input with the +// fewest keys and looks each one up in the first four inputs, also checking +// an array container for a shared value, before allocating anything for it. +// The remaining inputs are then intersected into the result one at a time. func FastAnd(bitmaps ...*Bitmap) *Bitmap { - if len(bitmaps) == 0 { + switch len(bitmaps) { + case 0: return NewBitmap() - } else if len(bitmaps) == 1 { + case 1: return bitmaps[0].Clone() + case 2: + return And(bitmaps[0], bitmaps[1]) } - answer := And(bitmaps[0], bitmaps[1]) - for _, bm := range bitmaps[2:] { - answer.And(bm) + driver := 0 + for i := 1; i < len(bitmaps); i++ { + if bitmaps[i].highlowcontainer.size() < bitmaps[driver].highlowcontainer.size() { + driver = i + } + } + dra := &bitmaps[driver].highlowcontainer + // The walk covers the first width inputs, each with a cursor that resumes + // where the previous key stopped. Past those, intersecting an input into + // the result costs less per key than walking it. + const width = 4 + n := min(len(bitmaps), width) + var pos [width]int + var cs [width + 1]container // the window and a driver past it + answer := NewBitmap() +keys: + for j := 0; j < dra.size(); j++ { + key := dra.getKeyAtIndex(j) + dc := dra.getContainerAtIndex(j) + // Only an array is checked for a value it shares with each input: a + // lookup per value that usually stops at the first. For a run or a + // bitmap the check costs about as much as the intersection. + _, check := dc.(*arrayContainer) + m := 0 + if !dc.isFull() { // a full container changes nothing + cs[0] = dc + m = 1 + } + for i, bm := range bitmaps[:n] { + if i == driver { + continue + } + ra := &bm.highlowcontainer + p := pos[i] + if p < ra.size() && ra.getKeyAtIndex(p) < key { + p = ra.advanceUntil(key, p) + } + pos[i] = p + if p == ra.size() { + break keys + } + if ra.getKeyAtIndex(p) != key { + continue keys + } + pos[i]++ + c := ra.getContainerAtIndex(p) + if c.isFull() { + continue + } + if check && !dc.intersects(c) { + continue keys + } + cs[m] = c + m++ + } + if c := andK(cs[:m]); c != nil { + if answer.IsEmpty() { + // Room for up to 64 of the driver's remaining keys at once. + k := min(dra.size()-j, 64) + answer.highlowcontainer = roaringArray{ + keys: make([]uint16, 0, k), + containers: make([]container, 0, k), + needCopyOnWrite: make([]bool, 0, k), + } + } + answer.highlowcontainer.appendContainer(key, c, false) + } + } + for i := n; i < len(bitmaps) && !answer.IsEmpty(); i++ { + if i != driver { + answer.And(bitmaps[i]) + } } return answer } +// andK intersects the containers of one key, none of them full, and returns +// nil when the intersection is empty. It may reorder cs. +func andK(cs []container) container { + switch len(cs) { + case 0: + return newRunContainer16Range(0, maxCapacity-1) + case 1: + return cs[0].clone() + } + // The smallest array goes first; the rest keep their order. + smallest := -1 + for i, c := range cs { + if x, ok := c.(*arrayContainer); ok && (smallest < 0 || x.getCardinality() < cs[smallest].getCardinality()) { + smallest = i + } + } + if smallest >= 0 { + a := cs[smallest] + copy(cs[1:smallest+1], cs[:smallest]) + cs[0] = a + return andKChain(cs) + } + n := 0 + for i, c := range cs { + if _, ok := c.(*runContainer16); ok { + cs[n], cs[i] = cs[i], cs[n] + n++ + } + } + if n == len(cs) { + return andKChain(cs) + } + return andKBitmaps(cs[n:], cs[:n]) +} + +// andKChain intersects cs in order: the first two into a new container, then +// the rest into it in place, so nothing bigger than the first result is built. +func andKChain(cs []container) container { + c := cs[0].and(cs[1]) + for i := 2; i < len(cs) && !c.isEmpty(); i++ { + c = c.iand(cs[i]) + } + if c.isEmpty() { + return nil + } + return c +} + +// andKBitmaps ANDs the bitmaps in a scratch bitmap on the stack, stopping as +// soon as the result is empty. Runs act as masks: only the words between +// their common start and end are ANDed, and the gaps inside a run are cleared +// afterwards. +func andKBitmaps(bms, runs []container) container { + lo, hi := 0, maxCapacity-1 + gaps := false + for _, c := range runs { + rc := c.(*runContainer16) + lo, hi = max(lo, int(rc.minimum())), min(hi, int(rc.maximum())) + gaps = gaps || len(rc.iv) > 1 + } + if lo > hi { + return nil + } + first, last := lo/64, hi/64 + var scratch [bitmapContainerSize]uint64 + w := scratch[first : last+1] + src := bms[0].(*bitmapContainer).bitmap[first : last+1] + card := uint64(0) + if len(bms) == 1 { + copy(w, src) + } + for _, c := range bms[1:] { + if card = andCardSlice(w, src, c.(*bitmapContainer).bitmap[first:last+1]); card == 0 { + return nil + } + src = w + } + if len(bms) == 1 || gaps || lo%64 != 0 || (hi+1)%64 != 0 { + resetBitmapRange(scratch[:], first*64, lo) + resetBitmapRange(scratch[:], hi+1, (last+1)*64) + for _, c := range runs { + clearBitmapGaps(scratch[:], c.(*runContainer16).iv, lo, hi+1) + } + if card = popcntSlice(w); card == 0 { + return nil + } + } + return containerFromWords(scratch[:], first, last, int(card)) +} + // FastOr computes the union between many bitmaps quickly, as opposed to having to call Or repeatedly. // It might also be faster than calling Or repeatedly. func FastOr(bitmaps ...*Bitmap) *Bitmap { diff --git a/fastand_kway_test.go b/fastand_kway_test.go new file mode 100644 index 00000000..4c1ce4e0 --- /dev/null +++ b/fastand_kway_test.go @@ -0,0 +1,292 @@ +package roaring + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +// kwayKey picks the container key for the k-th key of an input: inputs share +// their keys, but each leaves some out, the way real inputs do. +func kwayKey(rng *rand.Rand, k int) (uint64, bool) { + return uint64(k) << 16, rng.Intn(8) != 0 +} + +// kwayBitmap builds a bitmap over up to keys keys holding random values at +// the given density: arrays when sparse, bitmaps when dense. +func kwayBitmap(rng *rand.Rand, keys int, density float64) *Bitmap { + bm := New() + for k := 0; k < keys; k++ { + base, present := kwayKey(rng, k) + if !present { + continue + } + for i := 0; i < int(density*65536); i++ { + bm.Add(uint32(base) + uint32(rng.Intn(65536))) + } + } + return bm +} + +// kwayRuns builds a run-optimized bitmap of random ranges over up to keys keys. +func kwayRuns(rng *rand.Rand, keys int) *Bitmap { + bm := New() + for k := 0; k < keys; k++ { + base, present := kwayKey(rng, k) + if !present { + continue + } + for start := 0; start < 65536; start += 1000 + rng.Intn(3000) { + bm.AddRange(base+uint64(start), base+uint64(min(start+1+rng.Intn(800), 65536))) + } + } + bm.RunOptimize() + return bm +} + +// intervalBitmap holds one run container at key 0 made of the given +// half-open intervals. +func intervalBitmap(t *testing.T, intervals [][2]uint64) *Bitmap { + t.Helper() + bm := New() + for _, iv := range intervals { + bm.AddRange(iv[0], iv[1]) + } + bm.RunOptimize() + require.Equal(t, uint64(1), bm.Stats().RunContainers, "%v", intervals) + return bm +} + +func randomIntervals(rng *rand.Rand) [][2]uint64 { + var out [][2]uint64 + pos := uint64(rng.Intn(64)) + for n := 1 + rng.Intn(40); n > 0 && pos < 65536; n-- { + length := min(uint64(4+rng.Intn(3000)), 65536-pos) + out = append(out, [2]uint64{pos, pos + length}) + pos += length + uint64(rng.Intn(200)) + } + return out +} + +// requireFastAnd checks FastAnd against the pairwise definition, that the +// result is valid, and that it shares no storage with its inputs, and +// reports whether the result is empty. +func requireFastAnd(t *testing.T, inputs ...*Bitmap) bool { + t.Helper() + before := make([]*Bitmap, len(inputs)) + for i, in := range inputs { + before[i] = in.Clone() + } + want := pairwiseAnd(inputs) + got := FastAnd(inputs...) + require.NoError(t, got.Validate()) + require.True(t, got.Equals(want), "%d values, want %d", got.GetCardinality(), want.GetCardinality()) + empty := got.IsEmpty() + got.Flip(0, 1<<21) // rewrites every container the inputs could share + for i, in := range inputs { + require.True(t, in.Equals(before[i]), "input %d was modified through the result", i) + } + return empty +} + +func TestFastAndMatchesPairwise(t *testing.T) { + rng := rand.New(rand.NewSource(3)) + densities := []float64{0.0005, 0.01, 0.05, 0.3, 0.8, 0.98} + whole := New() + whole.AddRange(0, 4<<16) + nonEmpty := 0 + for trial := 0; trial < 400; trial++ { + inputs := make([]*Bitmap, 3+rng.Intn(10)) + for i := range inputs { + switch rng.Intn(6) { + case 0: + inputs[i] = kwayRuns(rng, 1+rng.Intn(4)) + case 1: + inputs[i] = intervalBitmap(t, randomIntervals(rng)) + case 2: + inputs[i] = whole + default: + inputs[i] = kwayBitmap(rng, 1+rng.Intn(4), densities[rng.Intn(len(densities))]) + } + } + if !requireFastAnd(t, inputs...) { + nonEmpty++ + } + } + require.Greater(t, nonEmpty, 100, "most trials must reach the intersection, not stop at a missing key") +} + +func TestFastAndEdges(t *testing.T) { + rng := rand.New(rand.NewSource(12)) + dense := func() *Bitmap { + bm := New() + for range 32768 { + bm.Add(uint32(rng.Intn(65536))) + } + bm.AddRange(0, 256) // so a mask's edges always have values to drop + return bm + } + full := intervalBitmap(t, [][2]uint64{{0, 65536}}) + // A range ending on a word boundary on one side only is masked on the other. + requireFastAnd(t, dense(), intervalBitmap(t, [][2]uint64{{64, 100}}), dense()) + requireFastAnd(t, dense(), intervalBitmap(t, [][2]uint64{{70, 128}}), dense()) + // Two ranges spanning whole words still mask the gap between them. + requireFastAnd(t, dense(), intervalBitmap(t, [][2]uint64{{0, 64}, {128, 192}}), dense()) + // A range narrower than a many-interval run's span clips it. + var manyShort [][2]uint64 + for p := uint64(0); p < 65536; p += 64 { + manyShort = append(manyShort, [2]uint64{p, p + 8}) + } + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{100, 40000}}), intervalBitmap(t, manyShort), dense(), dense()) + // Two ranges that each meet the bitmap but overlap only where it is + // empty, so the masked result is empty. + gap := New() + gap.Add(0) + gap.AddRange(128, 5300) + requireFastAnd(t, gap, intervalBitmap(t, [][2]uint64{{0, 128}}), intervalBitmap(t, [][2]uint64{{64, 192}})) + // Ranges that each meet the first one but have no common value end the + // key before a bitmap is read, as do disjoint runs. + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{0, 40000}}), intervalBitmap(t, [][2]uint64{{30000, 65536}}), intervalBitmap(t, [][2]uint64{{0, 20000}}), dense()) + requireFastAnd(t, intervalBitmap(t, [][2]uint64{{0, 1000}}), intervalBitmap(t, [][2]uint64{{2000, 3000}}), dense()) + // Whole-key runs change nothing. + y := New() + y.AddMany([]uint32{1, 5, 9, 70000}) + require.True(t, FastAnd(y, full, y).Equals(And(y, full))) + require.True(t, FastAnd(full, full, full).Equals(full)) +} + +// singletons returns n inputs holding the values 0 to n-1, one each: the +// second input already fails the check. +func singletons(n int) []*Bitmap { + inputs := make([]*Bitmap, n) + for i := range inputs { + inputs[i] = New() + inputs[i].Add(uint32(i)) + } + return inputs +} + +// kwayTriple returns three dense inputs whose pairs all overlap but whose +// intersection is empty. +func kwayTriple() (a, b, c *Bitmap) { + a, b, c = New(), New(), New() + for id := uint32(0); id < 4<<16; id++ { + switch id % 3 { + case 0: + a.Add(id) + b.Add(id) + case 1: + b.Add(id) + c.Add(id) + case 2: + a.Add(id) + c.Add(id) + } + } + return a, b, c +} + +func TestFastAndEmptyAllocatesNoContainer(t *testing.T) { + a, b, c := kwayTriple() + require.True(t, a.Intersects(b) && b.Intersects(c) && a.Intersects(c)) + require.True(t, FastAnd(a, b, c).IsEmpty()) + // Only the answer bitmap; never a container. + require.LessOrEqual(t, testing.AllocsPerRun(50, func() { FastAnd(a, b, c) }), 1.0) + + mask := intervalBitmap(t, [][2]uint64{{0, 65536}}) + evens, odds := New(), New() + for v := uint32(0); v < 65536; v += 2 { + evens.Add(v) + odds.Add(v + 1) + } + require.True(t, FastAnd(mask, evens, odds).IsEmpty()) + require.LessOrEqual(t, testing.AllocsPerRun(50, func() { FastAnd(mask, evens, odds) }), 1.0) + + // Nothing is allocated for the inputs past an empty result. + inputs := singletons(1000) + require.True(t, FastAnd(inputs...).IsEmpty()) + require.LessOrEqual(t, testing.AllocsPerRun(50, func() { FastAnd(inputs...) }), 1.0) +} + +// randomKey holds card random values on key. +func randomKey(rng *rand.Rand, card int, key uint32) *Bitmap { + bm := New() + for bm.GetCardinality() < uint64(card) { + bm.Add(key<<16 | uint32(rng.Intn(65536))) + } + return bm +} + +func pairwiseAnd(bms []*Bitmap) *Bitmap { + r := And(bms[0], bms[1]) + for _, bm := range bms[2:] { + r.And(bm) + } + return r +} + +// BenchmarkFastAndShapes runs FastAnd against pairwise And on the same +// inputs, one shape per container pairing plus many-input shapes; the first +// seven add a whole-key range like a query window. +func BenchmarkFastAndShapes(b *testing.B) { + rng := rand.New(rand.NewSource(9)) + window := New() + window.AddRange(0, 65536) + wide := New() + wide.AddRange(0, 8<<16) + spread := func(card int) *Bitmap { + bm := New() + for k := uint32(0); k < 8; k++ { + bm.Or(randomKey(rng, card, k)) + } + return bm + } + x, y, z := kwayTriple() + same := make([]*Bitmap, 128) + base := spread(30) + for i := range same { + same[i] = base.Clone() + } + // 1,024 inputs of 256 keys that all survive: the widest walk. + keys256 := New() + for k := uint32(0); k < 256; k++ { + for v := uint32(0); v < 64; v += 2 { + keys256.Add(k<<16 | v) + } + } + surviving := make([]*Bitmap, 1024) + for i := range surviving { + surviving[i] = keys256.Clone() + } + for _, tc := range []struct { + name string + inputs []*Bitmap + }{ + {"array30-bitmap", []*Bitmap{randomKey(rng, 30, 0), randomKey(rng, 33000, 0), window}}, + {"array4000-bitmap", []*Bitmap{randomKey(rng, 4000, 0), randomKey(rng, 33000, 0), window}}, + {"array30-runs", []*Bitmap{randomKey(rng, 30, 0), runBitmap(250, 16, 32), window}}, + {"array4000-runs", []*Bitmap{randomKey(rng, 4000, 0), runBitmap(250, 16, 32), window}}, + {"bitmap-bitmap", []*Bitmap{randomKey(rng, 33000, 0), randomKey(rng, 33000, 0), window}}, + {"runs-bitmap", []*Bitmap{runBitmap(250, 16, 32), randomKey(rng, 33000, 0), window}}, + {"runs-runs", []*Bitmap{runBitmap(250, 16, 32), runBitmap(1875, 16, 32), window}}, + {"jointly-empty", []*Bitmap{x, y, z}}, + {"eight-keys", []*Bitmap{spread(30), spread(33000), wide}}, + {"singletons-10000", singletons(10000)}, + {"equal-x128", same}, + {"surviving-1024x256", surviving}, + } { + b.Run(tc.name+"/fastand", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = FastAnd(tc.inputs...) + } + }) + b.Run(tc.name+"/pairwise", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = pairwiseAnd(tc.inputs) + } + }) + } +} diff --git a/parallel.go b/parallel.go index 01ba856f..9703912f 100644 --- a/parallel.go +++ b/parallel.go @@ -283,18 +283,8 @@ func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap { andFunc := func() { // Assumes only structs with >=2 containers are passed for input := range inputChan { - c := input.containers[0].and(input.containers[1]) - for _, next := range input.containers[2:] { - if c.isEmpty() { - break - } - c = c.iand(next) - } - - // Send a nil explicitly if the result of the intersection is an empty container - if c.isEmpty() { - c = nil - } + // A nil result tells the receiver the intersection is empty. + c := andKChain(input.containers) kx := keyedContainer{ input.key, diff --git a/real_data_benchmark_test.go b/real_data_benchmark_test.go index 7e47a323..f40a12a4 100644 --- a/real_data_benchmark_test.go +++ b/real_data_benchmark_test.go @@ -186,3 +186,9 @@ func BenchmarkRealDataFastOr(b *testing.B) { return FastOr(bitmaps...).GetCardinality() }) } + +func BenchmarkRealDataFastAnd(b *testing.B) { + benchmarkRealDataAggregate(b, func(bitmaps []*Bitmap) uint64 { + return FastAnd(bitmaps...).GetCardinality() + }) +} diff --git a/runbitmap_and_test.go b/runbitmap_and_test.go new file mode 100644 index 00000000..bb3f5abf --- /dev/null +++ b/runbitmap_and_test.go @@ -0,0 +1,274 @@ +package roaring + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +// randomRun builds a canonical run container: up to n sorted intervals of +// at most maxLen values separated by gaps of at most maxGap, optionally +// touching 0 and 65535. +func randomRun(rng *rand.Rand, n, maxLen, maxGap int, touchEdges bool) *runContainer16 { + rc := &runContainer16{} + pos := 0 + if !touchEdges { + pos = rng.Intn(100) + } + for i := 0; i < n && pos < 65536; i++ { + length := min(1+rng.Intn(1+rng.Intn(maxLen)), 65536-pos) + rc.iv = append(rc.iv, interval16{start: uint16(pos), length: uint16(length - 1)}) + pos += length + 1 + rng.Intn(maxGap) + } + if last := int(rc.maximum()); touchEdges && last != 65535 { + if start := 65535 - rng.Intn(500); last+2 <= start { + rc.iv = append(rc.iv, interval16{start: uint16(start), length: uint16(65535 - start)}) + } + } + return rc +} + +// randomBitmapContainer ANDs ands random words per position, so the density +// is 1/2^ands. +func randomBitmapContainer(rng *rand.Rand, ands int) *bitmapContainer { + bc := newBitmapContainer() + for i := range bc.bitmap { + w := rng.Uint64() + for k := 1; k < ands; k++ { + w &= rng.Uint64() + } + bc.bitmap[i] = w + } + bc.computeCardinality() + return bc +} + +func requireAndResult(t *testing.T, want, got container) { + t.Helper() + require.True(t, want.equals(got), "want %d values, got %d", want.getCardinality(), got.getCardinality()) + _, isArray := got.(*arrayContainer) + require.Equal(t, got.getCardinality() <= arrayDefaultMaxSize, isArray, "container kind must follow its cardinality") +} + +// The reference is the old path: materialize the run, then the unchanged +// bitmap-bitmap intersection. Runs of one to hundreds of intervals check +// both the per-interval and the scratch paths, against every density. +func TestRunAndBitmapContainer(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + regular := func(stride, length int) *runContainer16 { + rc := &runContainer16{} + for start := 0; start+length <= 65536; start += stride { + rc.iv = append(rc.iv, interval16{start: uint16(start), length: uint16(length - 1)}) + } + return rc + } + runs := []*runContainer16{regular(32, 8), regular(64, 1), regular(3, 2), regular(200, 100)} + for i := 0; i < 120; i++ { + runs = append(runs, randomRun(rng, 1+rng.Intn(40), 3000, 2000, i%5 == 0), randomRun(rng, 65+rng.Intn(400), 60, 120, false)) + } + // Only values in the gaps of regular(32, 8), so that run's result is empty. + gaps := newBitmapContainer() + for v := 0; v < 65536; v++ { + if v%32 >= 8 { + gaps.iadd(uint16(v)) + } + } + for i, rc := range runs { + for _, bc := range []*bitmapContainer{randomBitmapContainer(rng, 6), randomBitmapContainer(rng, 2), randomBitmapContainer(rng, 1), newBitmapContainerwithRange(0, 65535), gaps} { + want := newBitmapContainerFromRun(rc).andBitmap(bc) + requireAndResult(t, want, rc.and(bc)) + requireAndResult(t, want, bc.and(rc)) + requireAndResult(t, want, bc.clone().iand(rc)) + require.Equal(t, !want.isEmpty(), rc.intersects(bc), "run %d", i) + } + } +} + +func TestRunAndBitmapContainerEdges(t *testing.T) { + empty := newBitmapContainer() + full := newBitmapContainerwithRange(0, 65535) + single := &runContainer16{iv: []interval16{{start: 100, length: 9}}} + edges := &runContainer16{iv: []interval16{{start: 0, length: 0}, {start: 65535, length: 0}}} + atThreshold := &runContainer16{iv: []interval16{{start: 0, length: arrayDefaultMaxSize - 1}}} + pastThreshold := &runContainer16{iv: []interval16{{start: 0, length: arrayDefaultMaxSize}}} + for _, rc := range []*runContainer16{single, edges, atThreshold, pastThreshold} { + for _, bc := range []*bitmapContainer{empty, full} { + want := newBitmapContainerFromRun(rc).andBitmap(bc) + requireAndResult(t, want, rc.and(bc)) + requireAndResult(t, want, bc.clone().iand(rc)) + } + } + // Exactly 4096 values through the scratch path: every even value below + // 16384 against the values below 8192. + evens := &runContainer16{} + for v := 0; v < 4*arrayDefaultMaxSize; v += 2 { + evens.iv = append(evens.iv, interval16{start: uint16(v)}) + } + half := newBitmapContainerwithRange(0, 2*arrayDefaultMaxSize-1) + requireAndResult(t, newBitmapContainerFromRun(evens).andBitmap(half), evens.and(half)) +} + +// An empty intersection allocates only its empty array container, and an +// in-place And with a full run and Intersects allocate nothing; the old paths +// allocated an 8 KiB bitmap first. +func TestRunAndBitmapContainerAllocations(t *testing.T) { + rc := &runContainer16{iv: []interval16{{start: 0, length: 999}, {start: 30000, length: 999}}} + bc := newBitmapContainerwithRange(2000, 20000) + require.LessOrEqual(t, testing.AllocsPerRun(100, func() { + if !rc.and(bc).isEmpty() { + t.Fatal("expected empty") + } + }), 1.0) + // The clone costs two allocations, the empty result one. + require.LessOrEqual(t, testing.AllocsPerRun(100, func() { + _ = bc.clone().iand(rc) + }), 3.0) + full := newRunContainer16Range(0, MaxUint16) + require.Zero(t, testing.AllocsPerRun(100, func() { _ = bc.iand(full) })) + require.Zero(t, testing.AllocsPerRun(100, func() { _ = rc.iand(full) })) + for _, c := range []container{bc, newArrayContainerRange(2000, 3000), &runContainer16{iv: []interval16{{start: 5000, length: 100}}}} { + require.Zero(t, testing.AllocsPerRun(100, func() { _ = rc.intersects(c) }), "%T", c) + } +} + +// The reference is the old definition, !rc.and(c).isEmpty(). +func TestRunContainerIntersects(t *testing.T) { + rng := rand.New(rand.NewSource(11)) + for trial := 0; trial < 400; trial++ { + rc := randomRun(rng, 1+rng.Intn(30), 3000, 2000, trial%7 == 0) + ac := newArrayContainer() + for range rng.Intn(300) { + ac.iadd(uint16(rng.Intn(65536))) + } + others := []container{randomBitmapContainer(rng, 1+trial%3*3), ac, randomRun(rng, 1+rng.Intn(30), 3000, 2000, trial%5 == 0)} + for _, c := range others { + want := !rc.and(c).isEmpty() + require.Equal(t, want, rc.intersects(c), "trial %d run vs %T", trial, c) + require.Equal(t, want, c.intersects(rc), "trial %d %T vs run", trial, c) + } + } + // A few values against many intervals take the search branch; the + // reference is the same definition. + many := randomRun(rng, 300, 60, 120, false) + require.Greater(t, len(many.iv), 200) + for trial := 0; trial < 200; trial++ { + few := newArrayContainer() + for range 1 + rng.Intn(6) { + few.iadd(uint16(rng.Intn(65536))) + } + want := !many.and(few).isEmpty() + require.Equal(t, want, many.intersects(few), "trial %d few vs many", trial) + require.Equal(t, want, few.intersects(many), "trial %d many vs few", trial) + } + // Edges: empty array, touching intervals, single values at both ends. + empty := newArrayContainer() + edges := &runContainer16{iv: []interval16{{start: 0, length: 0}, {start: 65535, length: 0}}} + require.False(t, edges.intersects(empty)) + one := newArrayContainer() + one.iadd(65535) + require.True(t, edges.intersects(one)) + touching := &runContainer16{iv: []interval16{{start: 1, length: 65533}}} + require.False(t, edges.intersects(touching)) + require.True(t, edges.intersects(newBitmapContainerwithRange(0, 0))) +} + +// runBitmap holds one run container at key 0 covering n intervals of the +// given length, spaced stride apart. +func runBitmap(n, length, stride int) *Bitmap { + bm := New() + for i := 0; i < n; i++ { + bm.AddRange(uint64(i*stride), uint64(i*stride+length)) + } + bm.RunOptimize() + return bm +} + +// BenchmarkRunAndBitmap intersects a run-optimized bitmap with a dense one +// over a single key through the public API, one case per result kind, plus +// the in-place form, alone and with the copy it needs, and the Intersects test. +func BenchmarkRunAndBitmap(b *testing.B) { + rng := rand.New(rand.NewSource(7)) + short := runBitmap(20, 300, 3000) // 6000 values + long := runBitmap(4, 10000, 16000) // 40000 values + many := runBitmap(1024, 8, 64) // 8192 values in 1024 intervals + gaps, quarter, half, dense := New(), New(), New(), New() + for v := uint32(0); v < 65536; v++ { + if !short.Contains(v) && rng.Intn(2) == 0 { + gaps.Add(v) + } + if rng.Intn(4) == 0 { + quarter.Add(v) + } + if rng.Intn(2) == 0 { + half.Add(v) + } + if rng.Intn(10) != 0 { + dense.Add(v) + } + } + for _, tc := range []struct { + name string + x, y *Bitmap + }{ + {"and/empty", short, gaps}, + {"and/array", short, half}, + {"and/bitmap", long, dense}, + {"and/many-intervals/array", many, quarter}, + {"and/many-intervals/bitmap", many, dense}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = And(tc.x, tc.y) + } + }) + } + for _, tc := range []struct { + name string + x *Bitmap + }{ + {"iand/bitmap", long}, + {"iand/many-intervals", many}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + var xs []*Bitmap + for b.Loop() { + if len(xs) == 0 { + // Fresh copies, made outside the timer: one And turns a copy into its result. + b.StopTimer() + for range 64 { + xs = append(xs, dense.Clone()) + } + b.StartTimer() + } + xs[len(xs)-1].And(tc.x) + xs = xs[:len(xs)-1] + } + }) + b.Run("clone+"+tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + dense.Clone().And(tc.x) + } + }) + } + few := New() + few.AddMany([]uint32{100, 30000, 60000}) + for _, tc := range []struct { + name string + x, y *Bitmap + }{ + {"intersects/disjoint", short, gaps}, + {"intersects/overlapping", short, half}, + {"intersects/few-values-many-intervals", many, few}, + } { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = tc.x.Intersects(tc.y) + } + }) + } +} diff --git a/runcontainer.go b/runcontainer.go index 7c369b75..f83c8021 100644 --- a/runcontainer.go +++ b/runcontainer.go @@ -1855,10 +1855,63 @@ func (rc *runContainer16) andCardinality(a container) int { panic("unsupported container type") } -// andBitmapContainer finds the intersection of rc and b. +const runAndScratchIntervals = 64 + +// andInScratch reports whether ANDing rc with a bitmap is cheaper in a +// scratch bitmap than interval by interval: past runAndScratchIntervals +// intervals, the measured crossover, if the run covers more than 4096 values +// or the vector fill makes reading the scratch back cheap. +func (rc *runContainer16) andInScratch() bool { + return len(rc.iv) > runAndScratchIntervals && (useVectorFill || rc.getCardinality() > arrayDefaultMaxSize) +} + +// andBitmapContainer intersects rc with bc, counting before allocating. func (rc *runContainer16) andBitmapContainer(bc *bitmapContainer) container { - bc2 := newBitmapContainerFromRun(rc) - return bc2.andBitmap(bc) + if rc.andInScratch() { + return rc.andBitmapContainerScratch(bc) + } + card := rc.andBitmapContainerCardinality(bc) + if card > arrayDefaultMaxSize { + answer := newBitmapContainer() + for i := range rc.iv { + copyBitmapRange(answer.bitmap, bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + answer.cardinality = card + return answer + } + return rc.andBitmapArray(bc, card) +} + +// andBitmapArray returns the card values of bc inside rc's intervals as an +// array container. +func (rc *runContainer16) andBitmapArray(bc *bitmapContainer, card int) *arrayContainer { + if useVectorFill && card >= arrayVectorFillMinCardinality { + var runBits [bitmapContainerSize]uint64 + for i := range rc.iv { + setBitmapRange(runBits[:], int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + ac := newArrayContainerSize(card) + fillArrayAND(ac.content, runBits[:], bc.bitmap) + return ac + } + ac := newArrayContainerCapacity(card) + for i := range rc.iv { + ac.content = appendBitmapRange(ac.content, bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + return ac +} + +func (rc *runContainer16) andBitmapContainerScratch(bc *bitmapContainer) container { + first, last := int(rc.iv[0].start)/64, int(rc.maximum())/64 + var scratch [bitmapContainerSize]uint64 + for i := range rc.iv { + copyBitmapRange(scratch[:], bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + card := int(popcntSlice(scratch[first : last+1])) + if card == 0 { + return newArrayContainerCapacity(0) + } + return containerFromWords(scratch[:], first, last, card) } func (rc *runContainer16) andArrayCardinality(ac *arrayContainer) int { @@ -1894,6 +1947,9 @@ func (rc *runContainer16) iand(a container) container { if rc.isFull() { return a.clone() } + if a.isFull() { + return rc + } switch c := a.(type) { case *runContainer16: // Important: there is no reason to believe that the @@ -2418,9 +2474,56 @@ func (rc *runContainer16) lazyOR(a container) container { } func (rc *runContainer16) intersects(a container) bool { - // TODO: optimize by doing inplace/less allocation - isect := rc.and(a) - return !isect.isEmpty() + switch c := a.(type) { + case *bitmapContainer: + for i := range rc.iv { + if c.intersectsRange(uint(rc.iv[i].start), uint(rc.iv[i].last())+1) { + return true + } + } + return false + case *arrayContainer: + if 2*len(c.content)*bits.Len(uint(len(rc.iv))) < len(rc.iv) { + // Few values against many intervals: a search each beats + // walking every interval. + for _, v := range c.content { + if rc.contains(v) { + return true + } + } + return false + } + // Otherwise andArray's merge, stopping at the first shared value. + for i, pos := 0, 0; i < len(rc.iv) && pos < len(c.content); { + v := c.content[pos] + for rc.iv[i].last() < v { + i++ + if i == len(rc.iv) { + return false + } + } + if rc.iv[i].start <= v { + return true + } + pos = advanceUntil(c.content, pos, len(c.content), rc.iv[i].start) + } + return false + case *runContainer16: + i, j := 0, 0 + for i < len(rc.iv) && j < len(c.iv) { + x, y := rc.iv[i], c.iv[j] + if haveOverlap16(x, y) { + return true + } + if x.last() < y.last() { + i++ + } else { + j++ + } + } + return false + } + panic("unsupported container type") } func (rc *runContainer16) xor(a container) container { diff --git a/util.go b/util.go index e49ad29c..ac133947 100644 --- a/util.go +++ b/util.go @@ -152,6 +152,76 @@ func flipBitmapRange(bitmap []uint64, start int, end int) { bitmap[endword] ^= ^uint64(0) >> (uint(-end) % 64) } +// copyBitmapRange copies the bits of src within [start, end) into dst, which +// must be zero there. +func copyBitmapRange(dst, src []uint64, start int, end int) { + if start >= end { + return + } + firstword := start / 64 + endword := (end - 1) / 64 + if firstword == endword { + dst[firstword] |= src[firstword] & (^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64)) + return + } + dst[firstword] |= src[firstword] & (^uint64(0) << uint(start%64)) + copy(dst[firstword+1:endword], src[firstword+1:endword]) + dst[endword] |= src[endword] & (^uint64(0) >> (uint(-end) % 64)) +} + +// appendBitmapRange appends the positions of the bits set in [start, end). +func appendBitmapRange(out []uint16, bitmap []uint64, start int, end int) []uint16 { + if start >= end { + return out + } + firstword := start / 64 + endword := (end - 1) / 64 + for i := firstword; i <= endword; i++ { + w := bitmap[i] + if i == firstword { + w &= ^uint64(0) << uint(start%64) + } + if i == endword { + w &= ^uint64(0) >> (uint(-end) % 64) + } + for w != 0 { + out = append(out, uint16(i*64+bits.TrailingZeros64(w))) + w &= w - 1 + } + } + return out +} + +// clearBitmapGaps clears the bits in [from, to) that no interval of iv covers. +func clearBitmapGaps(bitmap []uint64, iv []interval16, from, to int) { + for _, x := range iv { + if int(x.start) >= to { + break + } + resetBitmapRange(bitmap, from, int(x.start)) + from = max(from, int(x.last())+1) + } + resetBitmapRange(bitmap, from, to) +} + +// containerFromWords builds an array or bitmap container from the card bits +// set in words [first, last] of w, which spans a whole key and is zero +// outside them. +func containerFromWords(w []uint64, first, last, card int) container { + if card <= arrayDefaultMaxSize { + if useVectorFill && card >= arrayVectorFillMinCardinality { + return newArrayContainerFromBitmap(&bitmapContainer{cardinality: card, bitmap: w}) + } + ac := newArrayContainerCapacity(card) + ac.content = appendBitmapRange(ac.content, w, first*64, (last+1)*64) + return ac + } + bc := newBitmapContainer() + copy(bc.bitmap[first:last+1], w[first:last+1]) + bc.cardinality = card + return bc +} + func resetBitmapRange(bitmap []uint64, start int, end int) { if start >= end { return @@ -163,9 +233,7 @@ func resetBitmapRange(bitmap []uint64, start int, end int) { return } bitmap[firstword] &= ^(^uint64(0) << uint(start%64)) - for i := firstword + 1; i < endword; i++ { - bitmap[i] = 0 - } + clear(bitmap[firstword+1 : endword]) bitmap[endword] &= ^(^uint64(0) >> (uint(-end) % 64)) }