fastaggregation: count before allocating in FastAnd

bitmapcontainer.go

@@ -318,6 +318,11 @@ func bitmapEquals(a, b []uint64) bool {
 // is the array-to-bitmap conversion point and leaves a comfortable margin.
 const bitmapContainerVectorFillMinCardinality = 4096
 
+// arrayVectorFillMinCardinality is where decoding a whole key with the vector
+// fill beats decoding only the words that hold the values. Measured crossover
+// is near 1000 values.
+const arrayVectorFillMinCardinality = 1024
+
 func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) int {
 	if useVectorFill && bc.cardinality >= bitmapContainerVectorFillMinCardinality {
 		return fillLeastSignificant16bitsVector(bc.bitmap, x, i, mask)
@@ -809,16 +814,28 @@ func (bc *bitmapContainer) iand(a container) container {
 		return bc.iandBitmap(x)
 	case *runContainer16:
 		if x.isFull() {
-			return bc.clone()
+			return bc
 		}
 		return bc.iandRun16(x)
 	}
 	panic("unsupported container type")
 }
 
 func (bc *bitmapContainer) iandRun16(rc *runContainer16) container {
-	rcb := newBitmapContainerFromRun(rc)
-	return bc.iandBitmap(rcb)
+	if rc.andInScratch() {
+		var scratch [bitmapContainerSize]uint64
+		for i := range rc.iv {
+			setBitmapRange(scratch[:], int(rc.iv[i].start), int(rc.iv[i].last())+1)
+		}
+		return bc.iandBitmap(&bitmapContainer{bitmap: scratch[:]})
+	}
+	card := rc.andBitmapContainerCardinality(bc)
+	if card <= arrayDefaultMaxSize {
+		return rc.andBitmapArray(bc, card)
+	}
+	clearBitmapGaps(bc.bitmap, rc.iv, 0, maxCapacity)
+	bc.cardinality = card
+	return bc
 }
 
 func (bc *bitmapContainer) iandArray(ac *arrayContainer) container {
@@ -828,15 +845,13 @@ func (bc *bitmapContainer) iandArray(ac *arrayContainer) container {
 
 func (bc *bitmapContainer) andArray(value2 *arrayContainer) *arrayContainer {
 	answer := newArrayContainerCapacity(len(value2.content))
-	answer.content = answer.content[:cap(answer.content)]
-	c := value2.getCardinality()
+	out, bitmap := answer.content[:cap(answer.content)], bc.bitmap
 	pos := 0
-	for k := 0; k < c; k++ {
-		v := value2.content[k]
-		answer.content[pos] = v
-		pos += int(bc.bitValue(v))
+	for _, v := range value2.content {
+		out[pos] = v
+		pos += int((bitmap[v>>6] >> (v & 63)) & 1)
 	}
-	answer.content = answer.content[:pos]
+	answer.content = out[:pos]
 	return answer
 }
 
@@ -850,6 +865,26 @@ func (bc *bitmapContainer) andArrayCardinality(value2 *arrayContainer) int {
 	return pos
 }
 
+func (bc *bitmapContainer) intersectsRange(start, end uint) bool {
+	if start >= end {
+		return false
+	}
+	firstword, endword := start/64, (end-1)/64
+	lo, hi := ^uint64(0)<<(start%64), ^uint64(0)>>((64-end)&63)
+	if firstword == endword {
+		return bc.bitmap[firstword]&lo&hi != 0
+	}
+	if bc.bitmap[firstword]&lo != 0 || bc.bitmap[endword]&hi != 0 {
+		return true
+	}
+	for _, w := range bc.bitmap[firstword+1 : endword] {
+		if w != 0 {
+			return true
+		}
+	}
+	return false
+}
+
 func (bc *bitmapContainer) getCardinalityInRange(start, end uint) int {
 	if start >= end {
 		return 0

fastaggregation.go

@@ -125,26 +125,188 @@ func (x1 *Bitmap) repairAfterLazy() {
 	}
 }
 
-// FastAnd computes the intersection between many bitmaps quickly
-// Compared to the And function, it can take many bitmaps as input, thus saving the trouble
-// of manually calling "And" many times.
+// FastAnd computes the intersection between many bitmaps quickly.
+// Compared to the And function, it can take many bitmaps as input.
 //
-// Performance hints: if you have very large and tiny bitmaps,
-// it may be beneficial performance-wise to put a tiny bitmap
-// in first position.
+// With three or more inputs, FastAnd walks the keys of the input with the
+// fewest keys and looks each one up in the first four inputs, also checking
+// an array container for a shared value, before allocating anything for it.
+// The remaining inputs are then intersected into the result one at a time.
 func FastAnd(bitmaps ...*Bitmap) *Bitmap {
-	if len(bitmaps) == 0 {
+	switch len(bitmaps) {
+	case 0:
 		return NewBitmap()
-	} else if len(bitmaps) == 1 {
+	case 1:
 		return bitmaps[0].Clone()
+	case 2:
+		return And(bitmaps[0], bitmaps[1])
 	}
-	answer := And(bitmaps[0], bitmaps[1])
-	for _, bm := range bitmaps[2:] {
-		answer.And(bm)
+	driver := 0
+	for i := 1; i < len(bitmaps); i++ {
+		if bitmaps[i].highlowcontainer.size() < bitmaps[driver].highlowcontainer.size() {
+			driver = i
+		}
+	}
+	dra := &bitmaps[driver].highlowcontainer
+	// The walk covers the first width inputs, each with a cursor that resumes
+	// where the previous key stopped. Past those, intersecting an input into
+	// the result costs less per key than walking it.
+	const width = 4
+	n := min(len(bitmaps), width)
+	var pos [width]int
+	var cs [width + 1]container // the window and a driver past it
+	answer := NewBitmap()
+keys:
+	for j := 0; j < dra.size(); j++ {
+		key := dra.getKeyAtIndex(j)
+		dc := dra.getContainerAtIndex(j)
+		// Only an array is checked for a value it shares with each input: a
+		// lookup per value that usually stops at the first. For a run or a
+		// bitmap the check costs about as much as the intersection.
+		_, check := dc.(*arrayContainer)
+		m := 0
+		if !dc.isFull() { // a full container changes nothing
+			cs[0] = dc
+			m = 1
+		}
+		for i, bm := range bitmaps[:n] {
+			if i == driver {
+				continue
+			}
+			ra := &bm.highlowcontainer
+			p := pos[i]
+			if p < ra.size() && ra.getKeyAtIndex(p) < key {
+				p = ra.advanceUntil(key, p)
+			}
+			pos[i] = p
+			if p == ra.size() {
+				break keys
+			}
+			if ra.getKeyAtIndex(p) != key {
+				continue keys
+			}
+			pos[i]++
+			c := ra.getContainerAtIndex(p)
+			if c.isFull() {
+				continue
+			}
+			if check && !dc.intersects(c) {
+				continue keys
+			}
+			cs[m] = c
+			m++
+		}
+		if c := andK(cs[:m]); c != nil {
+			if answer.IsEmpty() {
+				// Room for up to 64 of the driver's remaining keys at once.
+				k := min(dra.size()-j, 64)
+				answer.highlowcontainer = roaringArray{
+					keys:            make([]uint16, 0, k),
+					containers:      make([]container, 0, k),
+					needCopyOnWrite: make([]bool, 0, k),
+				}
+			}
+			answer.highlowcontainer.appendContainer(key, c, false)
+		}
+	}
+	for i := n; i < len(bitmaps) && !answer.IsEmpty(); i++ {
+		if i != driver {
+			answer.And(bitmaps[i])
+		}
 	}
 	return answer
 }
 
+// andK intersects the containers of one key, none of them full, and returns
+// nil when the intersection is empty. It may reorder cs.
+func andK(cs []container) container {
+	switch len(cs) {
+	case 0:
+		return newRunContainer16Range(0, maxCapacity-1)
+	case 1:
+		return cs[0].clone()
+	}
+	// The smallest array goes first; the rest keep their order.
+	smallest := -1
+	for i, c := range cs {
+		if x, ok := c.(*arrayContainer); ok && (smallest < 0 || x.getCardinality() < cs[smallest].getCardinality()) {
+			smallest = i
+		}
+	}
+	if smallest >= 0 {
+		a := cs[smallest]
+		copy(cs[1:smallest+1], cs[:smallest])
+		cs[0] = a
+		return andKChain(cs)
+	}
+	n := 0
+	for i, c := range cs {
+		if _, ok := c.(*runContainer16); ok {
+			cs[n], cs[i] = cs[i], cs[n]
+			n++
+		}
+	}
+	if n == len(cs) {
+		return andKChain(cs)
+	}
+	return andKBitmaps(cs[n:], cs[:n])
+}
+
+// andKChain intersects cs in order: the first two into a new container, then
+// the rest into it in place, so nothing bigger than the first result is built.
+func andKChain(cs []container) container {
+	c := cs[0].and(cs[1])
+	for i := 2; i < len(cs) && !c.isEmpty(); i++ {
+		c = c.iand(cs[i])
+	}
+	if c.isEmpty() {
+		return nil
+	}
+	return c
+}
+
+// andKBitmaps ANDs the bitmaps in a scratch bitmap on the stack, stopping as
+// soon as the result is empty. Runs act as masks: only the words between
+// their common start and end are ANDed, and the gaps inside a run are cleared
+// afterwards.
+func andKBitmaps(bms, runs []container) container {
+	lo, hi := 0, maxCapacity-1
+	gaps := false
+	for _, c := range runs {
+		rc := c.(*runContainer16)
+		lo, hi = max(lo, int(rc.minimum())), min(hi, int(rc.maximum()))
+		gaps = gaps || len(rc.iv) > 1
+	}
+	if lo > hi {
+		return nil
+	}
+	first, last := lo/64, hi/64
+	var scratch [bitmapContainerSize]uint64
+	w := scratch[first : last+1]
+	src := bms[0].(*bitmapContainer).bitmap[first : last+1]
+	card := uint64(0)
+	if len(bms) == 1 {
+		copy(w, src)
+	}
+	for _, c := range bms[1:] {
+		if card = andCardSlice(w, src, c.(*bitmapContainer).bitmap[first:last+1]); card == 0 {
+			return nil
+		}
+		src = w
+	}
+	if len(bms) == 1 || gaps || lo%64 != 0 || (hi+1)%64 != 0 {
+		resetBitmapRange(scratch[:], first*64, lo)
+		resetBitmapRange(scratch[:], hi+1, (last+1)*64)
+		for _, c := range runs {
+			clearBitmapGaps(scratch[:], c.(*runContainer16).iv, lo, hi+1)
+		}
+		if card = popcntSlice(w); card == 0 {
+			return nil
+		}
+	}
+	return containerFromWords(scratch[:], first, last, int(card))
+}
+
 // FastOr computes the union between many bitmaps quickly, as opposed to having to call Or repeatedly.
 // It might also be faster than calling Or repeatedly.
 func FastOr(bitmaps ...*Bitmap) *Bitmap {