diff --git a/docs/conf.py b/docs/conf.py index ef16dfd..9ca0caf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # built documents. # # The short X.Y version. -version = '1.1.0' +version = '1.2.0' # The full version, including alpha/beta/rc tags. release = version diff --git a/pyroaring/croaring_version.pxi b/pyroaring/croaring_version.pxi index 034abd2..2540596 100644 --- a/pyroaring/croaring_version.pxi +++ b/pyroaring/croaring_version.pxi @@ -1 +1 @@ -__croaring_version__ = "v4.6.1" \ No newline at end of file +__croaring_version__ = "v5.2.2" \ No newline at end of file diff --git a/pyroaring/roaring.c b/pyroaring/roaring.c index abc1833..953421b 100644 --- a/pyroaring/roaring.c +++ b/pyroaring/roaring.c @@ -1,5 +1,5 @@ // !!! DO NOT EDIT - THIS IS AN AUTO-GENERATED FILE !!! -// Created by amalgamation.sh on 2026-03-09T14:59:44Z +// Created by amalgamation.sh on 2026-09-17T17:49:11Z /* * The CRoaring project is under a dual license (Apache/MIT). @@ -62,138 +62,6 @@ #endif #include "roaring.h" /* include public API definitions */ -/* begin file include/roaring/containers/perfparameters.h */ -#ifndef PERFPARAMETERS_H_ -#define PERFPARAMETERS_H_ - -#include - -#ifdef __cplusplus -extern "C" { -namespace roaring { -namespace internal { -#endif - -/** -During lazy computations, we can transform array containers into bitset -containers as -long as we can expect them to have ARRAY_LAZY_LOWERBOUND values. -*/ -enum { ARRAY_LAZY_LOWERBOUND = 1024 }; - -/* default initial size of a run container - setting it to zero delays the malloc.*/ -enum { RUN_DEFAULT_INIT_SIZE = 0 }; - -/* default initial size of an array container - setting it to zero delays the malloc */ -enum { ARRAY_DEFAULT_INIT_SIZE = 0 }; - -/* automatic bitset conversion during lazy or */ -#ifndef LAZY_OR_BITSET_CONVERSION -#define LAZY_OR_BITSET_CONVERSION true -#endif - -/* automatically attempt to convert a bitset to a full run during lazy - * evaluation */ -#ifndef LAZY_OR_BITSET_CONVERSION_TO_FULL -#define LAZY_OR_BITSET_CONVERSION_TO_FULL true -#endif - -/* automatically attempt to convert a bitset to a full run */ -#ifndef OR_BITSET_CONVERSION_TO_FULL -#define OR_BITSET_CONVERSION_TO_FULL true -#endif - -#ifdef __cplusplus -} -} -} // extern "C" { namespace roaring { namespace internal { -#endif - -#endif -/* end file include/roaring/containers/perfparameters.h */ -/* begin file include/roaring/utilasm.h */ -/* - * utilasm.h - * - */ - -#ifndef INCLUDE_UTILASM_H_ -#define INCLUDE_UTILASM_H_ - - -#ifdef __cplusplus -extern "C" { -namespace roaring { -#endif - -#if defined(CROARING_INLINE_ASM) -#define CROARING_ASMBITMANIPOPTIMIZATION // optimization flag - -#define ASM_SHIFT_RIGHT(srcReg, bitsReg, destReg) \ - __asm volatile("shrx %1, %2, %0" \ - : "=r"(destReg) \ - : /* write */ \ - "r"(bitsReg), /* read only */ \ - "r"(srcReg) /* read only */ \ - ) - -#define ASM_INPLACESHIFT_RIGHT(srcReg, bitsReg) \ - __asm volatile("shrx %1, %0, %0" \ - : "+r"(srcReg) \ - : /* read/write */ \ - "r"(bitsReg) /* read only */ \ - ) - -#define ASM_SHIFT_LEFT(srcReg, bitsReg, destReg) \ - __asm volatile("shlx %1, %2, %0" \ - : "=r"(destReg) \ - : /* write */ \ - "r"(bitsReg), /* read only */ \ - "r"(srcReg) /* read only */ \ - ) -// set bit at position testBit within testByte to 1 and -// copy cmovDst to cmovSrc if that bit was previously clear -#define ASM_SET_BIT_INC_WAS_CLEAR(testByte, testBit, count) \ - __asm volatile( \ - "bts %2, %0\n" \ - "sbb $-1, %1\n" \ - : "+r"(testByte), /* read/write */ \ - "+r"(count) \ - : /* read/write */ \ - "r"(testBit) /* read only */ \ - ) - -#define ASM_CLEAR_BIT_DEC_WAS_SET(testByte, testBit, count) \ - __asm volatile( \ - "btr %2, %0\n" \ - "sbb $0, %1\n" \ - : "+r"(testByte), /* read/write */ \ - "+r"(count) \ - : /* read/write */ \ - "r"(testBit) /* read only */ \ - ) - -#define ASM_BT64(testByte, testBit, count) \ - __asm volatile( \ - "bt %2,%1\n" \ - "sbb %0,%0" /*could use setb */ \ - : "=r"(count) \ - : /* write */ \ - "r"(testByte), /* read only */ \ - "r"(testBit) /* read only */ \ - ) - -#endif - -#ifdef __cplusplus -} -} // extern "C" { namespace roaring { -#endif - -#endif /* INCLUDE_UTILASM_H_ */ -/* end file include/roaring/utilasm.h */ /* begin file include/roaring/art/art.h */ #ifndef ART_ART_H #define ART_ART_H @@ -2078,9 +1946,19 @@ static inline uint32_t unique(uint16_t *out, uint32_t len) { return pos; } -// use with qsort, could be avoided -static int uint16_compare(const void *a, const void *b) { - return (*(uint16_t *)a - *(uint16_t *)b); +// Sort a very short run of uint16 values in place. The callers below feed this +// at most 16 values; calling qsort() for that costs more in glibc merge-sort +// setup, function-pointer compares and memmove traffic than the sort itself. +static inline void sort_uint16_short(uint16_t *z, uint32_t n) { + for (uint32_t i = 1; i < n; i++) { + uint16_t v = z[i]; + uint32_t j = i; + while (j > 0 && z[j - 1] > v) { + z[j] = z[j - 1]; + j--; + } + z[j] = v; + } } CROARING_TARGET_AVX2 @@ -2149,7 +2027,7 @@ uint32_t union_vector16(const uint16_t *array1, uint32_t length1, memcpy(buffer + leftoversize, array1 + 8 * pos1, (length1 - 8 * len1) * sizeof(uint16_t)); leftoversize += length1 - 8 * len1; - qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + sort_uint16_short(buffer, leftoversize); leftoversize = unique(buffer, leftoversize); len += (uint32_t)union_uint16(buffer, leftoversize, array2 + 8 * pos2, @@ -2158,7 +2036,7 @@ uint32_t union_vector16(const uint16_t *array1, uint32_t length1, memcpy(buffer + leftoversize, array2 + 8 * pos2, (length2 - 8 * len2) * sizeof(uint16_t)); leftoversize += length2 - 8 * len2; - qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + sort_uint16_short(buffer, leftoversize); leftoversize = unique(buffer, leftoversize); len += (uint32_t)union_uint16(buffer, leftoversize, array1 + 8 * pos1, length1 - 8 * pos1, output); @@ -2172,6 +2050,193 @@ CROARING_UNTARGET_AVX2 * */ +/** + * Start of the AVX-512 16-bit union code. + * + * union_vector16 above merges 8 lanes at a time with an odd-even transposition + * network: 8 dependent min/max stages per 8 output values, which measures at + * ~2.3 cycles per input element -- no better than the scalar union_uint16. + * + * With AVX-512 we can merge 32+32 lanes with a Batcher bitonic network: one + * reverse plus 5 compare-exchange stages per sorted half, i.e. ~4x fewer + * operations per output value. This needs cross-lane 16-bit permutes (vpermw, + * vpermt2w) and a 16-bit compress store (vpcompressw); AVX2 has none of these, + * so the algorithm is genuinely AVX-512-only rather than a wider rerun of the + * SSE code. + */ +#if CROARING_COMPILER_SUPPORTS_AVX512 + +CROARING_TARGET_AVX512 + +// A compare-exchange at distance d pairs lane i with lane i^d and keeps the +// smaller value in whichever lane has its d-bit clear. `hi` selects the lanes +// whose d-bit is set. +static inline __m512i avx512_cx16(__m512i v, __m512i t, __mmask32 hi) { + return _mm512_mask_mov_epi16(_mm512_min_epu16(v, t), hi, + _mm512_max_epu16(v, t)); +} + +// Sort a bitonic 32-lane sequence into ascending order. Each distance has a +// dedicated cheap shuffle, so no stage needs a full vpermw. +static inline __m512i avx512_bitonic_sort32(__m512i v) { + v = avx512_cx16(v, _mm512_shuffle_i64x2(v, v, 0x4E), 0xFFFF0000u); // d=16 + v = avx512_cx16(v, _mm512_shuffle_i64x2(v, v, 0xB1), 0xFF00FF00u); // d=8 + v = avx512_cx16(v, _mm512_shuffle_epi32(v, 0x4E), 0xF0F0F0F0u); // d=4 + v = avx512_cx16(v, _mm512_shuffle_epi32(v, 0xB1), 0xCCCCCCCCu); // d=2 + v = avx512_cx16(v, _mm512_rol_epi32(v, 16), 0xAAAAAAAAu); // d=1 + return v; +} + +// Merge two ascending 32-lane vectors: *lo receives the 32 smallest values in +// ascending order, *hi the 32 largest, also ascending. +static inline void avx512_bitonic_merge32(__m512i a, __m512i b, __m512i *lo, + __m512i *hi) { + static const uint16_t revtab[32] = { + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + __m512i br = _mm512_permutexvar_epi16( + _mm512_loadu_si512((const __m512i *)revtab), b); + *lo = avx512_bitonic_sort32(_mm512_min_epu16(a, br)); + *hi = avx512_bitonic_sort32(_mm512_max_epu16(a, br)); +} + +// Write the values of the ascending vector `v` that differ from their +// predecessor, where the predecessor of lane 0 is *last. Updates *last to the +// largest value emitted and returns how many values were written. +static inline int avx512_emit_unique16(__m512i v, uint16_t *out, + uint16_t *last) { + static const uint16_t shift1[32] = { + 32, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}; + __m512i prev = _mm512_permutex2var_epi16( + v, _mm512_loadu_si512((const __m512i *)shift1), + _mm512_set1_epi16((short)*last)); + __mmask32 keep = _mm512_cmpneq_epi16_mask(v, prev); + _mm512_mask_compressstoreu_epi16(out, keep, v); + *last = (uint16_t)_mm_extract_epi16(_mm512_extracti32x4_epi32(v, 3), 7); + return (int)roaring_hamming(keep); +} + +/** + * A one-pass AVX-512 union of two sorted uint16 arrays. + * + * As with union_vector16, the caller must guarantee that `output` has room for + * size_1 + size_2 values. Partial overlap of `output` with the inputs is + * tolerated in exactly the same way union_vector16 tolerates it (see the long + * comment in array_run_container_inplace_union): after consuming p1 + p2 whole + * 32-lane blocks this routine has written at most 32*(p1+p2) - 32 values, + * because one merged block is always held back in `vmax`. The output pointer + * therefore stays strictly behind the read pointers. + * + * That bound is exactly tight rather than merely satisfied: in the worst case + * the final merge below writes its last value to precisely the slot the last + * input value was read from. Anything that reduces how much the tail holds back + * -- shrinking `pending`, emitting `vmax` sooner -- silently breaks aliased + * callers, so re-derive the bound before changing the buffering here. + */ +uint32_t avx512_union_uint16(const uint16_t *array1, uint32_t length1, + const uint16_t *array2, uint32_t length2, + uint16_t *output) { + const uint32_t W = 32; // lanes per 512-bit register + if (length1 < W || length2 < W) { + // Not enough on one side to fill a block. union_vector16 keeps + // vectorizing at 8-lane granularity, so it still beats the scalar + // merge once there is a worthwhile amount of data on the other side; + // below that its own fixed overhead dominates. + if (length1 + length2 >= 64) { + return union_vector16(array1, length1, array2, length2, output); + } + return (uint32_t)union_uint16(array1, length1, array2, length2, output); + } + const uint32_t blocks1 = length1 / W, blocks2 = length2 / W; + uint32_t p1 = 0, p2 = 0; + uint16_t *out = output; + __m512i vmin, vmax; + + avx512_bitonic_merge32(_mm512_loadu_si512((const __m512i *)array1), + _mm512_loadu_si512((const __m512i *)array2), &vmin, + &vmax); + p1 = 1; + p2 = 1; + // Lane 0 of the very first block has no predecessor. Seeding `last` with a + // value that differs from it in one bit keeps the first value. + uint16_t last = + (uint16_t)(_mm_extract_epi16(_mm512_castsi512_si128(vmin), 0) ^ 1); + out += avx512_emit_unique16(vmin, out, &last); + + while (p1 < blocks1 && p2 < blocks2) { + // Which side advances is essentially unpredictable, so select the + // source pointer without branching. + const uint16_t *pa = array1 + W * p1; + const uint16_t *pb = array2 + W * p2; + const uint32_t take1 = (*pa <= *pb) ? 1 : 0; + const __m512i v = + _mm512_loadu_si512((const __m512i *)(take1 ? pa : pb)); + p1 += take1; + p2 += 1 - take1; + avx512_bitonic_merge32(v, vmax, &vmin, &vmax); + out += avx512_emit_unique16(vmin, out, &last); + } + + // The ragged tail: the values still held in vmax, plus at most W-1 leftover + // values from the exhausted side, plus the rest of the other side. Two + // two-way merges branch-predict far better than one three-way merge. + // + // vmax can hold the same value twice (once from each input), so it goes + // through avx512_emit_unique16 rather than a raw store: that dedups within + // the block and against `last` at the same time. Every remaining tail value + // is >= last and the tail is sorted, so for the two scalar inputs only the + // very first value can still duplicate `last`. + uint16_t pending[32]; + uint16_t pending_last = last; + size_t npending = + (size_t)avx512_emit_unique16(vmax, pending, &pending_last); + + const uint16_t *rest1 = array1 + W * p1, *rest2 = array2 + W * p2; + size_t nrest1 = length1 - W * p1, nrest2 = length2 - W * p2; + const uint16_t *shortrest, *longrest; + size_t nshort, nlong; + if (p1 == blocks1) { + shortrest = rest1; + nshort = nrest1; + longrest = rest2; + nlong = nrest2; + } else { + shortrest = rest2; + nshort = nrest2; + longrest = rest1; + nlong = nrest1; + } + if (nshort > 0 && shortrest[0] == last) { + shortrest++; + nshort--; + } + if (nlong > 0 && longrest[0] == last) { + longrest++; + nlong--; + } + + uint16_t merged[2 * 32]; // <= W pending + (W-1) leftover + size_t nmerged = union_uint16(pending, npending, shortrest, nshort, merged); + // When the small side held fewer than two blocks the loop above ran at most + // once, so `longrest` can still be most of the larger input. Finishing that + // scalar would give away more than the block loop won, hence the same + // 8-lane handoff as in the short-input case. + if (nlong >= 64) { + out += union_vector16(merged, (uint32_t)nmerged, longrest, + (uint32_t)nlong, out); + } else { + out += union_uint16(merged, nmerged, longrest, nlong, out); + } + return (uint32_t)(out - output); +} +CROARING_UNTARGET_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + +/** + * End of the AVX-512 16-bit union code. + */ + /** * Start of SIMD 16-bit XOR code */ @@ -2290,7 +2355,7 @@ uint32_t xor_vector16(const uint16_t *array1, uint32_t length1, (length2 - 8 * pos2) * sizeof(uint16_t)); len += (length2 - 8 * pos2); } else { - qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + sort_uint16_short(buffer, leftoversize); leftoversize = unique_xor(buffer, leftoversize); len += xor_uint16(buffer, leftoversize, array2 + 8 * pos2, length2 - 8 * pos2, output); @@ -2304,7 +2369,7 @@ uint32_t xor_vector16(const uint16_t *array1, uint32_t length1, (length1 - 8 * pos1) * sizeof(uint16_t)); len += (length1 - 8 * pos1); } else { - qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + sort_uint16_short(buffer, leftoversize); leftoversize = unique_xor(buffer, leftoversize); len += xor_uint16(buffer, leftoversize, array1 + 8 * pos1, length1 - 8 * pos1, output); @@ -2416,7 +2481,20 @@ size_t fast_union_uint16(const uint16_t *set_1, size_t size_1, const uint16_t *set_2, size_t size_2, uint16_t *buffer) { #if CROARING_IS_X64 - if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + const unsigned support = (unsigned)croaring_hardware_support(); +#if CROARING_COMPILER_SUPPORTS_AVX512 + if (support & ROARING_SUPPORTS_AVX512) { + // compute union with smallest array first + if (size_1 < size_2) { + return avx512_union_uint16(set_1, (uint32_t)size_1, set_2, + (uint32_t)size_2, buffer); + } else { + return avx512_union_uint16(set_2, (uint32_t)size_2, set_1, + (uint32_t)size_1, buffer); + } + } +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + if (support & ROARING_SUPPORTS_AVX2) { // compute union with smallest array first if (size_1 < size_2) { return union_vector16(set_1, (uint32_t)size_1, set_2, @@ -2605,7 +2683,6 @@ CROARING_UNTARGET_AVX512 #endif/* end file src/array_util.c */ /* begin file src/art/art.c */ #include -#include #include #include @@ -4146,7 +4223,7 @@ static void art_node_print_type(art_ref_t ref) { } } -void art_node_printf(const art_t *art, art_ref_t ref, uint8_t depth) { +static void art_node_printf(const art_t *art, art_ref_t ref, uint8_t depth) { if (art_is_leaf(ref)) { printf("{ type: Leaf, key: "); art_leaf_t *leaf = (art_leaf_t *)art_deref(art, ref); @@ -5002,1643 +5079,1682 @@ size_t art_frozen_view(const char *buf, size_t maxbytes, art_t *art) { } // namespace internal #endif /* end file src/art/art.c */ -/* begin file src/bitset_util.c */ -#include +/* begin file src/bitset.c */ +#include #include #include #include #include -#if CROARING_IS_X64 -#ifndef CROARING_COMPILER_SUPPORTS_AVX512 -#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." -#endif // CROARING_COMPILER_SUPPORTS_AVX512 -#endif -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wuninitialized" -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif #ifdef __cplusplus -using namespace ::roaring::internal; extern "C" { namespace roaring { -namespace api { +namespace internal { #endif -#if CROARING_IS_X64 -static uint8_t lengthTable[256] = { - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, - 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, - 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, - 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, - 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, - 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; -#endif +extern inline void bitset_print(const bitset_t *b); +extern inline bool bitset_for_each(const bitset_t *b, bitset_iterator iterator, + void *ptr); +extern inline size_t bitset_next_set_bits(const bitset_t *bitset, + size_t *buffer, size_t capacity, + size_t *startfrom); +extern inline void bitset_set_to_value(bitset_t *bitset, size_t i, bool flag); +extern inline bool bitset_next_set_bit(const bitset_t *bitset, size_t *i); +extern inline void bitset_set(bitset_t *bitset, size_t i); +extern inline bool bitset_get(const bitset_t *bitset, size_t i); +extern inline size_t bitset_size_in_words(const bitset_t *bitset); +extern inline size_t bitset_size_in_bits(const bitset_t *bitset); +extern inline size_t bitset_size_in_bytes(const bitset_t *bitset); -#if CROARING_IS_X64 -ALIGNED(32) -static uint32_t vecDecodeTable[256][8] = { - {0, 0, 0, 0, 0, 0, 0, 0}, /* 0x00 (00000000) */ - {1, 0, 0, 0, 0, 0, 0, 0}, /* 0x01 (00000001) */ - {2, 0, 0, 0, 0, 0, 0, 0}, /* 0x02 (00000010) */ - {1, 2, 0, 0, 0, 0, 0, 0}, /* 0x03 (00000011) */ - {3, 0, 0, 0, 0, 0, 0, 0}, /* 0x04 (00000100) */ - {1, 3, 0, 0, 0, 0, 0, 0}, /* 0x05 (00000101) */ - {2, 3, 0, 0, 0, 0, 0, 0}, /* 0x06 (00000110) */ - {1, 2, 3, 0, 0, 0, 0, 0}, /* 0x07 (00000111) */ - {4, 0, 0, 0, 0, 0, 0, 0}, /* 0x08 (00001000) */ - {1, 4, 0, 0, 0, 0, 0, 0}, /* 0x09 (00001001) */ - {2, 4, 0, 0, 0, 0, 0, 0}, /* 0x0A (00001010) */ - {1, 2, 4, 0, 0, 0, 0, 0}, /* 0x0B (00001011) */ - {3, 4, 0, 0, 0, 0, 0, 0}, /* 0x0C (00001100) */ - {1, 3, 4, 0, 0, 0, 0, 0}, /* 0x0D (00001101) */ - {2, 3, 4, 0, 0, 0, 0, 0}, /* 0x0E (00001110) */ - {1, 2, 3, 4, 0, 0, 0, 0}, /* 0x0F (00001111) */ - {5, 0, 0, 0, 0, 0, 0, 0}, /* 0x10 (00010000) */ - {1, 5, 0, 0, 0, 0, 0, 0}, /* 0x11 (00010001) */ - {2, 5, 0, 0, 0, 0, 0, 0}, /* 0x12 (00010010) */ - {1, 2, 5, 0, 0, 0, 0, 0}, /* 0x13 (00010011) */ - {3, 5, 0, 0, 0, 0, 0, 0}, /* 0x14 (00010100) */ - {1, 3, 5, 0, 0, 0, 0, 0}, /* 0x15 (00010101) */ - {2, 3, 5, 0, 0, 0, 0, 0}, /* 0x16 (00010110) */ - {1, 2, 3, 5, 0, 0, 0, 0}, /* 0x17 (00010111) */ - {4, 5, 0, 0, 0, 0, 0, 0}, /* 0x18 (00011000) */ - {1, 4, 5, 0, 0, 0, 0, 0}, /* 0x19 (00011001) */ - {2, 4, 5, 0, 0, 0, 0, 0}, /* 0x1A (00011010) */ - {1, 2, 4, 5, 0, 0, 0, 0}, /* 0x1B (00011011) */ - {3, 4, 5, 0, 0, 0, 0, 0}, /* 0x1C (00011100) */ - {1, 3, 4, 5, 0, 0, 0, 0}, /* 0x1D (00011101) */ - {2, 3, 4, 5, 0, 0, 0, 0}, /* 0x1E (00011110) */ - {1, 2, 3, 4, 5, 0, 0, 0}, /* 0x1F (00011111) */ - {6, 0, 0, 0, 0, 0, 0, 0}, /* 0x20 (00100000) */ - {1, 6, 0, 0, 0, 0, 0, 0}, /* 0x21 (00100001) */ - {2, 6, 0, 0, 0, 0, 0, 0}, /* 0x22 (00100010) */ - {1, 2, 6, 0, 0, 0, 0, 0}, /* 0x23 (00100011) */ - {3, 6, 0, 0, 0, 0, 0, 0}, /* 0x24 (00100100) */ - {1, 3, 6, 0, 0, 0, 0, 0}, /* 0x25 (00100101) */ - {2, 3, 6, 0, 0, 0, 0, 0}, /* 0x26 (00100110) */ - {1, 2, 3, 6, 0, 0, 0, 0}, /* 0x27 (00100111) */ - {4, 6, 0, 0, 0, 0, 0, 0}, /* 0x28 (00101000) */ - {1, 4, 6, 0, 0, 0, 0, 0}, /* 0x29 (00101001) */ - {2, 4, 6, 0, 0, 0, 0, 0}, /* 0x2A (00101010) */ - {1, 2, 4, 6, 0, 0, 0, 0}, /* 0x2B (00101011) */ - {3, 4, 6, 0, 0, 0, 0, 0}, /* 0x2C (00101100) */ - {1, 3, 4, 6, 0, 0, 0, 0}, /* 0x2D (00101101) */ - {2, 3, 4, 6, 0, 0, 0, 0}, /* 0x2E (00101110) */ - {1, 2, 3, 4, 6, 0, 0, 0}, /* 0x2F (00101111) */ - {5, 6, 0, 0, 0, 0, 0, 0}, /* 0x30 (00110000) */ - {1, 5, 6, 0, 0, 0, 0, 0}, /* 0x31 (00110001) */ - {2, 5, 6, 0, 0, 0, 0, 0}, /* 0x32 (00110010) */ - {1, 2, 5, 6, 0, 0, 0, 0}, /* 0x33 (00110011) */ - {3, 5, 6, 0, 0, 0, 0, 0}, /* 0x34 (00110100) */ - {1, 3, 5, 6, 0, 0, 0, 0}, /* 0x35 (00110101) */ - {2, 3, 5, 6, 0, 0, 0, 0}, /* 0x36 (00110110) */ - {1, 2, 3, 5, 6, 0, 0, 0}, /* 0x37 (00110111) */ - {4, 5, 6, 0, 0, 0, 0, 0}, /* 0x38 (00111000) */ - {1, 4, 5, 6, 0, 0, 0, 0}, /* 0x39 (00111001) */ - {2, 4, 5, 6, 0, 0, 0, 0}, /* 0x3A (00111010) */ - {1, 2, 4, 5, 6, 0, 0, 0}, /* 0x3B (00111011) */ - {3, 4, 5, 6, 0, 0, 0, 0}, /* 0x3C (00111100) */ - {1, 3, 4, 5, 6, 0, 0, 0}, /* 0x3D (00111101) */ - {2, 3, 4, 5, 6, 0, 0, 0}, /* 0x3E (00111110) */ - {1, 2, 3, 4, 5, 6, 0, 0}, /* 0x3F (00111111) */ - {7, 0, 0, 0, 0, 0, 0, 0}, /* 0x40 (01000000) */ - {1, 7, 0, 0, 0, 0, 0, 0}, /* 0x41 (01000001) */ - {2, 7, 0, 0, 0, 0, 0, 0}, /* 0x42 (01000010) */ - {1, 2, 7, 0, 0, 0, 0, 0}, /* 0x43 (01000011) */ - {3, 7, 0, 0, 0, 0, 0, 0}, /* 0x44 (01000100) */ - {1, 3, 7, 0, 0, 0, 0, 0}, /* 0x45 (01000101) */ - {2, 3, 7, 0, 0, 0, 0, 0}, /* 0x46 (01000110) */ - {1, 2, 3, 7, 0, 0, 0, 0}, /* 0x47 (01000111) */ - {4, 7, 0, 0, 0, 0, 0, 0}, /* 0x48 (01001000) */ - {1, 4, 7, 0, 0, 0, 0, 0}, /* 0x49 (01001001) */ - {2, 4, 7, 0, 0, 0, 0, 0}, /* 0x4A (01001010) */ - {1, 2, 4, 7, 0, 0, 0, 0}, /* 0x4B (01001011) */ - {3, 4, 7, 0, 0, 0, 0, 0}, /* 0x4C (01001100) */ - {1, 3, 4, 7, 0, 0, 0, 0}, /* 0x4D (01001101) */ - {2, 3, 4, 7, 0, 0, 0, 0}, /* 0x4E (01001110) */ - {1, 2, 3, 4, 7, 0, 0, 0}, /* 0x4F (01001111) */ - {5, 7, 0, 0, 0, 0, 0, 0}, /* 0x50 (01010000) */ - {1, 5, 7, 0, 0, 0, 0, 0}, /* 0x51 (01010001) */ - {2, 5, 7, 0, 0, 0, 0, 0}, /* 0x52 (01010010) */ - {1, 2, 5, 7, 0, 0, 0, 0}, /* 0x53 (01010011) */ - {3, 5, 7, 0, 0, 0, 0, 0}, /* 0x54 (01010100) */ - {1, 3, 5, 7, 0, 0, 0, 0}, /* 0x55 (01010101) */ - {2, 3, 5, 7, 0, 0, 0, 0}, /* 0x56 (01010110) */ - {1, 2, 3, 5, 7, 0, 0, 0}, /* 0x57 (01010111) */ - {4, 5, 7, 0, 0, 0, 0, 0}, /* 0x58 (01011000) */ - {1, 4, 5, 7, 0, 0, 0, 0}, /* 0x59 (01011001) */ - {2, 4, 5, 7, 0, 0, 0, 0}, /* 0x5A (01011010) */ - {1, 2, 4, 5, 7, 0, 0, 0}, /* 0x5B (01011011) */ - {3, 4, 5, 7, 0, 0, 0, 0}, /* 0x5C (01011100) */ - {1, 3, 4, 5, 7, 0, 0, 0}, /* 0x5D (01011101) */ - {2, 3, 4, 5, 7, 0, 0, 0}, /* 0x5E (01011110) */ - {1, 2, 3, 4, 5, 7, 0, 0}, /* 0x5F (01011111) */ - {6, 7, 0, 0, 0, 0, 0, 0}, /* 0x60 (01100000) */ - {1, 6, 7, 0, 0, 0, 0, 0}, /* 0x61 (01100001) */ - {2, 6, 7, 0, 0, 0, 0, 0}, /* 0x62 (01100010) */ - {1, 2, 6, 7, 0, 0, 0, 0}, /* 0x63 (01100011) */ - {3, 6, 7, 0, 0, 0, 0, 0}, /* 0x64 (01100100) */ - {1, 3, 6, 7, 0, 0, 0, 0}, /* 0x65 (01100101) */ - {2, 3, 6, 7, 0, 0, 0, 0}, /* 0x66 (01100110) */ - {1, 2, 3, 6, 7, 0, 0, 0}, /* 0x67 (01100111) */ - {4, 6, 7, 0, 0, 0, 0, 0}, /* 0x68 (01101000) */ - {1, 4, 6, 7, 0, 0, 0, 0}, /* 0x69 (01101001) */ - {2, 4, 6, 7, 0, 0, 0, 0}, /* 0x6A (01101010) */ - {1, 2, 4, 6, 7, 0, 0, 0}, /* 0x6B (01101011) */ - {3, 4, 6, 7, 0, 0, 0, 0}, /* 0x6C (01101100) */ - {1, 3, 4, 6, 7, 0, 0, 0}, /* 0x6D (01101101) */ - {2, 3, 4, 6, 7, 0, 0, 0}, /* 0x6E (01101110) */ - {1, 2, 3, 4, 6, 7, 0, 0}, /* 0x6F (01101111) */ - {5, 6, 7, 0, 0, 0, 0, 0}, /* 0x70 (01110000) */ - {1, 5, 6, 7, 0, 0, 0, 0}, /* 0x71 (01110001) */ - {2, 5, 6, 7, 0, 0, 0, 0}, /* 0x72 (01110010) */ - {1, 2, 5, 6, 7, 0, 0, 0}, /* 0x73 (01110011) */ - {3, 5, 6, 7, 0, 0, 0, 0}, /* 0x74 (01110100) */ - {1, 3, 5, 6, 7, 0, 0, 0}, /* 0x75 (01110101) */ - {2, 3, 5, 6, 7, 0, 0, 0}, /* 0x76 (01110110) */ - {1, 2, 3, 5, 6, 7, 0, 0}, /* 0x77 (01110111) */ - {4, 5, 6, 7, 0, 0, 0, 0}, /* 0x78 (01111000) */ - {1, 4, 5, 6, 7, 0, 0, 0}, /* 0x79 (01111001) */ - {2, 4, 5, 6, 7, 0, 0, 0}, /* 0x7A (01111010) */ - {1, 2, 4, 5, 6, 7, 0, 0}, /* 0x7B (01111011) */ - {3, 4, 5, 6, 7, 0, 0, 0}, /* 0x7C (01111100) */ - {1, 3, 4, 5, 6, 7, 0, 0}, /* 0x7D (01111101) */ - {2, 3, 4, 5, 6, 7, 0, 0}, /* 0x7E (01111110) */ - {1, 2, 3, 4, 5, 6, 7, 0}, /* 0x7F (01111111) */ - {8, 0, 0, 0, 0, 0, 0, 0}, /* 0x80 (10000000) */ - {1, 8, 0, 0, 0, 0, 0, 0}, /* 0x81 (10000001) */ - {2, 8, 0, 0, 0, 0, 0, 0}, /* 0x82 (10000010) */ - {1, 2, 8, 0, 0, 0, 0, 0}, /* 0x83 (10000011) */ - {3, 8, 0, 0, 0, 0, 0, 0}, /* 0x84 (10000100) */ - {1, 3, 8, 0, 0, 0, 0, 0}, /* 0x85 (10000101) */ - {2, 3, 8, 0, 0, 0, 0, 0}, /* 0x86 (10000110) */ - {1, 2, 3, 8, 0, 0, 0, 0}, /* 0x87 (10000111) */ - {4, 8, 0, 0, 0, 0, 0, 0}, /* 0x88 (10001000) */ - {1, 4, 8, 0, 0, 0, 0, 0}, /* 0x89 (10001001) */ - {2, 4, 8, 0, 0, 0, 0, 0}, /* 0x8A (10001010) */ - {1, 2, 4, 8, 0, 0, 0, 0}, /* 0x8B (10001011) */ - {3, 4, 8, 0, 0, 0, 0, 0}, /* 0x8C (10001100) */ - {1, 3, 4, 8, 0, 0, 0, 0}, /* 0x8D (10001101) */ - {2, 3, 4, 8, 0, 0, 0, 0}, /* 0x8E (10001110) */ - {1, 2, 3, 4, 8, 0, 0, 0}, /* 0x8F (10001111) */ - {5, 8, 0, 0, 0, 0, 0, 0}, /* 0x90 (10010000) */ - {1, 5, 8, 0, 0, 0, 0, 0}, /* 0x91 (10010001) */ - {2, 5, 8, 0, 0, 0, 0, 0}, /* 0x92 (10010010) */ - {1, 2, 5, 8, 0, 0, 0, 0}, /* 0x93 (10010011) */ - {3, 5, 8, 0, 0, 0, 0, 0}, /* 0x94 (10010100) */ - {1, 3, 5, 8, 0, 0, 0, 0}, /* 0x95 (10010101) */ - {2, 3, 5, 8, 0, 0, 0, 0}, /* 0x96 (10010110) */ - {1, 2, 3, 5, 8, 0, 0, 0}, /* 0x97 (10010111) */ - {4, 5, 8, 0, 0, 0, 0, 0}, /* 0x98 (10011000) */ - {1, 4, 5, 8, 0, 0, 0, 0}, /* 0x99 (10011001) */ - {2, 4, 5, 8, 0, 0, 0, 0}, /* 0x9A (10011010) */ - {1, 2, 4, 5, 8, 0, 0, 0}, /* 0x9B (10011011) */ - {3, 4, 5, 8, 0, 0, 0, 0}, /* 0x9C (10011100) */ - {1, 3, 4, 5, 8, 0, 0, 0}, /* 0x9D (10011101) */ - {2, 3, 4, 5, 8, 0, 0, 0}, /* 0x9E (10011110) */ - {1, 2, 3, 4, 5, 8, 0, 0}, /* 0x9F (10011111) */ - {6, 8, 0, 0, 0, 0, 0, 0}, /* 0xA0 (10100000) */ - {1, 6, 8, 0, 0, 0, 0, 0}, /* 0xA1 (10100001) */ - {2, 6, 8, 0, 0, 0, 0, 0}, /* 0xA2 (10100010) */ - {1, 2, 6, 8, 0, 0, 0, 0}, /* 0xA3 (10100011) */ - {3, 6, 8, 0, 0, 0, 0, 0}, /* 0xA4 (10100100) */ - {1, 3, 6, 8, 0, 0, 0, 0}, /* 0xA5 (10100101) */ - {2, 3, 6, 8, 0, 0, 0, 0}, /* 0xA6 (10100110) */ - {1, 2, 3, 6, 8, 0, 0, 0}, /* 0xA7 (10100111) */ - {4, 6, 8, 0, 0, 0, 0, 0}, /* 0xA8 (10101000) */ - {1, 4, 6, 8, 0, 0, 0, 0}, /* 0xA9 (10101001) */ - {2, 4, 6, 8, 0, 0, 0, 0}, /* 0xAA (10101010) */ - {1, 2, 4, 6, 8, 0, 0, 0}, /* 0xAB (10101011) */ - {3, 4, 6, 8, 0, 0, 0, 0}, /* 0xAC (10101100) */ - {1, 3, 4, 6, 8, 0, 0, 0}, /* 0xAD (10101101) */ - {2, 3, 4, 6, 8, 0, 0, 0}, /* 0xAE (10101110) */ - {1, 2, 3, 4, 6, 8, 0, 0}, /* 0xAF (10101111) */ - {5, 6, 8, 0, 0, 0, 0, 0}, /* 0xB0 (10110000) */ - {1, 5, 6, 8, 0, 0, 0, 0}, /* 0xB1 (10110001) */ - {2, 5, 6, 8, 0, 0, 0, 0}, /* 0xB2 (10110010) */ - {1, 2, 5, 6, 8, 0, 0, 0}, /* 0xB3 (10110011) */ - {3, 5, 6, 8, 0, 0, 0, 0}, /* 0xB4 (10110100) */ - {1, 3, 5, 6, 8, 0, 0, 0}, /* 0xB5 (10110101) */ - {2, 3, 5, 6, 8, 0, 0, 0}, /* 0xB6 (10110110) */ - {1, 2, 3, 5, 6, 8, 0, 0}, /* 0xB7 (10110111) */ - {4, 5, 6, 8, 0, 0, 0, 0}, /* 0xB8 (10111000) */ - {1, 4, 5, 6, 8, 0, 0, 0}, /* 0xB9 (10111001) */ - {2, 4, 5, 6, 8, 0, 0, 0}, /* 0xBA (10111010) */ - {1, 2, 4, 5, 6, 8, 0, 0}, /* 0xBB (10111011) */ - {3, 4, 5, 6, 8, 0, 0, 0}, /* 0xBC (10111100) */ - {1, 3, 4, 5, 6, 8, 0, 0}, /* 0xBD (10111101) */ - {2, 3, 4, 5, 6, 8, 0, 0}, /* 0xBE (10111110) */ - {1, 2, 3, 4, 5, 6, 8, 0}, /* 0xBF (10111111) */ - {7, 8, 0, 0, 0, 0, 0, 0}, /* 0xC0 (11000000) */ - {1, 7, 8, 0, 0, 0, 0, 0}, /* 0xC1 (11000001) */ - {2, 7, 8, 0, 0, 0, 0, 0}, /* 0xC2 (11000010) */ - {1, 2, 7, 8, 0, 0, 0, 0}, /* 0xC3 (11000011) */ - {3, 7, 8, 0, 0, 0, 0, 0}, /* 0xC4 (11000100) */ - {1, 3, 7, 8, 0, 0, 0, 0}, /* 0xC5 (11000101) */ - {2, 3, 7, 8, 0, 0, 0, 0}, /* 0xC6 (11000110) */ - {1, 2, 3, 7, 8, 0, 0, 0}, /* 0xC7 (11000111) */ - {4, 7, 8, 0, 0, 0, 0, 0}, /* 0xC8 (11001000) */ - {1, 4, 7, 8, 0, 0, 0, 0}, /* 0xC9 (11001001) */ - {2, 4, 7, 8, 0, 0, 0, 0}, /* 0xCA (11001010) */ - {1, 2, 4, 7, 8, 0, 0, 0}, /* 0xCB (11001011) */ - {3, 4, 7, 8, 0, 0, 0, 0}, /* 0xCC (11001100) */ - {1, 3, 4, 7, 8, 0, 0, 0}, /* 0xCD (11001101) */ - {2, 3, 4, 7, 8, 0, 0, 0}, /* 0xCE (11001110) */ - {1, 2, 3, 4, 7, 8, 0, 0}, /* 0xCF (11001111) */ - {5, 7, 8, 0, 0, 0, 0, 0}, /* 0xD0 (11010000) */ - {1, 5, 7, 8, 0, 0, 0, 0}, /* 0xD1 (11010001) */ - {2, 5, 7, 8, 0, 0, 0, 0}, /* 0xD2 (11010010) */ - {1, 2, 5, 7, 8, 0, 0, 0}, /* 0xD3 (11010011) */ - {3, 5, 7, 8, 0, 0, 0, 0}, /* 0xD4 (11010100) */ - {1, 3, 5, 7, 8, 0, 0, 0}, /* 0xD5 (11010101) */ - {2, 3, 5, 7, 8, 0, 0, 0}, /* 0xD6 (11010110) */ - {1, 2, 3, 5, 7, 8, 0, 0}, /* 0xD7 (11010111) */ - {4, 5, 7, 8, 0, 0, 0, 0}, /* 0xD8 (11011000) */ - {1, 4, 5, 7, 8, 0, 0, 0}, /* 0xD9 (11011001) */ - {2, 4, 5, 7, 8, 0, 0, 0}, /* 0xDA (11011010) */ - {1, 2, 4, 5, 7, 8, 0, 0}, /* 0xDB (11011011) */ - {3, 4, 5, 7, 8, 0, 0, 0}, /* 0xDC (11011100) */ - {1, 3, 4, 5, 7, 8, 0, 0}, /* 0xDD (11011101) */ - {2, 3, 4, 5, 7, 8, 0, 0}, /* 0xDE (11011110) */ - {1, 2, 3, 4, 5, 7, 8, 0}, /* 0xDF (11011111) */ - {6, 7, 8, 0, 0, 0, 0, 0}, /* 0xE0 (11100000) */ - {1, 6, 7, 8, 0, 0, 0, 0}, /* 0xE1 (11100001) */ - {2, 6, 7, 8, 0, 0, 0, 0}, /* 0xE2 (11100010) */ - {1, 2, 6, 7, 8, 0, 0, 0}, /* 0xE3 (11100011) */ - {3, 6, 7, 8, 0, 0, 0, 0}, /* 0xE4 (11100100) */ - {1, 3, 6, 7, 8, 0, 0, 0}, /* 0xE5 (11100101) */ - {2, 3, 6, 7, 8, 0, 0, 0}, /* 0xE6 (11100110) */ - {1, 2, 3, 6, 7, 8, 0, 0}, /* 0xE7 (11100111) */ - {4, 6, 7, 8, 0, 0, 0, 0}, /* 0xE8 (11101000) */ - {1, 4, 6, 7, 8, 0, 0, 0}, /* 0xE9 (11101001) */ - {2, 4, 6, 7, 8, 0, 0, 0}, /* 0xEA (11101010) */ - {1, 2, 4, 6, 7, 8, 0, 0}, /* 0xEB (11101011) */ - {3, 4, 6, 7, 8, 0, 0, 0}, /* 0xEC (11101100) */ - {1, 3, 4, 6, 7, 8, 0, 0}, /* 0xED (11101101) */ - {2, 3, 4, 6, 7, 8, 0, 0}, /* 0xEE (11101110) */ - {1, 2, 3, 4, 6, 7, 8, 0}, /* 0xEF (11101111) */ - {5, 6, 7, 8, 0, 0, 0, 0}, /* 0xF0 (11110000) */ - {1, 5, 6, 7, 8, 0, 0, 0}, /* 0xF1 (11110001) */ - {2, 5, 6, 7, 8, 0, 0, 0}, /* 0xF2 (11110010) */ - {1, 2, 5, 6, 7, 8, 0, 0}, /* 0xF3 (11110011) */ - {3, 5, 6, 7, 8, 0, 0, 0}, /* 0xF4 (11110100) */ - {1, 3, 5, 6, 7, 8, 0, 0}, /* 0xF5 (11110101) */ - {2, 3, 5, 6, 7, 8, 0, 0}, /* 0xF6 (11110110) */ - {1, 2, 3, 5, 6, 7, 8, 0}, /* 0xF7 (11110111) */ - {4, 5, 6, 7, 8, 0, 0, 0}, /* 0xF8 (11111000) */ - {1, 4, 5, 6, 7, 8, 0, 0}, /* 0xF9 (11111001) */ - {2, 4, 5, 6, 7, 8, 0, 0}, /* 0xFA (11111010) */ - {1, 2, 4, 5, 6, 7, 8, 0}, /* 0xFB (11111011) */ - {3, 4, 5, 6, 7, 8, 0, 0}, /* 0xFC (11111100) */ - {1, 3, 4, 5, 6, 7, 8, 0}, /* 0xFD (11111101) */ - {2, 3, 4, 5, 6, 7, 8, 0}, /* 0xFE (11111110) */ - {1, 2, 3, 4, 5, 6, 7, 8} /* 0xFF (11111111) */ -}; - -#endif // #if CROARING_IS_X64 - -#if CROARING_IS_X64 -// same as vecDecodeTable but in 16 bits -ALIGNED(32) -static uint16_t vecDecodeTable_uint16[256][8] = { - {0, 0, 0, 0, 0, 0, 0, 0}, /* 0x00 (00000000) */ - {1, 0, 0, 0, 0, 0, 0, 0}, /* 0x01 (00000001) */ - {2, 0, 0, 0, 0, 0, 0, 0}, /* 0x02 (00000010) */ - {1, 2, 0, 0, 0, 0, 0, 0}, /* 0x03 (00000011) */ - {3, 0, 0, 0, 0, 0, 0, 0}, /* 0x04 (00000100) */ - {1, 3, 0, 0, 0, 0, 0, 0}, /* 0x05 (00000101) */ - {2, 3, 0, 0, 0, 0, 0, 0}, /* 0x06 (00000110) */ - {1, 2, 3, 0, 0, 0, 0, 0}, /* 0x07 (00000111) */ - {4, 0, 0, 0, 0, 0, 0, 0}, /* 0x08 (00001000) */ - {1, 4, 0, 0, 0, 0, 0, 0}, /* 0x09 (00001001) */ - {2, 4, 0, 0, 0, 0, 0, 0}, /* 0x0A (00001010) */ - {1, 2, 4, 0, 0, 0, 0, 0}, /* 0x0B (00001011) */ - {3, 4, 0, 0, 0, 0, 0, 0}, /* 0x0C (00001100) */ - {1, 3, 4, 0, 0, 0, 0, 0}, /* 0x0D (00001101) */ - {2, 3, 4, 0, 0, 0, 0, 0}, /* 0x0E (00001110) */ - {1, 2, 3, 4, 0, 0, 0, 0}, /* 0x0F (00001111) */ - {5, 0, 0, 0, 0, 0, 0, 0}, /* 0x10 (00010000) */ - {1, 5, 0, 0, 0, 0, 0, 0}, /* 0x11 (00010001) */ - {2, 5, 0, 0, 0, 0, 0, 0}, /* 0x12 (00010010) */ - {1, 2, 5, 0, 0, 0, 0, 0}, /* 0x13 (00010011) */ - {3, 5, 0, 0, 0, 0, 0, 0}, /* 0x14 (00010100) */ - {1, 3, 5, 0, 0, 0, 0, 0}, /* 0x15 (00010101) */ - {2, 3, 5, 0, 0, 0, 0, 0}, /* 0x16 (00010110) */ - {1, 2, 3, 5, 0, 0, 0, 0}, /* 0x17 (00010111) */ - {4, 5, 0, 0, 0, 0, 0, 0}, /* 0x18 (00011000) */ - {1, 4, 5, 0, 0, 0, 0, 0}, /* 0x19 (00011001) */ - {2, 4, 5, 0, 0, 0, 0, 0}, /* 0x1A (00011010) */ - {1, 2, 4, 5, 0, 0, 0, 0}, /* 0x1B (00011011) */ - {3, 4, 5, 0, 0, 0, 0, 0}, /* 0x1C (00011100) */ - {1, 3, 4, 5, 0, 0, 0, 0}, /* 0x1D (00011101) */ - {2, 3, 4, 5, 0, 0, 0, 0}, /* 0x1E (00011110) */ - {1, 2, 3, 4, 5, 0, 0, 0}, /* 0x1F (00011111) */ - {6, 0, 0, 0, 0, 0, 0, 0}, /* 0x20 (00100000) */ - {1, 6, 0, 0, 0, 0, 0, 0}, /* 0x21 (00100001) */ - {2, 6, 0, 0, 0, 0, 0, 0}, /* 0x22 (00100010) */ - {1, 2, 6, 0, 0, 0, 0, 0}, /* 0x23 (00100011) */ - {3, 6, 0, 0, 0, 0, 0, 0}, /* 0x24 (00100100) */ - {1, 3, 6, 0, 0, 0, 0, 0}, /* 0x25 (00100101) */ - {2, 3, 6, 0, 0, 0, 0, 0}, /* 0x26 (00100110) */ - {1, 2, 3, 6, 0, 0, 0, 0}, /* 0x27 (00100111) */ - {4, 6, 0, 0, 0, 0, 0, 0}, /* 0x28 (00101000) */ - {1, 4, 6, 0, 0, 0, 0, 0}, /* 0x29 (00101001) */ - {2, 4, 6, 0, 0, 0, 0, 0}, /* 0x2A (00101010) */ - {1, 2, 4, 6, 0, 0, 0, 0}, /* 0x2B (00101011) */ - {3, 4, 6, 0, 0, 0, 0, 0}, /* 0x2C (00101100) */ - {1, 3, 4, 6, 0, 0, 0, 0}, /* 0x2D (00101101) */ - {2, 3, 4, 6, 0, 0, 0, 0}, /* 0x2E (00101110) */ - {1, 2, 3, 4, 6, 0, 0, 0}, /* 0x2F (00101111) */ - {5, 6, 0, 0, 0, 0, 0, 0}, /* 0x30 (00110000) */ - {1, 5, 6, 0, 0, 0, 0, 0}, /* 0x31 (00110001) */ - {2, 5, 6, 0, 0, 0, 0, 0}, /* 0x32 (00110010) */ - {1, 2, 5, 6, 0, 0, 0, 0}, /* 0x33 (00110011) */ - {3, 5, 6, 0, 0, 0, 0, 0}, /* 0x34 (00110100) */ - {1, 3, 5, 6, 0, 0, 0, 0}, /* 0x35 (00110101) */ - {2, 3, 5, 6, 0, 0, 0, 0}, /* 0x36 (00110110) */ - {1, 2, 3, 5, 6, 0, 0, 0}, /* 0x37 (00110111) */ - {4, 5, 6, 0, 0, 0, 0, 0}, /* 0x38 (00111000) */ - {1, 4, 5, 6, 0, 0, 0, 0}, /* 0x39 (00111001) */ - {2, 4, 5, 6, 0, 0, 0, 0}, /* 0x3A (00111010) */ - {1, 2, 4, 5, 6, 0, 0, 0}, /* 0x3B (00111011) */ - {3, 4, 5, 6, 0, 0, 0, 0}, /* 0x3C (00111100) */ - {1, 3, 4, 5, 6, 0, 0, 0}, /* 0x3D (00111101) */ - {2, 3, 4, 5, 6, 0, 0, 0}, /* 0x3E (00111110) */ - {1, 2, 3, 4, 5, 6, 0, 0}, /* 0x3F (00111111) */ - {7, 0, 0, 0, 0, 0, 0, 0}, /* 0x40 (01000000) */ - {1, 7, 0, 0, 0, 0, 0, 0}, /* 0x41 (01000001) */ - {2, 7, 0, 0, 0, 0, 0, 0}, /* 0x42 (01000010) */ - {1, 2, 7, 0, 0, 0, 0, 0}, /* 0x43 (01000011) */ - {3, 7, 0, 0, 0, 0, 0, 0}, /* 0x44 (01000100) */ - {1, 3, 7, 0, 0, 0, 0, 0}, /* 0x45 (01000101) */ - {2, 3, 7, 0, 0, 0, 0, 0}, /* 0x46 (01000110) */ - {1, 2, 3, 7, 0, 0, 0, 0}, /* 0x47 (01000111) */ - {4, 7, 0, 0, 0, 0, 0, 0}, /* 0x48 (01001000) */ - {1, 4, 7, 0, 0, 0, 0, 0}, /* 0x49 (01001001) */ - {2, 4, 7, 0, 0, 0, 0, 0}, /* 0x4A (01001010) */ - {1, 2, 4, 7, 0, 0, 0, 0}, /* 0x4B (01001011) */ - {3, 4, 7, 0, 0, 0, 0, 0}, /* 0x4C (01001100) */ - {1, 3, 4, 7, 0, 0, 0, 0}, /* 0x4D (01001101) */ - {2, 3, 4, 7, 0, 0, 0, 0}, /* 0x4E (01001110) */ - {1, 2, 3, 4, 7, 0, 0, 0}, /* 0x4F (01001111) */ - {5, 7, 0, 0, 0, 0, 0, 0}, /* 0x50 (01010000) */ - {1, 5, 7, 0, 0, 0, 0, 0}, /* 0x51 (01010001) */ - {2, 5, 7, 0, 0, 0, 0, 0}, /* 0x52 (01010010) */ - {1, 2, 5, 7, 0, 0, 0, 0}, /* 0x53 (01010011) */ - {3, 5, 7, 0, 0, 0, 0, 0}, /* 0x54 (01010100) */ - {1, 3, 5, 7, 0, 0, 0, 0}, /* 0x55 (01010101) */ - {2, 3, 5, 7, 0, 0, 0, 0}, /* 0x56 (01010110) */ - {1, 2, 3, 5, 7, 0, 0, 0}, /* 0x57 (01010111) */ - {4, 5, 7, 0, 0, 0, 0, 0}, /* 0x58 (01011000) */ - {1, 4, 5, 7, 0, 0, 0, 0}, /* 0x59 (01011001) */ - {2, 4, 5, 7, 0, 0, 0, 0}, /* 0x5A (01011010) */ - {1, 2, 4, 5, 7, 0, 0, 0}, /* 0x5B (01011011) */ - {3, 4, 5, 7, 0, 0, 0, 0}, /* 0x5C (01011100) */ - {1, 3, 4, 5, 7, 0, 0, 0}, /* 0x5D (01011101) */ - {2, 3, 4, 5, 7, 0, 0, 0}, /* 0x5E (01011110) */ - {1, 2, 3, 4, 5, 7, 0, 0}, /* 0x5F (01011111) */ - {6, 7, 0, 0, 0, 0, 0, 0}, /* 0x60 (01100000) */ - {1, 6, 7, 0, 0, 0, 0, 0}, /* 0x61 (01100001) */ - {2, 6, 7, 0, 0, 0, 0, 0}, /* 0x62 (01100010) */ - {1, 2, 6, 7, 0, 0, 0, 0}, /* 0x63 (01100011) */ - {3, 6, 7, 0, 0, 0, 0, 0}, /* 0x64 (01100100) */ - {1, 3, 6, 7, 0, 0, 0, 0}, /* 0x65 (01100101) */ - {2, 3, 6, 7, 0, 0, 0, 0}, /* 0x66 (01100110) */ - {1, 2, 3, 6, 7, 0, 0, 0}, /* 0x67 (01100111) */ - {4, 6, 7, 0, 0, 0, 0, 0}, /* 0x68 (01101000) */ - {1, 4, 6, 7, 0, 0, 0, 0}, /* 0x69 (01101001) */ - {2, 4, 6, 7, 0, 0, 0, 0}, /* 0x6A (01101010) */ - {1, 2, 4, 6, 7, 0, 0, 0}, /* 0x6B (01101011) */ - {3, 4, 6, 7, 0, 0, 0, 0}, /* 0x6C (01101100) */ - {1, 3, 4, 6, 7, 0, 0, 0}, /* 0x6D (01101101) */ - {2, 3, 4, 6, 7, 0, 0, 0}, /* 0x6E (01101110) */ - {1, 2, 3, 4, 6, 7, 0, 0}, /* 0x6F (01101111) */ - {5, 6, 7, 0, 0, 0, 0, 0}, /* 0x70 (01110000) */ - {1, 5, 6, 7, 0, 0, 0, 0}, /* 0x71 (01110001) */ - {2, 5, 6, 7, 0, 0, 0, 0}, /* 0x72 (01110010) */ - {1, 2, 5, 6, 7, 0, 0, 0}, /* 0x73 (01110011) */ - {3, 5, 6, 7, 0, 0, 0, 0}, /* 0x74 (01110100) */ - {1, 3, 5, 6, 7, 0, 0, 0}, /* 0x75 (01110101) */ - {2, 3, 5, 6, 7, 0, 0, 0}, /* 0x76 (01110110) */ - {1, 2, 3, 5, 6, 7, 0, 0}, /* 0x77 (01110111) */ - {4, 5, 6, 7, 0, 0, 0, 0}, /* 0x78 (01111000) */ - {1, 4, 5, 6, 7, 0, 0, 0}, /* 0x79 (01111001) */ - {2, 4, 5, 6, 7, 0, 0, 0}, /* 0x7A (01111010) */ - {1, 2, 4, 5, 6, 7, 0, 0}, /* 0x7B (01111011) */ - {3, 4, 5, 6, 7, 0, 0, 0}, /* 0x7C (01111100) */ - {1, 3, 4, 5, 6, 7, 0, 0}, /* 0x7D (01111101) */ - {2, 3, 4, 5, 6, 7, 0, 0}, /* 0x7E (01111110) */ - {1, 2, 3, 4, 5, 6, 7, 0}, /* 0x7F (01111111) */ - {8, 0, 0, 0, 0, 0, 0, 0}, /* 0x80 (10000000) */ - {1, 8, 0, 0, 0, 0, 0, 0}, /* 0x81 (10000001) */ - {2, 8, 0, 0, 0, 0, 0, 0}, /* 0x82 (10000010) */ - {1, 2, 8, 0, 0, 0, 0, 0}, /* 0x83 (10000011) */ - {3, 8, 0, 0, 0, 0, 0, 0}, /* 0x84 (10000100) */ - {1, 3, 8, 0, 0, 0, 0, 0}, /* 0x85 (10000101) */ - {2, 3, 8, 0, 0, 0, 0, 0}, /* 0x86 (10000110) */ - {1, 2, 3, 8, 0, 0, 0, 0}, /* 0x87 (10000111) */ - {4, 8, 0, 0, 0, 0, 0, 0}, /* 0x88 (10001000) */ - {1, 4, 8, 0, 0, 0, 0, 0}, /* 0x89 (10001001) */ - {2, 4, 8, 0, 0, 0, 0, 0}, /* 0x8A (10001010) */ - {1, 2, 4, 8, 0, 0, 0, 0}, /* 0x8B (10001011) */ - {3, 4, 8, 0, 0, 0, 0, 0}, /* 0x8C (10001100) */ - {1, 3, 4, 8, 0, 0, 0, 0}, /* 0x8D (10001101) */ - {2, 3, 4, 8, 0, 0, 0, 0}, /* 0x8E (10001110) */ - {1, 2, 3, 4, 8, 0, 0, 0}, /* 0x8F (10001111) */ - {5, 8, 0, 0, 0, 0, 0, 0}, /* 0x90 (10010000) */ - {1, 5, 8, 0, 0, 0, 0, 0}, /* 0x91 (10010001) */ - {2, 5, 8, 0, 0, 0, 0, 0}, /* 0x92 (10010010) */ - {1, 2, 5, 8, 0, 0, 0, 0}, /* 0x93 (10010011) */ - {3, 5, 8, 0, 0, 0, 0, 0}, /* 0x94 (10010100) */ - {1, 3, 5, 8, 0, 0, 0, 0}, /* 0x95 (10010101) */ - {2, 3, 5, 8, 0, 0, 0, 0}, /* 0x96 (10010110) */ - {1, 2, 3, 5, 8, 0, 0, 0}, /* 0x97 (10010111) */ - {4, 5, 8, 0, 0, 0, 0, 0}, /* 0x98 (10011000) */ - {1, 4, 5, 8, 0, 0, 0, 0}, /* 0x99 (10011001) */ - {2, 4, 5, 8, 0, 0, 0, 0}, /* 0x9A (10011010) */ - {1, 2, 4, 5, 8, 0, 0, 0}, /* 0x9B (10011011) */ - {3, 4, 5, 8, 0, 0, 0, 0}, /* 0x9C (10011100) */ - {1, 3, 4, 5, 8, 0, 0, 0}, /* 0x9D (10011101) */ - {2, 3, 4, 5, 8, 0, 0, 0}, /* 0x9E (10011110) */ - {1, 2, 3, 4, 5, 8, 0, 0}, /* 0x9F (10011111) */ - {6, 8, 0, 0, 0, 0, 0, 0}, /* 0xA0 (10100000) */ - {1, 6, 8, 0, 0, 0, 0, 0}, /* 0xA1 (10100001) */ - {2, 6, 8, 0, 0, 0, 0, 0}, /* 0xA2 (10100010) */ - {1, 2, 6, 8, 0, 0, 0, 0}, /* 0xA3 (10100011) */ - {3, 6, 8, 0, 0, 0, 0, 0}, /* 0xA4 (10100100) */ - {1, 3, 6, 8, 0, 0, 0, 0}, /* 0xA5 (10100101) */ - {2, 3, 6, 8, 0, 0, 0, 0}, /* 0xA6 (10100110) */ - {1, 2, 3, 6, 8, 0, 0, 0}, /* 0xA7 (10100111) */ - {4, 6, 8, 0, 0, 0, 0, 0}, /* 0xA8 (10101000) */ - {1, 4, 6, 8, 0, 0, 0, 0}, /* 0xA9 (10101001) */ - {2, 4, 6, 8, 0, 0, 0, 0}, /* 0xAA (10101010) */ - {1, 2, 4, 6, 8, 0, 0, 0}, /* 0xAB (10101011) */ - {3, 4, 6, 8, 0, 0, 0, 0}, /* 0xAC (10101100) */ - {1, 3, 4, 6, 8, 0, 0, 0}, /* 0xAD (10101101) */ - {2, 3, 4, 6, 8, 0, 0, 0}, /* 0xAE (10101110) */ - {1, 2, 3, 4, 6, 8, 0, 0}, /* 0xAF (10101111) */ - {5, 6, 8, 0, 0, 0, 0, 0}, /* 0xB0 (10110000) */ - {1, 5, 6, 8, 0, 0, 0, 0}, /* 0xB1 (10110001) */ - {2, 5, 6, 8, 0, 0, 0, 0}, /* 0xB2 (10110010) */ - {1, 2, 5, 6, 8, 0, 0, 0}, /* 0xB3 (10110011) */ - {3, 5, 6, 8, 0, 0, 0, 0}, /* 0xB4 (10110100) */ - {1, 3, 5, 6, 8, 0, 0, 0}, /* 0xB5 (10110101) */ - {2, 3, 5, 6, 8, 0, 0, 0}, /* 0xB6 (10110110) */ - {1, 2, 3, 5, 6, 8, 0, 0}, /* 0xB7 (10110111) */ - {4, 5, 6, 8, 0, 0, 0, 0}, /* 0xB8 (10111000) */ - {1, 4, 5, 6, 8, 0, 0, 0}, /* 0xB9 (10111001) */ - {2, 4, 5, 6, 8, 0, 0, 0}, /* 0xBA (10111010) */ - {1, 2, 4, 5, 6, 8, 0, 0}, /* 0xBB (10111011) */ - {3, 4, 5, 6, 8, 0, 0, 0}, /* 0xBC (10111100) */ - {1, 3, 4, 5, 6, 8, 0, 0}, /* 0xBD (10111101) */ - {2, 3, 4, 5, 6, 8, 0, 0}, /* 0xBE (10111110) */ - {1, 2, 3, 4, 5, 6, 8, 0}, /* 0xBF (10111111) */ - {7, 8, 0, 0, 0, 0, 0, 0}, /* 0xC0 (11000000) */ - {1, 7, 8, 0, 0, 0, 0, 0}, /* 0xC1 (11000001) */ - {2, 7, 8, 0, 0, 0, 0, 0}, /* 0xC2 (11000010) */ - {1, 2, 7, 8, 0, 0, 0, 0}, /* 0xC3 (11000011) */ - {3, 7, 8, 0, 0, 0, 0, 0}, /* 0xC4 (11000100) */ - {1, 3, 7, 8, 0, 0, 0, 0}, /* 0xC5 (11000101) */ - {2, 3, 7, 8, 0, 0, 0, 0}, /* 0xC6 (11000110) */ - {1, 2, 3, 7, 8, 0, 0, 0}, /* 0xC7 (11000111) */ - {4, 7, 8, 0, 0, 0, 0, 0}, /* 0xC8 (11001000) */ - {1, 4, 7, 8, 0, 0, 0, 0}, /* 0xC9 (11001001) */ - {2, 4, 7, 8, 0, 0, 0, 0}, /* 0xCA (11001010) */ - {1, 2, 4, 7, 8, 0, 0, 0}, /* 0xCB (11001011) */ - {3, 4, 7, 8, 0, 0, 0, 0}, /* 0xCC (11001100) */ - {1, 3, 4, 7, 8, 0, 0, 0}, /* 0xCD (11001101) */ - {2, 3, 4, 7, 8, 0, 0, 0}, /* 0xCE (11001110) */ - {1, 2, 3, 4, 7, 8, 0, 0}, /* 0xCF (11001111) */ - {5, 7, 8, 0, 0, 0, 0, 0}, /* 0xD0 (11010000) */ - {1, 5, 7, 8, 0, 0, 0, 0}, /* 0xD1 (11010001) */ - {2, 5, 7, 8, 0, 0, 0, 0}, /* 0xD2 (11010010) */ - {1, 2, 5, 7, 8, 0, 0, 0}, /* 0xD3 (11010011) */ - {3, 5, 7, 8, 0, 0, 0, 0}, /* 0xD4 (11010100) */ - {1, 3, 5, 7, 8, 0, 0, 0}, /* 0xD5 (11010101) */ - {2, 3, 5, 7, 8, 0, 0, 0}, /* 0xD6 (11010110) */ - {1, 2, 3, 5, 7, 8, 0, 0}, /* 0xD7 (11010111) */ - {4, 5, 7, 8, 0, 0, 0, 0}, /* 0xD8 (11011000) */ - {1, 4, 5, 7, 8, 0, 0, 0}, /* 0xD9 (11011001) */ - {2, 4, 5, 7, 8, 0, 0, 0}, /* 0xDA (11011010) */ - {1, 2, 4, 5, 7, 8, 0, 0}, /* 0xDB (11011011) */ - {3, 4, 5, 7, 8, 0, 0, 0}, /* 0xDC (11011100) */ - {1, 3, 4, 5, 7, 8, 0, 0}, /* 0xDD (11011101) */ - {2, 3, 4, 5, 7, 8, 0, 0}, /* 0xDE (11011110) */ - {1, 2, 3, 4, 5, 7, 8, 0}, /* 0xDF (11011111) */ - {6, 7, 8, 0, 0, 0, 0, 0}, /* 0xE0 (11100000) */ - {1, 6, 7, 8, 0, 0, 0, 0}, /* 0xE1 (11100001) */ - {2, 6, 7, 8, 0, 0, 0, 0}, /* 0xE2 (11100010) */ - {1, 2, 6, 7, 8, 0, 0, 0}, /* 0xE3 (11100011) */ - {3, 6, 7, 8, 0, 0, 0, 0}, /* 0xE4 (11100100) */ - {1, 3, 6, 7, 8, 0, 0, 0}, /* 0xE5 (11100101) */ - {2, 3, 6, 7, 8, 0, 0, 0}, /* 0xE6 (11100110) */ - {1, 2, 3, 6, 7, 8, 0, 0}, /* 0xE7 (11100111) */ - {4, 6, 7, 8, 0, 0, 0, 0}, /* 0xE8 (11101000) */ - {1, 4, 6, 7, 8, 0, 0, 0}, /* 0xE9 (11101001) */ - {2, 4, 6, 7, 8, 0, 0, 0}, /* 0xEA (11101010) */ - {1, 2, 4, 6, 7, 8, 0, 0}, /* 0xEB (11101011) */ - {3, 4, 6, 7, 8, 0, 0, 0}, /* 0xEC (11101100) */ - {1, 3, 4, 6, 7, 8, 0, 0}, /* 0xED (11101101) */ - {2, 3, 4, 6, 7, 8, 0, 0}, /* 0xEE (11101110) */ - {1, 2, 3, 4, 6, 7, 8, 0}, /* 0xEF (11101111) */ - {5, 6, 7, 8, 0, 0, 0, 0}, /* 0xF0 (11110000) */ - {1, 5, 6, 7, 8, 0, 0, 0}, /* 0xF1 (11110001) */ - {2, 5, 6, 7, 8, 0, 0, 0}, /* 0xF2 (11110010) */ - {1, 2, 5, 6, 7, 8, 0, 0}, /* 0xF3 (11110011) */ - {3, 5, 6, 7, 8, 0, 0, 0}, /* 0xF4 (11110100) */ - {1, 3, 5, 6, 7, 8, 0, 0}, /* 0xF5 (11110101) */ - {2, 3, 5, 6, 7, 8, 0, 0}, /* 0xF6 (11110110) */ - {1, 2, 3, 5, 6, 7, 8, 0}, /* 0xF7 (11110111) */ - {4, 5, 6, 7, 8, 0, 0, 0}, /* 0xF8 (11111000) */ - {1, 4, 5, 6, 7, 8, 0, 0}, /* 0xF9 (11111001) */ - {2, 4, 5, 6, 7, 8, 0, 0}, /* 0xFA (11111010) */ - {1, 2, 4, 5, 6, 7, 8, 0}, /* 0xFB (11111011) */ - {3, 4, 5, 6, 7, 8, 0, 0}, /* 0xFC (11111100) */ - {1, 3, 4, 5, 6, 7, 8, 0}, /* 0xFD (11111101) */ - {2, 3, 4, 5, 6, 7, 8, 0}, /* 0xFE (11111110) */ - {1, 2, 3, 4, 5, 6, 7, 8} /* 0xFF (11111111) */ -}; - -#endif - -#if CROARING_IS_X64 -#if CROARING_COMPILER_SUPPORTS_AVX512 -CROARING_TARGET_AVX512 -const uint8_t vbmi2_table[64] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63}; -size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, - uint32_t *vout, size_t outcapacity, - uint32_t base) { - uint32_t *out = (uint32_t *)vout; - uint32_t *initout = out; - uint32_t *safeout = out + outcapacity; - __m512i base_v = _mm512_set1_epi32(base); - __m512i index_table = _mm512_loadu_si512(vbmi2_table); - size_t i = 0; - - for (; (i < length) && ((out + 64) < safeout); i += 1) { - uint64_t v = words[i]; - __m512i vec = _mm512_maskz_compress_epi8(v, index_table); - - uint8_t advance = (uint8_t)roaring_hamming(v); - - __m512i vbase = - _mm512_add_epi32(base_v, _mm512_set1_epi32((int)(i * 64))); - __m512i r1 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec, 0)); - __m512i r2 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec, 1)); - __m512i r3 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec, 2)); - __m512i r4 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec, 3)); - - r1 = _mm512_add_epi32(r1, vbase); - r2 = _mm512_add_epi32(r2, vbase); - r3 = _mm512_add_epi32(r3, vbase); - r4 = _mm512_add_epi32(r4, vbase); - _mm512_storeu_si512((__m512i *)out, r1); - _mm512_storeu_si512((__m512i *)(out + 16), r2); - _mm512_storeu_si512((__m512i *)(out + 32), r3); - _mm512_storeu_si512((__m512i *)(out + 48), r4); - - out += advance; +/* Create a new bitset. Return NULL in case of failure. */ +bitset_t *bitset_create(void) { + bitset_t *bitset = NULL; + /* Allocate the bitset itself. */ + if ((bitset = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { + return NULL; } + bitset->array = NULL; + bitset->arraysize = 0; + bitset->capacity = 0; + return bitset; +} - base += i * 64; - - for (; (i < length) && (out < safeout); ++i) { - uint64_t w = words[i]; - while ((w != 0) && (out < safeout)) { - int r = - roaring_trailing_zeroes(w); // on x64, should compile to TZCNT - uint32_t val = r + base; - memcpy(out, &val, - sizeof(uint32_t)); // should be compiled as a MOV on x64 - out++; - w &= (w - 1); - } - base += 64; +/* Create a new bitset able to contain size bits. Return NULL in case of + * failure. */ +bitset_t *bitset_create_with_capacity(size_t size) { + bitset_t *bitset = NULL; + /* Allocate the bitset itself. */ + if ((bitset = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { + return NULL; } - - return out - initout; + bitset->arraysize = + (size + sizeof(uint64_t) * 8 - 1) / (sizeof(uint64_t) * 8); + bitset->capacity = bitset->arraysize; + if ((bitset->array = (uint64_t *)roaring_calloc( + bitset->arraysize, sizeof(uint64_t))) == NULL) { + roaring_free(bitset); + return NULL; + } + return bitset; } -// Reference: -// https://lemire.me/blog/2022/05/10/faster-bitset-decoding-using-intel-avx-512/ -size_t bitset_extract_setbits_avx512_uint16(const uint64_t *array, - size_t length, uint16_t *vout, - size_t capacity, uint16_t base) { - uint16_t *out = (uint16_t *)vout; - uint16_t *initout = out; - uint16_t *safeout = vout + capacity; - - __m512i base_v = _mm512_set1_epi16(base); - __m512i index_table = _mm512_loadu_si512(vbmi2_table); - size_t i = 0; - - for (; (i < length) && ((out + 64) < safeout); i++) { - uint64_t v = array[i]; - __m512i vec = _mm512_maskz_compress_epi8(v, index_table); - - uint8_t advance = (uint8_t)roaring_hamming(v); - - __m512i vbase = - _mm512_add_epi16(base_v, _mm512_set1_epi16((short)(i * 64))); - __m512i r1 = _mm512_cvtepi8_epi16(_mm512_extracti32x8_epi32(vec, 0)); - __m512i r2 = _mm512_cvtepi8_epi16(_mm512_extracti32x8_epi32(vec, 1)); - - r1 = _mm512_add_epi16(r1, vbase); - r2 = _mm512_add_epi16(r2, vbase); - - _mm512_storeu_si512((__m512i *)out, r1); - _mm512_storeu_si512((__m512i *)(out + 32), r2); - out += advance; +/* Create a copy */ +bitset_t *bitset_copy(const bitset_t *bitset) { + bitset_t *copy = NULL; + /* Allocate the bitset itself. */ + if ((copy = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { + return NULL; + } + memcpy(copy, bitset, sizeof(bitset_t)); + copy->capacity = copy->arraysize; + if ((copy->array = (uint64_t *)roaring_malloc(sizeof(uint64_t) * + bitset->arraysize)) == NULL) { + roaring_free(copy); + return NULL; } + memcpy(copy->array, bitset->array, sizeof(uint64_t) * bitset->arraysize); + return copy; +} - base += i * 64; +void bitset_clear(bitset_t *bitset) { + memset(bitset->array, 0, sizeof(uint64_t) * bitset->arraysize); +} - for (; (i < length) && (out < safeout); ++i) { - uint64_t w = array[i]; - while ((w != 0) && (out < safeout)) { - int r = - roaring_trailing_zeroes(w); // on x64, should compile to TZCNT - uint32_t val = r + base; - memcpy(out, &val, sizeof(uint16_t)); - out++; - w &= (w - 1); +void bitset_fill(bitset_t *bitset) { + memset(bitset->array, 0xff, sizeof(uint64_t) * bitset->arraysize); +} + +void bitset_shift_left(bitset_t *bitset, size_t s) { + size_t extra_words = s / 64; + int inword_shift = s % 64; + size_t as = bitset->arraysize; + if (inword_shift == 0) { + bitset_resize(bitset, as + extra_words, false); + // could be done with a memmove + for (size_t i = as + extra_words; i > extra_words; i--) { + bitset->array[i - 1] = bitset->array[i - 1 - extra_words]; } - base += 64; + } else { + bitset_resize(bitset, as + extra_words + 1, true); + bitset->array[as + extra_words] = + bitset->array[as - 1] >> (64 - inword_shift); + for (size_t i = as + extra_words; i >= extra_words + 2; i--) { + bitset->array[i - 1] = + (bitset->array[i - 1 - extra_words] << inword_shift) | + (bitset->array[i - 2 - extra_words] >> (64 - inword_shift)); + } + bitset->array[extra_words] = bitset->array[0] << inword_shift; + } + for (size_t i = 0; i < extra_words; i++) { + bitset->array[i] = 0; } - - return out - initout; } -CROARING_UNTARGET_AVX512 -#endif -CROARING_TARGET_AVX2 -size_t bitset_extract_setbits_avx2(const uint64_t *words, size_t length, - uint32_t *out, size_t outcapacity, - uint32_t base) { - uint32_t *initout = out; - __m256i baseVec = _mm256_set1_epi32(base - 1); - __m256i incVec = _mm256_set1_epi32(64); - __m256i add8 = _mm256_set1_epi32(8); - uint32_t *safeout = out + outcapacity; - size_t i = 0; - for (; (i < length) && (out + 64 <= safeout); ++i) { - uint64_t w = words[i]; - if (w == 0) { - baseVec = _mm256_add_epi32(baseVec, incVec); - } else { - for (int k = 0; k < 4; ++k) { - uint8_t byteA = (uint8_t)w; - uint8_t byteB = (uint8_t)(w >> 8); - w >>= 16; - __m256i vecA = - _mm256_loadu_si256((const __m256i *)vecDecodeTable[byteA]); - __m256i vecB = - _mm256_loadu_si256((const __m256i *)vecDecodeTable[byteB]); - uint8_t advanceA = lengthTable[byteA]; - uint8_t advanceB = lengthTable[byteB]; - vecA = _mm256_add_epi32(baseVec, vecA); - baseVec = _mm256_add_epi32(baseVec, add8); - vecB = _mm256_add_epi32(baseVec, vecB); - baseVec = _mm256_add_epi32(baseVec, add8); - _mm256_storeu_si256((__m256i *)out, vecA); - out += advanceA; - _mm256_storeu_si256((__m256i *)out, vecB); - out += advanceB; - } +void bitset_shift_right(bitset_t *bitset, size_t s) { + size_t extra_words = s / 64; + int inword_shift = s % 64; + size_t as = bitset->arraysize; + if (inword_shift == 0) { + // could be done with a memmove + for (size_t i = 0; i < as - extra_words; i++) { + bitset->array[i] = bitset->array[i + extra_words]; } - } - base += i * 64; - for (; (i < length) && (out < safeout); ++i) { - uint64_t w = words[i]; - while ((w != 0) && (out < safeout)) { - int r = - roaring_trailing_zeroes(w); // on x64, should compile to TZCNT - uint32_t val = r + base; - memcpy(out, &val, - sizeof(uint32_t)); // should be compiled as a MOV on x64 - out++; - w &= (w - 1); + bitset_resize(bitset, as - extra_words, false); + + } else { + for (size_t i = 0; i + extra_words + 1 < as; i++) { + bitset->array[i] = + (bitset->array[i + extra_words] >> inword_shift) | + (bitset->array[i + extra_words + 1] << (64 - inword_shift)); } - base += 64; + bitset->array[as - extra_words - 1] = + (bitset->array[as - 1] >> inword_shift); + bitset_resize(bitset, as - extra_words, false); } - return out - initout; } -CROARING_UNTARGET_AVX2 -#endif // CROARING_IS_X64 -size_t bitset_extract_setbits(const uint64_t *words, size_t length, - uint32_t *out, uint32_t base) { - int outpos = 0; - for (size_t i = 0; i < length; ++i) { - uint64_t w = words[i]; - while (w != 0) { - int r = - roaring_trailing_zeroes(w); // on x64, should compile to TZCNT - uint32_t val = r + base; - memcpy(out + outpos, &val, - sizeof(uint32_t)); // should be compiled as a MOV on x64 - outpos++; - w &= (w - 1); - } - base += 64; +/* Free memory. */ +void bitset_free(bitset_t *bitset) { + if (bitset == NULL) { + return; } - return outpos; + roaring_free(bitset->array); + roaring_free(bitset); } -size_t bitset_extract_intersection_setbits_uint16( - const uint64_t *__restrict__ words1, const uint64_t *__restrict__ words2, - size_t length, uint16_t *out, uint16_t base) { - int outpos = 0; - for (size_t i = 0; i < length; ++i) { - uint64_t w = words1[i] & words2[i]; - while (w != 0) { - int r = roaring_trailing_zeroes(w); - out[outpos++] = (uint16_t)(r + base); - w &= (w - 1); +/* Resize the bitset so that it can support newarraysize * 64 bits. Return true + * in case of success, false for failure. */ +bool bitset_resize(bitset_t *bitset, size_t newarraysize, bool padwithzeroes) { + if (newarraysize > SIZE_MAX / 64) { + return false; + } + size_t smallest = + newarraysize < bitset->arraysize ? newarraysize : bitset->arraysize; + if (bitset->capacity < newarraysize) { + uint64_t *newarray; + size_t newcapacity = bitset->capacity; + if (newcapacity == 0) { + newcapacity = 1; } - base += 64; + while (newcapacity < newarraysize) { + newcapacity *= 2; + } + if ((newarray = (uint64_t *)roaring_realloc( + bitset->array, sizeof(uint64_t) * newcapacity)) == NULL) { + return false; + } + bitset->capacity = newcapacity; + bitset->array = newarray; } - return outpos; + if (padwithzeroes && (newarraysize > smallest)) + memset(bitset->array + smallest, 0, + sizeof(uint64_t) * (newarraysize - smallest)); + bitset->arraysize = newarraysize; + return true; // success! } -#if CROARING_IS_X64 -/* - * Given a bitset containing "length" 64-bit words, write out the position - * of all the set bits to "out" as 16-bit integers, values start at "base" (can - *be set to zero). - * - * The "out" pointer should be sufficient to store the actual number of bits - *set. - * - * Returns how many values were actually decoded. - * - * This function uses SSE decoding. - */ -CROARING_TARGET_AVX2 -size_t bitset_extract_setbits_sse_uint16(const uint64_t *words, size_t length, - uint16_t *out, size_t outcapacity, - uint16_t base) { - uint16_t *initout = out; - __m128i baseVec = _mm_set1_epi16(base - 1); - __m128i incVec = _mm_set1_epi16(64); - __m128i add8 = _mm_set1_epi16(8); - uint16_t *safeout = out + outcapacity; - const int numberofbytes = 2; // process two bytes at a time - size_t i = 0; - for (; (i < length) && (out + numberofbytes * 8 <= safeout); ++i) { - uint64_t w = words[i]; - if (w == 0) { - baseVec = _mm_add_epi16(baseVec, incVec); - } else { - for (int k = 0; k < 4; ++k) { - uint8_t byteA = (uint8_t)w; - uint8_t byteB = (uint8_t)(w >> 8); - w >>= 16; - __m128i vecA = _mm_loadu_si128( - (const __m128i *)vecDecodeTable_uint16[byteA]); - __m128i vecB = _mm_loadu_si128( - (const __m128i *)vecDecodeTable_uint16[byteB]); - uint8_t advanceA = lengthTable[byteA]; - uint8_t advanceB = lengthTable[byteB]; - vecA = _mm_add_epi16(baseVec, vecA); - baseVec = _mm_add_epi16(baseVec, add8); - vecB = _mm_add_epi16(baseVec, vecB); - baseVec = _mm_add_epi16(baseVec, add8); - _mm_storeu_si128((__m128i *)out, vecA); - out += advanceA; - _mm_storeu_si128((__m128i *)out, vecB); - out += advanceB; - } - } +size_t bitset_count(const bitset_t *bitset) { + size_t card = 0; + size_t k = 0; + for (; k + 7 < bitset->arraysize; k += 8) { + card += roaring_hamming(bitset->array[k]); + card += roaring_hamming(bitset->array[k + 1]); + card += roaring_hamming(bitset->array[k + 2]); + card += roaring_hamming(bitset->array[k + 3]); + card += roaring_hamming(bitset->array[k + 4]); + card += roaring_hamming(bitset->array[k + 5]); + card += roaring_hamming(bitset->array[k + 6]); + card += roaring_hamming(bitset->array[k + 7]); } - base += (uint16_t)(i * 64); - for (; (i < length) && (out < safeout); ++i) { - uint64_t w = words[i]; - while ((w != 0) && (out < safeout)) { - int r = roaring_trailing_zeroes(w); - *out = (uint16_t)(r + base); - out++; - w &= (w - 1); - } - base += 64; + for (; k + 3 < bitset->arraysize; k += 4) { + card += roaring_hamming(bitset->array[k]); + card += roaring_hamming(bitset->array[k + 1]); + card += roaring_hamming(bitset->array[k + 2]); + card += roaring_hamming(bitset->array[k + 3]); } - return out - initout; + for (; k < bitset->arraysize; k++) { + card += roaring_hamming(bitset->array[k]); + } + return card; } -CROARING_UNTARGET_AVX2 -#endif -/* - * Given a bitset containing "length" 64-bit words, write out the position - * of all the set bits to "out", values start at "base" (can be set to zero). - * - * The "out" pointer should be sufficient to store the actual number of bits - *set. - * - * Returns how many values were actually decoded. - */ -size_t bitset_extract_setbits_uint16(const uint64_t *words, size_t length, - uint16_t *out, uint16_t base) { - int outpos = 0; - for (size_t i = 0; i < length; ++i) { - uint64_t w = words[i]; - while (w != 0) { - int r = roaring_trailing_zeroes(w); - out[outpos++] = (uint16_t)(r + base); - w &= (w - 1); +bool bitset_inplace_union(bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + for (size_t k = 0; k < minlength; ++k) { + b1->array[k] |= b2->array[k]; + } + if (b2->arraysize > b1->arraysize) { + size_t oldsize = b1->arraysize; + if (!bitset_resize(b1, b2->arraysize, false)) return false; + memcpy(b1->array + oldsize, b2->array + oldsize, + (b2->arraysize - oldsize) * sizeof(uint64_t)); + } + return true; +} + +bool bitset_empty(const bitset_t *bitset) { + for (size_t k = 0; k < bitset->arraysize; k++) { + if (bitset->array[k] != 0) { + return false; } - base += 64; } - return outpos; + return true; } -#if defined(CROARING_ASMBITMANIPOPTIMIZATION) && defined(CROARING_IS_X64) +size_t bitset_minimum(const bitset_t *bitset) { + for (size_t k = 0; k < bitset->arraysize; k++) { + uint64_t w = bitset->array[k]; + if (w != 0) { + return roaring_trailing_zeroes(w) + k * 64; + } + } + return SIZE_MAX; +} -static inline uint64_t _asm_bitset_set_list_withcard(uint64_t *words, - uint64_t card, - const uint16_t *list, - uint64_t length) { - uint64_t offset, load, pos; - uint64_t shift = 6; - const uint16_t *end = list + length; - if (!length) return card; - // TODO: could unroll for performance, see bitset_set_list - // bts is not available as an intrinsic in GCC - __asm volatile( - "1:\n" - "movzwq (%[list]), %[pos]\n" - "shrx %[shift], %[pos], %[offset]\n" - "mov (%[words],%[offset],8), %[load]\n" - "bts %[pos], %[load]\n" - "mov %[load], (%[words],%[offset],8)\n" - "sbb $-1, %[card]\n" - "add $2, %[list]\n" - "cmp %[list], %[end]\n" - "jnz 1b" - : [card] "+&r"(card), [list] "+&r"(list), [load] "=&r"(load), - [pos] "=&r"(pos), [offset] "=&r"(offset) - : [end] "r"(end), [words] "r"(words), [shift] "r"(shift)); - return card; +bool bitset_grow(bitset_t *bitset, size_t newarraysize) { + if (newarraysize < bitset->arraysize) { + return false; + } + if (newarraysize > SIZE_MAX / 64) { + return false; + } + if (bitset->capacity < newarraysize) { + uint64_t *newarray; + size_t newcapacity = (UINT64_C(0xFFFFFFFFFFFFFFFF) >> + roaring_leading_zeroes(newarraysize)) + + 1; + while (newcapacity < newarraysize) { + newcapacity *= 2; + } + if ((newarray = (uint64_t *)roaring_realloc( + bitset->array, sizeof(uint64_t) * newcapacity)) == NULL) { + return false; + } + bitset->capacity = newcapacity; + bitset->array = newarray; + } + memset(bitset->array + bitset->arraysize, 0, + sizeof(uint64_t) * (newarraysize - bitset->arraysize)); + bitset->arraysize = newarraysize; + return true; // success! } -static inline void _asm_bitset_set_list(uint64_t *words, const uint16_t *list, - uint64_t length) { - uint64_t pos; - const uint16_t *end = list + length; +size_t bitset_maximum(const bitset_t *bitset) { + for (size_t k = bitset->arraysize; k > 0; k--) { + uint64_t w = bitset->array[k - 1]; + if (w != 0) { + return 63 - roaring_leading_zeroes(w) + (k - 1) * 64; + } + } + return 0; +} - uint64_t shift = 6; - uint64_t offset; - uint64_t load; - for (; list + 3 < end; list += 4) { - pos = list[0]; - __asm volatile( - "shrx %[shift], %[pos], %[offset]\n" - "mov (%[words],%[offset],8), %[load]\n" - "bts %[pos], %[load]\n" - "mov %[load], (%[words],%[offset],8)" - : [load] "=&r"(load), [offset] "=&r"(offset) - : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); - pos = list[1]; - __asm volatile( - "shrx %[shift], %[pos], %[offset]\n" - "mov (%[words],%[offset],8), %[load]\n" - "bts %[pos], %[load]\n" - "mov %[load], (%[words],%[offset],8)" - : [load] "=&r"(load), [offset] "=&r"(offset) - : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); - pos = list[2]; - __asm volatile( - "shrx %[shift], %[pos], %[offset]\n" - "mov (%[words],%[offset],8), %[load]\n" - "bts %[pos], %[load]\n" - "mov %[load], (%[words],%[offset],8)" - : [load] "=&r"(load), [offset] "=&r"(offset) - : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); - pos = list[3]; - __asm volatile( - "shrx %[shift], %[pos], %[offset]\n" - "mov (%[words],%[offset],8), %[load]\n" - "bts %[pos], %[load]\n" - "mov %[load], (%[words],%[offset],8)" - : [load] "=&r"(load), [offset] "=&r"(offset) - : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); +/* Returns true if bitsets share no common elements, false otherwise. + * + * Performs early-out if common element found. */ +bool bitsets_disjoint(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + + for (size_t k = 0; k < minlength; k++) { + if ((b1->array[k] & b2->array[k]) != 0) return false; } + return true; +} - while (list != end) { - pos = list[0]; - __asm volatile( - "shrx %[shift], %[pos], %[offset]\n" - "mov (%[words],%[offset],8), %[load]\n" - "bts %[pos], %[load]\n" - "mov %[load], (%[words],%[offset],8)" - : [load] "=&r"(load), [offset] "=&r"(offset) - : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); - list++; +/* Returns true if bitsets contain at least 1 common element, false if they are + * disjoint. + * + * Performs early-out if common element found. */ +bool bitsets_intersect(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + + for (size_t k = 0; k < minlength; k++) { + if ((b1->array[k] & b2->array[k]) != 0) return true; } + return false; } -static inline uint64_t _asm_bitset_clear_list(uint64_t *words, uint64_t card, - const uint16_t *list, - uint64_t length) { - uint64_t offset, load, pos; - uint64_t shift = 6; - const uint16_t *end = list + length; - if (!length) return card; - // btr is not available as an intrinsic in GCC - __asm volatile( - "1:\n" - "movzwq (%[list]), %[pos]\n" - "shrx %[shift], %[pos], %[offset]\n" - "mov (%[words],%[offset],8), %[load]\n" - "btr %[pos], %[load]\n" - "mov %[load], (%[words],%[offset],8)\n" - "sbb $0, %[card]\n" - "add $2, %[list]\n" - "cmp %[list], %[end]\n" - "jnz 1b" - : [card] "+&r"(card), [list] "+&r"(list), [load] "=&r"(load), - [pos] "=&r"(pos), [offset] "=&r"(offset) - : [end] "r"(end), [words] "r"(words), [shift] "r"(shift) - : - /* clobbers */ "memory"); - return card; +/* Returns true if b has any bits set in or after b->array[starting_loc]. */ +static bool any_bits_set(const bitset_t *b, size_t starting_loc) { + if (starting_loc >= b->arraysize) { + return false; + } + for (size_t k = starting_loc; k < b->arraysize; k++) { + if (b->array[k] != 0) return true; + } + return false; } -static inline uint64_t _scalar_bitset_clear_list(uint64_t *words, uint64_t card, - const uint16_t *list, - uint64_t length) { - uint64_t offset, load, newload, pos, index; - const uint16_t *end = list + length; - while (list != end) { - pos = *(const uint16_t *)list; - offset = pos >> 6; - index = pos % 64; - load = words[offset]; - newload = load & ~(UINT64_C(1) << index); - card -= (load ^ newload) >> index; - words[offset] = newload; - list++; +/* Returns true if b1 has all of b2's bits set. + * + * Performs early out if a bit is found in b2 that is not found in b1. */ +bool bitset_contains_all(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t min_size = b1->arraysize; + if (b1->arraysize > b2->arraysize) { + min_size = b2->arraysize; } - return card; + for (size_t k = 0; k < min_size; k++) { + if ((b1->array[k] & b2->array[k]) != b2->array[k]) { + return false; + } + } + if (b2->arraysize > b1->arraysize) { + /* Need to check if b2 has any bits set beyond b1's array */ + return !any_bits_set(b2, b1->arraysize); + } + return true; } -static inline uint64_t _scalar_bitset_set_list_withcard(uint64_t *words, - uint64_t card, - const uint16_t *list, - uint64_t length) { - uint64_t offset, load, newload, pos, index; - const uint16_t *end = list + length; - while (list != end) { - pos = *list; - offset = pos >> 6; - index = pos % 64; - load = words[offset]; - newload = load | (UINT64_C(1) << index); - card += (load ^ newload) >> index; - words[offset] = newload; - list++; +size_t bitset_union_count(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t answer = 0; + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + for (; k + 3 < minlength; k += 4) { + answer += roaring_hamming(b1->array[k] | b2->array[k]); + answer += roaring_hamming(b1->array[k + 1] | b2->array[k + 1]); + answer += roaring_hamming(b1->array[k + 2] | b2->array[k + 2]); + answer += roaring_hamming(b1->array[k + 3] | b2->array[k + 3]); } - return card; + for (; k < minlength; ++k) { + answer += roaring_hamming(b1->array[k] | b2->array[k]); + } + if (b2->arraysize > b1->arraysize) { + // k is equal to b1->arraysize + for (; k + 3 < b2->arraysize; k += 4) { + answer += roaring_hamming(b2->array[k]); + answer += roaring_hamming(b2->array[k + 1]); + answer += roaring_hamming(b2->array[k + 2]); + answer += roaring_hamming(b2->array[k + 3]); + } + for (; k < b2->arraysize; ++k) { + answer += roaring_hamming(b2->array[k]); + } + } else { + // k is equal to b2->arraysize + for (; k + 3 < b1->arraysize; k += 4) { + answer += roaring_hamming(b1->array[k]); + answer += roaring_hamming(b1->array[k + 1]); + answer += roaring_hamming(b1->array[k + 2]); + answer += roaring_hamming(b1->array[k + 3]); + } + for (; k < b1->arraysize; ++k) { + answer += roaring_hamming(b1->array[k]); + } + } + return answer; } -static inline void _scalar_bitset_set_list(uint64_t *words, - const uint16_t *list, - uint64_t length) { - uint64_t offset, load, newload, pos, index; - const uint16_t *end = list + length; - while (list != end) { - pos = *list; - offset = pos >> 6; - index = pos % 64; - load = words[offset]; - newload = load | (UINT64_C(1) << index); - words[offset] = newload; - list++; +void bitset_inplace_intersection(bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + for (; k < minlength; ++k) { + b1->array[k] &= b2->array[k]; + } + for (; k < b1->arraysize; ++k) { + b1->array[k] = 0; // memset could, maybe, be a tiny bit faster } } -uint64_t bitset_clear_list(uint64_t *words, uint64_t card, const uint16_t *list, - uint64_t length) { - if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { - return _asm_bitset_clear_list(words, card, list, length); - } else { - return _scalar_bitset_clear_list(words, card, list, length); +size_t bitset_intersection_count(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t answer = 0; + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + for (size_t k = 0; k < minlength; ++k) { + answer += roaring_hamming(b1->array[k] & b2->array[k]); } + return answer; } -uint64_t bitset_set_list_withcard(uint64_t *words, uint64_t card, - const uint16_t *list, uint64_t length) { - if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { - return _asm_bitset_set_list_withcard(words, card, list, length); - } else { - return _scalar_bitset_set_list_withcard(words, card, list, length); +void bitset_inplace_difference(bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + for (; k < minlength; ++k) { + b1->array[k] &= ~(b2->array[k]); } } -void bitset_set_list(uint64_t *words, const uint16_t *list, uint64_t length) { - if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { - _asm_bitset_set_list(words, list, length); - } else { - _scalar_bitset_set_list(words, list, length); +size_t bitset_difference_count(const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + size_t answer = 0; + for (; k < minlength; ++k) { + answer += roaring_hamming(b1->array[k] & ~(b2->array[k])); + } + for (; k < b1->arraysize; ++k) { + answer += roaring_hamming(b1->array[k]); } + return answer; } -#else -uint64_t bitset_clear_list(uint64_t *words, uint64_t card, const uint16_t *list, - uint64_t length) { - uint64_t offset, load, newload, pos, index; - const uint16_t *end = list + length; - while (list != end) { - pos = *(const uint16_t *)list; - offset = pos >> 6; - index = pos % 64; - load = words[offset]; - newload = load & ~(UINT64_C(1) << index); - card -= (load ^ newload) >> index; - words[offset] = newload; - list++; + +bool bitset_inplace_symmetric_difference( + bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + for (; k < minlength; ++k) { + b1->array[k] ^= b2->array[k]; } - return card; + if (b2->arraysize > b1->arraysize) { + size_t oldsize = b1->arraysize; + if (!bitset_resize(b1, b2->arraysize, false)) return false; + memcpy(b1->array + oldsize, b2->array + oldsize, + (b2->arraysize - oldsize) * sizeof(uint64_t)); + } + return true; } -uint64_t bitset_set_list_withcard(uint64_t *words, uint64_t card, - const uint16_t *list, uint64_t length) { - uint64_t offset, load, newload, pos, index; - const uint16_t *end = list + length; - while (list != end) { - pos = *list; - offset = pos >> 6; - index = pos % 64; - load = words[offset]; - newload = load | (UINT64_C(1) << index); - card += (load ^ newload) >> index; - words[offset] = newload; - list++; +size_t bitset_symmetric_difference_count( + const bitset_t *CROARING_CBITSET_RESTRICT b1, + const bitset_t *CROARING_CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + size_t answer = 0; + for (; k < minlength; ++k) { + answer += roaring_hamming(b1->array[k] ^ b2->array[k]); } - return card; + if (b2->arraysize > b1->arraysize) { + for (; k < b2->arraysize; ++k) { + answer += roaring_hamming(b2->array[k]); + } + } else { + for (; k < b1->arraysize; ++k) { + answer += roaring_hamming(b1->array[k]); + } + } + return answer; } -void bitset_set_list(uint64_t *words, const uint16_t *list, uint64_t length) { - uint64_t offset, load, newload, pos, index; - const uint16_t *end = list + length; - while (list != end) { - pos = *list; - offset = pos >> 6; - index = pos % 64; - load = words[offset]; - newload = load | (UINT64_C(1) << index); - words[offset] = newload; - list++; +bool bitset_trim(bitset_t *bitset) { + size_t newsize = bitset->arraysize; + while (newsize > 0) { + if (bitset->array[newsize - 1] == 0) + newsize -= 1; + else + break; + } + if (bitset->capacity == newsize) return true; // nothing to do + uint64_t *newarray; + if ((newarray = (uint64_t *)roaring_realloc( + bitset->array, sizeof(uint64_t) * newsize)) == NULL) { + return false; } + bitset->array = newarray; + bitset->capacity = newsize; + bitset->arraysize = newsize; + return true; } +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { #endif +/* end file src/bitset.c */ +/* begin file src/bitset_util.c */ +#include +#include +#include +#include +#include -/* flip specified bits */ -/* TODO: consider whether worthwhile to make an asm version */ -uint64_t bitset_flip_list_withcard(uint64_t *words, uint64_t card, - const uint16_t *list, uint64_t length) { - uint64_t offset, load, newload, pos, index; - const uint16_t *end = list + length; - while (list != end) { - pos = *list; - offset = pos >> 6; - index = pos % 64; - load = words[offset]; - newload = load ^ (UINT64_C(1) << index); - // todo: is a branch here all that bad? - card += - (1 - 2 * (((UINT64_C(1) << index) & load) >> index)); // +1 or -1 - words[offset] = newload; - list++; - } - return card; -} +#if CROARING_IS_X64 +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif +#ifdef __cplusplus +using namespace ::roaring::internal; +extern "C" { +namespace roaring { +namespace api { +#endif -void bitset_flip_list(uint64_t *words, const uint16_t *list, uint64_t length) { - uint64_t offset, load, newload, pos, index; - const uint16_t *end = list + length; - while (list != end) { - pos = *list; - offset = pos >> 6; - index = pos % 64; - load = words[offset]; - newload = load ^ (UINT64_C(1) << index); - words[offset] = newload; - list++; - } -} +#if CROARING_IS_X64 +static uint8_t lengthTable[256] = { + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, + 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, + 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, + 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, + 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, + 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; +#endif + +#if CROARING_IS_X64 +ALIGNED(32) +static uint32_t vecDecodeTable[256][8] = { + {0, 0, 0, 0, 0, 0, 0, 0}, /* 0x00 (00000000) */ + {1, 0, 0, 0, 0, 0, 0, 0}, /* 0x01 (00000001) */ + {2, 0, 0, 0, 0, 0, 0, 0}, /* 0x02 (00000010) */ + {1, 2, 0, 0, 0, 0, 0, 0}, /* 0x03 (00000011) */ + {3, 0, 0, 0, 0, 0, 0, 0}, /* 0x04 (00000100) */ + {1, 3, 0, 0, 0, 0, 0, 0}, /* 0x05 (00000101) */ + {2, 3, 0, 0, 0, 0, 0, 0}, /* 0x06 (00000110) */ + {1, 2, 3, 0, 0, 0, 0, 0}, /* 0x07 (00000111) */ + {4, 0, 0, 0, 0, 0, 0, 0}, /* 0x08 (00001000) */ + {1, 4, 0, 0, 0, 0, 0, 0}, /* 0x09 (00001001) */ + {2, 4, 0, 0, 0, 0, 0, 0}, /* 0x0A (00001010) */ + {1, 2, 4, 0, 0, 0, 0, 0}, /* 0x0B (00001011) */ + {3, 4, 0, 0, 0, 0, 0, 0}, /* 0x0C (00001100) */ + {1, 3, 4, 0, 0, 0, 0, 0}, /* 0x0D (00001101) */ + {2, 3, 4, 0, 0, 0, 0, 0}, /* 0x0E (00001110) */ + {1, 2, 3, 4, 0, 0, 0, 0}, /* 0x0F (00001111) */ + {5, 0, 0, 0, 0, 0, 0, 0}, /* 0x10 (00010000) */ + {1, 5, 0, 0, 0, 0, 0, 0}, /* 0x11 (00010001) */ + {2, 5, 0, 0, 0, 0, 0, 0}, /* 0x12 (00010010) */ + {1, 2, 5, 0, 0, 0, 0, 0}, /* 0x13 (00010011) */ + {3, 5, 0, 0, 0, 0, 0, 0}, /* 0x14 (00010100) */ + {1, 3, 5, 0, 0, 0, 0, 0}, /* 0x15 (00010101) */ + {2, 3, 5, 0, 0, 0, 0, 0}, /* 0x16 (00010110) */ + {1, 2, 3, 5, 0, 0, 0, 0}, /* 0x17 (00010111) */ + {4, 5, 0, 0, 0, 0, 0, 0}, /* 0x18 (00011000) */ + {1, 4, 5, 0, 0, 0, 0, 0}, /* 0x19 (00011001) */ + {2, 4, 5, 0, 0, 0, 0, 0}, /* 0x1A (00011010) */ + {1, 2, 4, 5, 0, 0, 0, 0}, /* 0x1B (00011011) */ + {3, 4, 5, 0, 0, 0, 0, 0}, /* 0x1C (00011100) */ + {1, 3, 4, 5, 0, 0, 0, 0}, /* 0x1D (00011101) */ + {2, 3, 4, 5, 0, 0, 0, 0}, /* 0x1E (00011110) */ + {1, 2, 3, 4, 5, 0, 0, 0}, /* 0x1F (00011111) */ + {6, 0, 0, 0, 0, 0, 0, 0}, /* 0x20 (00100000) */ + {1, 6, 0, 0, 0, 0, 0, 0}, /* 0x21 (00100001) */ + {2, 6, 0, 0, 0, 0, 0, 0}, /* 0x22 (00100010) */ + {1, 2, 6, 0, 0, 0, 0, 0}, /* 0x23 (00100011) */ + {3, 6, 0, 0, 0, 0, 0, 0}, /* 0x24 (00100100) */ + {1, 3, 6, 0, 0, 0, 0, 0}, /* 0x25 (00100101) */ + {2, 3, 6, 0, 0, 0, 0, 0}, /* 0x26 (00100110) */ + {1, 2, 3, 6, 0, 0, 0, 0}, /* 0x27 (00100111) */ + {4, 6, 0, 0, 0, 0, 0, 0}, /* 0x28 (00101000) */ + {1, 4, 6, 0, 0, 0, 0, 0}, /* 0x29 (00101001) */ + {2, 4, 6, 0, 0, 0, 0, 0}, /* 0x2A (00101010) */ + {1, 2, 4, 6, 0, 0, 0, 0}, /* 0x2B (00101011) */ + {3, 4, 6, 0, 0, 0, 0, 0}, /* 0x2C (00101100) */ + {1, 3, 4, 6, 0, 0, 0, 0}, /* 0x2D (00101101) */ + {2, 3, 4, 6, 0, 0, 0, 0}, /* 0x2E (00101110) */ + {1, 2, 3, 4, 6, 0, 0, 0}, /* 0x2F (00101111) */ + {5, 6, 0, 0, 0, 0, 0, 0}, /* 0x30 (00110000) */ + {1, 5, 6, 0, 0, 0, 0, 0}, /* 0x31 (00110001) */ + {2, 5, 6, 0, 0, 0, 0, 0}, /* 0x32 (00110010) */ + {1, 2, 5, 6, 0, 0, 0, 0}, /* 0x33 (00110011) */ + {3, 5, 6, 0, 0, 0, 0, 0}, /* 0x34 (00110100) */ + {1, 3, 5, 6, 0, 0, 0, 0}, /* 0x35 (00110101) */ + {2, 3, 5, 6, 0, 0, 0, 0}, /* 0x36 (00110110) */ + {1, 2, 3, 5, 6, 0, 0, 0}, /* 0x37 (00110111) */ + {4, 5, 6, 0, 0, 0, 0, 0}, /* 0x38 (00111000) */ + {1, 4, 5, 6, 0, 0, 0, 0}, /* 0x39 (00111001) */ + {2, 4, 5, 6, 0, 0, 0, 0}, /* 0x3A (00111010) */ + {1, 2, 4, 5, 6, 0, 0, 0}, /* 0x3B (00111011) */ + {3, 4, 5, 6, 0, 0, 0, 0}, /* 0x3C (00111100) */ + {1, 3, 4, 5, 6, 0, 0, 0}, /* 0x3D (00111101) */ + {2, 3, 4, 5, 6, 0, 0, 0}, /* 0x3E (00111110) */ + {1, 2, 3, 4, 5, 6, 0, 0}, /* 0x3F (00111111) */ + {7, 0, 0, 0, 0, 0, 0, 0}, /* 0x40 (01000000) */ + {1, 7, 0, 0, 0, 0, 0, 0}, /* 0x41 (01000001) */ + {2, 7, 0, 0, 0, 0, 0, 0}, /* 0x42 (01000010) */ + {1, 2, 7, 0, 0, 0, 0, 0}, /* 0x43 (01000011) */ + {3, 7, 0, 0, 0, 0, 0, 0}, /* 0x44 (01000100) */ + {1, 3, 7, 0, 0, 0, 0, 0}, /* 0x45 (01000101) */ + {2, 3, 7, 0, 0, 0, 0, 0}, /* 0x46 (01000110) */ + {1, 2, 3, 7, 0, 0, 0, 0}, /* 0x47 (01000111) */ + {4, 7, 0, 0, 0, 0, 0, 0}, /* 0x48 (01001000) */ + {1, 4, 7, 0, 0, 0, 0, 0}, /* 0x49 (01001001) */ + {2, 4, 7, 0, 0, 0, 0, 0}, /* 0x4A (01001010) */ + {1, 2, 4, 7, 0, 0, 0, 0}, /* 0x4B (01001011) */ + {3, 4, 7, 0, 0, 0, 0, 0}, /* 0x4C (01001100) */ + {1, 3, 4, 7, 0, 0, 0, 0}, /* 0x4D (01001101) */ + {2, 3, 4, 7, 0, 0, 0, 0}, /* 0x4E (01001110) */ + {1, 2, 3, 4, 7, 0, 0, 0}, /* 0x4F (01001111) */ + {5, 7, 0, 0, 0, 0, 0, 0}, /* 0x50 (01010000) */ + {1, 5, 7, 0, 0, 0, 0, 0}, /* 0x51 (01010001) */ + {2, 5, 7, 0, 0, 0, 0, 0}, /* 0x52 (01010010) */ + {1, 2, 5, 7, 0, 0, 0, 0}, /* 0x53 (01010011) */ + {3, 5, 7, 0, 0, 0, 0, 0}, /* 0x54 (01010100) */ + {1, 3, 5, 7, 0, 0, 0, 0}, /* 0x55 (01010101) */ + {2, 3, 5, 7, 0, 0, 0, 0}, /* 0x56 (01010110) */ + {1, 2, 3, 5, 7, 0, 0, 0}, /* 0x57 (01010111) */ + {4, 5, 7, 0, 0, 0, 0, 0}, /* 0x58 (01011000) */ + {1, 4, 5, 7, 0, 0, 0, 0}, /* 0x59 (01011001) */ + {2, 4, 5, 7, 0, 0, 0, 0}, /* 0x5A (01011010) */ + {1, 2, 4, 5, 7, 0, 0, 0}, /* 0x5B (01011011) */ + {3, 4, 5, 7, 0, 0, 0, 0}, /* 0x5C (01011100) */ + {1, 3, 4, 5, 7, 0, 0, 0}, /* 0x5D (01011101) */ + {2, 3, 4, 5, 7, 0, 0, 0}, /* 0x5E (01011110) */ + {1, 2, 3, 4, 5, 7, 0, 0}, /* 0x5F (01011111) */ + {6, 7, 0, 0, 0, 0, 0, 0}, /* 0x60 (01100000) */ + {1, 6, 7, 0, 0, 0, 0, 0}, /* 0x61 (01100001) */ + {2, 6, 7, 0, 0, 0, 0, 0}, /* 0x62 (01100010) */ + {1, 2, 6, 7, 0, 0, 0, 0}, /* 0x63 (01100011) */ + {3, 6, 7, 0, 0, 0, 0, 0}, /* 0x64 (01100100) */ + {1, 3, 6, 7, 0, 0, 0, 0}, /* 0x65 (01100101) */ + {2, 3, 6, 7, 0, 0, 0, 0}, /* 0x66 (01100110) */ + {1, 2, 3, 6, 7, 0, 0, 0}, /* 0x67 (01100111) */ + {4, 6, 7, 0, 0, 0, 0, 0}, /* 0x68 (01101000) */ + {1, 4, 6, 7, 0, 0, 0, 0}, /* 0x69 (01101001) */ + {2, 4, 6, 7, 0, 0, 0, 0}, /* 0x6A (01101010) */ + {1, 2, 4, 6, 7, 0, 0, 0}, /* 0x6B (01101011) */ + {3, 4, 6, 7, 0, 0, 0, 0}, /* 0x6C (01101100) */ + {1, 3, 4, 6, 7, 0, 0, 0}, /* 0x6D (01101101) */ + {2, 3, 4, 6, 7, 0, 0, 0}, /* 0x6E (01101110) */ + {1, 2, 3, 4, 6, 7, 0, 0}, /* 0x6F (01101111) */ + {5, 6, 7, 0, 0, 0, 0, 0}, /* 0x70 (01110000) */ + {1, 5, 6, 7, 0, 0, 0, 0}, /* 0x71 (01110001) */ + {2, 5, 6, 7, 0, 0, 0, 0}, /* 0x72 (01110010) */ + {1, 2, 5, 6, 7, 0, 0, 0}, /* 0x73 (01110011) */ + {3, 5, 6, 7, 0, 0, 0, 0}, /* 0x74 (01110100) */ + {1, 3, 5, 6, 7, 0, 0, 0}, /* 0x75 (01110101) */ + {2, 3, 5, 6, 7, 0, 0, 0}, /* 0x76 (01110110) */ + {1, 2, 3, 5, 6, 7, 0, 0}, /* 0x77 (01110111) */ + {4, 5, 6, 7, 0, 0, 0, 0}, /* 0x78 (01111000) */ + {1, 4, 5, 6, 7, 0, 0, 0}, /* 0x79 (01111001) */ + {2, 4, 5, 6, 7, 0, 0, 0}, /* 0x7A (01111010) */ + {1, 2, 4, 5, 6, 7, 0, 0}, /* 0x7B (01111011) */ + {3, 4, 5, 6, 7, 0, 0, 0}, /* 0x7C (01111100) */ + {1, 3, 4, 5, 6, 7, 0, 0}, /* 0x7D (01111101) */ + {2, 3, 4, 5, 6, 7, 0, 0}, /* 0x7E (01111110) */ + {1, 2, 3, 4, 5, 6, 7, 0}, /* 0x7F (01111111) */ + {8, 0, 0, 0, 0, 0, 0, 0}, /* 0x80 (10000000) */ + {1, 8, 0, 0, 0, 0, 0, 0}, /* 0x81 (10000001) */ + {2, 8, 0, 0, 0, 0, 0, 0}, /* 0x82 (10000010) */ + {1, 2, 8, 0, 0, 0, 0, 0}, /* 0x83 (10000011) */ + {3, 8, 0, 0, 0, 0, 0, 0}, /* 0x84 (10000100) */ + {1, 3, 8, 0, 0, 0, 0, 0}, /* 0x85 (10000101) */ + {2, 3, 8, 0, 0, 0, 0, 0}, /* 0x86 (10000110) */ + {1, 2, 3, 8, 0, 0, 0, 0}, /* 0x87 (10000111) */ + {4, 8, 0, 0, 0, 0, 0, 0}, /* 0x88 (10001000) */ + {1, 4, 8, 0, 0, 0, 0, 0}, /* 0x89 (10001001) */ + {2, 4, 8, 0, 0, 0, 0, 0}, /* 0x8A (10001010) */ + {1, 2, 4, 8, 0, 0, 0, 0}, /* 0x8B (10001011) */ + {3, 4, 8, 0, 0, 0, 0, 0}, /* 0x8C (10001100) */ + {1, 3, 4, 8, 0, 0, 0, 0}, /* 0x8D (10001101) */ + {2, 3, 4, 8, 0, 0, 0, 0}, /* 0x8E (10001110) */ + {1, 2, 3, 4, 8, 0, 0, 0}, /* 0x8F (10001111) */ + {5, 8, 0, 0, 0, 0, 0, 0}, /* 0x90 (10010000) */ + {1, 5, 8, 0, 0, 0, 0, 0}, /* 0x91 (10010001) */ + {2, 5, 8, 0, 0, 0, 0, 0}, /* 0x92 (10010010) */ + {1, 2, 5, 8, 0, 0, 0, 0}, /* 0x93 (10010011) */ + {3, 5, 8, 0, 0, 0, 0, 0}, /* 0x94 (10010100) */ + {1, 3, 5, 8, 0, 0, 0, 0}, /* 0x95 (10010101) */ + {2, 3, 5, 8, 0, 0, 0, 0}, /* 0x96 (10010110) */ + {1, 2, 3, 5, 8, 0, 0, 0}, /* 0x97 (10010111) */ + {4, 5, 8, 0, 0, 0, 0, 0}, /* 0x98 (10011000) */ + {1, 4, 5, 8, 0, 0, 0, 0}, /* 0x99 (10011001) */ + {2, 4, 5, 8, 0, 0, 0, 0}, /* 0x9A (10011010) */ + {1, 2, 4, 5, 8, 0, 0, 0}, /* 0x9B (10011011) */ + {3, 4, 5, 8, 0, 0, 0, 0}, /* 0x9C (10011100) */ + {1, 3, 4, 5, 8, 0, 0, 0}, /* 0x9D (10011101) */ + {2, 3, 4, 5, 8, 0, 0, 0}, /* 0x9E (10011110) */ + {1, 2, 3, 4, 5, 8, 0, 0}, /* 0x9F (10011111) */ + {6, 8, 0, 0, 0, 0, 0, 0}, /* 0xA0 (10100000) */ + {1, 6, 8, 0, 0, 0, 0, 0}, /* 0xA1 (10100001) */ + {2, 6, 8, 0, 0, 0, 0, 0}, /* 0xA2 (10100010) */ + {1, 2, 6, 8, 0, 0, 0, 0}, /* 0xA3 (10100011) */ + {3, 6, 8, 0, 0, 0, 0, 0}, /* 0xA4 (10100100) */ + {1, 3, 6, 8, 0, 0, 0, 0}, /* 0xA5 (10100101) */ + {2, 3, 6, 8, 0, 0, 0, 0}, /* 0xA6 (10100110) */ + {1, 2, 3, 6, 8, 0, 0, 0}, /* 0xA7 (10100111) */ + {4, 6, 8, 0, 0, 0, 0, 0}, /* 0xA8 (10101000) */ + {1, 4, 6, 8, 0, 0, 0, 0}, /* 0xA9 (10101001) */ + {2, 4, 6, 8, 0, 0, 0, 0}, /* 0xAA (10101010) */ + {1, 2, 4, 6, 8, 0, 0, 0}, /* 0xAB (10101011) */ + {3, 4, 6, 8, 0, 0, 0, 0}, /* 0xAC (10101100) */ + {1, 3, 4, 6, 8, 0, 0, 0}, /* 0xAD (10101101) */ + {2, 3, 4, 6, 8, 0, 0, 0}, /* 0xAE (10101110) */ + {1, 2, 3, 4, 6, 8, 0, 0}, /* 0xAF (10101111) */ + {5, 6, 8, 0, 0, 0, 0, 0}, /* 0xB0 (10110000) */ + {1, 5, 6, 8, 0, 0, 0, 0}, /* 0xB1 (10110001) */ + {2, 5, 6, 8, 0, 0, 0, 0}, /* 0xB2 (10110010) */ + {1, 2, 5, 6, 8, 0, 0, 0}, /* 0xB3 (10110011) */ + {3, 5, 6, 8, 0, 0, 0, 0}, /* 0xB4 (10110100) */ + {1, 3, 5, 6, 8, 0, 0, 0}, /* 0xB5 (10110101) */ + {2, 3, 5, 6, 8, 0, 0, 0}, /* 0xB6 (10110110) */ + {1, 2, 3, 5, 6, 8, 0, 0}, /* 0xB7 (10110111) */ + {4, 5, 6, 8, 0, 0, 0, 0}, /* 0xB8 (10111000) */ + {1, 4, 5, 6, 8, 0, 0, 0}, /* 0xB9 (10111001) */ + {2, 4, 5, 6, 8, 0, 0, 0}, /* 0xBA (10111010) */ + {1, 2, 4, 5, 6, 8, 0, 0}, /* 0xBB (10111011) */ + {3, 4, 5, 6, 8, 0, 0, 0}, /* 0xBC (10111100) */ + {1, 3, 4, 5, 6, 8, 0, 0}, /* 0xBD (10111101) */ + {2, 3, 4, 5, 6, 8, 0, 0}, /* 0xBE (10111110) */ + {1, 2, 3, 4, 5, 6, 8, 0}, /* 0xBF (10111111) */ + {7, 8, 0, 0, 0, 0, 0, 0}, /* 0xC0 (11000000) */ + {1, 7, 8, 0, 0, 0, 0, 0}, /* 0xC1 (11000001) */ + {2, 7, 8, 0, 0, 0, 0, 0}, /* 0xC2 (11000010) */ + {1, 2, 7, 8, 0, 0, 0, 0}, /* 0xC3 (11000011) */ + {3, 7, 8, 0, 0, 0, 0, 0}, /* 0xC4 (11000100) */ + {1, 3, 7, 8, 0, 0, 0, 0}, /* 0xC5 (11000101) */ + {2, 3, 7, 8, 0, 0, 0, 0}, /* 0xC6 (11000110) */ + {1, 2, 3, 7, 8, 0, 0, 0}, /* 0xC7 (11000111) */ + {4, 7, 8, 0, 0, 0, 0, 0}, /* 0xC8 (11001000) */ + {1, 4, 7, 8, 0, 0, 0, 0}, /* 0xC9 (11001001) */ + {2, 4, 7, 8, 0, 0, 0, 0}, /* 0xCA (11001010) */ + {1, 2, 4, 7, 8, 0, 0, 0}, /* 0xCB (11001011) */ + {3, 4, 7, 8, 0, 0, 0, 0}, /* 0xCC (11001100) */ + {1, 3, 4, 7, 8, 0, 0, 0}, /* 0xCD (11001101) */ + {2, 3, 4, 7, 8, 0, 0, 0}, /* 0xCE (11001110) */ + {1, 2, 3, 4, 7, 8, 0, 0}, /* 0xCF (11001111) */ + {5, 7, 8, 0, 0, 0, 0, 0}, /* 0xD0 (11010000) */ + {1, 5, 7, 8, 0, 0, 0, 0}, /* 0xD1 (11010001) */ + {2, 5, 7, 8, 0, 0, 0, 0}, /* 0xD2 (11010010) */ + {1, 2, 5, 7, 8, 0, 0, 0}, /* 0xD3 (11010011) */ + {3, 5, 7, 8, 0, 0, 0, 0}, /* 0xD4 (11010100) */ + {1, 3, 5, 7, 8, 0, 0, 0}, /* 0xD5 (11010101) */ + {2, 3, 5, 7, 8, 0, 0, 0}, /* 0xD6 (11010110) */ + {1, 2, 3, 5, 7, 8, 0, 0}, /* 0xD7 (11010111) */ + {4, 5, 7, 8, 0, 0, 0, 0}, /* 0xD8 (11011000) */ + {1, 4, 5, 7, 8, 0, 0, 0}, /* 0xD9 (11011001) */ + {2, 4, 5, 7, 8, 0, 0, 0}, /* 0xDA (11011010) */ + {1, 2, 4, 5, 7, 8, 0, 0}, /* 0xDB (11011011) */ + {3, 4, 5, 7, 8, 0, 0, 0}, /* 0xDC (11011100) */ + {1, 3, 4, 5, 7, 8, 0, 0}, /* 0xDD (11011101) */ + {2, 3, 4, 5, 7, 8, 0, 0}, /* 0xDE (11011110) */ + {1, 2, 3, 4, 5, 7, 8, 0}, /* 0xDF (11011111) */ + {6, 7, 8, 0, 0, 0, 0, 0}, /* 0xE0 (11100000) */ + {1, 6, 7, 8, 0, 0, 0, 0}, /* 0xE1 (11100001) */ + {2, 6, 7, 8, 0, 0, 0, 0}, /* 0xE2 (11100010) */ + {1, 2, 6, 7, 8, 0, 0, 0}, /* 0xE3 (11100011) */ + {3, 6, 7, 8, 0, 0, 0, 0}, /* 0xE4 (11100100) */ + {1, 3, 6, 7, 8, 0, 0, 0}, /* 0xE5 (11100101) */ + {2, 3, 6, 7, 8, 0, 0, 0}, /* 0xE6 (11100110) */ + {1, 2, 3, 6, 7, 8, 0, 0}, /* 0xE7 (11100111) */ + {4, 6, 7, 8, 0, 0, 0, 0}, /* 0xE8 (11101000) */ + {1, 4, 6, 7, 8, 0, 0, 0}, /* 0xE9 (11101001) */ + {2, 4, 6, 7, 8, 0, 0, 0}, /* 0xEA (11101010) */ + {1, 2, 4, 6, 7, 8, 0, 0}, /* 0xEB (11101011) */ + {3, 4, 6, 7, 8, 0, 0, 0}, /* 0xEC (11101100) */ + {1, 3, 4, 6, 7, 8, 0, 0}, /* 0xED (11101101) */ + {2, 3, 4, 6, 7, 8, 0, 0}, /* 0xEE (11101110) */ + {1, 2, 3, 4, 6, 7, 8, 0}, /* 0xEF (11101111) */ + {5, 6, 7, 8, 0, 0, 0, 0}, /* 0xF0 (11110000) */ + {1, 5, 6, 7, 8, 0, 0, 0}, /* 0xF1 (11110001) */ + {2, 5, 6, 7, 8, 0, 0, 0}, /* 0xF2 (11110010) */ + {1, 2, 5, 6, 7, 8, 0, 0}, /* 0xF3 (11110011) */ + {3, 5, 6, 7, 8, 0, 0, 0}, /* 0xF4 (11110100) */ + {1, 3, 5, 6, 7, 8, 0, 0}, /* 0xF5 (11110101) */ + {2, 3, 5, 6, 7, 8, 0, 0}, /* 0xF6 (11110110) */ + {1, 2, 3, 5, 6, 7, 8, 0}, /* 0xF7 (11110111) */ + {4, 5, 6, 7, 8, 0, 0, 0}, /* 0xF8 (11111000) */ + {1, 4, 5, 6, 7, 8, 0, 0}, /* 0xF9 (11111001) */ + {2, 4, 5, 6, 7, 8, 0, 0}, /* 0xFA (11111010) */ + {1, 2, 4, 5, 6, 7, 8, 0}, /* 0xFB (11111011) */ + {3, 4, 5, 6, 7, 8, 0, 0}, /* 0xFC (11111100) */ + {1, 3, 4, 5, 6, 7, 8, 0}, /* 0xFD (11111101) */ + {2, 3, 4, 5, 6, 7, 8, 0}, /* 0xFE (11111110) */ + {1, 2, 3, 4, 5, 6, 7, 8} /* 0xFF (11111111) */ +}; -#ifdef __cplusplus -} -} -} // extern "C" { namespace roaring { namespace api { -#endif -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif -/* end file src/bitset_util.c */ -/* begin file src/bitset.c */ -#include -#include -#include -#include -#include +#endif // #if CROARING_IS_X64 +#if CROARING_IS_X64 +// same as vecDecodeTable but in 16 bits +ALIGNED(32) +static uint16_t vecDecodeTable_uint16[256][8] = { + {0, 0, 0, 0, 0, 0, 0, 0}, /* 0x00 (00000000) */ + {1, 0, 0, 0, 0, 0, 0, 0}, /* 0x01 (00000001) */ + {2, 0, 0, 0, 0, 0, 0, 0}, /* 0x02 (00000010) */ + {1, 2, 0, 0, 0, 0, 0, 0}, /* 0x03 (00000011) */ + {3, 0, 0, 0, 0, 0, 0, 0}, /* 0x04 (00000100) */ + {1, 3, 0, 0, 0, 0, 0, 0}, /* 0x05 (00000101) */ + {2, 3, 0, 0, 0, 0, 0, 0}, /* 0x06 (00000110) */ + {1, 2, 3, 0, 0, 0, 0, 0}, /* 0x07 (00000111) */ + {4, 0, 0, 0, 0, 0, 0, 0}, /* 0x08 (00001000) */ + {1, 4, 0, 0, 0, 0, 0, 0}, /* 0x09 (00001001) */ + {2, 4, 0, 0, 0, 0, 0, 0}, /* 0x0A (00001010) */ + {1, 2, 4, 0, 0, 0, 0, 0}, /* 0x0B (00001011) */ + {3, 4, 0, 0, 0, 0, 0, 0}, /* 0x0C (00001100) */ + {1, 3, 4, 0, 0, 0, 0, 0}, /* 0x0D (00001101) */ + {2, 3, 4, 0, 0, 0, 0, 0}, /* 0x0E (00001110) */ + {1, 2, 3, 4, 0, 0, 0, 0}, /* 0x0F (00001111) */ + {5, 0, 0, 0, 0, 0, 0, 0}, /* 0x10 (00010000) */ + {1, 5, 0, 0, 0, 0, 0, 0}, /* 0x11 (00010001) */ + {2, 5, 0, 0, 0, 0, 0, 0}, /* 0x12 (00010010) */ + {1, 2, 5, 0, 0, 0, 0, 0}, /* 0x13 (00010011) */ + {3, 5, 0, 0, 0, 0, 0, 0}, /* 0x14 (00010100) */ + {1, 3, 5, 0, 0, 0, 0, 0}, /* 0x15 (00010101) */ + {2, 3, 5, 0, 0, 0, 0, 0}, /* 0x16 (00010110) */ + {1, 2, 3, 5, 0, 0, 0, 0}, /* 0x17 (00010111) */ + {4, 5, 0, 0, 0, 0, 0, 0}, /* 0x18 (00011000) */ + {1, 4, 5, 0, 0, 0, 0, 0}, /* 0x19 (00011001) */ + {2, 4, 5, 0, 0, 0, 0, 0}, /* 0x1A (00011010) */ + {1, 2, 4, 5, 0, 0, 0, 0}, /* 0x1B (00011011) */ + {3, 4, 5, 0, 0, 0, 0, 0}, /* 0x1C (00011100) */ + {1, 3, 4, 5, 0, 0, 0, 0}, /* 0x1D (00011101) */ + {2, 3, 4, 5, 0, 0, 0, 0}, /* 0x1E (00011110) */ + {1, 2, 3, 4, 5, 0, 0, 0}, /* 0x1F (00011111) */ + {6, 0, 0, 0, 0, 0, 0, 0}, /* 0x20 (00100000) */ + {1, 6, 0, 0, 0, 0, 0, 0}, /* 0x21 (00100001) */ + {2, 6, 0, 0, 0, 0, 0, 0}, /* 0x22 (00100010) */ + {1, 2, 6, 0, 0, 0, 0, 0}, /* 0x23 (00100011) */ + {3, 6, 0, 0, 0, 0, 0, 0}, /* 0x24 (00100100) */ + {1, 3, 6, 0, 0, 0, 0, 0}, /* 0x25 (00100101) */ + {2, 3, 6, 0, 0, 0, 0, 0}, /* 0x26 (00100110) */ + {1, 2, 3, 6, 0, 0, 0, 0}, /* 0x27 (00100111) */ + {4, 6, 0, 0, 0, 0, 0, 0}, /* 0x28 (00101000) */ + {1, 4, 6, 0, 0, 0, 0, 0}, /* 0x29 (00101001) */ + {2, 4, 6, 0, 0, 0, 0, 0}, /* 0x2A (00101010) */ + {1, 2, 4, 6, 0, 0, 0, 0}, /* 0x2B (00101011) */ + {3, 4, 6, 0, 0, 0, 0, 0}, /* 0x2C (00101100) */ + {1, 3, 4, 6, 0, 0, 0, 0}, /* 0x2D (00101101) */ + {2, 3, 4, 6, 0, 0, 0, 0}, /* 0x2E (00101110) */ + {1, 2, 3, 4, 6, 0, 0, 0}, /* 0x2F (00101111) */ + {5, 6, 0, 0, 0, 0, 0, 0}, /* 0x30 (00110000) */ + {1, 5, 6, 0, 0, 0, 0, 0}, /* 0x31 (00110001) */ + {2, 5, 6, 0, 0, 0, 0, 0}, /* 0x32 (00110010) */ + {1, 2, 5, 6, 0, 0, 0, 0}, /* 0x33 (00110011) */ + {3, 5, 6, 0, 0, 0, 0, 0}, /* 0x34 (00110100) */ + {1, 3, 5, 6, 0, 0, 0, 0}, /* 0x35 (00110101) */ + {2, 3, 5, 6, 0, 0, 0, 0}, /* 0x36 (00110110) */ + {1, 2, 3, 5, 6, 0, 0, 0}, /* 0x37 (00110111) */ + {4, 5, 6, 0, 0, 0, 0, 0}, /* 0x38 (00111000) */ + {1, 4, 5, 6, 0, 0, 0, 0}, /* 0x39 (00111001) */ + {2, 4, 5, 6, 0, 0, 0, 0}, /* 0x3A (00111010) */ + {1, 2, 4, 5, 6, 0, 0, 0}, /* 0x3B (00111011) */ + {3, 4, 5, 6, 0, 0, 0, 0}, /* 0x3C (00111100) */ + {1, 3, 4, 5, 6, 0, 0, 0}, /* 0x3D (00111101) */ + {2, 3, 4, 5, 6, 0, 0, 0}, /* 0x3E (00111110) */ + {1, 2, 3, 4, 5, 6, 0, 0}, /* 0x3F (00111111) */ + {7, 0, 0, 0, 0, 0, 0, 0}, /* 0x40 (01000000) */ + {1, 7, 0, 0, 0, 0, 0, 0}, /* 0x41 (01000001) */ + {2, 7, 0, 0, 0, 0, 0, 0}, /* 0x42 (01000010) */ + {1, 2, 7, 0, 0, 0, 0, 0}, /* 0x43 (01000011) */ + {3, 7, 0, 0, 0, 0, 0, 0}, /* 0x44 (01000100) */ + {1, 3, 7, 0, 0, 0, 0, 0}, /* 0x45 (01000101) */ + {2, 3, 7, 0, 0, 0, 0, 0}, /* 0x46 (01000110) */ + {1, 2, 3, 7, 0, 0, 0, 0}, /* 0x47 (01000111) */ + {4, 7, 0, 0, 0, 0, 0, 0}, /* 0x48 (01001000) */ + {1, 4, 7, 0, 0, 0, 0, 0}, /* 0x49 (01001001) */ + {2, 4, 7, 0, 0, 0, 0, 0}, /* 0x4A (01001010) */ + {1, 2, 4, 7, 0, 0, 0, 0}, /* 0x4B (01001011) */ + {3, 4, 7, 0, 0, 0, 0, 0}, /* 0x4C (01001100) */ + {1, 3, 4, 7, 0, 0, 0, 0}, /* 0x4D (01001101) */ + {2, 3, 4, 7, 0, 0, 0, 0}, /* 0x4E (01001110) */ + {1, 2, 3, 4, 7, 0, 0, 0}, /* 0x4F (01001111) */ + {5, 7, 0, 0, 0, 0, 0, 0}, /* 0x50 (01010000) */ + {1, 5, 7, 0, 0, 0, 0, 0}, /* 0x51 (01010001) */ + {2, 5, 7, 0, 0, 0, 0, 0}, /* 0x52 (01010010) */ + {1, 2, 5, 7, 0, 0, 0, 0}, /* 0x53 (01010011) */ + {3, 5, 7, 0, 0, 0, 0, 0}, /* 0x54 (01010100) */ + {1, 3, 5, 7, 0, 0, 0, 0}, /* 0x55 (01010101) */ + {2, 3, 5, 7, 0, 0, 0, 0}, /* 0x56 (01010110) */ + {1, 2, 3, 5, 7, 0, 0, 0}, /* 0x57 (01010111) */ + {4, 5, 7, 0, 0, 0, 0, 0}, /* 0x58 (01011000) */ + {1, 4, 5, 7, 0, 0, 0, 0}, /* 0x59 (01011001) */ + {2, 4, 5, 7, 0, 0, 0, 0}, /* 0x5A (01011010) */ + {1, 2, 4, 5, 7, 0, 0, 0}, /* 0x5B (01011011) */ + {3, 4, 5, 7, 0, 0, 0, 0}, /* 0x5C (01011100) */ + {1, 3, 4, 5, 7, 0, 0, 0}, /* 0x5D (01011101) */ + {2, 3, 4, 5, 7, 0, 0, 0}, /* 0x5E (01011110) */ + {1, 2, 3, 4, 5, 7, 0, 0}, /* 0x5F (01011111) */ + {6, 7, 0, 0, 0, 0, 0, 0}, /* 0x60 (01100000) */ + {1, 6, 7, 0, 0, 0, 0, 0}, /* 0x61 (01100001) */ + {2, 6, 7, 0, 0, 0, 0, 0}, /* 0x62 (01100010) */ + {1, 2, 6, 7, 0, 0, 0, 0}, /* 0x63 (01100011) */ + {3, 6, 7, 0, 0, 0, 0, 0}, /* 0x64 (01100100) */ + {1, 3, 6, 7, 0, 0, 0, 0}, /* 0x65 (01100101) */ + {2, 3, 6, 7, 0, 0, 0, 0}, /* 0x66 (01100110) */ + {1, 2, 3, 6, 7, 0, 0, 0}, /* 0x67 (01100111) */ + {4, 6, 7, 0, 0, 0, 0, 0}, /* 0x68 (01101000) */ + {1, 4, 6, 7, 0, 0, 0, 0}, /* 0x69 (01101001) */ + {2, 4, 6, 7, 0, 0, 0, 0}, /* 0x6A (01101010) */ + {1, 2, 4, 6, 7, 0, 0, 0}, /* 0x6B (01101011) */ + {3, 4, 6, 7, 0, 0, 0, 0}, /* 0x6C (01101100) */ + {1, 3, 4, 6, 7, 0, 0, 0}, /* 0x6D (01101101) */ + {2, 3, 4, 6, 7, 0, 0, 0}, /* 0x6E (01101110) */ + {1, 2, 3, 4, 6, 7, 0, 0}, /* 0x6F (01101111) */ + {5, 6, 7, 0, 0, 0, 0, 0}, /* 0x70 (01110000) */ + {1, 5, 6, 7, 0, 0, 0, 0}, /* 0x71 (01110001) */ + {2, 5, 6, 7, 0, 0, 0, 0}, /* 0x72 (01110010) */ + {1, 2, 5, 6, 7, 0, 0, 0}, /* 0x73 (01110011) */ + {3, 5, 6, 7, 0, 0, 0, 0}, /* 0x74 (01110100) */ + {1, 3, 5, 6, 7, 0, 0, 0}, /* 0x75 (01110101) */ + {2, 3, 5, 6, 7, 0, 0, 0}, /* 0x76 (01110110) */ + {1, 2, 3, 5, 6, 7, 0, 0}, /* 0x77 (01110111) */ + {4, 5, 6, 7, 0, 0, 0, 0}, /* 0x78 (01111000) */ + {1, 4, 5, 6, 7, 0, 0, 0}, /* 0x79 (01111001) */ + {2, 4, 5, 6, 7, 0, 0, 0}, /* 0x7A (01111010) */ + {1, 2, 4, 5, 6, 7, 0, 0}, /* 0x7B (01111011) */ + {3, 4, 5, 6, 7, 0, 0, 0}, /* 0x7C (01111100) */ + {1, 3, 4, 5, 6, 7, 0, 0}, /* 0x7D (01111101) */ + {2, 3, 4, 5, 6, 7, 0, 0}, /* 0x7E (01111110) */ + {1, 2, 3, 4, 5, 6, 7, 0}, /* 0x7F (01111111) */ + {8, 0, 0, 0, 0, 0, 0, 0}, /* 0x80 (10000000) */ + {1, 8, 0, 0, 0, 0, 0, 0}, /* 0x81 (10000001) */ + {2, 8, 0, 0, 0, 0, 0, 0}, /* 0x82 (10000010) */ + {1, 2, 8, 0, 0, 0, 0, 0}, /* 0x83 (10000011) */ + {3, 8, 0, 0, 0, 0, 0, 0}, /* 0x84 (10000100) */ + {1, 3, 8, 0, 0, 0, 0, 0}, /* 0x85 (10000101) */ + {2, 3, 8, 0, 0, 0, 0, 0}, /* 0x86 (10000110) */ + {1, 2, 3, 8, 0, 0, 0, 0}, /* 0x87 (10000111) */ + {4, 8, 0, 0, 0, 0, 0, 0}, /* 0x88 (10001000) */ + {1, 4, 8, 0, 0, 0, 0, 0}, /* 0x89 (10001001) */ + {2, 4, 8, 0, 0, 0, 0, 0}, /* 0x8A (10001010) */ + {1, 2, 4, 8, 0, 0, 0, 0}, /* 0x8B (10001011) */ + {3, 4, 8, 0, 0, 0, 0, 0}, /* 0x8C (10001100) */ + {1, 3, 4, 8, 0, 0, 0, 0}, /* 0x8D (10001101) */ + {2, 3, 4, 8, 0, 0, 0, 0}, /* 0x8E (10001110) */ + {1, 2, 3, 4, 8, 0, 0, 0}, /* 0x8F (10001111) */ + {5, 8, 0, 0, 0, 0, 0, 0}, /* 0x90 (10010000) */ + {1, 5, 8, 0, 0, 0, 0, 0}, /* 0x91 (10010001) */ + {2, 5, 8, 0, 0, 0, 0, 0}, /* 0x92 (10010010) */ + {1, 2, 5, 8, 0, 0, 0, 0}, /* 0x93 (10010011) */ + {3, 5, 8, 0, 0, 0, 0, 0}, /* 0x94 (10010100) */ + {1, 3, 5, 8, 0, 0, 0, 0}, /* 0x95 (10010101) */ + {2, 3, 5, 8, 0, 0, 0, 0}, /* 0x96 (10010110) */ + {1, 2, 3, 5, 8, 0, 0, 0}, /* 0x97 (10010111) */ + {4, 5, 8, 0, 0, 0, 0, 0}, /* 0x98 (10011000) */ + {1, 4, 5, 8, 0, 0, 0, 0}, /* 0x99 (10011001) */ + {2, 4, 5, 8, 0, 0, 0, 0}, /* 0x9A (10011010) */ + {1, 2, 4, 5, 8, 0, 0, 0}, /* 0x9B (10011011) */ + {3, 4, 5, 8, 0, 0, 0, 0}, /* 0x9C (10011100) */ + {1, 3, 4, 5, 8, 0, 0, 0}, /* 0x9D (10011101) */ + {2, 3, 4, 5, 8, 0, 0, 0}, /* 0x9E (10011110) */ + {1, 2, 3, 4, 5, 8, 0, 0}, /* 0x9F (10011111) */ + {6, 8, 0, 0, 0, 0, 0, 0}, /* 0xA0 (10100000) */ + {1, 6, 8, 0, 0, 0, 0, 0}, /* 0xA1 (10100001) */ + {2, 6, 8, 0, 0, 0, 0, 0}, /* 0xA2 (10100010) */ + {1, 2, 6, 8, 0, 0, 0, 0}, /* 0xA3 (10100011) */ + {3, 6, 8, 0, 0, 0, 0, 0}, /* 0xA4 (10100100) */ + {1, 3, 6, 8, 0, 0, 0, 0}, /* 0xA5 (10100101) */ + {2, 3, 6, 8, 0, 0, 0, 0}, /* 0xA6 (10100110) */ + {1, 2, 3, 6, 8, 0, 0, 0}, /* 0xA7 (10100111) */ + {4, 6, 8, 0, 0, 0, 0, 0}, /* 0xA8 (10101000) */ + {1, 4, 6, 8, 0, 0, 0, 0}, /* 0xA9 (10101001) */ + {2, 4, 6, 8, 0, 0, 0, 0}, /* 0xAA (10101010) */ + {1, 2, 4, 6, 8, 0, 0, 0}, /* 0xAB (10101011) */ + {3, 4, 6, 8, 0, 0, 0, 0}, /* 0xAC (10101100) */ + {1, 3, 4, 6, 8, 0, 0, 0}, /* 0xAD (10101101) */ + {2, 3, 4, 6, 8, 0, 0, 0}, /* 0xAE (10101110) */ + {1, 2, 3, 4, 6, 8, 0, 0}, /* 0xAF (10101111) */ + {5, 6, 8, 0, 0, 0, 0, 0}, /* 0xB0 (10110000) */ + {1, 5, 6, 8, 0, 0, 0, 0}, /* 0xB1 (10110001) */ + {2, 5, 6, 8, 0, 0, 0, 0}, /* 0xB2 (10110010) */ + {1, 2, 5, 6, 8, 0, 0, 0}, /* 0xB3 (10110011) */ + {3, 5, 6, 8, 0, 0, 0, 0}, /* 0xB4 (10110100) */ + {1, 3, 5, 6, 8, 0, 0, 0}, /* 0xB5 (10110101) */ + {2, 3, 5, 6, 8, 0, 0, 0}, /* 0xB6 (10110110) */ + {1, 2, 3, 5, 6, 8, 0, 0}, /* 0xB7 (10110111) */ + {4, 5, 6, 8, 0, 0, 0, 0}, /* 0xB8 (10111000) */ + {1, 4, 5, 6, 8, 0, 0, 0}, /* 0xB9 (10111001) */ + {2, 4, 5, 6, 8, 0, 0, 0}, /* 0xBA (10111010) */ + {1, 2, 4, 5, 6, 8, 0, 0}, /* 0xBB (10111011) */ + {3, 4, 5, 6, 8, 0, 0, 0}, /* 0xBC (10111100) */ + {1, 3, 4, 5, 6, 8, 0, 0}, /* 0xBD (10111101) */ + {2, 3, 4, 5, 6, 8, 0, 0}, /* 0xBE (10111110) */ + {1, 2, 3, 4, 5, 6, 8, 0}, /* 0xBF (10111111) */ + {7, 8, 0, 0, 0, 0, 0, 0}, /* 0xC0 (11000000) */ + {1, 7, 8, 0, 0, 0, 0, 0}, /* 0xC1 (11000001) */ + {2, 7, 8, 0, 0, 0, 0, 0}, /* 0xC2 (11000010) */ + {1, 2, 7, 8, 0, 0, 0, 0}, /* 0xC3 (11000011) */ + {3, 7, 8, 0, 0, 0, 0, 0}, /* 0xC4 (11000100) */ + {1, 3, 7, 8, 0, 0, 0, 0}, /* 0xC5 (11000101) */ + {2, 3, 7, 8, 0, 0, 0, 0}, /* 0xC6 (11000110) */ + {1, 2, 3, 7, 8, 0, 0, 0}, /* 0xC7 (11000111) */ + {4, 7, 8, 0, 0, 0, 0, 0}, /* 0xC8 (11001000) */ + {1, 4, 7, 8, 0, 0, 0, 0}, /* 0xC9 (11001001) */ + {2, 4, 7, 8, 0, 0, 0, 0}, /* 0xCA (11001010) */ + {1, 2, 4, 7, 8, 0, 0, 0}, /* 0xCB (11001011) */ + {3, 4, 7, 8, 0, 0, 0, 0}, /* 0xCC (11001100) */ + {1, 3, 4, 7, 8, 0, 0, 0}, /* 0xCD (11001101) */ + {2, 3, 4, 7, 8, 0, 0, 0}, /* 0xCE (11001110) */ + {1, 2, 3, 4, 7, 8, 0, 0}, /* 0xCF (11001111) */ + {5, 7, 8, 0, 0, 0, 0, 0}, /* 0xD0 (11010000) */ + {1, 5, 7, 8, 0, 0, 0, 0}, /* 0xD1 (11010001) */ + {2, 5, 7, 8, 0, 0, 0, 0}, /* 0xD2 (11010010) */ + {1, 2, 5, 7, 8, 0, 0, 0}, /* 0xD3 (11010011) */ + {3, 5, 7, 8, 0, 0, 0, 0}, /* 0xD4 (11010100) */ + {1, 3, 5, 7, 8, 0, 0, 0}, /* 0xD5 (11010101) */ + {2, 3, 5, 7, 8, 0, 0, 0}, /* 0xD6 (11010110) */ + {1, 2, 3, 5, 7, 8, 0, 0}, /* 0xD7 (11010111) */ + {4, 5, 7, 8, 0, 0, 0, 0}, /* 0xD8 (11011000) */ + {1, 4, 5, 7, 8, 0, 0, 0}, /* 0xD9 (11011001) */ + {2, 4, 5, 7, 8, 0, 0, 0}, /* 0xDA (11011010) */ + {1, 2, 4, 5, 7, 8, 0, 0}, /* 0xDB (11011011) */ + {3, 4, 5, 7, 8, 0, 0, 0}, /* 0xDC (11011100) */ + {1, 3, 4, 5, 7, 8, 0, 0}, /* 0xDD (11011101) */ + {2, 3, 4, 5, 7, 8, 0, 0}, /* 0xDE (11011110) */ + {1, 2, 3, 4, 5, 7, 8, 0}, /* 0xDF (11011111) */ + {6, 7, 8, 0, 0, 0, 0, 0}, /* 0xE0 (11100000) */ + {1, 6, 7, 8, 0, 0, 0, 0}, /* 0xE1 (11100001) */ + {2, 6, 7, 8, 0, 0, 0, 0}, /* 0xE2 (11100010) */ + {1, 2, 6, 7, 8, 0, 0, 0}, /* 0xE3 (11100011) */ + {3, 6, 7, 8, 0, 0, 0, 0}, /* 0xE4 (11100100) */ + {1, 3, 6, 7, 8, 0, 0, 0}, /* 0xE5 (11100101) */ + {2, 3, 6, 7, 8, 0, 0, 0}, /* 0xE6 (11100110) */ + {1, 2, 3, 6, 7, 8, 0, 0}, /* 0xE7 (11100111) */ + {4, 6, 7, 8, 0, 0, 0, 0}, /* 0xE8 (11101000) */ + {1, 4, 6, 7, 8, 0, 0, 0}, /* 0xE9 (11101001) */ + {2, 4, 6, 7, 8, 0, 0, 0}, /* 0xEA (11101010) */ + {1, 2, 4, 6, 7, 8, 0, 0}, /* 0xEB (11101011) */ + {3, 4, 6, 7, 8, 0, 0, 0}, /* 0xEC (11101100) */ + {1, 3, 4, 6, 7, 8, 0, 0}, /* 0xED (11101101) */ + {2, 3, 4, 6, 7, 8, 0, 0}, /* 0xEE (11101110) */ + {1, 2, 3, 4, 6, 7, 8, 0}, /* 0xEF (11101111) */ + {5, 6, 7, 8, 0, 0, 0, 0}, /* 0xF0 (11110000) */ + {1, 5, 6, 7, 8, 0, 0, 0}, /* 0xF1 (11110001) */ + {2, 5, 6, 7, 8, 0, 0, 0}, /* 0xF2 (11110010) */ + {1, 2, 5, 6, 7, 8, 0, 0}, /* 0xF3 (11110011) */ + {3, 5, 6, 7, 8, 0, 0, 0}, /* 0xF4 (11110100) */ + {1, 3, 5, 6, 7, 8, 0, 0}, /* 0xF5 (11110101) */ + {2, 3, 5, 6, 7, 8, 0, 0}, /* 0xF6 (11110110) */ + {1, 2, 3, 5, 6, 7, 8, 0}, /* 0xF7 (11110111) */ + {4, 5, 6, 7, 8, 0, 0, 0}, /* 0xF8 (11111000) */ + {1, 4, 5, 6, 7, 8, 0, 0}, /* 0xF9 (11111001) */ + {2, 4, 5, 6, 7, 8, 0, 0}, /* 0xFA (11111010) */ + {1, 2, 4, 5, 6, 7, 8, 0}, /* 0xFB (11111011) */ + {3, 4, 5, 6, 7, 8, 0, 0}, /* 0xFC (11111100) */ + {1, 3, 4, 5, 6, 7, 8, 0}, /* 0xFD (11111101) */ + {2, 3, 4, 5, 6, 7, 8, 0}, /* 0xFE (11111110) */ + {1, 2, 3, 4, 5, 6, 7, 8} /* 0xFF (11111111) */ +}; -#ifdef __cplusplus -extern "C" { -namespace roaring { -namespace internal { #endif -extern inline void bitset_print(const bitset_t *b); -extern inline bool bitset_for_each(const bitset_t *b, bitset_iterator iterator, - void *ptr); -extern inline size_t bitset_next_set_bits(const bitset_t *bitset, - size_t *buffer, size_t capacity, - size_t *startfrom); -extern inline void bitset_set_to_value(bitset_t *bitset, size_t i, bool flag); -extern inline bool bitset_next_set_bit(const bitset_t *bitset, size_t *i); -extern inline void bitset_set(bitset_t *bitset, size_t i); -extern inline bool bitset_get(const bitset_t *bitset, size_t i); -extern inline size_t bitset_size_in_words(const bitset_t *bitset); -extern inline size_t bitset_size_in_bits(const bitset_t *bitset); -extern inline size_t bitset_size_in_bytes(const bitset_t *bitset); +#if CROARING_IS_X64 +#if CROARING_COMPILER_SUPPORTS_AVX512 +CROARING_TARGET_AVX512 +const uint8_t vbmi2_table[64] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63}; +// Reference: +// https://lemire.me/blog/2022/05/10/faster-bitset-decoding-using-intel-avx-512/ +// +// One VPCOMPRESSB (AVX-512 VBMI2) turns a whole 64-bit word into the (up to +// 64) byte-sized positions of its set bits in a single shot; the positions are +// then widened 16 at a time with VPMOVZXBD and added to a running base vector. +// +// Two details matter for speed. Each block is stored under a mask derived from +// BZHI(-1, popcount), so a store writes exactly the values it produced: the +// caller needs no padding past the values it asked for, and no store bandwidth +// goes to values nobody wants. And only the ceil(popcount/16) blocks that +// carry a value are computed at all -- bitset containers are usually well +// under half full, so the later blocks are almost always skipped. +size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, + uint32_t *vout, size_t outcapacity, + uint32_t base) { + if (outcapacity == 0) return 0; + uint32_t *out = vout; + uint32_t *initout = out; + uint32_t *safeout = out + outcapacity; + // base_v holds the value of bit 0 of the word being decoded. + __m512i base_v = _mm512_set1_epi32((int)base); + const __m512i inc_v = _mm512_set1_epi32(64); + const __m512i index_table = _mm512_loadu_si512(vbmi2_table); + size_t i = 0; -/* Create a new bitset. Return NULL in case of failure. */ -bitset_t *bitset_create(void) { - bitset_t *bitset = NULL; - /* Allocate the bitset itself. */ - if ((bitset = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { - return NULL; + for (; i < length; i++) { + uint64_t v = words[i]; + if (v != 0) { + size_t advance = (size_t)roaring_hamming(v); + if (out + advance > safeout) break; + __m512i vec = _mm512_maskz_compress_epi8((__mmask64)v, index_table); + // BZHI(-1, advance): advance is in [1, 64] here, so the shift is + // well defined. + uint64_t bits = ~UINT64_C(0) >> (64 - advance); + _mm512_mask_storeu_epi32( + out, (__mmask16)bits, + _mm512_add_epi32( + base_v, _mm512_cvtepu8_epi32(_mm512_castsi512_si128(vec)))); + if (advance > 16) { + _mm512_mask_storeu_epi32( + out + 16, (__mmask16)(bits >> 16), + _mm512_add_epi32(base_v, + _mm512_cvtepu8_epi32( + _mm512_extracti32x4_epi32(vec, 1)))); + if (advance > 32) { + _mm512_mask_storeu_epi32( + out + 32, (__mmask16)(bits >> 32), + _mm512_add_epi32( + base_v, _mm512_cvtepu8_epi32( + _mm512_extracti32x4_epi32(vec, 2)))); + if (advance > 48) { + _mm512_mask_storeu_epi32( + out + 48, (__mmask16)(bits >> 48), + _mm512_add_epi32( + base_v, + _mm512_cvtepu8_epi32( + _mm512_extracti32x4_epi32(vec, 3)))); + } + } + } + out += advance; + } + base_v = _mm512_add_epi32(base_v, inc_v); } - bitset->array = NULL; - bitset->arraysize = 0; - bitset->capacity = 0; - return bitset; -} -/* Create a new bitset able to contain size bits. Return NULL in case of - * failure. */ -bitset_t *bitset_create_with_capacity(size_t size) { - bitset_t *bitset = NULL; - /* Allocate the bitset itself. */ - if ((bitset = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { - return NULL; - } - bitset->arraysize = - (size + sizeof(uint64_t) * 8 - 1) / (sizeof(uint64_t) * 8); - bitset->capacity = bitset->arraysize; - if ((bitset->array = (uint64_t *)roaring_calloc( - bitset->arraysize, sizeof(uint64_t))) == NULL) { - roaring_free(bitset); - return NULL; - } - return bitset; -} + base += (uint32_t)(i * 64); -/* Create a copy */ -bitset_t *bitset_copy(const bitset_t *bitset) { - bitset_t *copy = NULL; - /* Allocate the bitset itself. */ - if ((copy = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { - return NULL; - } - memcpy(copy, bitset, sizeof(bitset_t)); - copy->capacity = copy->arraysize; - if ((copy->array = (uint64_t *)roaring_malloc(sizeof(uint64_t) * - bitset->arraysize)) == NULL) { - roaring_free(copy); - return NULL; + for (; (i < length) && (out < safeout); ++i) { + uint64_t w = words[i]; + while ((w != 0) && (out < safeout)) { + int r = + roaring_trailing_zeroes(w); // on x64, should compile to TZCNT + uint32_t val = r + base; + memcpy(out, &val, + sizeof(uint32_t)); // should be compiled as a MOV on x64 + out++; + w &= (w - 1); + } + base += 64; } - memcpy(copy->array, bitset->array, sizeof(uint64_t) * bitset->arraysize); - return copy; -} -void bitset_clear(bitset_t *bitset) { - memset(bitset->array, 0, sizeof(uint64_t) * bitset->arraysize); -} - -void bitset_fill(bitset_t *bitset) { - memset(bitset->array, 0xff, sizeof(uint64_t) * bitset->arraysize); + return out - initout; } -void bitset_shift_left(bitset_t *bitset, size_t s) { - size_t extra_words = s / 64; - int inword_shift = s % 64; - size_t as = bitset->arraysize; - if (inword_shift == 0) { - bitset_resize(bitset, as + extra_words, false); - // could be done with a memmove - for (size_t i = as + extra_words; i > extra_words; i--) { - bitset->array[i - 1] = bitset->array[i - 1 - extra_words]; - } - } else { - bitset_resize(bitset, as + extra_words + 1, true); - bitset->array[as + extra_words] = - bitset->array[as - 1] >> (64 - inword_shift); - for (size_t i = as + extra_words; i >= extra_words + 2; i--) { - bitset->array[i - 1] = - (bitset->array[i - 1 - extra_words] << inword_shift) | - (bitset->array[i - 2 - extra_words] >> (64 - inword_shift)); - } - bitset->array[extra_words] = bitset->array[0] << inword_shift; - } - for (size_t i = 0; i < extra_words; i++) { - bitset->array[i] = 0; - } -} +// Same kernel as bitset_extract_setbits_avx512, writing 16-bit values, so the +// blocks hold 32 values instead of 16. This one is only worth calling on +// reasonably dense input: see array_container_from_bitset. +size_t bitset_extract_setbits_avx512_uint16(const uint64_t *array, + size_t length, uint16_t *vout, + size_t capacity, uint16_t base) { + if (capacity == 0) return 0; + uint16_t *out = vout; + uint16_t *initout = out; + uint16_t *safeout = vout + capacity; -void bitset_shift_right(bitset_t *bitset, size_t s) { - size_t extra_words = s / 64; - int inword_shift = s % 64; - size_t as = bitset->arraysize; - if (inword_shift == 0) { - // could be done with a memmove - for (size_t i = 0; i < as - extra_words; i++) { - bitset->array[i] = bitset->array[i + extra_words]; - } - bitset_resize(bitset, as - extra_words, false); + __m512i base_v = _mm512_set1_epi16((short)base); + const __m512i inc_v = _mm512_set1_epi16(64); + const __m512i index_table = _mm512_loadu_si512(vbmi2_table); + size_t i = 0; - } else { - for (size_t i = 0; i + extra_words + 1 < as; i++) { - bitset->array[i] = - (bitset->array[i + extra_words] >> inword_shift) | - (bitset->array[i + extra_words + 1] << (64 - inword_shift)); + for (; i < length; i++) { + uint64_t v = array[i]; + if (v != 0) { + size_t advance = (size_t)roaring_hamming(v); + if (out + advance > safeout) break; + __m512i vec = _mm512_maskz_compress_epi8((__mmask64)v, index_table); + uint64_t bits = ~UINT64_C(0) >> (64 - advance); + _mm512_mask_storeu_epi16( + out, (__mmask32)bits, + _mm512_add_epi16( + base_v, _mm512_cvtepu8_epi16(_mm512_castsi512_si256(vec)))); + if (advance > 32) { + _mm512_mask_storeu_epi16( + out + 32, (__mmask32)(bits >> 32), + _mm512_add_epi16(base_v, + _mm512_cvtepu8_epi16( + _mm512_extracti64x4_epi64(vec, 1)))); + } + out += advance; } - bitset->array[as - extra_words - 1] = - (bitset->array[as - 1] >> inword_shift); - bitset_resize(bitset, as - extra_words, false); + base_v = _mm512_add_epi16(base_v, inc_v); } -} -/* Free memory. */ -void bitset_free(bitset_t *bitset) { - if (bitset == NULL) { - return; - } - roaring_free(bitset->array); - roaring_free(bitset); -} + base = (uint16_t)(base + i * 64); -/* Resize the bitset so that it can support newarraysize * 64 bits. Return true - * in case of success, false for failure. */ -bool bitset_resize(bitset_t *bitset, size_t newarraysize, bool padwithzeroes) { - if (newarraysize > SIZE_MAX / 64) { - return false; - } - size_t smallest = - newarraysize < bitset->arraysize ? newarraysize : bitset->arraysize; - if (bitset->capacity < newarraysize) { - uint64_t *newarray; - size_t newcapacity = bitset->capacity; - if (newcapacity == 0) { - newcapacity = 1; - } - while (newcapacity < newarraysize) { - newcapacity *= 2; - } - if ((newarray = (uint64_t *)roaring_realloc( - bitset->array, sizeof(uint64_t) * newcapacity)) == NULL) { - return false; + for (; (i < length) && (out < safeout); ++i) { + uint64_t w = array[i]; + while ((w != 0) && (out < safeout)) { + int r = + roaring_trailing_zeroes(w); // on x64, should compile to TZCNT + *out = (uint16_t)(r + base); + out++; + w &= (w - 1); } - bitset->capacity = newcapacity; - bitset->array = newarray; + base += 64; } - if (padwithzeroes && (newarraysize > smallest)) - memset(bitset->array + smallest, 0, - sizeof(uint64_t) * (newarraysize - smallest)); - bitset->arraysize = newarraysize; - return true; // success! -} -size_t bitset_count(const bitset_t *bitset) { - size_t card = 0; - size_t k = 0; - for (; k + 7 < bitset->arraysize; k += 8) { - card += roaring_hamming(bitset->array[k]); - card += roaring_hamming(bitset->array[k + 1]); - card += roaring_hamming(bitset->array[k + 2]); - card += roaring_hamming(bitset->array[k + 3]); - card += roaring_hamming(bitset->array[k + 4]); - card += roaring_hamming(bitset->array[k + 5]); - card += roaring_hamming(bitset->array[k + 6]); - card += roaring_hamming(bitset->array[k + 7]); - } - for (; k + 3 < bitset->arraysize; k += 4) { - card += roaring_hamming(bitset->array[k]); - card += roaring_hamming(bitset->array[k + 1]); - card += roaring_hamming(bitset->array[k + 2]); - card += roaring_hamming(bitset->array[k + 3]); - } - for (; k < bitset->arraysize; k++) { - card += roaring_hamming(bitset->array[k]); - } - return card; + return out - initout; } +CROARING_UNTARGET_AVX512 +#endif -bool bitset_inplace_union(bitset_t *CROARING_CBITSET_RESTRICT b1, - const bitset_t *CROARING_CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - for (size_t k = 0; k < minlength; ++k) { - b1->array[k] |= b2->array[k]; +CROARING_TARGET_AVX2 +size_t bitset_extract_setbits_avx2(const uint64_t *words, size_t length, + uint32_t *out, size_t outcapacity, + uint32_t base) { + if (outcapacity == 0) return 0; + uint32_t *initout = out; + __m256i baseVec = _mm256_set1_epi32(base - 1); + __m256i incVec = _mm256_set1_epi32(64); + __m256i add8 = _mm256_set1_epi32(8); + uint32_t *safeout = out + outcapacity; + size_t i = 0; + for (; (i < length) && (out + 64 <= safeout); ++i) { + uint64_t w = words[i]; + if (w == 0) { + baseVec = _mm256_add_epi32(baseVec, incVec); + } else { + for (int k = 0; k < 4; ++k) { + uint8_t byteA = (uint8_t)w; + uint8_t byteB = (uint8_t)(w >> 8); + w >>= 16; + __m256i vecA = + _mm256_loadu_si256((const __m256i *)vecDecodeTable[byteA]); + __m256i vecB = + _mm256_loadu_si256((const __m256i *)vecDecodeTable[byteB]); + uint8_t advanceA = lengthTable[byteA]; + uint8_t advanceB = lengthTable[byteB]; + vecA = _mm256_add_epi32(baseVec, vecA); + baseVec = _mm256_add_epi32(baseVec, add8); + vecB = _mm256_add_epi32(baseVec, vecB); + baseVec = _mm256_add_epi32(baseVec, add8); + _mm256_storeu_si256((__m256i *)out, vecA); + out += advanceA; + _mm256_storeu_si256((__m256i *)out, vecB); + out += advanceB; + } + } } - if (b2->arraysize > b1->arraysize) { - size_t oldsize = b1->arraysize; - if (!bitset_resize(b1, b2->arraysize, false)) return false; - memcpy(b1->array + oldsize, b2->array + oldsize, - (b2->arraysize - oldsize) * sizeof(uint64_t)); + base += i * 64; + for (; (i < length) && (out < safeout); ++i) { + uint64_t w = words[i]; + while ((w != 0) && (out < safeout)) { + int r = + roaring_trailing_zeroes(w); // on x64, should compile to TZCNT + uint32_t val = r + base; + memcpy(out, &val, + sizeof(uint32_t)); // should be compiled as a MOV on x64 + out++; + w &= (w - 1); + } + base += 64; } - return true; + return out - initout; } +CROARING_UNTARGET_AVX2 +#endif // CROARING_IS_X64 -bool bitset_empty(const bitset_t *bitset) { - for (size_t k = 0; k < bitset->arraysize; k++) { - if (bitset->array[k] != 0) { - return false; +size_t bitset_extract_setbits(const uint64_t *words, size_t length, + uint32_t *out, uint32_t base) { + int outpos = 0; + for (size_t i = 0; i < length; ++i) { + uint64_t w = words[i]; + while (w != 0) { + int r = + roaring_trailing_zeroes(w); // on x64, should compile to TZCNT + uint32_t val = r + base; + memcpy(out + outpos, &val, + sizeof(uint32_t)); // should be compiled as a MOV on x64 + outpos++; + w &= (w - 1); } + base += 64; } - return true; + return outpos; } -size_t bitset_minimum(const bitset_t *bitset) { - for (size_t k = 0; k < bitset->arraysize; k++) { - uint64_t w = bitset->array[k]; - if (w != 0) { - return roaring_trailing_zeroes(w) + k * 64; +size_t bitset_extract_intersection_setbits_uint16( + const uint64_t *__restrict__ words1, const uint64_t *__restrict__ words2, + size_t length, uint16_t *out, uint16_t base) { + int outpos = 0; + for (size_t i = 0; i < length; ++i) { + uint64_t w = words1[i] & words2[i]; + while (w != 0) { + int r = roaring_trailing_zeroes(w); + out[outpos++] = (uint16_t)(r + base); + w &= (w - 1); } + base += 64; } - return SIZE_MAX; + return outpos; } -bool bitset_grow(bitset_t *bitset, size_t newarraysize) { - if (newarraysize < bitset->arraysize) { - return false; - } - if (newarraysize > SIZE_MAX / 64) { - return false; - } - if (bitset->capacity < newarraysize) { - uint64_t *newarray; - size_t newcapacity = (UINT64_C(0xFFFFFFFFFFFFFFFF) >> - roaring_leading_zeroes(newarraysize)) + - 1; - while (newcapacity < newarraysize) { - newcapacity *= 2; +#if CROARING_IS_X64 +/* + * Given a bitset containing "length" 64-bit words, write out the position + * of all the set bits to "out" as 16-bit integers, values start at "base" (can + *be set to zero). + * + * The "out" pointer should be sufficient to store the actual number of bits + *set. + * + * Returns how many values were actually decoded. + * + * This function uses SSE decoding. + */ +CROARING_TARGET_AVX2 +size_t bitset_extract_setbits_sse_uint16(const uint64_t *words, size_t length, + uint16_t *out, size_t outcapacity, + uint16_t base) { + if (outcapacity == 0) return 0; + uint16_t *initout = out; + __m128i baseVec = _mm_set1_epi16(base - 1); + __m128i incVec = _mm_set1_epi16(64); + __m128i add8 = _mm_set1_epi16(8); + uint16_t *safeout = out + outcapacity; + const int numberofbytes = 2; // process two bytes at a time + size_t i = 0; + for (; (i < length) && (out + numberofbytes * 8 <= safeout); ++i) { + uint64_t w = words[i]; + if (w == 0) { + baseVec = _mm_add_epi16(baseVec, incVec); + } else { + for (int k = 0; k < 4; ++k) { + uint8_t byteA = (uint8_t)w; + uint8_t byteB = (uint8_t)(w >> 8); + w >>= 16; + __m128i vecA = _mm_loadu_si128( + (const __m128i *)vecDecodeTable_uint16[byteA]); + __m128i vecB = _mm_loadu_si128( + (const __m128i *)vecDecodeTable_uint16[byteB]); + uint8_t advanceA = lengthTable[byteA]; + uint8_t advanceB = lengthTable[byteB]; + vecA = _mm_add_epi16(baseVec, vecA); + baseVec = _mm_add_epi16(baseVec, add8); + vecB = _mm_add_epi16(baseVec, vecB); + baseVec = _mm_add_epi16(baseVec, add8); + _mm_storeu_si128((__m128i *)out, vecA); + out += advanceA; + _mm_storeu_si128((__m128i *)out, vecB); + out += advanceB; + } } - if ((newarray = (uint64_t *)roaring_realloc( - bitset->array, sizeof(uint64_t) * newcapacity)) == NULL) { - return false; + } + base += (uint16_t)(i * 64); + for (; (i < length) && (out < safeout); ++i) { + uint64_t w = words[i]; + while ((w != 0) && (out < safeout)) { + int r = roaring_trailing_zeroes(w); + *out = (uint16_t)(r + base); + out++; + w &= (w - 1); } - bitset->capacity = newcapacity; - bitset->array = newarray; + base += 64; } - memset(bitset->array + bitset->arraysize, 0, - sizeof(uint64_t) * (newarraysize - bitset->arraysize)); - bitset->arraysize = newarraysize; - return true; // success! + return out - initout; } +CROARING_UNTARGET_AVX2 +#endif -size_t bitset_maximum(const bitset_t *bitset) { - for (size_t k = bitset->arraysize; k > 0; k--) { - uint64_t w = bitset->array[k - 1]; - if (w != 0) { - return 63 - roaring_leading_zeroes(w) + (k - 1) * 64; +/* + * Given a bitset containing "length" 64-bit words, write out the position + * of all the set bits to "out", values start at "base" (can be set to zero). + * + * The "out" pointer should be sufficient to store the actual number of bits + *set. + * + * Returns how many values were actually decoded. + */ +size_t bitset_extract_setbits_uint16(const uint64_t *words, size_t length, + uint16_t *out, uint16_t base) { + int outpos = 0; + for (size_t i = 0; i < length; ++i) { + uint64_t w = words[i]; + while (w != 0) { + int r = roaring_trailing_zeroes(w); + out[outpos++] = (uint16_t)(r + base); + w &= (w - 1); } + base += 64; } - return 0; + return outpos; } -/* Returns true if bitsets share no common elements, false otherwise. - * - * Performs early-out if common element found. */ -bool bitsets_disjoint(const bitset_t *CROARING_CBITSET_RESTRICT b1, - const bitset_t *CROARING_CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; +#if defined(CROARING_ASMBITMANIPOPTIMIZATION) && defined(CROARING_IS_X64) - for (size_t k = 0; k < minlength; k++) { - if ((b1->array[k] & b2->array[k]) != 0) return false; +static inline uint64_t _asm_bitset_set_list_withcard(uint64_t *words, + uint64_t card, + const uint16_t *list, + uint64_t length) { + uint64_t offset, load, pos; + uint64_t shift = 6; + const uint16_t *end = list + length; + if (!length) return card; + // TODO: could unroll for performance, see bitset_set_list + // bts is not available as an intrinsic in GCC + __asm volatile( + "1:\n" + "movzwq (%[list]), %[pos]\n" + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "bts %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)\n" + "sbb $-1, %[card]\n" + "add $2, %[list]\n" + "cmp %[list], %[end]\n" + "jnz 1b" + : [card] "+&r"(card), [list] "+&r"(list), [load] "=&r"(load), + [pos] "=&r"(pos), [offset] "=&r"(offset) + : [end] "r"(end), [words] "r"(words), [shift] "r"(shift)); + return card; +} + +static inline void _asm_bitset_set_list(uint64_t *words, const uint16_t *list, + uint64_t length) { + uint64_t pos; + const uint16_t *end = list + length; + + uint64_t shift = 6; + uint64_t offset; + uint64_t load; + for (; list + 3 < end; list += 4) { + pos = list[0]; + __asm volatile( + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "bts %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)" + : [load] "=&r"(load), [offset] "=&r"(offset) + : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); + pos = list[1]; + __asm volatile( + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "bts %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)" + : [load] "=&r"(load), [offset] "=&r"(offset) + : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); + pos = list[2]; + __asm volatile( + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "bts %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)" + : [load] "=&r"(load), [offset] "=&r"(offset) + : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); + pos = list[3]; + __asm volatile( + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "bts %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)" + : [load] "=&r"(load), [offset] "=&r"(offset) + : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); } - return true; + + while (list != end) { + pos = list[0]; + __asm volatile( + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "bts %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)" + : [load] "=&r"(load), [offset] "=&r"(offset) + : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); + list++; + } +} + +static inline uint64_t _asm_bitset_clear_list(uint64_t *words, uint64_t card, + const uint16_t *list, + uint64_t length) { + uint64_t offset, load, pos; + uint64_t shift = 6; + const uint16_t *end = list + length; + if (!length) return card; + // btr is not available as an intrinsic in GCC + __asm volatile( + "1:\n" + "movzwq (%[list]), %[pos]\n" + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "btr %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)\n" + "sbb $0, %[card]\n" + "add $2, %[list]\n" + "cmp %[list], %[end]\n" + "jnz 1b" + : [card] "+&r"(card), [list] "+&r"(list), [load] "=&r"(load), + [pos] "=&r"(pos), [offset] "=&r"(offset) + : [end] "r"(end), [words] "r"(words), [shift] "r"(shift) + : + /* clobbers */ "memory"); + return card; } -/* Returns true if bitsets contain at least 1 common element, false if they are - * disjoint. - * - * Performs early-out if common element found. */ -bool bitsets_intersect(const bitset_t *CROARING_CBITSET_RESTRICT b1, - const bitset_t *CROARING_CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - - for (size_t k = 0; k < minlength; k++) { - if ((b1->array[k] & b2->array[k]) != 0) return true; +static inline uint64_t _scalar_bitset_clear_list(uint64_t *words, uint64_t card, + const uint16_t *list, + uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *(const uint16_t *)list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load & ~(UINT64_C(1) << index); + card -= (load ^ newload) >> index; + words[offset] = newload; + list++; } - return false; + return card; } -/* Returns true if b has any bits set in or after b->array[starting_loc]. */ -static bool any_bits_set(const bitset_t *b, size_t starting_loc) { - if (starting_loc >= b->arraysize) { - return false; - } - for (size_t k = starting_loc; k < b->arraysize; k++) { - if (b->array[k] != 0) return true; +static inline uint64_t _scalar_bitset_set_list_withcard(uint64_t *words, + uint64_t card, + const uint16_t *list, + uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load | (UINT64_C(1) << index); + card += (load ^ newload) >> index; + words[offset] = newload; + list++; } - return false; + return card; } -/* Returns true if b1 has all of b2's bits set. - * - * Performs early out if a bit is found in b2 that is not found in b1. */ -bool bitset_contains_all(const bitset_t *CROARING_CBITSET_RESTRICT b1, - const bitset_t *CROARING_CBITSET_RESTRICT b2) { - size_t min_size = b1->arraysize; - if (b1->arraysize > b2->arraysize) { - min_size = b2->arraysize; - } - for (size_t k = 0; k < min_size; k++) { - if ((b1->array[k] & b2->array[k]) != b2->array[k]) { - return false; - } - } - if (b2->arraysize > b1->arraysize) { - /* Need to check if b2 has any bits set beyond b1's array */ - return !any_bits_set(b2, b1->arraysize); +static inline void _scalar_bitset_set_list(uint64_t *words, + const uint16_t *list, + uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load | (UINT64_C(1) << index); + words[offset] = newload; + list++; } - return true; } -size_t bitset_union_count(const bitset_t *CROARING_CBITSET_RESTRICT b1, - const bitset_t *CROARING_CBITSET_RESTRICT b2) { - size_t answer = 0; - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - size_t k = 0; - for (; k + 3 < minlength; k += 4) { - answer += roaring_hamming(b1->array[k] | b2->array[k]); - answer += roaring_hamming(b1->array[k + 1] | b2->array[k + 1]); - answer += roaring_hamming(b1->array[k + 2] | b2->array[k + 2]); - answer += roaring_hamming(b1->array[k + 3] | b2->array[k + 3]); - } - for (; k < minlength; ++k) { - answer += roaring_hamming(b1->array[k] | b2->array[k]); - } - if (b2->arraysize > b1->arraysize) { - // k is equal to b1->arraysize - for (; k + 3 < b2->arraysize; k += 4) { - answer += roaring_hamming(b2->array[k]); - answer += roaring_hamming(b2->array[k + 1]); - answer += roaring_hamming(b2->array[k + 2]); - answer += roaring_hamming(b2->array[k + 3]); - } - for (; k < b2->arraysize; ++k) { - answer += roaring_hamming(b2->array[k]); - } +uint64_t bitset_clear_list(uint64_t *words, uint64_t card, const uint16_t *list, + uint64_t length) { + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + return _asm_bitset_clear_list(words, card, list, length); } else { - // k is equal to b2->arraysize - for (; k + 3 < b1->arraysize; k += 4) { - answer += roaring_hamming(b1->array[k]); - answer += roaring_hamming(b1->array[k + 1]); - answer += roaring_hamming(b1->array[k + 2]); - answer += roaring_hamming(b1->array[k + 3]); - } - for (; k < b1->arraysize; ++k) { - answer += roaring_hamming(b1->array[k]); - } - } - return answer; -} - -void bitset_inplace_intersection(bitset_t *CROARING_CBITSET_RESTRICT b1, - const bitset_t *CROARING_CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - size_t k = 0; - for (; k < minlength; ++k) { - b1->array[k] &= b2->array[k]; - } - for (; k < b1->arraysize; ++k) { - b1->array[k] = 0; // memset could, maybe, be a tiny bit faster + return _scalar_bitset_clear_list(words, card, list, length); } } -size_t bitset_intersection_count(const bitset_t *CROARING_CBITSET_RESTRICT b1, - const bitset_t *CROARING_CBITSET_RESTRICT b2) { - size_t answer = 0; - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - for (size_t k = 0; k < minlength; ++k) { - answer += roaring_hamming(b1->array[k] & b2->array[k]); +uint64_t bitset_set_list_withcard(uint64_t *words, uint64_t card, + const uint16_t *list, uint64_t length) { + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + return _asm_bitset_set_list_withcard(words, card, list, length); + } else { + return _scalar_bitset_set_list_withcard(words, card, list, length); } - return answer; } -void bitset_inplace_difference(bitset_t *CROARING_CBITSET_RESTRICT b1, - const bitset_t *CROARING_CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - size_t k = 0; - for (; k < minlength; ++k) { - b1->array[k] &= ~(b2->array[k]); +void bitset_set_list(uint64_t *words, const uint16_t *list, uint64_t length) { + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + _asm_bitset_set_list(words, list, length); + } else { + _scalar_bitset_set_list(words, list, length); } } - -size_t bitset_difference_count(const bitset_t *CROARING_CBITSET_RESTRICT b1, - const bitset_t *CROARING_CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - size_t k = 0; - size_t answer = 0; - for (; k < minlength; ++k) { - answer += roaring_hamming(b1->array[k] & ~(b2->array[k])); - } - for (; k < b1->arraysize; ++k) { - answer += roaring_hamming(b1->array[k]); +#else +uint64_t bitset_clear_list(uint64_t *words, uint64_t card, const uint16_t *list, + uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *(const uint16_t *)list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load & ~(UINT64_C(1) << index); + card -= (load ^ newload) >> index; + words[offset] = newload; + list++; } - return answer; + return card; } -bool bitset_inplace_symmetric_difference( - bitset_t *CROARING_CBITSET_RESTRICT b1, - const bitset_t *CROARING_CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - size_t k = 0; - for (; k < minlength; ++k) { - b1->array[k] ^= b2->array[k]; - } - if (b2->arraysize > b1->arraysize) { - size_t oldsize = b1->arraysize; - if (!bitset_resize(b1, b2->arraysize, false)) return false; - memcpy(b1->array + oldsize, b2->array + oldsize, - (b2->arraysize - oldsize) * sizeof(uint64_t)); +uint64_t bitset_set_list_withcard(uint64_t *words, uint64_t card, + const uint16_t *list, uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load | (UINT64_C(1) << index); + card += (load ^ newload) >> index; + words[offset] = newload; + list++; } - return true; + return card; } -size_t bitset_symmetric_difference_count( - const bitset_t *CROARING_CBITSET_RESTRICT b1, - const bitset_t *CROARING_CBITSET_RESTRICT b2) { - size_t minlength = - b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; - size_t k = 0; - size_t answer = 0; - for (; k < minlength; ++k) { - answer += roaring_hamming(b1->array[k] ^ b2->array[k]); - } - if (b2->arraysize > b1->arraysize) { - for (; k < b2->arraysize; ++k) { - answer += roaring_hamming(b2->array[k]); - } - } else { - for (; k < b1->arraysize; ++k) { - answer += roaring_hamming(b1->array[k]); - } +void bitset_set_list(uint64_t *words, const uint16_t *list, uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load | (UINT64_C(1) << index); + words[offset] = newload; + list++; } - return answer; } -bool bitset_trim(bitset_t *bitset) { - size_t newsize = bitset->arraysize; - while (newsize > 0) { - if (bitset->array[newsize - 1] == 0) - newsize -= 1; - else - break; +#endif + +/* flip specified bits */ +/* TODO: consider whether worthwhile to make an asm version */ + +uint64_t bitset_flip_list_withcard(uint64_t *words, uint64_t card, + const uint16_t *list, uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load ^ (UINT64_C(1) << index); + // todo: is a branch here all that bad? + card += + (1 - 2 * (((UINT64_C(1) << index) & load) >> index)); // +1 or -1 + words[offset] = newload; + list++; } - if (bitset->capacity == newsize) return true; // nothing to do - uint64_t *newarray; - if ((newarray = (uint64_t *)roaring_realloc( - bitset->array, sizeof(uint64_t) * newsize)) == NULL) { - return false; + return card; +} + +void bitset_flip_list(uint64_t *words, const uint16_t *list, uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load ^ (UINT64_C(1) << index); + words[offset] = newload; + list++; } - bitset->array = newarray; - bitset->capacity = newsize; - bitset->arraysize = newsize; - return true; } #ifdef __cplusplus } } -} // extern "C" { namespace roaring { namespace internal { +} // extern "C" { namespace roaring { namespace api { #endif -/* end file src/bitset.c */ +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif +/* end file src/bitset_util.c */ /* begin file src/containers/array.c */ /* * array.c @@ -6751,17 +6867,19 @@ void array_container_offset(const array_container_t *c, container_t **loc, if (loc && lo_cap) { lo = array_container_create_given_capacity(lo_cap); for (int i = 0; i < lo_cap; ++i) { - array_container_add(lo, c->array[i] + offset); + lo->array[i] = c->array[i] + offset; } + lo->cardinality = lo_cap; *loc = (container_t *)lo; } hi_cap = c->cardinality - lo_cap; if (hic && hi_cap) { hi = array_container_create_given_capacity(hi_cap); - for (int i = lo_cap; i < c->cardinality; ++i) { - array_container_add(hi, c->array[i] + offset); + for (int i = 0; i < hi_cap; ++i) { + hi->array[i] = c->array[lo_cap + i] + offset; } + hi->cardinality = hi_cap; *hic = (container_t *)hi; } } @@ -7148,7 +7266,14 @@ int32_t array_container_number_of_runs(const array_container_t *ac) { * */ int32_t array_container_write(const array_container_t *container, char *buf) { +#if CROARING_IS_BIG_ENDIAN + for (int32_t i = 0; i < container->cardinality; ++i) { + uint16_t v_le = croaring_htole16(container->array[i]); + memcpy(buf + i * sizeof(uint16_t), &v_le, sizeof(uint16_t)); + } +#else memcpy(buf, container->array, container->cardinality * sizeof(uint16_t)); +#endif return array_container_size_in_bytes(container); } @@ -7181,7 +7306,15 @@ int32_t array_container_read(int32_t cardinality, array_container_t *container, array_container_grow(container, cardinality, false); } container->cardinality = cardinality; +#if CROARING_IS_BIG_ENDIAN + for (int32_t i = 0; i < cardinality; ++i) { + uint16_t v_le; + memcpy(&v_le, buf + i * sizeof(uint16_t), sizeof(uint16_t)); + container->array[i] = croaring_letoh16(v_le); + } +#else memcpy(container->array, buf, container->cardinality * sizeof(uint16_t)); +#endif return array_container_size_in_bytes(container); } @@ -7264,8 +7397,7 @@ void bitset_container_set_all(bitset_container_t *bitset) { bitset->cardinality = (1 << 16); } -/* Create a new bitset. Return NULL in case of failure. */ -bitset_container_t *bitset_container_create(void) { +static bitset_container_t *bitset_container_allocate(void) { bitset_container_t *bitset = (bitset_container_t *)roaring_malloc(sizeof(bitset_container_t)); @@ -7290,7 +7422,23 @@ bitset_container_t *bitset_container_create(void) { roaring_free(bitset); return NULL; } - bitset_container_clear(bitset); + return bitset; +} + +/* Create a new bitset. Return NULL in case of failure. */ +bitset_container_t *bitset_container_create(void) { + bitset_container_t *bitset = bitset_container_allocate(); + if (bitset) { + bitset_container_clear(bitset); + } + return bitset; +} + +bitset_container_t *bitset_container_create_uninitialized(void) { + bitset_container_t *bitset = bitset_container_allocate(); + if (bitset) { + bitset->cardinality = 0; + } return bitset; } @@ -7715,36 +7863,36 @@ CROARING_UNTARGET_AVX512 i < BITSET_CONTAINER_SIZE_IN_WORDS / (CROARING_WORDS_IN_AVX2_REG); \ i += innerloop) { \ __m256i A1, A2, AO; \ - A1 = _mm256_lddqu_si256((const __m256i *)(words_1)); \ - A2 = _mm256_lddqu_si256((const __m256i *)(words_2)); \ + A1 = _mm256_loadu_si256((const __m256i *)(words_1)); \ + A2 = _mm256_loadu_si256((const __m256i *)(words_2)); \ AO = avx_intrinsic(A2, A1); \ _mm256_storeu_si256((__m256i *)out, AO); \ - A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 32)); \ - A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 32)); \ + A1 = _mm256_loadu_si256((const __m256i *)(words_1 + 32)); \ + A2 = _mm256_loadu_si256((const __m256i *)(words_2 + 32)); \ AO = avx_intrinsic(A2, A1); \ _mm256_storeu_si256((__m256i *)(out + 32), AO); \ - A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 64)); \ - A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 64)); \ + A1 = _mm256_loadu_si256((const __m256i *)(words_1 + 64)); \ + A2 = _mm256_loadu_si256((const __m256i *)(words_2 + 64)); \ AO = avx_intrinsic(A2, A1); \ _mm256_storeu_si256((__m256i *)(out + 64), AO); \ - A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 96)); \ - A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 96)); \ + A1 = _mm256_loadu_si256((const __m256i *)(words_1 + 96)); \ + A2 = _mm256_loadu_si256((const __m256i *)(words_2 + 96)); \ AO = avx_intrinsic(A2, A1); \ _mm256_storeu_si256((__m256i *)(out + 96), AO); \ - A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 128)); \ - A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 128)); \ + A1 = _mm256_loadu_si256((const __m256i *)(words_1 + 128)); \ + A2 = _mm256_loadu_si256((const __m256i *)(words_2 + 128)); \ AO = avx_intrinsic(A2, A1); \ _mm256_storeu_si256((__m256i *)(out + 128), AO); \ - A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 160)); \ - A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 160)); \ + A1 = _mm256_loadu_si256((const __m256i *)(words_1 + 160)); \ + A2 = _mm256_loadu_si256((const __m256i *)(words_2 + 160)); \ AO = avx_intrinsic(A2, A1); \ _mm256_storeu_si256((__m256i *)(out + 160), AO); \ - A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 192)); \ - A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 192)); \ + A1 = _mm256_loadu_si256((const __m256i *)(words_1 + 192)); \ + A2 = _mm256_loadu_si256((const __m256i *)(words_2 + 192)); \ AO = avx_intrinsic(A2, A1); \ _mm256_storeu_si256((__m256i *)(out + 192), AO); \ - A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 224)); \ - A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 224)); \ + A1 = _mm256_loadu_si256((const __m256i *)(words_1 + 224)); \ + A2 = _mm256_loadu_si256((const __m256i *)(words_2 + 224)); \ AO = avx_intrinsic(A2, A1); \ _mm256_storeu_si256((__m256i *)(out + 224), AO); \ out += 256; \ @@ -8142,7 +8290,11 @@ int bitset_container_to_uint32_array( #if CROARING_IS_X64 int support = croaring_hardware_support(); #if CROARING_COMPILER_SUPPORTS_AVX512 - if(( support & ROARING_SUPPORTS_AVX512 ) && (bc->cardinality >= 8192)) // heuristic + // Unlike the AVX2 kernel, the AVX-512 one needs no cardinality heuristic: + // it skips empty words and only stores the blocks that carry a value, so it + // beat the scalar loop at every cardinality measured from 16 to 65536 (on + // an Emerald Rapids Xeon, 1.3x at 512 values, 6x at 4096, 7.5x at 8192). + if( support & ROARING_SUPPORTS_AVX512 ) return (int) bitset_extract_setbits_avx512(bc->words, BITSET_CONTAINER_SIZE_IN_WORDS, out, bc->cardinality, base); else @@ -8253,7 +8405,14 @@ int bitset_container_number_of_runs(bitset_container_t *bc) { int32_t bitset_container_write(const bitset_container_t *container, char *buf) { +#if CROARING_IS_BIG_ENDIAN + for (int32_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; ++i) { + uint64_t w_le = croaring_htole64(container->words[i]); + memcpy(buf + i * sizeof(uint64_t), &w_le, sizeof(uint64_t)); + } +#else memcpy(buf, container->words, BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t)); +#endif return bitset_container_size_in_bytes(container); } @@ -8261,7 +8420,15 @@ int32_t bitset_container_write(const bitset_container_t *container, int32_t bitset_container_read(int32_t cardinality, bitset_container_t *container, const char *buf) { container->cardinality = cardinality; +#if CROARING_IS_BIG_ENDIAN + for (int32_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; ++i) { + uint64_t w_le; + memcpy(&w_le, buf + i * sizeof(uint64_t), sizeof(uint64_t)); + container->words[i] = croaring_letoh64(w_le); + } +#else memcpy(container->words, buf, BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t)); +#endif return bitset_container_size_in_bytes(container); } @@ -8747,12 +8914,13 @@ container_t *shared_container_extract_copy(shared_container_t *sc, assert(sc->typecode != SHARED_CONTAINER_TYPE); *typecode = sc->typecode; container_t *answer; - if (croaring_refcount_dec(&sc->counter)) { + if (croaring_refcount_is_unique(&sc->counter)) { answer = sc->container; sc->container = NULL; // paranoid roaring_free(sc); } else { answer = container_clone(sc->container, *typecode); + shared_container_free(sc); } assert(*typecode != SHARED_CONTAINER_TYPE); return answer; @@ -8806,6 +8974,7 @@ extern inline container_t *container_andnot(const container_t *c1, uint8_t type2, uint8_t *result_type); +CROARING_ALLOW_UNALIGNED roaring_container_iterator_t container_init_iterator(const container_t *c, uint8_t typecode, uint16_t *value) { @@ -8845,6 +9014,7 @@ roaring_container_iterator_t container_init_iterator(const container_t *c, } } +CROARING_ALLOW_UNALIGNED roaring_container_iterator_t container_init_iterator_last(const container_t *c, uint8_t typecode, uint16_t *value) { @@ -8920,15 +9090,199 @@ bool container_iterator_lower_bound(const container_t *c, uint8_t typecode, default: assert(false); roaring_unreachable; - return false; + return false; + } +} + +CROARING_ALLOW_UNALIGNED +bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint32_t high16, uint32_t *buf, + uint32_t count, uint32_t *consumed, + uint16_t *value_out) { + *consumed = 0; + if (count == 0) { + return false; + } + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = it->index / 64; + uint64_t word = + bc->words[wordindex] & (UINT64_MAX << (it->index % 64)); + do { + // Read set bits. + while (word != 0 && *consumed < count) { + *buf = high16 | + (wordindex * 64 + roaring_trailing_zeroes(word)); + word = word & (word - 1); + buf++; + (*consumed)++; + } + // Skip unset bits. + while (word == 0 && + wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { + wordindex++; + word = bc->words[wordindex]; + } + } while (word != 0 && *consumed < count); + + if (word != 0) { + it->index = wordindex * 64 + roaring_trailing_zeroes(word); + *value_out = it->index; + return true; + } + return false; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint32_t num_values = + minimum_uint32(ac->cardinality - it->index, count); + // Hoist so GCC can vectorize the uint16->uint32 widen-or. + const uint16_t *src = ac->array + it->index; + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high16 | src[i]; + } + *consumed += num_values; + it->index += num_values; + if (it->index < ac->cardinality) { + *value_out = ac->array[it->index]; + return true; + } + return false; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + do { + uint32_t largest_run_value = + rc->runs[it->index].value + rc->runs[it->index].length; + uint32_t num_values = minimum_uint32( + largest_run_value - *value_out + 1, count - *consumed); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high16 | (*value_out + i); + } + *value_out += num_values; + buf += num_values; + *consumed += num_values; + + // We check for `value == 0` because `it->value += num_values` + // can overflow when `value == UINT16_MAX`, and `count > + // length`. In this case `value` will overflow to 0. + if (*value_out > largest_run_value || *value_out == 0) { + it->index++; + if (it->index < rc->n_runs) { + *value_out = rc->runs[it->index].value; + } else { + return false; + } + } + } while (*consumed < count); + return true; + } + default: + assert(false); + roaring_unreachable; + return 0; + } +} + +CROARING_ALLOW_UNALIGNED +bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint64_t high48, uint64_t *buf, + uint32_t count, uint32_t *consumed, + uint16_t *value_out) { + *consumed = 0; + if (count == 0) { + return false; + } + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = it->index / 64; + uint64_t word = + bc->words[wordindex] & (UINT64_MAX << (it->index % 64)); + do { + // Read set bits. + while (word != 0 && *consumed < count) { + *buf = high48 | + (wordindex * 64 + roaring_trailing_zeroes(word)); + word = word & (word - 1); + buf++; + (*consumed)++; + } + // Skip unset bits. + while (word == 0 && + wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { + wordindex++; + word = bc->words[wordindex]; + } + } while (word != 0 && *consumed < count); + + if (word != 0) { + it->index = wordindex * 64 + roaring_trailing_zeroes(word); + *value_out = it->index; + return true; + } + return false; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint32_t num_values = + minimum_uint32(ac->cardinality - it->index, count); + // Hoist so GCC can vectorize the uint16->uint64 widen-or. + const uint16_t *src = ac->array + it->index; + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high48 | src[i]; + } + *consumed += num_values; + it->index += num_values; + if (it->index < ac->cardinality) { + *value_out = ac->array[it->index]; + return true; + } + return false; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + do { + uint32_t largest_run_value = + rc->runs[it->index].value + rc->runs[it->index].length; + uint32_t num_values = minimum_uint32( + largest_run_value - *value_out + 1, count - *consumed); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high48 | (*value_out + i); + } + *value_out += num_values; + buf += num_values; + *consumed += num_values; + + // We check for `value == 0` because `it->value += num_values` + // can overflow when `value == UINT16_MAX`, and `count > + // length`. In this case `value` will overflow to 0. + if (*value_out > largest_run_value || *value_out == 0) { + it->index++; + if (it->index < rc->n_runs) { + *value_out = rc->runs[it->index].value; + } else { + return false; + } + } + } while (*consumed < count); + return true; + } + default: + assert(false); + roaring_unreachable; + return 0; } } -bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, - roaring_container_iterator_t *it, - uint32_t high16, uint32_t *buf, - uint32_t count, uint32_t *consumed, - uint16_t *value_out) { +CROARING_ALLOW_UNALIGNED +bool container_iterator_read_backward_into_uint32( + const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, + uint32_t high16, uint32_t *buf, uint32_t count, uint32_t *consumed, + uint16_t *value_out) { *consumed = 0; if (count == 0) { return false; @@ -8938,26 +9292,26 @@ bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, const bitset_container_t *bc = const_CAST_bitset(c); uint32_t wordindex = it->index / 64; uint64_t word = - bc->words[wordindex] & (UINT64_MAX << (it->index % 64)); + bc->words[wordindex] & (UINT64_MAX >> (63 - (it->index % 64))); do { // Read set bits. while (word != 0 && *consumed < count) { - *buf = high16 | - (wordindex * 64 + roaring_trailing_zeroes(word)); - word = word & (word - 1); + uint32_t bit = 63 - roaring_leading_zeroes(word); + *buf = high16 | (wordindex * 64 + bit); + word &= ~(UINT64_C(1) << bit); buf++; (*consumed)++; } // Skip unset bits. - while (word == 0 && - wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { - wordindex++; + while (word == 0 && wordindex > 0) { + wordindex--; word = bc->words[wordindex]; } } while (word != 0 && *consumed < count); if (word != 0) { - it->index = wordindex * 64 + roaring_trailing_zeroes(word); + it->index = + wordindex * 64 + (63 - roaring_leading_zeroes(word)); *value_out = it->index; return true; } @@ -8966,13 +9320,15 @@ bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, case ARRAY_CONTAINER_TYPE: { const array_container_t *ac = const_CAST_array(c); uint32_t num_values = - minimum_uint32(ac->cardinality - it->index, count); + minimum_uint32((uint32_t)(it->index + 1), count); + // Walk backwards so GCC can vectorize the uint16->uint32 widen-or. + const uint16_t *src = ac->array + it->index + 1; for (uint32_t i = 0; i < num_values; i++) { - buf[i] = high16 | ac->array[it->index + i]; + buf[i] = high16 | *--src; } *consumed += num_values; - it->index += num_values; - if (it->index < ac->cardinality) { + it->index -= num_values; + if (it->index >= 0) { *value_out = ac->array[it->index]; return true; } @@ -8981,24 +9337,25 @@ bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, case RUN_CONTAINER_TYPE: { const run_container_t *rc = const_CAST_run(c); do { - uint32_t largest_run_value = - rc->runs[it->index].value + rc->runs[it->index].length; - uint32_t num_values = minimum_uint32( - largest_run_value - *value_out + 1, count - *consumed); + uint32_t run_start = rc->runs[it->index].value; + uint32_t num_values = minimum_uint32(*value_out - run_start + 1, + count - *consumed); for (uint32_t i = 0; i < num_values; i++) { - buf[i] = high16 | (*value_out + i); + buf[i] = high16 | (*value_out - i); } - *value_out += num_values; + *value_out -= num_values; buf += num_values; *consumed += num_values; - // We check for `value == 0` because `it->value += num_values` - // can overflow when `value == UINT16_MAX`, and `count > - // length`. In this case `value` will overflow to 0. - if (*value_out > largest_run_value || *value_out == 0) { - it->index++; - if (it->index < rc->n_runs) { - *value_out = rc->runs[it->index].value; + // We check for `value == UINT16_MAX` because + // `*value_out -= num_values` can underflow when + // `value == 0` (run_start == 0). In this case `value` + // will underflow to UINT16_MAX. + if (*value_out < run_start || *value_out == UINT16_MAX) { + it->index--; + if (it->index >= 0) { + *value_out = rc->runs[it->index].value + + rc->runs[it->index].length; } else { return false; } @@ -9013,11 +9370,11 @@ bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, } } -bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, - roaring_container_iterator_t *it, - uint64_t high48, uint64_t *buf, - uint32_t count, uint32_t *consumed, - uint16_t *value_out) { +CROARING_ALLOW_UNALIGNED +bool container_iterator_read_backward_into_uint64( + const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, + uint64_t high48, uint64_t *buf, uint32_t count, uint32_t *consumed, + uint16_t *value_out) { *consumed = 0; if (count == 0) { return false; @@ -9027,26 +9384,26 @@ bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, const bitset_container_t *bc = const_CAST_bitset(c); uint32_t wordindex = it->index / 64; uint64_t word = - bc->words[wordindex] & (UINT64_MAX << (it->index % 64)); + bc->words[wordindex] & (UINT64_MAX >> (63 - (it->index % 64))); do { // Read set bits. while (word != 0 && *consumed < count) { - *buf = high48 | - (wordindex * 64 + roaring_trailing_zeroes(word)); - word = word & (word - 1); + uint32_t bit = 63 - roaring_leading_zeroes(word); + *buf = high48 | (wordindex * 64 + bit); + word &= ~(UINT64_C(1) << bit); buf++; (*consumed)++; } // Skip unset bits. - while (word == 0 && - wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { - wordindex++; + while (word == 0 && wordindex > 0) { + wordindex--; word = bc->words[wordindex]; } } while (word != 0 && *consumed < count); if (word != 0) { - it->index = wordindex * 64 + roaring_trailing_zeroes(word); + it->index = + wordindex * 64 + (63 - roaring_leading_zeroes(word)); *value_out = it->index; return true; } @@ -9055,13 +9412,15 @@ bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, case ARRAY_CONTAINER_TYPE: { const array_container_t *ac = const_CAST_array(c); uint32_t num_values = - minimum_uint32(ac->cardinality - it->index, count); + minimum_uint32((uint32_t)(it->index + 1), count); + // Walk backwards so GCC can vectorize the uint16->uint64 widen-or. + const uint16_t *src = ac->array + it->index + 1; for (uint32_t i = 0; i < num_values; i++) { - buf[i] = high48 | ac->array[it->index + i]; + buf[i] = high48 | *--src; } *consumed += num_values; - it->index += num_values; - if (it->index < ac->cardinality) { + it->index -= num_values; + if (it->index >= 0) { *value_out = ac->array[it->index]; return true; } @@ -9070,24 +9429,21 @@ bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, case RUN_CONTAINER_TYPE: { const run_container_t *rc = const_CAST_run(c); do { - uint32_t largest_run_value = - rc->runs[it->index].value + rc->runs[it->index].length; - uint32_t num_values = minimum_uint32( - largest_run_value - *value_out + 1, count - *consumed); + uint32_t run_start = rc->runs[it->index].value; + uint32_t num_values = minimum_uint32(*value_out - run_start + 1, + count - *consumed); for (uint32_t i = 0; i < num_values; i++) { - buf[i] = high48 | (*value_out + i); + buf[i] = high48 | (*value_out - i); } - *value_out += num_values; + *value_out -= num_values; buf += num_values; *consumed += num_values; - // We check for `value == 0` because `it->value += num_values` - // can overflow when `value == UINT16_MAX`, and `count > - // length`. In this case `value` will overflow to 0. - if (*value_out > largest_run_value || *value_out == 0) { - it->index++; - if (it->index < rc->n_runs) { - *value_out = rc->runs[it->index].value; + if (*value_out < run_start || *value_out == UINT16_MAX) { + it->index--; + if (it->index >= 0) { + *value_out = rc->runs[it->index].value + + rc->runs[it->index].length; } else { return false; } @@ -9301,6 +9657,171 @@ bool container_iterator_skip_backward(const container_t *c, uint8_t typecode, return has_value; } +uint16_t container_iterator_find_run_end(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value, bool *has_more) { + switch (typecode) { + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + uint16_t run_end = + rc->runs[it->index].value + rc->runs[it->index].length; + it->index++; + if (it->index < rc->n_runs) { + *has_more = true; + *value = rc->runs[it->index].value; + } else { + *has_more = false; + } + return run_end; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint16_t v = *value; + while (it->index + 1 < ac->cardinality && + ac->array[it->index + 1] == (uint16_t)(v + 1)) { + it->index++; + v++; + } + it->index++; + if (it->index < ac->cardinality) { + *has_more = true; + *value = ac->array[it->index]; + } else { + *has_more = false; + } + return v; + } + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t pos = (uint32_t)*value + 1; + uint16_t run_end; + if (pos >= (1 << 16)) { + *has_more = false; + return UINT16_MAX; + } + uint32_t wordindex = pos / 64; + uint64_t word = ~bc->words[wordindex] & (UINT64_MAX << (pos % 64)); + while (word == 0 && + wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { + wordindex++; + word = ~bc->words[wordindex]; + } + if (word != 0) { + run_end = (uint16_t)(wordindex * 64 + + roaring_trailing_zeroes(word) - 1); + } else { + run_end = UINT16_MAX; + } + uint32_t next_pos = (uint32_t)run_end + 1; + if (next_pos >= (1 << 16)) { + *has_more = false; + } else { + wordindex = next_pos / 64; + word = bc->words[wordindex] & (UINT64_MAX << (next_pos % 64)); + while (word == 0 && + wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { + wordindex++; + word = bc->words[wordindex]; + } + if (word != 0) { + *has_more = true; + it->index = wordindex * 64 + roaring_trailing_zeroes(word); + *value = (uint16_t)it->index; + } else { + *has_more = false; + } + } + return run_end; + } + default: + assert(false); + roaring_unreachable; + return 0; + } +} + +uint16_t container_iterator_find_run_start(const container_t *c, + uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value, bool *has_more) { + switch (typecode) { + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + uint16_t run_start = rc->runs[it->index].value; + it->index--; + if (it->index >= 0) { + *has_more = true; + *value = rc->runs[it->index].value + rc->runs[it->index].length; + } else { + *has_more = false; + } + return run_start; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint16_t v = *value; + while (it->index > 0 && + ac->array[it->index - 1] == (uint16_t)(v - 1)) { + it->index--; + v--; + } + it->index--; + if (it->index >= 0) { + *has_more = true; + *value = ac->array[it->index]; + } else { + *has_more = false; + } + return v; + } + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + if (*value == 0) { + *has_more = false; + return 0; + } + uint32_t pos = (uint32_t)*value - 1; + int32_t wordindex = (int32_t)(pos / 64); + uint64_t word = + ~bc->words[wordindex] & (UINT64_MAX >> (63 - (pos % 64))); + while (word == 0 && --wordindex >= 0) { + word = ~bc->words[wordindex]; + } + uint16_t run_start; + if (word != 0) { + run_start = (uint16_t)(wordindex * 64 + + (63 - roaring_leading_zeroes(word)) + 1); + } else { + run_start = 0; + } + if (run_start == 0) { + *has_more = false; + } else { + int32_t prev_pos = (int32_t)run_start - 1; + wordindex = prev_pos / 64; + word = bc->words[wordindex] & + (UINT64_MAX >> (63 - (prev_pos % 64))); + while (word == 0 && --wordindex >= 0) { + word = bc->words[wordindex]; + } + if (word != 0) { + *has_more = true; + it->index = + wordindex * 64 + (63 - roaring_leading_zeroes(word)); + *value = (uint16_t)it->index; + } else { + *has_more = false; + } + } + return run_start; + } + default: + assert(false); + roaring_unreachable; + return 0; + } +} + #ifdef __cplusplus } } @@ -9366,7 +9887,14 @@ array_container_t *array_container_from_bitset(const bitset_container_t *bits) { result->cardinality = bits->cardinality; #if CROARING_IS_X64 #if CROARING_COMPILER_SUPPORTS_AVX512 - if (croaring_hardware_support() & ROARING_SUPPORTS_AVX512) { + // Every caller first checks that the cardinality fits an array container + // (<= DEFAULT_MAX_SIZE), so the input here is sparse: under 6.25% density, + // a couple of set bits per word. + // Below ~1024 values the vector decoder does not make back its per-word + // cost and the scalar loop is as fast or faster; from 1024 up it wins + // (1.2x at 1024, 1.4x at 4096, measured on an Emerald Rapids Xeon). + if ((bits->cardinality >= 1024) && + (croaring_hardware_support() & ROARING_SUPPORTS_AVX512)) { bitset_extract_setbits_avx512_uint16( bits->words, BITSET_CONTAINER_SIZE_IN_WORDS, result->array, bits->cardinality, 0); @@ -12581,9 +13109,21 @@ bool run_container_validate(const run_container_t *run, const char **reason) { int32_t run_container_write(const run_container_t *container, char *buf) { uint16_t cast_16 = container->n_runs; - memcpy(buf, &cast_16, sizeof(uint16_t)); + uint16_t n_runs_le = croaring_htole16(cast_16); + memcpy(buf, &n_runs_le, sizeof(uint16_t)); +#if CROARING_IS_BIG_ENDIAN + char *out = buf + sizeof(uint16_t); + for (int32_t i = 0; i < container->n_runs; ++i) { + uint16_t v_le = croaring_htole16(container->runs[i].value); + uint16_t l_le = croaring_htole16(container->runs[i].length); + memcpy(out, &v_le, sizeof(uint16_t)); + memcpy(out + sizeof(uint16_t), &l_le, sizeof(uint16_t)); + out += sizeof(rle16_t); + } +#else memcpy(buf + sizeof(uint16_t), container->runs, container->n_runs * sizeof(rle16_t)); +#endif return run_container_size_in_bytes(container); } @@ -12592,12 +13132,24 @@ int32_t run_container_read(int32_t cardinality, run_container_t *container, (void)cardinality; uint16_t cast_16; memcpy(&cast_16, buf, sizeof(uint16_t)); - container->n_runs = cast_16; + container->n_runs = croaring_letoh16(cast_16); if (container->n_runs > container->capacity) run_container_grow(container, container->n_runs, false); if (container->n_runs > 0) { +#if CROARING_IS_BIG_ENDIAN + const char *in = buf + sizeof(uint16_t); + for (int32_t i = 0; i < container->n_runs; ++i) { + uint16_t v_le, l_le; + memcpy(&v_le, in, sizeof(uint16_t)); + memcpy(&l_le, in + sizeof(uint16_t), sizeof(uint16_t)); + container->runs[i].value = croaring_letoh16(v_le); + container->runs[i].length = croaring_letoh16(l_le); + in += sizeof(rle16_t); + } +#else memcpy(container->runs, buf + sizeof(uint16_t), container->n_runs * sizeof(rle16_t)); +#endif } return run_container_size_in_bytes(container); } @@ -12792,9 +13344,11 @@ int run_container_get_index(const run_container_t *container, uint16_t x) { return -1; } } +#define CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY 0 #if defined(CROARING_IS_X64) && CROARING_COMPILER_SUPPORTS_AVX512 +#if CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY CROARING_TARGET_AVX512 CROARING_ALLOW_UNALIGNED /* Get the cardinality of `run'. Requires an actual computation. */ @@ -12821,6 +13375,7 @@ static inline int _avx512_run_container_cardinality( } CROARING_UNTARGET_AVX512 +#endif // CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY CROARING_TARGET_AVX2 CROARING_ALLOW_UNALIGNED @@ -12836,7 +13391,7 @@ static inline int _avx2_run_container_cardinality(const run_container_t *run) { if (n_runs > step) { __m256i total = _mm256_setzero_si256(); for (; k + step <= n_runs; k += step) { - __m256i ymm1 = _mm256_lddqu_si256((const __m256i *)(runs + k)); + __m256i ymm1 = _mm256_loadu_si256((const __m256i *)(runs + k)); __m256i justlengths = _mm256_srli_epi32(ymm1, 16); total = _mm256_add_epi32(total, justlengths); } @@ -12913,7 +13468,6 @@ static inline int _scalar_run_container_cardinality( int run_container_cardinality(const run_container_t *run) { // Empirically AVX-512 is not always faster than AVX2 -#define CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY 0 #if CROARING_COMPILER_SUPPORTS_AVX512 && \ CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY if (croaring_hardware_support() & ROARING_SUPPORTS_AVX512) { @@ -13061,11 +13615,17 @@ POSSIBILITY OF SUCH DAMAGE. #endif // _MSC_VER == 1938 #endif // __clang__ +#ifdef __FILC__ +#include +#endif + // We need portability.h to be included first, see // https://github.com/RoaringBitmap/CRoaring/issues/394 #if CROARING_REGULAR_VISUAL_STUDIO #include -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#elif CROARING_IS_X64 && \ + ((defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || \ + defined(__FILC__)) #include #endif // CROARING_REGULAR_VISUAL_STUDIO @@ -13104,7 +13664,7 @@ unsigned int CROARING_AVX512_REQUIRED = CROARING_AVX512VBMI2 | CROARING_AVX512BITALG | CROARING_AVX512VPOPCNTDQ); #endif -#if defined(__x86_64__) || defined(_M_AMD64) // x64 +#if CROARING_IS_X64 // x64 static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) { @@ -13115,7 +13675,8 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, *ebx = cpu_info[1]; *ecx = cpu_info[2]; *edx = cpu_info[3]; -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || \ + defined(__FILC__) uint32_t level = *eax; __get_cpuid(level, eax, ebx, ecx, edx); #else @@ -13131,6 +13692,8 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, static inline uint64_t xgetbv(void) { #if defined(_MSC_VER) return _xgetbv(0); +#elif defined(__FILC__) + return zxgetbv(); #else uint32_t xcr0_lo, xcr0_hi; __asm__("xgetbv\n\t" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); @@ -13249,7 +13812,7 @@ static inline uint32_t dynamic_croaring_detect_supported_architectures(void) { #endif // end SIMD extension detection code -#if defined(__x86_64__) || defined(_M_AMD64) // x64 +#if CROARING_IS_X64 // x64 #if CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_CPP static inline uint32_t croaring_detect_supported_architectures(void) { @@ -13331,7 +13894,7 @@ int croaring_hardware_support(void) { } #endif -#endif // defined(__x86_64__) || defined(_M_AMD64) // x64 +#endif // CROARING_IS_X64 // x64 #ifdef __cplusplus } } @@ -13391,972 +13954,1404 @@ void* roaring_realloc(void* p, size_t new_sz) { return global_memory_hook.realloc(p, new_sz); } -void* roaring_calloc(size_t n_elements, size_t element_size) { - return global_memory_hook.calloc(n_elements, element_size); +void* roaring_calloc(size_t n_elements, size_t element_size) { + return global_memory_hook.calloc(n_elements, element_size); +} + +void roaring_free(void* p) { global_memory_hook.free(p); } + +void* roaring_aligned_malloc(size_t alignment, size_t size) { + return global_memory_hook.aligned_malloc(alignment, size); +} + +void roaring_aligned_free(void* p) { global_memory_hook.aligned_free(p); } +/* end file src/memory.c */ +/* begin file src/roaring.c */ +#include +#include +#include +#include +#include +#include +#include + + +// Include after roaring.h + +#ifdef __cplusplus +using namespace ::roaring::internal; + +extern "C" { +namespace roaring { +namespace api { +#endif + +#define CROARING_SERIALIZATION_ARRAY_UINT32 1 +#define CROARING_SERIALIZATION_CONTAINER 2 +extern inline bool roaring_bitmap_contains(const roaring_bitmap_t *r, + uint32_t val); +extern inline int roaring_trailing_zeroes(unsigned long long input_num); +extern inline int roaring_leading_zeroes(unsigned long long input_num); +extern inline void roaring_bitmap_init_cleared(roaring_bitmap_t *r); +extern inline bool roaring_bitmap_get_copy_on_write(const roaring_bitmap_t *r); +extern inline void roaring_bitmap_set_copy_on_write(roaring_bitmap_t *r, + bool cow); +extern inline roaring_bitmap_t *roaring_bitmap_create(void); +extern inline void roaring_bitmap_add_range(roaring_bitmap_t *r, uint64_t min, + uint64_t max); +extern inline void roaring_bitmap_remove_range(roaring_bitmap_t *r, + uint64_t min, uint64_t max); + +static inline bool is_cow(const roaring_bitmap_t *r) { + return r->high_low_container.flags & ROARING_FLAG_COW; +} +static inline bool is_frozen(const roaring_bitmap_t *r) { + return r->high_low_container.flags & ROARING_FLAG_FROZEN; +} + +// this is like roaring_bitmap_add, but it populates pointer arguments in such a +// way +// that we can recover the container touched, which, in turn can be used to +// accelerate some functions (when you repeatedly need to add to the same +// container) +static inline container_t *containerptr_roaring_bitmap_add(roaring_bitmap_t *r, + uint32_t val, + uint8_t *type, + int *index) { + roaring_array_t *ra = &r->high_low_container; + + uint16_t hb = val >> 16; + const int i = ra_get_index(ra, hb); + if (i >= 0) { + ra_unshare_container_at_index(ra, (uint16_t)i); + container_t *c = ra_get_container_at_index(ra, (uint16_t)i, type); + uint8_t new_type = *type; + container_t *c2 = container_add(c, val & 0xFFFF, *type, &new_type); + *index = i; + if (c2 != c) { + container_free(c, *type); + ra_set_container_at_index(ra, i, c2, new_type); + *type = new_type; + return c2; + } else { + return c; + } + } else { + array_container_t *new_ac = array_container_create(); + container_t *c = + container_add(new_ac, val & 0xFFFF, ARRAY_CONTAINER_TYPE, type); + // we could just assume that it stays an array container + ra_insert_new_key_value_at(ra, -i - 1, hb, c, *type); + *index = -i - 1; + return c; + } +} + +roaring_bitmap_t *roaring_bitmap_create_with_capacity(uint32_t cap) { + roaring_bitmap_t *ans = + (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); + if (!ans) { + return NULL; + } + bool is_ok = ra_init_with_capacity(&ans->high_low_container, cap); + if (!is_ok) { + roaring_free(ans); + return NULL; + } + return ans; +} + +bool roaring_bitmap_init_with_capacity(roaring_bitmap_t *r, uint32_t cap) { + return ra_init_with_capacity(&r->high_low_container, cap); } -void roaring_free(void* p) { global_memory_hook.free(p); } - -void* roaring_aligned_malloc(size_t alignment, size_t size) { - return global_memory_hook.aligned_malloc(alignment, size); +static inline void add_bulk_impl(roaring_bitmap_t *r, + roaring_bulk_context_t *context, + uint32_t val) { + uint16_t key = val >> 16; + if (context->container == NULL || context->key != key) { + uint8_t typecode; + int idx; + context->container = + containerptr_roaring_bitmap_add(r, val, &typecode, &idx); + context->typecode = typecode; + context->idx = idx; + context->key = key; + } else { + // no need to seek the container, it is at hand + // because we already have the container at hand, we can do the + // insertion directly, bypassing the roaring_bitmap_add call + uint8_t new_typecode; + container_t *container2 = container_add( + context->container, val & 0xFFFF, context->typecode, &new_typecode); + if (container2 != context->container) { + // rare instance when we need to change the container type + container_free(context->container, context->typecode); + ra_set_container_at_index(&r->high_low_container, context->idx, + container2, new_typecode); + context->typecode = new_typecode; + context->container = container2; + } + } } -void roaring_aligned_free(void* p) { global_memory_hook.aligned_free(p); } -/* end file src/memory.c */ -/* begin file src/roaring_array.c */ -#include -#include -#include -#include -#include -#include - +void roaring_bitmap_add_many(roaring_bitmap_t *r, size_t n_args, + const uint32_t *vals) { + uint32_t val; + const uint32_t *start = vals; + const uint32_t *end = vals + n_args; + const uint32_t *current_val = start; -#ifdef __cplusplus -extern "C" { -namespace roaring { -namespace internal { -#endif + if (n_args == 0) { + return; + } -// Convention: [0,ra->size) all elements are initialized -// [ra->size, ra->allocation_size) is junk and contains nothing needing freeing + uint8_t typecode; + int idx; + container_t *container; + val = *current_val; + container = containerptr_roaring_bitmap_add(r, val, &typecode, &idx); + roaring_bulk_context_t context = {container, idx, (uint16_t)(val >> 16), + typecode}; -extern inline int32_t ra_get_size(const roaring_array_t *ra); -extern inline int32_t ra_get_index(const roaring_array_t *ra, uint16_t x); + for (; current_val != end; current_val++) { + memcpy(&val, current_val, sizeof(val)); + add_bulk_impl(r, &context, val); + } +} -extern inline container_t *ra_get_container_at_index(const roaring_array_t *ra, - uint16_t i, - uint8_t *typecode); +void roaring_bitmap_add_bulk(roaring_bitmap_t *r, + roaring_bulk_context_t *context, uint32_t val) { + add_bulk_impl(r, context, val); +} -extern inline void ra_unshare_container_at_index(roaring_array_t *ra, - uint16_t i); +bool roaring_bitmap_contains_bulk(const roaring_bitmap_t *r, + roaring_bulk_context_t *context, + uint32_t val) { + uint16_t key = val >> 16; + if (context->container == NULL || context->key != key) { + int32_t start_idx = -1; + if (context->container != NULL && context->key < key) { + start_idx = context->idx; + } + int idx = ra_advance_until(&r->high_low_container, key, start_idx); + if (idx == ra_get_size(&r->high_low_container)) { + return false; + } + uint8_t typecode; + context->container = ra_get_container_at_index( + &r->high_low_container, (uint16_t)idx, &typecode); + context->typecode = typecode; + context->idx = idx; + context->key = + ra_get_key_at_index(&r->high_low_container, (uint16_t)idx); + // ra_advance_until finds the next key >= the target, we found a later + // container. + if (context->key != key) { + return false; + } + } + // context is now set up + return container_contains(context->container, val & 0xFFFF, + context->typecode); +} -extern inline void ra_replace_key_and_container_at_index(roaring_array_t *ra, - int32_t i, - uint16_t key, - container_t *c, - uint8_t typecode); +roaring_bitmap_t *roaring_bitmap_of_ptr(size_t n_args, const uint32_t *vals) { + roaring_bitmap_t *answer = roaring_bitmap_create(); + roaring_bitmap_add_many(answer, n_args, vals); + return answer; +} -extern inline void ra_set_container_at_index(const roaring_array_t *ra, - int32_t i, container_t *c, - uint8_t typecode); +roaring_bitmap_t *roaring_bitmap_of(size_t n_args, ...) { + // todo: could be greatly optimized but we do not expect this call to ever + // include long lists + roaring_bitmap_t *answer = roaring_bitmap_create(); + roaring_bulk_context_t context = CROARING_ZERO_INITIALIZER; + va_list ap; + va_start(ap, n_args); + for (size_t i = 0; i < n_args; i++) { + uint32_t val = va_arg(ap, uint32_t); + roaring_bitmap_add_bulk(answer, &context, val); + } + va_end(ap); + return answer; +} -static bool realloc_array(roaring_array_t *ra, int32_t new_capacity) { - // - // Note: not implemented using C's realloc(), because the memory layout is - // Struct-of-Arrays vs. Array-of-Structs: - // https://github.com/RoaringBitmap/CRoaring/issues/256 +static inline uint64_t minimum_uint64(uint64_t a, uint64_t b) { + return (a < b) ? a : b; +} - if (new_capacity == 0) { - roaring_free(ra->containers); - ra->containers = NULL; - ra->keys = NULL; - ra->typecodes = NULL; - ra->allocation_size = 0; - return true; +roaring_bitmap_t *roaring_bitmap_from_range(uint64_t min, uint64_t max, + uint32_t step) { + if (max >= UINT64_C(0x100000000)) { + max = UINT64_C(0x100000000); } - const size_t memoryneeded = - new_capacity * - (sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t)); - void *bigalloc = roaring_malloc(memoryneeded); - if (!bigalloc) return false; - void *oldbigalloc = ra->containers; - container_t **newcontainers = (container_t **)bigalloc; - uint16_t *newkeys = (uint16_t *)(newcontainers + new_capacity); - uint8_t *newtypecodes = (uint8_t *)(newkeys + new_capacity); - assert((char *)(newtypecodes + new_capacity) == - (char *)bigalloc + memoryneeded); - if (ra->size > 0) { - memcpy(newcontainers, ra->containers, sizeof(container_t *) * ra->size); - memcpy(newkeys, ra->keys, sizeof(uint16_t) * ra->size); - memcpy(newtypecodes, ra->typecodes, sizeof(uint8_t) * ra->size); + if (step == 0) return NULL; + if (max <= min) return NULL; + roaring_bitmap_t *answer = roaring_bitmap_create(); + if (step >= (1 << 16)) { + for (uint32_t value = (uint32_t)min; value < max; value += step) { + roaring_bitmap_add(answer, value); + } + return answer; } - ra->containers = newcontainers; - ra->keys = newkeys; - ra->typecodes = newtypecodes; - ra->allocation_size = new_capacity; - roaring_free(oldbigalloc); - return true; + uint64_t min_tmp = min; + do { + uint32_t key = (uint32_t)min_tmp >> 16; + uint32_t container_min = min_tmp & 0xFFFF; + uint32_t container_max = + (uint32_t)minimum_uint64(max - (key << 16), 1 << 16); + uint8_t type; + container_t *container = container_from_range( + &type, container_min, container_max, (uint16_t)step); + ra_append(&answer->high_low_container, (uint16_t)key, container, type); + uint32_t gap = container_max - container_min + step - 1; + min_tmp += gap - (gap % step); + } while (min_tmp < max); + // cardinality of bitmap will be ((uint64_t) max - min + step - 1 ) / step + return answer; } -bool ra_init_with_capacity(roaring_array_t *new_ra, uint32_t cap) { - if (!new_ra) return false; - ra_init(new_ra); - - // Containers hold 64Ki elements, so 64Ki containers is enough to hold - // `0x10000 * 0x10000` (all 2^32) elements - if (cap > 0x10000) { - cap = 0x10000; +void roaring_bitmap_add_range_closed(roaring_bitmap_t *r, uint32_t min, + uint32_t max) { + if (min > max) { + return; } - if (cap > 0) { - void *bigalloc = roaring_malloc( - cap * (sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t))); - if (bigalloc == NULL) return false; - new_ra->containers = (container_t **)bigalloc; - new_ra->keys = (uint16_t *)(new_ra->containers + cap); - new_ra->typecodes = (uint8_t *)(new_ra->keys + cap); - // Narrowing is safe because of above check - new_ra->allocation_size = (int32_t)cap; + roaring_array_t *ra = &r->high_low_container; + + uint32_t min_key = min >> 16; + uint32_t max_key = max >> 16; + + int32_t num_required_containers = max_key - min_key + 1; + int32_t suffix_length = + count_greater(ra->keys, ra->size, (uint16_t)max_key); + int32_t prefix_length = + count_less(ra->keys, ra->size - suffix_length, (uint16_t)min_key); + int32_t common_length = ra->size - prefix_length - suffix_length; + + if (num_required_containers > common_length) { + ra_shift_tail(ra, suffix_length, + num_required_containers - common_length); } - return true; -} -int ra_shrink_to_fit(roaring_array_t *ra) { - int savings = (ra->allocation_size - ra->size) * - (sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t)); - if (!realloc_array(ra, ra->size)) { - return 0; + int32_t src = prefix_length + common_length - 1; + int32_t dst = ra->size - suffix_length - 1; + for (uint32_t key = max_key; key != min_key - 1; + key--) { // beware of min_key==0 + uint32_t container_min = (min_key == key) ? (min & 0xffff) : 0; + uint32_t container_max = (max_key == key) ? (max & 0xffff) : 0xffff; + container_t *new_container; + uint8_t new_type; + + if (src >= 0 && ra->keys[src] == key) { + ra_unshare_container_at_index(ra, (uint16_t)src); + new_container = + container_add_range(ra->containers[src], ra->typecodes[src], + container_min, container_max, &new_type); + if (new_container != ra->containers[src]) { + container_free(ra->containers[src], ra->typecodes[src]); + } + src--; + } else { + new_container = container_from_range(&new_type, container_min, + container_max + 1, 1); + } + ra_replace_key_and_container_at_index(ra, dst, (uint16_t)key, + new_container, new_type); + dst--; } - ra->allocation_size = ra->size; - return savings; } -void ra_init(roaring_array_t *new_ra) { - if (!new_ra) { +void roaring_bitmap_remove_range_closed(roaring_bitmap_t *r, uint32_t min, + uint32_t max) { + if (min > max) { return; } - new_ra->keys = NULL; - new_ra->containers = NULL; - new_ra->typecodes = NULL; - new_ra->allocation_size = 0; - new_ra->size = 0; - new_ra->flags = 0; -} + roaring_array_t *ra = &r->high_low_container; -bool ra_overwrite(const roaring_array_t *source, roaring_array_t *dest, - bool copy_on_write) { - ra_clear_containers(dest); // we are going to overwrite them - if (source->size == 0) { // Note: can't call memcpy(NULL), even w/size - dest->size = 0; // <--- This is important. - return true; // output was just cleared, so they match - } - if (dest->allocation_size < source->size) { - if (!realloc_array(dest, source->size)) { - return false; - } - } - dest->size = source->size; - memcpy(dest->keys, source->keys, dest->size * sizeof(uint16_t)); - // we go through the containers, turning them into shared containers... - if (copy_on_write) { - for (int32_t i = 0; i < dest->size; ++i) { - source->containers[i] = get_copy_of_container( - source->containers[i], &source->typecodes[i], copy_on_write); + uint32_t min_key = min >> 16; + uint32_t max_key = max >> 16; + + int32_t src = count_less(ra->keys, ra->size, (uint16_t)min_key); + int32_t dst = src; + while (src < ra->size && ra->keys[src] <= max_key) { + uint32_t container_min = + (min_key == ra->keys[src]) ? (min & 0xffff) : 0; + uint32_t container_max = + (max_key == ra->keys[src]) ? (max & 0xffff) : 0xffff; + ra_unshare_container_at_index(ra, (uint16_t)src); + container_t *new_container; + uint8_t new_type; + new_container = + container_remove_range(ra->containers[src], ra->typecodes[src], + container_min, container_max, &new_type); + if (new_container != ra->containers[src]) { + container_free(ra->containers[src], ra->typecodes[src]); } - // we do a shallow copy to the other bitmap - memcpy(dest->containers, source->containers, - dest->size * sizeof(container_t *)); - memcpy(dest->typecodes, source->typecodes, - dest->size * sizeof(uint8_t)); - } else { - memcpy(dest->typecodes, source->typecodes, - dest->size * sizeof(uint8_t)); - for (int32_t i = 0; i < dest->size; i++) { - dest->containers[i] = - container_clone(source->containers[i], source->typecodes[i]); - if (dest->containers[i] == NULL) { - for (int32_t j = 0; j < i; j++) { - container_free(dest->containers[j], dest->typecodes[j]); - } - ra_clear_without_containers(dest); - return false; - } + if (new_container) { + ra_replace_key_and_container_at_index(ra, dst, ra->keys[src], + new_container, new_type); + dst++; } + src++; } - return true; -} - -void ra_clear_containers(roaring_array_t *ra) { - for (int32_t i = 0; i < ra->size; ++i) { - container_free(ra->containers[i], ra->typecodes[i]); + if (src > dst) { + ra_shift_tail(ra, ra->size - src, dst - src); } } -void ra_reset(roaring_array_t *ra) { - ra_clear_containers(ra); - ra->size = 0; - ra_shrink_to_fit(ra); -} - -void ra_clear_without_containers(roaring_array_t *ra) { - roaring_free( - ra->containers); // keys and typecodes are allocated with containers - ra->size = 0; - ra->allocation_size = 0; - ra->containers = NULL; - ra->keys = NULL; - ra->typecodes = NULL; -} +void roaring_bitmap_printf(const roaring_bitmap_t *r) { + const roaring_array_t *ra = &r->high_low_container; -void ra_clear(roaring_array_t *ra) { - ra_clear_containers(ra); - ra_clear_without_containers(ra); -} + printf("{"); + for (int i = 0; i < ra->size; ++i) { + container_printf_as_uint32_array(ra->containers[i], ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16); -bool extend_array(roaring_array_t *ra, int32_t k) { - int32_t desired_size = ra->size + k; - const int32_t max_containers = 65536; - assert(desired_size <= max_containers); - if (desired_size > ra->allocation_size) { - int32_t new_capacity = - (ra->size < 1024) ? 2 * desired_size : 5 * desired_size / 4; - if (new_capacity > max_containers) { - new_capacity = max_containers; + if (i + 1 < ra->size) { + printf(","); } - - return realloc_array(ra, new_capacity); } - return true; + printf("}"); } -void ra_append(roaring_array_t *ra, uint16_t key, container_t *c, - uint8_t typecode) { - extend_array(ra, 1); - const int32_t pos = ra->size; - - ra->keys[pos] = key; - ra->containers[pos] = c; - ra->typecodes[pos] = typecode; - ra->size++; -} +void roaring_bitmap_printf_describe(const roaring_bitmap_t *r) { + const roaring_array_t *ra = &r->high_low_container; -void ra_append_copy(roaring_array_t *ra, const roaring_array_t *sa, - uint16_t index, bool copy_on_write) { - extend_array(ra, 1); - const int32_t pos = ra->size; + printf("{"); + for (int i = 0; i < ra->size; ++i) { + printf("%d: %s (%d)", ra->keys[i], + get_full_container_name(ra->containers[i], ra->typecodes[i]), + container_get_cardinality(ra->containers[i], ra->typecodes[i])); + if (ra->typecodes[i] == SHARED_CONTAINER_TYPE) { + printf("(shared count = %" PRIu32 " )", + croaring_refcount_get( + &(CAST_shared(ra->containers[i])->counter))); + } - // old contents is junk that does not need freeing - ra->keys[pos] = sa->keys[index]; - // the shared container will be in two bitmaps - if (copy_on_write) { - sa->containers[index] = get_copy_of_container( - sa->containers[index], &sa->typecodes[index], copy_on_write); - ra->containers[pos] = sa->containers[index]; - ra->typecodes[pos] = sa->typecodes[index]; - } else { - ra->containers[pos] = - container_clone(sa->containers[index], sa->typecodes[index]); - ra->typecodes[pos] = sa->typecodes[index]; + if (i + 1 < ra->size) { + printf(", "); + } } - ra->size++; + printf("}"); } -void ra_append_copies_until(roaring_array_t *ra, const roaring_array_t *sa, - uint16_t stopping_key, bool copy_on_write) { - for (int32_t i = 0; i < sa->size; ++i) { - if (sa->keys[i] >= stopping_key) break; - ra_append_copy(ra, sa, (uint16_t)i, copy_on_write); +/** + * (For advanced users.) + * Collect statistics about the bitmap + */ +void roaring_bitmap_statistics(const roaring_bitmap_t *r, + roaring_statistics_t *stat) { + const roaring_array_t *ra = &r->high_low_container; + + memset(stat, 0, sizeof(*stat)); + stat->n_containers = ra->size; + stat->min_value = roaring_bitmap_minimum(r); + stat->max_value = roaring_bitmap_maximum(r); + + for (int i = 0; i < ra->size; ++i) { + uint8_t truetype = + get_container_type(ra->containers[i], ra->typecodes[i]); + uint32_t card = + container_get_cardinality(ra->containers[i], ra->typecodes[i]); + uint32_t sbytes = + container_size_in_bytes(ra->containers[i], ra->typecodes[i]); + stat->cardinality += card; + switch (truetype) { + case BITSET_CONTAINER_TYPE: + stat->n_bitset_containers++; + stat->n_values_bitset_containers += card; + stat->n_bytes_bitset_containers += sbytes; + break; + case ARRAY_CONTAINER_TYPE: + stat->n_array_containers++; + stat->n_values_array_containers += card; + stat->n_bytes_array_containers += sbytes; + break; + case RUN_CONTAINER_TYPE: + stat->n_run_containers++; + stat->n_values_run_containers += card; + stat->n_bytes_run_containers += sbytes; + break; + default: + assert(false); + roaring_unreachable; + } } } -void ra_append_copy_range(roaring_array_t *ra, const roaring_array_t *sa, - int32_t start_index, int32_t end_index, - bool copy_on_write) { - extend_array(ra, end_index - start_index); - for (int32_t i = start_index; i < end_index; ++i) { - const int32_t pos = ra->size; - ra->keys[pos] = sa->keys[i]; - if (copy_on_write) { - sa->containers[i] = get_copy_of_container( - sa->containers[i], &sa->typecodes[i], copy_on_write); - ra->containers[pos] = sa->containers[i]; - ra->typecodes[pos] = sa->typecodes[i]; - } else { - ra->containers[pos] = - container_clone(sa->containers[i], sa->typecodes[i]); - ra->typecodes[pos] = sa->typecodes[i]; +bool roaring_contains_shared(const roaring_bitmap_t *r) { + const roaring_array_t *ra = &r->high_low_container; + for (int i = 0; i < ra->size; ++i) { + if (ra->typecodes[i] == SHARED_CONTAINER_TYPE) { + return true; } - ra->size++; } + return false; } -void ra_append_copies_after(roaring_array_t *ra, const roaring_array_t *sa, - uint16_t before_start, bool copy_on_write) { - int start_location = ra_get_index(sa, before_start); - if (start_location >= 0) - ++start_location; - else - start_location = -start_location - 1; - ra_append_copy_range(ra, sa, start_location, sa->size, copy_on_write); +bool roaring_unshare_all(roaring_bitmap_t *r) { + const roaring_array_t *ra = &r->high_low_container; + bool unshared = false; + for (int i = 0; i < ra->size; ++i) { + uint8_t typecode = ra->typecodes[i]; + if (typecode == SHARED_CONTAINER_TYPE) { + ra->containers[i] = get_writable_copy_if_shared(ra->containers[i], + &ra->typecodes[i]); + unshared = true; + } + } + return unshared; } -void ra_append_move_range(roaring_array_t *ra, roaring_array_t *sa, - int32_t start_index, int32_t end_index) { - extend_array(ra, end_index - start_index); +/* + * Checks that: + * - Array containers are sorted and contain no duplicates + * - Range containers are sorted and contain no overlapping ranges + * - Roaring containers are sorted by key and there are no duplicate keys + * - The correct container type is use for each container (e.g. bitmaps aren't + * used for small containers) + * - Shared containers are only used when the bitmap is COW + */ +bool roaring_bitmap_internal_validate(const roaring_bitmap_t *r, + const char **reason) { + const char *reason_local; + if (reason == NULL) { + // Always allow assigning through *reason + reason = &reason_local; + } + *reason = NULL; + const roaring_array_t *ra = &r->high_low_container; + if (ra->size < 0) { + *reason = "negative size"; + return false; + } + if (ra->allocation_size < 0) { + *reason = "negative allocation size"; + return false; + } + if (ra->size > ra->allocation_size) { + *reason = "more containers than allocated space"; + return false; + } + if (ra->flags & ~(ROARING_FLAG_COW | ROARING_FLAG_FROZEN)) { + *reason = "invalid flags"; + return false; + } + if (ra->size == 0) { + return true; + } - for (int32_t i = start_index; i < end_index; ++i) { - const int32_t pos = ra->size; + if (ra->keys == NULL) { + *reason = "keys is NULL"; + return false; + } + if (ra->typecodes == NULL) { + *reason = "typecodes is NULL"; + return false; + } + if (ra->containers == NULL) { + *reason = "containers is NULL"; + return false; + } - ra->keys[pos] = sa->keys[i]; - ra->containers[pos] = sa->containers[i]; - ra->typecodes[pos] = sa->typecodes[i]; - ra->size++; + uint32_t prev_key = ra->keys[0]; + for (int32_t i = 1; i < ra->size; ++i) { + if (ra->keys[i] <= prev_key) { + *reason = "keys not strictly increasing"; + return false; + } + prev_key = ra->keys[i]; } -} -void ra_append_range(roaring_array_t *ra, roaring_array_t *sa, - int32_t start_index, int32_t end_index, - bool copy_on_write) { - extend_array(ra, end_index - start_index); + bool cow = roaring_bitmap_get_copy_on_write(r); - for (int32_t i = start_index; i < end_index; ++i) { - const int32_t pos = ra->size; - ra->keys[pos] = sa->keys[i]; - if (copy_on_write) { - sa->containers[i] = get_copy_of_container( - sa->containers[i], &sa->typecodes[i], copy_on_write); - ra->containers[pos] = sa->containers[i]; - ra->typecodes[pos] = sa->typecodes[i]; - } else { - ra->containers[pos] = - container_clone(sa->containers[i], sa->typecodes[i]); - ra->typecodes[pos] = sa->typecodes[i]; + for (int32_t i = 0; i < ra->size; ++i) { + if (ra->typecodes[i] == SHARED_CONTAINER_TYPE && !cow) { + *reason = "shared container in non-COW bitmap"; + return false; + } + if (!container_internal_validate(ra->containers[i], ra->typecodes[i], + reason)) { + // reason should already be set + if (*reason == NULL) { + *reason = "container failed to validate but no reason given"; + } + return false; } - ra->size++; } -} -container_t *ra_get_container(roaring_array_t *ra, uint16_t x, - uint8_t *typecode) { - int i = binarySearch(ra->keys, (int32_t)ra->size, x); - if (i < 0) return NULL; - *typecode = ra->typecodes[i]; - return ra->containers[i]; + return true; } -extern inline container_t *ra_get_container_at_index(const roaring_array_t *ra, - uint16_t i, - uint8_t *typecode); - -extern inline uint16_t ra_get_key_at_index(const roaring_array_t *ra, - uint16_t i); - -extern inline int32_t ra_get_index(const roaring_array_t *ra, uint16_t x); - -extern inline int32_t ra_advance_until(const roaring_array_t *ra, uint16_t x, - int32_t pos); - -// everything skipped over is freed -int32_t ra_advance_until_freeing(roaring_array_t *ra, uint16_t x, int32_t pos) { - while (pos < ra->size && ra->keys[pos] < x) { - container_free(ra->containers[pos], ra->typecodes[pos]); - ++pos; +roaring_bitmap_t *roaring_bitmap_copy(const roaring_bitmap_t *r) { + roaring_bitmap_t *ans = + (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); + if (!ans) { + return NULL; } - return pos; -} - -void ra_insert_new_key_value_at(roaring_array_t *ra, int32_t i, uint16_t key, - container_t *c, uint8_t typecode) { - extend_array(ra, 1); - // May be an optimization opportunity with DIY memmove - memmove(&(ra->keys[i + 1]), &(ra->keys[i]), - sizeof(uint16_t) * (ra->size - i)); - memmove(&(ra->containers[i + 1]), &(ra->containers[i]), - sizeof(container_t *) * (ra->size - i)); - memmove(&(ra->typecodes[i + 1]), &(ra->typecodes[i]), - sizeof(uint8_t) * (ra->size - i)); - ra->keys[i] = key; - ra->containers[i] = c; - ra->typecodes[i] = typecode; - ra->size++; + if (!ra_init_with_capacity( // allocation of list of containers can fail + &ans->high_low_container, r->high_low_container.size)) { + roaring_free(ans); + return NULL; + } + if (!ra_overwrite( // memory allocation of individual containers may fail + &r->high_low_container, &ans->high_low_container, is_cow(r))) { + roaring_bitmap_free(ans); // overwrite should leave in freeable state + return NULL; + } + roaring_bitmap_set_copy_on_write(ans, is_cow(r)); + return ans; } -// note: Java routine set things to 0, enabling GC. -// Java called it "resize" but it was always used to downsize. -// Allowing upsize would break the conventions about -// valid containers below ra->size. - -void ra_downsize(roaring_array_t *ra, int32_t new_length) { - assert(new_length <= ra->size); - ra->size = new_length; +bool roaring_bitmap_overwrite(roaring_bitmap_t *dest, + const roaring_bitmap_t *src) { + roaring_bitmap_set_copy_on_write(dest, is_cow(src)); + return ra_overwrite(&src->high_low_container, &dest->high_low_container, + is_cow(src)); } -void ra_remove_at_index(roaring_array_t *ra, int32_t i) { - memmove(&(ra->containers[i]), &(ra->containers[i + 1]), - sizeof(container_t *) * (ra->size - i - 1)); - memmove(&(ra->keys[i]), &(ra->keys[i + 1]), - sizeof(uint16_t) * (ra->size - i - 1)); - memmove(&(ra->typecodes[i]), &(ra->typecodes[i + 1]), - sizeof(uint8_t) * (ra->size - i - 1)); - ra->size--; +void roaring_bitmap_free(const roaring_bitmap_t *r) { + if (r == NULL) { + return; + } + if (!is_frozen(r)) { + ra_clear((roaring_array_t *)&r->high_low_container); + } + roaring_free((roaring_bitmap_t *)r); } -void ra_remove_at_index_and_free(roaring_array_t *ra, int32_t i) { - container_free(ra->containers[i], ra->typecodes[i]); - ra_remove_at_index(ra, i); +void roaring_bitmap_clear(roaring_bitmap_t *r) { + ra_reset(&r->high_low_container); } -// used in inplace andNot only, to slide left the containers from -// the mutated RoaringBitmap that are after the largest container of -// the argument RoaringBitmap. In use it should be followed by a call to -// downsize. -// -void ra_copy_range(roaring_array_t *ra, uint32_t begin, uint32_t end, - uint32_t new_begin) { - assert(begin <= end); - assert(new_begin < begin); - - const int range = end - begin; - - // We ensure to previously have freed overwritten containers - // that are not copied elsewhere - - memmove(&(ra->containers[new_begin]), &(ra->containers[begin]), - sizeof(container_t *) * range); - memmove(&(ra->keys[new_begin]), &(ra->keys[begin]), - sizeof(uint16_t) * range); - memmove(&(ra->typecodes[new_begin]), &(ra->typecodes[begin]), - sizeof(uint8_t) * range); -} +void roaring_bitmap_add(roaring_bitmap_t *r, uint32_t val) { + roaring_array_t *ra = &r->high_low_container; -void ra_shift_tail(roaring_array_t *ra, int32_t count, int32_t distance) { - if (distance > 0) { - extend_array(ra, distance); + const uint16_t hb = val >> 16; + const int i = ra_get_index(ra, hb); + uint8_t typecode; + if (i >= 0) { + ra_unshare_container_at_index(ra, (uint16_t)i); + container_t *container = + ra_get_container_at_index(ra, (uint16_t)i, &typecode); + uint8_t newtypecode = typecode; + container_t *container2 = + container_add(container, val & 0xFFFF, typecode, &newtypecode); + if (container2 != container) { + container_free(container, typecode); + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); + } + } else { + array_container_t *newac = array_container_create(); + container_t *container = + container_add(newac, val & 0xFFFF, ARRAY_CONTAINER_TYPE, &typecode); + // we could just assume that it stays an array container + ra_insert_new_key_value_at(&r->high_low_container, -i - 1, hb, + container, typecode); } - int32_t srcpos = ra->size - count; - int32_t dstpos = srcpos + distance; - memmove(&(ra->keys[dstpos]), &(ra->keys[srcpos]), sizeof(uint16_t) * count); - memmove(&(ra->containers[dstpos]), &(ra->containers[srcpos]), - sizeof(container_t *) * count); - memmove(&(ra->typecodes[dstpos]), &(ra->typecodes[srcpos]), - sizeof(uint8_t) * count); - ra->size += distance; } -void ra_to_uint32_array(const roaring_array_t *ra, uint32_t *ans) { - size_t ctr = 0; - for (int32_t i = 0; i < ra->size; ++i) { - int num_added = container_to_uint32_array( - ans + ctr, ra->containers[i], ra->typecodes[i], - ((uint32_t)ra->keys[i]) << 16); - ctr += num_added; - } -} +bool roaring_bitmap_add_checked(roaring_bitmap_t *r, uint32_t val) { + const uint16_t hb = val >> 16; + const int i = ra_get_index(&r->high_low_container, hb); + uint8_t typecode; + bool result = false; + if (i >= 0) { + ra_unshare_container_at_index(&r->high_low_container, (uint16_t)i); + container_t *container = ra_get_container_at_index( + &r->high_low_container, (uint16_t)i, &typecode); -bool ra_has_run_container(const roaring_array_t *ra) { - for (int32_t k = 0; k < ra->size; ++k) { - if (get_container_type(ra->containers[k], ra->typecodes[k]) == - RUN_CONTAINER_TYPE) - return true; - } - return false; -} + const int oldCardinality = + container_get_cardinality(container, typecode); -uint32_t ra_portable_header_size(const roaring_array_t *ra) { - if (ra_has_run_container(ra)) { - if (ra->size < - NO_OFFSET_THRESHOLD) { // for small bitmaps, we omit the offsets - return 4 + (ra->size + 7) / 8 + 4 * ra->size; + uint8_t newtypecode = typecode; + container_t *container2 = + container_add(container, val & 0xFFFF, typecode, &newtypecode); + if (container2 != container) { + container_free(container, typecode); + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); + result = true; + } else { + const int newCardinality = + container_get_cardinality(container, newtypecode); + + result = oldCardinality != newCardinality; } - return 4 + (ra->size + 7) / 8 + - 8 * ra->size; // - 4 because we pack the size with the cookie } else { - return 4 + 4 + 8 * ra->size; + array_container_t *newac = array_container_create(); + container_t *container = + container_add(newac, val & 0xFFFF, ARRAY_CONTAINER_TYPE, &typecode); + // we could just assume that it stays an array container + ra_insert_new_key_value_at(&r->high_low_container, -i - 1, hb, + container, typecode); + result = true; } -} -size_t ra_portable_size_in_bytes(const roaring_array_t *ra) { - size_t count = ra_portable_header_size(ra); + return result; +} - for (int32_t k = 0; k < ra->size; ++k) { - count += container_size_in_bytes(ra->containers[k], ra->typecodes[k]); +void roaring_bitmap_remove(roaring_bitmap_t *r, uint32_t val) { + const uint16_t hb = val >> 16; + const int i = ra_get_index(&r->high_low_container, hb); + uint8_t typecode; + if (i >= 0) { + ra_unshare_container_at_index(&r->high_low_container, (uint16_t)i); + container_t *container = ra_get_container_at_index( + &r->high_low_container, (uint16_t)i, &typecode); + uint8_t newtypecode = typecode; + container_t *container2 = + container_remove(container, val & 0xFFFF, typecode, &newtypecode); + if (container2 != container) { + container_free(container, typecode); + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); + } + if (container_nonzero_cardinality(container2, newtypecode)) { + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); + } else { + ra_remove_at_index_and_free(&r->high_low_container, i); + } } - return count; } -// This function is endian-sensitive. -size_t ra_portable_serialize(const roaring_array_t *ra, char *buf) { - char *initbuf = buf; - uint32_t startOffset = 0; - bool hasrun = ra_has_run_container(ra); - if (hasrun) { - uint32_t cookie = SERIAL_COOKIE | ((uint32_t)(ra->size - 1) << 16); - memcpy(buf, &cookie, sizeof(cookie)); - buf += sizeof(cookie); - uint32_t s = (ra->size + 7) / 8; - memset(buf, 0, s); - for (int32_t i = 0; i < ra->size; ++i) { - if (get_container_type(ra->containers[i], ra->typecodes[i]) == - RUN_CONTAINER_TYPE) { - buf[i / 8] |= 1 << (i % 8); - } +bool roaring_bitmap_remove_checked(roaring_bitmap_t *r, uint32_t val) { + const uint16_t hb = val >> 16; + const int i = ra_get_index(&r->high_low_container, hb); + uint8_t typecode; + bool result = false; + if (i >= 0) { + ra_unshare_container_at_index(&r->high_low_container, (uint16_t)i); + container_t *container = ra_get_container_at_index( + &r->high_low_container, (uint16_t)i, &typecode); + + const int oldCardinality = + container_get_cardinality(container, typecode); + + uint8_t newtypecode = typecode; + container_t *container2 = + container_remove(container, val & 0xFFFF, typecode, &newtypecode); + if (container2 != container) { + container_free(container, typecode); + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); } - buf += s; - if (ra->size < NO_OFFSET_THRESHOLD) { - startOffset = 4 + 4 * ra->size + s; + + const int newCardinality = + container_get_cardinality(container2, newtypecode); + + if (newCardinality != 0) { + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); } else { - startOffset = 4 + 8 * ra->size + s; + ra_remove_at_index_and_free(&r->high_low_container, i); } - } else { // backwards compatibility - uint32_t cookie = SERIAL_COOKIE_NO_RUNCONTAINER; - - memcpy(buf, &cookie, sizeof(cookie)); - buf += sizeof(cookie); - memcpy(buf, &ra->size, sizeof(ra->size)); - buf += sizeof(ra->size); - startOffset = 4 + 4 + 4 * ra->size + 4 * ra->size; + result = oldCardinality != newCardinality; } - for (int32_t k = 0; k < ra->size; ++k) { - memcpy(buf, &ra->keys[k], sizeof(ra->keys[k])); - buf += sizeof(ra->keys[k]); - // get_cardinality returns a value in [1,1<<16], subtracting one - // we get [0,1<<16 - 1] which fits in 16 bits - uint16_t card = (uint16_t)(container_get_cardinality(ra->containers[k], - ra->typecodes[k]) - - 1); - memcpy(buf, &card, sizeof(card)); - buf += sizeof(card); + return result; +} + +void roaring_bitmap_remove_many(roaring_bitmap_t *r, size_t n_args, + const uint32_t *vals) { + if (n_args == 0 || r->high_low_container.size == 0) { + return; } - if ((!hasrun) || (ra->size >= NO_OFFSET_THRESHOLD)) { - // writing the containers offsets - for (int32_t k = 0; k < ra->size; k++) { - memcpy(buf, &startOffset, sizeof(startOffset)); - buf += sizeof(startOffset); - startOffset = - startOffset + - container_size_in_bytes(ra->containers[k], ra->typecodes[k]); + int32_t pos = + -1; // position of the container used in the previous iteration + for (size_t i = 0; i < n_args; i++) { + uint16_t key = (uint16_t)(vals[i] >> 16); + if (pos < 0 || key != r->high_low_container.keys[pos]) { + pos = ra_get_index(&r->high_low_container, key); + } + if (pos >= 0) { + uint8_t new_typecode; + container_t *new_container; + new_container = container_remove( + r->high_low_container.containers[pos], vals[i] & 0xffff, + r->high_low_container.typecodes[pos], &new_typecode); + if (new_container != r->high_low_container.containers[pos]) { + container_free(r->high_low_container.containers[pos], + r->high_low_container.typecodes[pos]); + ra_replace_key_and_container_at_index(&r->high_low_container, + pos, key, new_container, + new_typecode); + } + if (!container_nonzero_cardinality(new_container, new_typecode)) { + container_free(new_container, new_typecode); + ra_remove_at_index(&r->high_low_container, pos); + pos = -1; + } } } - for (int32_t k = 0; k < ra->size; ++k) { - buf += container_write(ra->containers[k], ra->typecodes[k], buf); - } - return buf - initbuf; } -// Quickly checks whether there is a serialized bitmap at the pointer, -// not exceeding size "maxbytes" in bytes. This function does not allocate -// memory dynamically. -// -// This function returns 0 if and only if no valid bitmap is found. -// Otherwise, it returns how many bytes are occupied. -// -size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { - size_t bytestotal = sizeof(int32_t); // for cookie - if (bytestotal > maxbytes) return 0; - uint32_t cookie; - memcpy(&cookie, buf, sizeof(int32_t)); - buf += sizeof(uint32_t); - if ((cookie & 0xFFFF) != SERIAL_COOKIE && - cookie != SERIAL_COOKIE_NO_RUNCONTAINER) { - return 0; - } - int32_t size; +// there should be some SIMD optimizations possible here +roaring_bitmap_t *roaring_bitmap_and(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + uint32_t neededcap = length1 > length2 ? length2 : length1; + roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(neededcap); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); - if ((cookie & 0xFFFF) == SERIAL_COOKIE) - size = (cookie >> 16) + 1; - else { - bytestotal += sizeof(int32_t); - if (bytestotal > maxbytes) return 0; - memcpy(&size, buf, sizeof(int32_t)); - buf += sizeof(uint32_t); - } - if (size > (1 << 16) || size < 0) { - return 0; + int pos1 = 0, pos2 = 0; + + while (pos1 < length1 && pos2 < length2) { + const uint16_t s1 = + ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + const uint16_t s2 = + ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + if (s1 == s2) { + uint8_t type1, type2; + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = container_and(c1, type1, c2, type2, &result_type); + + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&answer->high_low_container, s1, c, result_type); + } else { + container_free(c, result_type); // otherwise: memory leak! + } + ++pos1; + ++pos2; + } else if (s1 < s2) { // s1 < s2 + pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + } } - char *bitmapOfRunContainers = NULL; - bool hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE; - if (hasrun) { - int32_t s = (size + 7) / 8; - bytestotal += s; - if (bytestotal > maxbytes) return 0; - bitmapOfRunContainers = (char *)buf; - buf += s; + return answer; +} + +/** + * Compute the union of 'number' bitmaps. + */ +roaring_bitmap_t *roaring_bitmap_or_many(size_t number, + const roaring_bitmap_t **x) { + if (number == 0) { + return roaring_bitmap_create(); } - bytestotal += size * 2 * sizeof(uint16_t); - if (bytestotal > maxbytes) return 0; - const char *keyscards = buf; - buf += size * 2 * sizeof(uint16_t); - if ((!hasrun) || (size >= NO_OFFSET_THRESHOLD)) { - // skipping the offsets - bytestotal += size * 4; - if (bytestotal > maxbytes) return 0; - buf += size * 4; + if (number == 1) { + return roaring_bitmap_copy(x[0]); } - // Reading the containers - for (int32_t k = 0; k < size; ++k) { - uint16_t tmp; - memcpy(&tmp, keyscards + 4 * k + 2, sizeof(tmp)); - uint32_t thiscard = tmp + 1; - bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); - bool isrun = false; - if (hasrun) { - if ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) { - isbitmap = false; - isrun = true; - } - } - if (isbitmap) { - size_t containersize = - BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); - bytestotal += containersize; - if (bytestotal > maxbytes) return 0; - buf += containersize; - } else if (isrun) { - bytestotal += sizeof(uint16_t); - if (bytestotal > maxbytes) return 0; - uint16_t n_runs; - memcpy(&n_runs, buf, sizeof(uint16_t)); - buf += sizeof(uint16_t); - size_t containersize = n_runs * sizeof(rle16_t); - bytestotal += containersize; - if (bytestotal > maxbytes) return 0; - buf += containersize; - } else { - size_t containersize = thiscard * sizeof(uint16_t); - bytestotal += containersize; - if (bytestotal > maxbytes) return 0; - buf += containersize; - } + roaring_bitmap_t *answer = + roaring_bitmap_lazy_or(x[0], x[1], CROARING_LAZY_OR_BITSET_CONVERSION); + for (size_t i = 2; i < number; i++) { + roaring_bitmap_lazy_or_inplace(answer, x[i], CROARING_LAZY_OR_BITSET_CONVERSION); } - return bytestotal; + roaring_bitmap_repair_after_lazy(answer); + return answer; } -// This function populates answer from the content of buf (reading up to -// maxbytes bytes). The function returns false if a properly serialized bitmap -// cannot be found. If it returns true, readbytes is populated by how many bytes -// were read, we have that *readbytes <= maxbytes. -// -// This function is endian-sensitive. -bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, - const size_t maxbytes, size_t *readbytes) { - *readbytes = sizeof(int32_t); // for cookie - if (*readbytes > maxbytes) { - // Ran out of bytes while reading first 4 bytes. - return false; +/** + * Compute the xor of 'number' bitmaps. + */ +roaring_bitmap_t *roaring_bitmap_xor_many(size_t number, + const roaring_bitmap_t **x) { + if (number == 0) { + return roaring_bitmap_create(); } - uint32_t cookie; - memcpy(&cookie, buf, sizeof(int32_t)); - buf += sizeof(uint32_t); - if ((cookie & 0xFFFF) != SERIAL_COOKIE && - cookie != SERIAL_COOKIE_NO_RUNCONTAINER) { - // "I failed to find one of the right cookies. - return false; + if (number == 1) { + return roaring_bitmap_copy(x[0]); } - int32_t size; + roaring_bitmap_t *answer = roaring_bitmap_lazy_xor(x[0], x[1]); + for (size_t i = 2; i < number; i++) { + roaring_bitmap_lazy_xor_inplace(answer, x[i]); + } + roaring_bitmap_repair_after_lazy(answer); + return answer; +} - if ((cookie & 0xFFFF) == SERIAL_COOKIE) - size = (cookie >> 16) + 1; - else { - *readbytes += sizeof(int32_t); - if (*readbytes > maxbytes) { - // Ran out of bytes while reading second part of the cookie. - return false; +// inplace and (modifies its first argument). +void roaring_bitmap_and_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + if (x1 == x2) return; + int pos1 = 0, pos2 = 0, intersection_size = 0; + const int length1 = ra_get_size(&x1->high_low_container); + const int length2 = ra_get_size(&x2->high_low_container); + + // any skipped-over or newly emptied containers in x1 + // have to be freed. + while (pos1 < length1 && pos2 < length2) { + const uint16_t s1 = + ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + const uint16_t s2 = + ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + if (s1 == s2) { + uint8_t type1, type2, result_type; + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + + // We do the computation "in place" only when c1 is not a shared + // container. Rationale: using a shared container safely with in + // place computation would require making a copy and then doing the + // computation in place which is likely less efficient than avoiding + // in place entirely and always generating a new container. + container_t *c = + (type1 == SHARED_CONTAINER_TYPE) + ? container_and(c1, type1, c2, type2, &result_type) + : container_iand(c1, type1, c2, type2, &result_type); + + if (c != c1) { // in this instance a new container was created, and + // we need to free the old one + container_free(c1, type1); + } + if (container_nonzero_cardinality(c, result_type)) { + ra_replace_key_and_container_at_index(&x1->high_low_container, + intersection_size, s1, c, + result_type); + intersection_size++; + } else { + container_free(c, result_type); + } + ++pos1; + ++pos2; + } else if (s1 < s2) { + pos1 = ra_advance_until_freeing(&x1->high_low_container, s2, pos1); + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); } - memcpy(&size, buf, sizeof(int32_t)); - buf += sizeof(uint32_t); } - if (size < 0) { - // You cannot have a negative number of containers, the data must be - // corrupted. - return false; + + // if we ended early because x2 ran out, then all remaining in x1 should be + // freed + while (pos1 < length1) { + container_free(x1->high_low_container.containers[pos1], + x1->high_low_container.typecodes[pos1]); + ++pos1; } - if (size > (1 << 16)) { - // You cannot have so many containers, the data must be corrupted. - return false; + + // all containers after this have either been copied or freed + ra_downsize(&x1->high_low_container, intersection_size); +} + +roaring_bitmap_t *roaring_bitmap_or(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + if (0 == length1) { + return roaring_bitmap_copy(x2); } - const char *bitmapOfRunContainers = NULL; - bool hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE; - if (hasrun) { - int32_t s = (size + 7) / 8; - *readbytes += s; - if (*readbytes > maxbytes) { // data is corrupted? - // Ran out of bytes while reading run bitmap. - return false; + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = + roaring_bitmap_create_with_capacity(length1 + length2); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = container_or(c1, type1, c2, type2, &result_type); + + // since we assume that the initial containers are non-empty, the + // result here + // can only be non-empty + ra_append(&answer->high_low_container, s1, c, result_type); + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + // c1 = container_clone(c1, type1); + c1 = get_copy_of_container(c1, &type1, is_cow(x1)); + if (is_cow(x1)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c1, + type1); + } + ra_append(&answer->high_low_container, s1, c1, type1); + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + // c2 = container_clone(c2, type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } - bitmapOfRunContainers = buf; - buf += s; } - const char *keyscards = buf; + if (pos1 == length1) { + ra_append_copy_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2, + is_cow(x2)); + } else if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); + } + return answer; +} - *readbytes += size * 2 * sizeof(uint16_t); - if (*readbytes > maxbytes) { - // Ran out of bytes while reading key-cardinality array. - return false; +static void roaring_inplace_merge_bulk(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2, int dst, + int left, int right, bool is_xor) { + roaring_array_t *ra1 = &x1->high_low_container; + const roaring_array_t *ra2 = &x2->high_low_container; + const bool cow2 = is_cow(x2); + const int length1 = ra1->size; + const int length2 = ra2->size; + + int distinct = 0; + { + int l = left, r = right; + while (l < length1 && r < length2) { + uint16_t k1 = ra1->keys[l]; + uint16_t k2 = ra2->keys[r]; + if (k1 < k2) { + l++; + } else if (k1 > k2) { + r++; + } else { + l++; + r++; + } + distinct++; + } + distinct += (length1 - l) + (length2 - r); } - buf += size * 2 * sizeof(uint16_t); + const int total = dst + distinct; - bool is_ok = ra_init_with_capacity(answer, size); - if (!is_ok) { - // Failed to allocate memory for roaring array. Bailing out. - return false; + roaring_array_t merged; + ra_init_with_capacity(&merged, total > 0 ? (uint32_t)total : 1); + + for (int i = 0; i < dst; i++) { + ra_append(&merged, ra1->keys[i], ra1->containers[i], ra1->typecodes[i]); } - for (int32_t k = 0; k < size; ++k) { - uint16_t tmp; - memcpy(&tmp, keyscards + 4 * k, sizeof(tmp)); - answer->keys[k] = tmp; + uint8_t result_type = 0; + while (left < length1 && right < length2) { + uint16_t k1 = ra1->keys[left]; + uint16_t k2 = ra2->keys[right]; + if (k1 < k2) { + ra_append(&merged, k1, ra1->containers[left], ra1->typecodes[left]); + left++; + } else if (k1 > k2) { + uint8_t type2 = ra2->typecodes[right]; + container_t *c2 = + get_copy_of_container(ra2->containers[right], &type2, cow2); + if (cow2) { + ra_set_container_at_index(ra2, right, c2, type2); + } + ra_append(&merged, k2, c2, type2); + right++; + } else { + uint8_t type1 = ra1->typecodes[left]; + container_t *c1 = ra1->containers[left]; + uint8_t type2 = ra2->typecodes[right]; + container_t *c2 = ra2->containers[right]; + if (is_xor) { + container_t *c; + if (type1 == SHARED_CONTAINER_TYPE) { + c = container_xor(c1, type1, c2, type2, &result_type); + shared_container_free(CAST_shared(c1)); + } else { + c = container_ixor(c1, type1, c2, type2, &result_type); + } + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&merged, k1, c, result_type); + } else { + container_free(c, result_type); + } + } else { + if (container_is_full(c1, type1)) { + ra_append(&merged, k1, c1, type1); + } else { + container_t *c = + (type1 == SHARED_CONTAINER_TYPE) + ? container_or(c1, type1, c2, type2, &result_type) + : container_ior(c1, type1, c2, type2, &result_type); + if (c != c1) { + container_free(c1, type1); + } + ra_append(&merged, k1, c, result_type); + } + } + left++; + right++; + } } - if ((!hasrun) || (size >= NO_OFFSET_THRESHOLD)) { - *readbytes += size * 4; - if (*readbytes > maxbytes) { // data is corrupted? - // Ran out of bytes while reading offsets. - ra_clear(answer); // we need to clear the containers already - // allocated, and the roaring array - return false; + for (; left < length1; left++) { + ra_append(&merged, ra1->keys[left], ra1->containers[left], + ra1->typecodes[left]); + } + for (; right < length2; right++) { + uint8_t type2 = ra2->typecodes[right]; + container_t *c2 = + get_copy_of_container(ra2->containers[right], &type2, cow2); + if (cow2) { + ra_set_container_at_index(ra2, right, c2, type2); } + ra_append(&merged, ra2->keys[right], c2, type2); + } - // skipping the offsets - buf += size * 4; + merged.flags = ra1->flags; + ra_clear_without_containers(ra1); + *ra1 = merged; +} + +// inplace or (modifies its first argument). +void roaring_bitmap_or_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; + + if (0 == length2) return; + + if (0 == length1) { + roaring_bitmap_overwrite(x1, x2); + return; } - // Reading the containers - for (int32_t k = 0; k < size; ++k) { - uint16_t tmp; - memcpy(&tmp, keyscards + 4 * k + 2, sizeof(tmp)); - uint32_t thiscard = tmp + 1; - bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); - bool isrun = false; - if (hasrun) { - if ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) { - isbitmap = false; - isrun = true; + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + if (!container_is_full(c1, type1)) { + container_t *c2 = ra_get_container_at_index( + &x2->high_low_container, (uint16_t)pos2, &type2); + container_t *c = + (type1 == SHARED_CONTAINER_TYPE) + ? container_or(c1, type1, c2, type2, &result_type) + : container_ior(c1, type1, c2, type2, &result_type); + + if (c != c1) { // in this instance a new container was created, + // and we need to free the old one + container_free(c1, type1); + } + ra_set_container_at_index(&x1->high_low_container, pos1, c, + result_type); } + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + roaring_inplace_merge_bulk(x1, x2, pos1, pos1, pos2, false); + return; } - if (isbitmap) { - // we check that the read is allowed - size_t containersize = - BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); - *readbytes += containersize; - if (*readbytes > maxbytes) { - // Running out of bytes while reading a bitset container. - ra_clear(answer); // we need to clear the containers already - // allocated, and the roaring array - return false; - } - // it is now safe to read - bitset_container_t *c = bitset_container_create(); - if (c == NULL) { // memory allocation failure - // Failed to allocate memory for a bitset container. - ra_clear(answer); // we need to clear the containers already - // allocated, and the roaring array - return false; - } - answer->size++; - buf += bitset_container_read(thiscard, c, buf); - answer->containers[k] = c; - answer->typecodes[k] = BITSET_CONTAINER_TYPE; - } else if (isrun) { - // we check that the read is allowed - *readbytes += sizeof(uint16_t); - if (*readbytes > maxbytes) { - // Running out of bytes while reading a run container (header). - ra_clear(answer); // we need to clear the containers already - // allocated, and the roaring array - return false; - } - uint16_t n_runs; - memcpy(&n_runs, buf, sizeof(uint16_t)); - size_t containersize = n_runs * sizeof(rle16_t); - *readbytes += containersize; - if (*readbytes > maxbytes) { // data is corrupted? - // Running out of bytes while reading a run container. - ra_clear(answer); // we need to clear the containers already - // allocated, and the roaring array - return false; - } - // it is now safe to read + } + if (pos1 == length1) { + ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, + pos2, length2, is_cow(x2)); + } +} - run_container_t *c = run_container_create(); - if (c == NULL) { // memory allocation failure - // Failed to allocate memory for a run container. - ra_clear(answer); // we need to clear the containers already - // allocated, and the roaring array - return false; +roaring_bitmap_t *roaring_bitmap_xor(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + if (0 == length1) { + return roaring_bitmap_copy(x2); + } + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = + roaring_bitmap_create_with_capacity(length1 + length2); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = container_xor(c1, type1, c2, type2, &result_type); + + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&answer->high_low_container, s1, c, result_type); + } else { + container_free(c, result_type); } - answer->size++; - buf += run_container_read(thiscard, c, buf); - answer->containers[k] = c; - answer->typecodes[k] = RUN_CONTAINER_TYPE; - } else { - // we check that the read is allowed - size_t containersize = thiscard * sizeof(uint16_t); - *readbytes += containersize; - if (*readbytes > maxbytes) { // data is corrupted? - // Running out of bytes while reading an array container. - ra_clear(answer); // we need to clear the containers already - // allocated, and the roaring array - return false; + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + c1 = get_copy_of_container(c1, &type1, is_cow(x1)); + if (is_cow(x1)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c1, + type1); } - // it is now safe to read - array_container_t *c = - array_container_create_given_capacity(thiscard); - if (c == NULL) { // memory allocation failure - // Failed to allocate memory for an array container. - ra_clear(answer); // we need to clear the containers already - // allocated, and the roaring array - return false; + ra_append(&answer->high_low_container, s1, c1, type1); + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); } - answer->size++; - buf += array_container_read(thiscard, c, buf); - answer->containers[k] = c; - answer->typecodes[k] = ARRAY_CONTAINER_TYPE; + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } } - return true; + if (pos1 == length1) { + ra_append_copy_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2, + is_cow(x2)); + } else if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); + } + return answer; } -#ifdef __cplusplus -} -} -} // extern "C" { namespace roaring { namespace internal { -#endif -/* end file src/roaring_array.c */ -/* begin file src/roaring_priority_queue.c */ +// inplace xor (modifies its first argument). -#ifdef __cplusplus -using namespace ::roaring::internal; +void roaring_bitmap_xor_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + assert(x1 != x2); + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; -extern "C" { -namespace roaring { -namespace api { -#endif + if (0 == length2) return; -struct roaring_pq_element_s { - uint64_t size; - bool is_temporary; - roaring_bitmap_t *bitmap; -}; + if (0 == length1) { + roaring_bitmap_overwrite(x1, x2); + return; + } -typedef struct roaring_pq_element_s roaring_pq_element_t; + // XOR can have new containers inserted from x2, but can also + // lose containers when x1 and x2 are nonempty and identical. -struct roaring_pq_s { - roaring_pq_element_t *elements; - uint64_t size; -}; + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); -typedef struct roaring_pq_s roaring_pq_t; + // We do the computation "in place" only when c1 is not a shared + // container. Rationale: using a shared container safely with in + // place computation would require making a copy and then doing the + // computation in place which is likely less efficient than avoiding + // in place entirely and always generating a new container. -static inline bool compare(roaring_pq_element_t *t1, roaring_pq_element_t *t2) { - return t1->size < t2->size; -} + container_t *c; + if (type1 == SHARED_CONTAINER_TYPE) { + c = container_xor(c1, type1, c2, type2, &result_type); + shared_container_free(CAST_shared(c1)); // so release + } else { + c = container_ixor(c1, type1, c2, type2, &result_type); + } -static void pq_add(roaring_pq_t *pq, roaring_pq_element_t *t) { - uint64_t i = pq->size; - pq->elements[pq->size++] = *t; - while (i > 0) { - uint64_t p = (i - 1) >> 1; - roaring_pq_element_t ap = pq->elements[p]; - if (!compare(t, &ap)) break; - pq->elements[i] = ap; - i = p; - } - pq->elements[i] = *t; -} + if (container_nonzero_cardinality(c, result_type)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c, + result_type); + ++pos1; + } else { + container_free(c, result_type); + roaring_inplace_merge_bulk(x1, x2, pos1, pos1 + 1, pos2 + 1, + true); + return; + } -static void pq_free(roaring_pq_t *pq) { roaring_free(pq); } + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); -static void percolate_down(roaring_pq_t *pq, uint32_t i) { - uint32_t size = (uint32_t)pq->size; - uint32_t hsize = size >> 1; - roaring_pq_element_t ai = pq->elements[i]; - while (i < hsize) { - uint32_t l = (i << 1) + 1; - uint32_t r = l + 1; - roaring_pq_element_t bestc = pq->elements[l]; - if (r < size) { - if (compare(pq->elements + r, &bestc)) { - l = r; - bestc = pq->elements[r]; - } - } - if (!compare(&bestc, &ai)) { - break; + } else if (s1 < s2) { // s1 < s2 + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + roaring_inplace_merge_bulk(x1, x2, pos1, pos1, pos2, true); + return; } - pq->elements[i] = bestc; - i = l; } - pq->elements[i] = ai; + if (pos1 == length1) { + ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, + pos2, length2, is_cow(x2)); + } } -static roaring_pq_t *create_pq(const roaring_bitmap_t **arr, uint32_t length) { - size_t alloc_size = - sizeof(roaring_pq_t) + sizeof(roaring_pq_element_t) * length; - roaring_pq_t *answer = (roaring_pq_t *)roaring_malloc(alloc_size); - answer->elements = (roaring_pq_element_t *)(answer + 1); - answer->size = length; - for (uint32_t i = 0; i < length; i++) { - answer->elements[i].bitmap = (roaring_bitmap_t *)arr[i]; - answer->elements[i].is_temporary = false; - answer->elements[i].size = - roaring_bitmap_portable_size_in_bytes(arr[i]); +roaring_bitmap_t *roaring_bitmap_andnot(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + if (0 == length1) { + roaring_bitmap_t *empty_bitmap = roaring_bitmap_create(); + roaring_bitmap_set_copy_on_write(empty_bitmap, + is_cow(x1) || is_cow(x2)); + return empty_bitmap; } - for (int32_t i = (length >> 1); i >= 0; i--) { - percolate_down(answer, i); + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(length1); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = 0; + uint16_t s2 = 0; + while (true) { + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = + container_andnot(c1, type1, c2, type2, &result_type); + + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&answer->high_low_container, s1, c, result_type); + } else { + container_free(c, result_type); + } + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + } else if (s1 < s2) { // s1 < s2 + const int next_pos1 = + ra_advance_until(&x1->high_low_container, s2, pos1); + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, next_pos1, + is_cow(x1)); + // TODO : perhaps some of the copy_on_write should be based on + // answer rather than x1 (more stringent?). Many similar cases + pos1 = next_pos1; + if (pos1 == length1) break; + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + if (pos2 == length2) break; + } + } + if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); } return answer; } -static roaring_pq_element_t pq_poll(roaring_pq_t *pq) { - roaring_pq_element_t ans = *pq->elements; - if (pq->size > 1) { - pq->elements[0] = pq->elements[--pq->size]; - percolate_down(pq, 0); - } else - --pq->size; - // memmove(pq->elements,pq->elements+1,(pq->size-1)*sizeof(roaring_pq_element_t));--pq->size; - return ans; -} +// inplace andnot (modifies its first argument). + +void roaring_bitmap_andnot_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + assert(x1 != x2); -// this function consumes and frees the inputs -static roaring_bitmap_t *lazy_or_from_lazy_inputs(roaring_bitmap_t *x1, - roaring_bitmap_t *x2) { uint8_t result_type = 0; - const int length1 = ra_get_size(&x1->high_low_container), - length2 = ra_get_size(&x2->high_low_container); + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; + int intersection_size = 0; + + if (0 == length2) return; + if (0 == length1) { - roaring_bitmap_free(x1); - return x2; - } - if (0 == length2) { - roaring_bitmap_free(x2); - return x1; + roaring_bitmap_clear(x1); + return; } - uint32_t neededcap = length1 > length2 ? length2 : length1; - roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(neededcap); + int pos1 = 0, pos2 = 0; uint8_t type1, type2; uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); while (true) { - if (s1 == s2) { - // todo: unsharing can be inefficient as it may create a clone where - // none - // is needed, but it has the benefit of being easy to reason about. - - ra_unshare_container_at_index(&x1->high_low_container, - (uint16_t)pos1); - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - assert(type1 != SHARED_CONTAINER_TYPE); - - ra_unshare_container_at_index(&x2->high_low_container, - (uint16_t)pos2); + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); container_t *c2 = ra_get_container_at_index(&x2->high_low_container, (uint16_t)pos2, &type2); - assert(type2 != SHARED_CONTAINER_TYPE); + + // We do the computation "in place" only when c1 is not a shared + // container. Rationale: using a shared container safely with in + // place computation would require making a copy and then doing the + // computation in place which is likely less efficient than avoiding + // in place entirely and always generating a new container. container_t *c; + if (type1 == SHARED_CONTAINER_TYPE) { + c = container_andnot(c1, type1, c2, type2, &result_type); + shared_container_free(CAST_shared(c1)); // release + } else { + c = container_iandnot(c1, type1, c2, type2, &result_type); + } - if ((type2 == BITSET_CONTAINER_TYPE) && - (type1 != BITSET_CONTAINER_TYPE)) { - c = container_lazy_ior(c2, type2, c1, type1, &result_type); - container_free(c1, type1); - if (c != c2) { - container_free(c2, type2); - } + if (container_nonzero_cardinality(c, result_type)) { + ra_replace_key_and_container_at_index(&x1->high_low_container, + intersection_size++, s1, + c, result_type); } else { - c = container_lazy_ior(c1, type1, c2, type2, &result_type); - container_free(c2, type2); - if (c != c1) { - container_free(c1, type1); - } + container_free(c, result_type); } - // since we assume that the initial containers are non-empty, the - // result here - // can only be non-empty - ra_append(&answer->high_low_container, s1, c, result_type); + ++pos1; ++pos2; if (pos1 == length1) break; @@ -14365,1369 +15360,1131 @@ static roaring_bitmap_t *lazy_or_from_lazy_inputs(roaring_bitmap_t *x1, s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } else if (s1 < s2) { // s1 < s2 - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - ra_append(&answer->high_low_container, s1, c1, type1); + if (pos1 != intersection_size) { + container_t *c1 = ra_get_container_at_index( + &x1->high_low_container, (uint16_t)pos1, &type1); + + ra_replace_key_and_container_at_index( + &x1->high_low_container, intersection_size, s1, c1, type1); + } + intersection_size++; pos1++; if (pos1 == length1) break; s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - ra_append(&answer->high_low_container, s2, c2, type2); - pos2++; + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); if (pos2 == length2) break; s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } } - if (pos1 == length1) { - ra_append_move_range(&answer->high_low_container, - &x2->high_low_container, pos2, length2); - } else if (pos2 == length2) { - ra_append_move_range(&answer->high_low_container, - &x1->high_low_container, pos1, length1); - } - ra_clear_without_containers(&x1->high_low_container); - ra_clear_without_containers(&x2->high_low_container); - roaring_free(x1); - roaring_free(x2); - return answer; -} - -/** - * Compute the union of 'number' bitmaps using a heap. This can - * sometimes be faster than roaring_bitmap_or_many which uses - * a naive algorithm. Caller is responsible for freeing the - * result. - */ -roaring_bitmap_t *roaring_bitmap_or_many_heap(uint32_t number, - const roaring_bitmap_t **x) { - if (number == 0) { - return roaring_bitmap_create(); - } - if (number == 1) { - return roaring_bitmap_copy(x[0]); - } - roaring_pq_t *pq = create_pq(x, number); - while (pq->size > 1) { - roaring_pq_element_t x1 = pq_poll(pq); - roaring_pq_element_t x2 = pq_poll(pq); - - if (x1.is_temporary && x2.is_temporary) { - roaring_bitmap_t *newb = - lazy_or_from_lazy_inputs(x1.bitmap, x2.bitmap); - // should normally return a fresh new bitmap *except* that - // it can return x1.bitmap or x2.bitmap in degenerate cases - bool temporary = !((newb == x1.bitmap) && (newb == x2.bitmap)); - uint64_t bsize = roaring_bitmap_portable_size_in_bytes(newb); - roaring_pq_element_t newelement = { - .size = bsize, .is_temporary = temporary, .bitmap = newb}; - pq_add(pq, &newelement); - } else if (x2.is_temporary) { - roaring_bitmap_lazy_or_inplace(x2.bitmap, x1.bitmap, false); - x2.size = roaring_bitmap_portable_size_in_bytes(x2.bitmap); - pq_add(pq, &x2); - } else if (x1.is_temporary) { - roaring_bitmap_lazy_or_inplace(x1.bitmap, x2.bitmap, false); - x1.size = roaring_bitmap_portable_size_in_bytes(x1.bitmap); - - pq_add(pq, &x1); - } else { - roaring_bitmap_t *newb = - roaring_bitmap_lazy_or(x1.bitmap, x2.bitmap, false); - uint64_t bsize = roaring_bitmap_portable_size_in_bytes(newb); - roaring_pq_element_t newelement = { - .size = bsize, .is_temporary = true, .bitmap = newb}; - - pq_add(pq, &newelement); - } - } - roaring_pq_element_t X = pq_poll(pq); - roaring_bitmap_t *answer = X.bitmap; - roaring_bitmap_repair_after_lazy(answer); - pq_free(pq); - return answer; -} - -#ifdef __cplusplus -} -} -} // extern "C" { namespace roaring { namespace api { -#endif -/* end file src/roaring_priority_queue.c */ -/* begin file src/roaring.c */ -#include -#include -#include -#include -#include -#include -#include - - -// Include after roaring.h - -#ifdef __cplusplus -using namespace ::roaring::internal; - -extern "C" { -namespace roaring { -namespace api { -#endif - -#define CROARING_SERIALIZATION_ARRAY_UINT32 1 -#define CROARING_SERIALIZATION_CONTAINER 2 -extern inline bool roaring_bitmap_contains(const roaring_bitmap_t *r, - uint32_t val); -extern inline int roaring_trailing_zeroes(unsigned long long input_num); -extern inline int roaring_leading_zeroes(unsigned long long input_num); -extern inline void roaring_bitmap_init_cleared(roaring_bitmap_t *r); -extern inline bool roaring_bitmap_get_copy_on_write(const roaring_bitmap_t *r); -extern inline void roaring_bitmap_set_copy_on_write(roaring_bitmap_t *r, - bool cow); -extern inline roaring_bitmap_t *roaring_bitmap_create(void); -extern inline void roaring_bitmap_add_range(roaring_bitmap_t *r, uint64_t min, - uint64_t max); -extern inline void roaring_bitmap_remove_range(roaring_bitmap_t *r, - uint64_t min, uint64_t max); - -static inline bool is_cow(const roaring_bitmap_t *r) { - return r->high_low_container.flags & ROARING_FLAG_COW; -} -static inline bool is_frozen(const roaring_bitmap_t *r) { - return r->high_low_container.flags & ROARING_FLAG_FROZEN; -} - -// this is like roaring_bitmap_add, but it populates pointer arguments in such a -// way -// that we can recover the container touched, which, in turn can be used to -// accelerate some functions (when you repeatedly need to add to the same -// container) -static inline container_t *containerptr_roaring_bitmap_add(roaring_bitmap_t *r, - uint32_t val, - uint8_t *type, - int *index) { - roaring_array_t *ra = &r->high_low_container; - - uint16_t hb = val >> 16; - const int i = ra_get_index(ra, hb); - if (i >= 0) { - ra_unshare_container_at_index(ra, (uint16_t)i); - container_t *c = ra_get_container_at_index(ra, (uint16_t)i, type); - uint8_t new_type = *type; - container_t *c2 = container_add(c, val & 0xFFFF, *type, &new_type); - *index = i; - if (c2 != c) { - container_free(c, *type); - ra_set_container_at_index(ra, i, c2, new_type); - *type = new_type; - return c2; - } else { - return c; - } - } else { - array_container_t *new_ac = array_container_create(); - container_t *c = - container_add(new_ac, val & 0xFFFF, ARRAY_CONTAINER_TYPE, type); - // we could just assume that it stays an array container - ra_insert_new_key_value_at(ra, -i - 1, hb, c, *type); - *index = -i - 1; - return c; - } -} - -roaring_bitmap_t *roaring_bitmap_create_with_capacity(uint32_t cap) { - roaring_bitmap_t *ans = - (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); - if (!ans) { - return NULL; - } - bool is_ok = ra_init_with_capacity(&ans->high_low_container, cap); - if (!is_ok) { - roaring_free(ans); - return NULL; - } - return ans; -} - -bool roaring_bitmap_init_with_capacity(roaring_bitmap_t *r, uint32_t cap) { - return ra_init_with_capacity(&r->high_low_container, cap); -} -static inline void add_bulk_impl(roaring_bitmap_t *r, - roaring_bulk_context_t *context, - uint32_t val) { - uint16_t key = val >> 16; - if (context->container == NULL || context->key != key) { - uint8_t typecode; - int idx; - context->container = - containerptr_roaring_bitmap_add(r, val, &typecode, &idx); - context->typecode = typecode; - context->idx = idx; - context->key = key; - } else { - // no need to seek the container, it is at hand - // because we already have the container at hand, we can do the - // insertion directly, bypassing the roaring_bitmap_add call - uint8_t new_typecode; - container_t *container2 = container_add( - context->container, val & 0xFFFF, context->typecode, &new_typecode); - if (container2 != context->container) { - // rare instance when we need to change the container type - container_free(context->container, context->typecode); - ra_set_container_at_index(&r->high_low_container, context->idx, - container2, new_typecode); - context->typecode = new_typecode; - context->container = container2; + if (pos1 < length1) { + // all containers between intersection_size and + // pos1 are junk. However, they have either been moved + // (thus still referenced) or involved in an iandnot + // that will clean up all containers that could not be reused. + // Thus we should not free the junk containers between + // intersection_size and pos1. + if (pos1 > intersection_size) { + // left slide of remaining items + ra_copy_range(&x1->high_low_container, pos1, length1, + intersection_size); } + // else current placement is fine + intersection_size += (length1 - pos1); } + ra_downsize(&x1->high_low_container, intersection_size); } -void roaring_bitmap_add_many(roaring_bitmap_t *r, size_t n_args, - const uint32_t *vals) { - uint32_t val; - const uint32_t *start = vals; - const uint32_t *end = vals + n_args; - const uint32_t *current_val = start; +uint64_t roaring_bitmap_get_cardinality(const roaring_bitmap_t *r) { + const roaring_array_t *ra = &r->high_low_container; - if (n_args == 0) { - return; + uint64_t card = 0; + for (int i = 0; i < ra->size; ++i) + card += container_get_cardinality(ra->containers[i], ra->typecodes[i]); + return card; +} + +uint64_t roaring_bitmap_range_cardinality(const roaring_bitmap_t *r, + uint64_t range_start, + uint64_t range_end) { + if (range_start >= range_end || range_start > (uint64_t)UINT32_MAX + 1) { + return 0; } + return roaring_bitmap_range_cardinality_closed(r, (uint32_t)range_start, + (uint32_t)(range_end - 1)); +} - uint8_t typecode; - int idx; - container_t *container; - val = *current_val; - container = containerptr_roaring_bitmap_add(r, val, &typecode, &idx); - roaring_bulk_context_t context = {container, idx, (uint16_t)(val >> 16), - typecode}; +uint64_t roaring_bitmap_range_cardinality_closed(const roaring_bitmap_t *r, + uint32_t range_start, + uint32_t range_end) { + const roaring_array_t *ra = &r->high_low_container; - for (; current_val != end; current_val++) { - memcpy(&val, current_val, sizeof(val)); - add_bulk_impl(r, &context, val); + if (range_start > range_end) { + return 0; } -} -void roaring_bitmap_add_bulk(roaring_bitmap_t *r, - roaring_bulk_context_t *context, uint32_t val) { - add_bulk_impl(r, context, val); -} + // now we have: 0 <= range_start <= range_end <= UINT32_MAX -bool roaring_bitmap_contains_bulk(const roaring_bitmap_t *r, - roaring_bulk_context_t *context, - uint32_t val) { - uint16_t key = val >> 16; - if (context->container == NULL || context->key != key) { - int32_t start_idx = -1; - if (context->container != NULL && context->key < key) { - start_idx = context->idx; + uint16_t minhb = (uint16_t)(range_start >> 16); + uint16_t maxhb = (uint16_t)(range_end >> 16); + + uint64_t card = 0; + + int i = ra_get_index(ra, minhb); + if (i >= 0) { + if (minhb == maxhb) { + card += container_rank(ra->containers[i], ra->typecodes[i], + range_end & 0xffff); + } else { + card += + container_get_cardinality(ra->containers[i], ra->typecodes[i]); } - int idx = ra_advance_until(&r->high_low_container, key, start_idx); - if (idx == ra_get_size(&r->high_low_container)) { - return false; + if ((range_start & 0xffff) != 0) { + card -= container_rank(ra->containers[i], ra->typecodes[i], + (range_start & 0xffff) - 1); } - uint8_t typecode; - context->container = ra_get_container_at_index( - &r->high_low_container, (uint16_t)idx, &typecode); - context->typecode = typecode; - context->idx = idx; - context->key = - ra_get_key_at_index(&r->high_low_container, (uint16_t)idx); - // ra_advance_until finds the next key >= the target, we found a later - // container. - if (context->key != key) { - return false; + i++; + } else { + i = -i - 1; + } + + for (; i < ra->size; i++) { + uint16_t key = ra->keys[i]; + if (key < maxhb) { + card += + container_get_cardinality(ra->containers[i], ra->typecodes[i]); + } else if (key == maxhb) { + card += container_rank(ra->containers[i], ra->typecodes[i], + range_end & 0xffff); + break; + } else { + break; } } - // context is now set up - return container_contains(context->container, val & 0xFFFF, - context->typecode); + + return card; } -roaring_bitmap_t *roaring_bitmap_of_ptr(size_t n_args, const uint32_t *vals) { - roaring_bitmap_t *answer = roaring_bitmap_create(); - roaring_bitmap_add_many(answer, n_args, vals); - return answer; +bool roaring_bitmap_is_empty(const roaring_bitmap_t *r) { + return r->high_low_container.size == 0; } -roaring_bitmap_t *roaring_bitmap_of(size_t n_args, ...) { - // todo: could be greatly optimized but we do not expect this call to ever - // include long lists - roaring_bitmap_t *answer = roaring_bitmap_create(); - roaring_bulk_context_t context = CROARING_ZERO_INITIALIZER; - va_list ap; - va_start(ap, n_args); - for (size_t i = 0; i < n_args; i++) { - uint32_t val = va_arg(ap, uint32_t); - roaring_bitmap_add_bulk(answer, &context, val); +void roaring_bitmap_to_uint32_array(const roaring_bitmap_t *r, uint32_t *ans) { + ra_to_uint32_array(&r->high_low_container, ans); +} + +bool roaring_bitmap_range_uint32_array(const roaring_bitmap_t *r, size_t offset, + size_t limit, uint32_t *ans) { + roaring_uint32_iterator_t it; + roaring_iterator_init(r, &it); + roaring_uint32_iterator_skip(&it, offset); + roaring_uint32_iterator_read(&it, ans, limit); + + // This function always succeeds + return true; +} + +/** convert array and bitmap containers to run containers when it is more + * efficient; + * also convert from run containers when more space efficient. Returns + * true if the result has at least one run container. + */ +bool roaring_bitmap_run_optimize(roaring_bitmap_t *r) { + bool answer = false; + for (int i = 0; i < r->high_low_container.size; i++) { + uint8_t type_original, type_after; + ra_unshare_container_at_index( + &r->high_low_container, + (uint16_t)i); // TODO: this introduces extra cloning! + container_t *c = ra_get_container_at_index(&r->high_low_container, + (uint16_t)i, &type_original); + container_t *c1 = convert_run_optimize(c, type_original, &type_after); + if (type_after == RUN_CONTAINER_TYPE) { + answer = true; + } + ra_set_container_at_index(&r->high_low_container, i, c1, type_after); } - va_end(ap); return answer; } -static inline uint64_t minimum_uint64(uint64_t a, uint64_t b) { - return (a < b) ? a : b; +size_t roaring_bitmap_shrink_to_fit(roaring_bitmap_t *r) { + size_t answer = 0; + for (int i = 0; i < r->high_low_container.size; i++) { + uint8_t type_original; + container_t *c = ra_get_container_at_index(&r->high_low_container, + (uint16_t)i, &type_original); + answer += container_shrink_to_fit(c, type_original); + } + answer += ra_shrink_to_fit(&r->high_low_container); + return answer; } -roaring_bitmap_t *roaring_bitmap_from_range(uint64_t min, uint64_t max, - uint32_t step) { - if (max >= UINT64_C(0x100000000)) { - max = UINT64_C(0x100000000); - } - if (step == 0) return NULL; - if (max <= min) return NULL; - roaring_bitmap_t *answer = roaring_bitmap_create(); - if (step >= (1 << 16)) { - for (uint32_t value = (uint32_t)min; value < max; value += step) { - roaring_bitmap_add(answer, value); +/** + * Remove run-length encoding even when it is more space efficient + * return whether a change was applied + */ +bool roaring_bitmap_remove_run_compression(roaring_bitmap_t *r) { + bool answer = false; + for (int i = 0; i < r->high_low_container.size; i++) { + uint8_t type_original, type_after; + container_t *c = ra_get_container_at_index(&r->high_low_container, + (uint16_t)i, &type_original); + if (get_container_type(c, type_original) == RUN_CONTAINER_TYPE) { + answer = true; + if (type_original == SHARED_CONTAINER_TYPE) { + run_container_t *truec = CAST_run(CAST_shared(c)->container); + int32_t card = run_container_cardinality(truec); + container_t *c1 = convert_to_bitset_or_array_container( + truec, card, &type_after); + shared_container_free(CAST_shared(c)); // frees run as needed + ra_set_container_at_index(&r->high_low_container, i, c1, + type_after); + + } else { + int32_t card = run_container_cardinality(CAST_run(c)); + container_t *c1 = convert_to_bitset_or_array_container( + CAST_run(c), card, &type_after); + run_container_free(CAST_run(c)); + ra_set_container_at_index(&r->high_low_container, i, c1, + type_after); + } } - return answer; } - uint64_t min_tmp = min; - do { - uint32_t key = (uint32_t)min_tmp >> 16; - uint32_t container_min = min_tmp & 0xFFFF; - uint32_t container_max = - (uint32_t)minimum_uint64(max - (key << 16), 1 << 16); - uint8_t type; - container_t *container = container_from_range( - &type, container_min, container_max, (uint16_t)step); - ra_append(&answer->high_low_container, (uint16_t)key, container, type); - uint32_t gap = container_max - container_min + step - 1; - min_tmp += gap - (gap % step); - } while (min_tmp < max); - // cardinality of bitmap will be ((uint64_t) max - min + step - 1 ) / step return answer; } -void roaring_bitmap_add_range_closed(roaring_bitmap_t *r, uint32_t min, - uint32_t max) { - if (min > max) { - return; +size_t roaring_bitmap_serialize(const roaring_bitmap_t *r, char *buf) { + size_t portablesize = roaring_bitmap_portable_size_in_bytes(r); + uint64_t cardinality = roaring_bitmap_get_cardinality(r); + uint64_t sizeasarray = cardinality * sizeof(uint32_t) + sizeof(uint32_t); + if (portablesize < sizeasarray) { + buf[0] = CROARING_SERIALIZATION_CONTAINER; + return roaring_bitmap_portable_serialize(r, buf + 1) + 1; + } else { + buf[0] = CROARING_SERIALIZATION_ARRAY_UINT32; + uint32_t card_le = croaring_htole32((uint32_t)cardinality); + memcpy(buf + 1, &card_le, sizeof(uint32_t)); + uint32_t *out = (uint32_t *)(buf + 1 + sizeof(uint32_t)); + roaring_bitmap_to_uint32_array(r, out); +#if CROARING_IS_BIG_ENDIAN + for (uint64_t i = 0; i < cardinality; ++i) { + uint32_t v; + memcpy(&v, out + i, sizeof(uint32_t)); + v = croaring_htole32(v); + memcpy(out + i, &v, sizeof(uint32_t)); + } +#endif + return 1 + (size_t)sizeasarray; + } +} + +size_t roaring_bitmap_size_in_bytes(const roaring_bitmap_t *r) { + size_t portablesize = roaring_bitmap_portable_size_in_bytes(r); + uint64_t sizeasarray = + roaring_bitmap_get_cardinality(r) * sizeof(uint32_t) + sizeof(uint32_t); + return portablesize < sizeasarray ? portablesize + 1 + : (size_t)sizeasarray + 1; +} + +size_t roaring_bitmap_portable_size_in_bytes(const roaring_bitmap_t *r) { + return ra_portable_size_in_bytes(&r->high_low_container); +} + +roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, + size_t maxbytes) { + roaring_bitmap_t *ans = + (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); + if (ans == NULL) { + return NULL; + } + size_t bytesread; + bool is_ok = ra_portable_deserialize(&ans->high_low_container, buf, + maxbytes, &bytesread); + if (!is_ok) { + roaring_free(ans); + return NULL; + } + roaring_bitmap_set_copy_on_write(ans, false); + if (!is_ok) { + roaring_free(ans); + return NULL; } + return ans; +} - roaring_array_t *ra = &r->high_low_container; +roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf) { + return roaring_bitmap_portable_deserialize_safe(buf, SIZE_MAX); +} - uint32_t min_key = min >> 16; - uint32_t max_key = max >> 16; +size_t roaring_bitmap_portable_deserialize_size(const char *buf, + size_t maxbytes) { + return ra_portable_deserialize_size(buf, maxbytes); +} - int32_t num_required_containers = max_key - min_key + 1; - int32_t suffix_length = - count_greater(ra->keys, ra->size, (uint16_t)max_key); - int32_t prefix_length = - count_less(ra->keys, ra->size - suffix_length, (uint16_t)min_key); - int32_t common_length = ra->size - prefix_length - suffix_length; +size_t roaring_bitmap_portable_serialize(const roaring_bitmap_t *r, char *buf) { + return ra_portable_serialize(&r->high_low_container, buf); +} - if (num_required_containers > common_length) { - ra_shift_tail(ra, suffix_length, - num_required_containers - common_length); - } +roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf) { + const char *bufaschar = (const char *)buf; + if (bufaschar[0] == CROARING_SERIALIZATION_ARRAY_UINT32) { + /* This looks like a compressed set of uint32_t elements */ + uint32_t card; - int32_t src = prefix_length + common_length - 1; - int32_t dst = ra->size - suffix_length - 1; - for (uint32_t key = max_key; key != min_key - 1; - key--) { // beware of min_key==0 - uint32_t container_min = (min_key == key) ? (min & 0xffff) : 0; - uint32_t container_max = (max_key == key) ? (max & 0xffff) : 0xffff; - container_t *new_container; - uint8_t new_type; + memcpy(&card, bufaschar + 1, sizeof(uint32_t)); + card = croaring_letoh32(card); - if (src >= 0 && ra->keys[src] == key) { - ra_unshare_container_at_index(ra, (uint16_t)src); - new_container = - container_add_range(ra->containers[src], ra->typecodes[src], - container_min, container_max, &new_type); - if (new_container != ra->containers[src]) { - container_free(ra->containers[src], ra->typecodes[src]); - } - src--; - } else { - new_container = container_from_range(&new_type, container_min, - container_max + 1, 1); + const uint32_t *elems = + (const uint32_t *)(bufaschar + 1 + sizeof(uint32_t)); + + roaring_bitmap_t *bitmap = roaring_bitmap_create(); + if (bitmap == NULL) { + return NULL; } - ra_replace_key_and_container_at_index(ra, dst, (uint16_t)key, - new_container, new_type); - dst--; - } + roaring_bulk_context_t context = CROARING_ZERO_INITIALIZER; + for (uint32_t i = 0; i < card; i++) { + // elems may not be aligned, read with memcpy + uint32_t elem; + memcpy(&elem, elems + i, sizeof(elem)); + elem = croaring_letoh32(elem); + roaring_bitmap_add_bulk(bitmap, &context, elem); + } + return bitmap; + + } else if (bufaschar[0] == CROARING_SERIALIZATION_CONTAINER) { + return roaring_bitmap_portable_deserialize(bufaschar + 1); + } else + return (NULL); } -void roaring_bitmap_remove_range_closed(roaring_bitmap_t *r, uint32_t min, - uint32_t max) { - if (min > max) { - return; +roaring_bitmap_t *roaring_bitmap_deserialize_safe(const void *buf, + size_t maxbytes) { + if (maxbytes < 1) { + return NULL; } - roaring_array_t *ra = &r->high_low_container; + const char *bufaschar = (const char *)buf; + if (bufaschar[0] == CROARING_SERIALIZATION_ARRAY_UINT32) { + if (maxbytes < 1 + sizeof(uint32_t)) { + return NULL; + } - uint32_t min_key = min >> 16; - uint32_t max_key = max >> 16; + /* This looks like a compressed set of uint32_t elements */ + uint32_t card; + memcpy(&card, bufaschar + 1, sizeof(uint32_t)); + card = croaring_letoh32(card); - int32_t src = count_less(ra->keys, ra->size, (uint16_t)min_key); - int32_t dst = src; - while (src < ra->size && ra->keys[src] <= max_key) { - uint32_t container_min = - (min_key == ra->keys[src]) ? (min & 0xffff) : 0; - uint32_t container_max = - (max_key == ra->keys[src]) ? (max & 0xffff) : 0xffff; - ra_unshare_container_at_index(ra, (uint16_t)src); - container_t *new_container; - uint8_t new_type; - new_container = - container_remove_range(ra->containers[src], ra->typecodes[src], - container_min, container_max, &new_type); - if (new_container != ra->containers[src]) { - container_free(ra->containers[src], ra->typecodes[src]); + // Check the buffer is big enough to contain card uint32_t elements + if (maxbytes < 1 + sizeof(uint32_t) + card * sizeof(uint32_t)) { + return NULL; } - if (new_container) { - ra_replace_key_and_container_at_index(ra, dst, ra->keys[src], - new_container, new_type); - dst++; + + const uint32_t *elems = + (const uint32_t *)(bufaschar + 1 + sizeof(uint32_t)); + + roaring_bitmap_t *bitmap = roaring_bitmap_create(); + if (bitmap == NULL) { + return NULL; } - src++; - } - if (src > dst) { - ra_shift_tail(ra, ra->size - src, dst - src); - } + roaring_bulk_context_t context = CROARING_ZERO_INITIALIZER; + for (uint32_t i = 0; i < card; i++) { + // elems may not be aligned, read with memcpy + uint32_t elem; + memcpy((char *)&elem, (char *)(elems + i), sizeof(elem)); + elem = croaring_letoh32(elem); + roaring_bitmap_add_bulk(bitmap, &context, elem); + } + return bitmap; + + } else if (bufaschar[0] == CROARING_SERIALIZATION_CONTAINER) { + return roaring_bitmap_portable_deserialize_safe(bufaschar + 1, + maxbytes - 1); + } else + return (NULL); } -void roaring_bitmap_printf(const roaring_bitmap_t *r) { +bool roaring_iterate(const roaring_bitmap_t *r, roaring_iterator iterator, + void *ptr) { const roaring_array_t *ra = &r->high_low_container; - printf("{"); - for (int i = 0; i < ra->size; ++i) { - container_printf_as_uint32_array(ra->containers[i], ra->typecodes[i], - ((uint32_t)ra->keys[i]) << 16); - - if (i + 1 < ra->size) { - printf(","); + for (int i = 0; i < ra->size; ++i) + if (!container_iterate(ra->containers[i], ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16, iterator, ptr)) { + return false; } - } - printf("}"); + return true; } -void roaring_bitmap_printf_describe(const roaring_bitmap_t *r) { +bool roaring_iterate64(const roaring_bitmap_t *r, roaring_iterator64 iterator, + uint64_t high_bits, void *ptr) { const roaring_array_t *ra = &r->high_low_container; - printf("{"); - for (int i = 0; i < ra->size; ++i) { - printf("%d: %s (%d)", ra->keys[i], - get_full_container_name(ra->containers[i], ra->typecodes[i]), - container_get_cardinality(ra->containers[i], ra->typecodes[i])); - if (ra->typecodes[i] == SHARED_CONTAINER_TYPE) { - printf("(shared count = %" PRIu32 " )", - croaring_refcount_get( - &(CAST_shared(ra->containers[i])->counter))); - } - - if (i + 1 < ra->size) { - printf(", "); + for (int i = 0; i < ra->size; ++i) + if (!container_iterate64(ra->containers[i], ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16, iterator, + high_bits, ptr)) { + return false; } - } - printf("}"); + return true; } -/** - * (For advanced users.) - * Collect statistics about the bitmap - */ -void roaring_bitmap_statistics(const roaring_bitmap_t *r, - roaring_statistics_t *stat) { - const roaring_array_t *ra = &r->high_low_container; - - memset(stat, 0, sizeof(*stat)); - stat->n_containers = ra->size; - stat->min_value = roaring_bitmap_minimum(r); - stat->max_value = roaring_bitmap_maximum(r); +/**** + * begin roaring_uint32_iterator_t + *****/ - for (int i = 0; i < ra->size; ++i) { - uint8_t truetype = - get_container_type(ra->containers[i], ra->typecodes[i]); - uint32_t card = - container_get_cardinality(ra->containers[i], ra->typecodes[i]); - uint32_t sbytes = - container_size_in_bytes(ra->containers[i], ra->typecodes[i]); - stat->cardinality += card; - switch (truetype) { - case BITSET_CONTAINER_TYPE: - stat->n_bitset_containers++; - stat->n_values_bitset_containers += card; - stat->n_bytes_bitset_containers += sbytes; - break; - case ARRAY_CONTAINER_TYPE: - stat->n_array_containers++; - stat->n_values_array_containers += card; - stat->n_bytes_array_containers += sbytes; - break; - case RUN_CONTAINER_TYPE: - stat->n_run_containers++; - stat->n_values_run_containers += card; - stat->n_bytes_run_containers += sbytes; - break; - default: - assert(false); - roaring_unreachable; - } +/** + * Partially initializes the iterator. Leaves it in either state: + * 1. Invalid due to `has_value = false`, or + * 2. At a container, with the high bits set, `has_value = true`. + */ +CROARING_WARN_UNUSED static bool iter_new_container_partial_init( + roaring_uint32_iterator_t *newit) { + newit->current_value = 0; + if (newit->container_index >= newit->parent->high_low_container.size || + newit->container_index < 0) { + newit->current_value = UINT32_MAX; + return (newit->has_value = false); } + newit->has_value = true; + // we precompute container, typecode and highbits so that successive + // iterators do not have to grab them from odd memory locations + // and have to worry about the (easily predicted) container_unwrap_shared + // call. + newit->container = + newit->parent->high_low_container.containers[newit->container_index]; + newit->typecode = + newit->parent->high_low_container.typecodes[newit->container_index]; + newit->highbits = + ((uint32_t) + newit->parent->high_low_container.keys[newit->container_index]) + << 16; + newit->container = + container_unwrap_shared(newit->container, &(newit->typecode)); + return true; } -bool roaring_contains_shared(const roaring_bitmap_t *r) { - const roaring_array_t *ra = &r->high_low_container; - for (int i = 0; i < ra->size; ++i) { - if (ra->typecodes[i] == SHARED_CONTAINER_TYPE) { - return true; - } +/** + * Positions the iterator at the first value of the current container that the + * iterator points at, if available. + */ +CROARING_WARN_UNUSED static bool loadfirstvalue( + roaring_uint32_iterator_t *newit) { + if (iter_new_container_partial_init(newit)) { + uint16_t value = 0; + newit->container_it = + container_init_iterator(newit->container, newit->typecode, &value); + newit->current_value = newit->highbits | value; } - return false; + return newit->has_value; } -bool roaring_unshare_all(roaring_bitmap_t *r) { - const roaring_array_t *ra = &r->high_low_container; - bool unshared = false; - for (int i = 0; i < ra->size; ++i) { - uint8_t typecode = ra->typecodes[i]; - if (typecode == SHARED_CONTAINER_TYPE) { - ra->containers[i] = get_writable_copy_if_shared(ra->containers[i], - &ra->typecodes[i]); - unshared = true; - } +/** + * Positions the iterator at the last value of the current container that the + * iterator points at, if available. + */ +CROARING_WARN_UNUSED static bool loadlastvalue( + roaring_uint32_iterator_t *newit) { + if (iter_new_container_partial_init(newit)) { + uint16_t value = 0; + newit->container_it = container_init_iterator_last( + newit->container, newit->typecode, &value); + newit->current_value = newit->highbits | value; } - return unshared; + return newit->has_value; } -/* - * Checks that: - * - Array containers are sorted and contain no duplicates - * - Range containers are sorted and contain no overlapping ranges - * - Roaring containers are sorted by key and there are no duplicate keys - * - The correct container type is use for each container (e.g. bitmaps aren't - * used for small containers) - * - Shared containers are only used when the bitmap is COW +/** + * Positions the iterator at the smallest value that is larger than or equal to + * `val` within the current container that the iterator points at. Assumes such + * a value exists within the current container. */ -bool roaring_bitmap_internal_validate(const roaring_bitmap_t *r, - const char **reason) { - const char *reason_local; - if (reason == NULL) { - // Always allow assigning through *reason - reason = &reason_local; - } - *reason = NULL; - const roaring_array_t *ra = &r->high_low_container; - if (ra->size < 0) { - *reason = "negative size"; - return false; - } - if (ra->allocation_size < 0) { - *reason = "negative allocation size"; - return false; - } - if (ra->size > ra->allocation_size) { - *reason = "more containers than allocated space"; +CROARING_WARN_UNUSED static bool loadfirstvalue_largeorequal( + roaring_uint32_iterator_t *newit, uint32_t val) { + bool partial_init = iter_new_container_partial_init(newit); + assert(partial_init); + if (!partial_init) { return false; } - if (ra->flags & ~(ROARING_FLAG_COW | ROARING_FLAG_FROZEN)) { - *reason = "invalid flags"; + uint16_t value = 0; + newit->container_it = + container_init_iterator(newit->container, newit->typecode, &value); + bool found = container_iterator_lower_bound( + newit->container, newit->typecode, &newit->container_it, &value, + val & 0xFFFF); + assert(found); + if (!found) { return false; } - if (ra->size == 0) { - return true; - } + newit->current_value = newit->highbits | value; + return true; +} - if (ra->keys == NULL) { - *reason = "keys is NULL"; - return false; - } - if (ra->typecodes == NULL) { - *reason = "typecodes is NULL"; - return false; - } - if (ra->containers == NULL) { - *reason = "containers is NULL"; - return false; - } +void roaring_iterator_init(const roaring_bitmap_t *r, + roaring_uint32_iterator_t *newit) { + newit->parent = r; + newit->container_index = 0; + newit->has_value = loadfirstvalue(newit); +} - uint32_t prev_key = ra->keys[0]; - for (int32_t i = 1; i < ra->size; ++i) { - if (ra->keys[i] <= prev_key) { - *reason = "keys not strictly increasing"; - return false; - } - prev_key = ra->keys[i]; - } +void roaring_iterator_init_last(const roaring_bitmap_t *r, + roaring_uint32_iterator_t *newit) { + newit->parent = r; + newit->container_index = newit->parent->high_low_container.size - 1; + newit->has_value = loadlastvalue(newit); +} - bool cow = roaring_bitmap_get_copy_on_write(r); +roaring_uint32_iterator_t *roaring_iterator_create(const roaring_bitmap_t *r) { + roaring_uint32_iterator_t *newit = + (roaring_uint32_iterator_t *)roaring_malloc( + sizeof(roaring_uint32_iterator_t)); + if (newit == NULL) return NULL; + roaring_iterator_init(r, newit); + return newit; +} - for (int32_t i = 0; i < ra->size; ++i) { - if (ra->typecodes[i] == SHARED_CONTAINER_TYPE && !cow) { - *reason = "shared container in non-COW bitmap"; - return false; - } - if (!container_internal_validate(ra->containers[i], ra->typecodes[i], - reason)) { - // reason should already be set - if (*reason == NULL) { - *reason = "container failed to validate but no reason given"; - } - return false; +roaring_uint32_iterator_t *roaring_uint32_iterator_copy( + const roaring_uint32_iterator_t *it) { + roaring_uint32_iterator_t *newit = + (roaring_uint32_iterator_t *)roaring_malloc( + sizeof(roaring_uint32_iterator_t)); + memcpy(newit, it, sizeof(roaring_uint32_iterator_t)); + return newit; +} + +bool roaring_uint32_iterator_move_equalorlarger(roaring_uint32_iterator_t *it, + uint32_t val) { + uint16_t hb = val >> 16; + const int i = ra_get_index(&it->parent->high_low_container, hb); + if (i >= 0) { + uint32_t lowvalue = + container_maximum(it->parent->high_low_container.containers[i], + it->parent->high_low_container.typecodes[i]); + uint16_t lb = val & 0xFFFF; + if (lowvalue < lb) { + // will have to load first value of next container + it->container_index = i + 1; + } else { + // the value is necessarily within the range of the container + it->container_index = i; + it->has_value = loadfirstvalue_largeorequal(it, val); + return it->has_value; } + } else { + // there is no matching, so we are going for the next container + it->container_index = -i - 1; } - - return true; + it->has_value = loadfirstvalue(it); + return it->has_value; } -roaring_bitmap_t *roaring_bitmap_copy(const roaring_bitmap_t *r) { - roaring_bitmap_t *ans = - (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); - if (!ans) { - return NULL; +bool roaring_uint32_iterator_advance(roaring_uint32_iterator_t *it) { + if (it->container_index >= it->parent->high_low_container.size) { + return (it->has_value = false); } - if (!ra_init_with_capacity( // allocation of list of containers can fail - &ans->high_low_container, r->high_low_container.size)) { - roaring_free(ans); - return NULL; + if (it->container_index < 0) { + it->container_index = 0; + return (it->has_value = loadfirstvalue(it)); } - if (!ra_overwrite( // memory allocation of individual containers may fail - &r->high_low_container, &ans->high_low_container, is_cow(r))) { - roaring_bitmap_free(ans); // overwrite should leave in freeable state - return NULL; + uint16_t low16 = (uint16_t)it->current_value; + if (container_iterator_next(it->container, it->typecode, &it->container_it, + &low16)) { + it->current_value = it->highbits | low16; + return (it->has_value = true); } - roaring_bitmap_set_copy_on_write(ans, is_cow(r)); - return ans; -} - -bool roaring_bitmap_overwrite(roaring_bitmap_t *dest, - const roaring_bitmap_t *src) { - roaring_bitmap_set_copy_on_write(dest, is_cow(src)); - return ra_overwrite(&src->high_low_container, &dest->high_low_container, - is_cow(src)); + it->container_index++; + return (it->has_value = loadfirstvalue(it)); } -void roaring_bitmap_free(const roaring_bitmap_t *r) { - if (r == NULL) { - return; +bool roaring_uint32_iterator_previous(roaring_uint32_iterator_t *it) { + if (it->container_index < 0) { + return (it->has_value = false); } - if (!is_frozen(r)) { - ra_clear((roaring_array_t *)&r->high_low_container); + if (it->container_index >= it->parent->high_low_container.size) { + it->container_index = it->parent->high_low_container.size - 1; + return (it->has_value = loadlastvalue(it)); + } + uint16_t low16 = (uint16_t)it->current_value; + if (container_iterator_prev(it->container, it->typecode, &it->container_it, + &low16)) { + it->current_value = it->highbits | low16; + return (it->has_value = true); } - roaring_free((roaring_bitmap_t *)r); -} - -void roaring_bitmap_clear(roaring_bitmap_t *r) { - ra_reset(&r->high_low_container); + it->container_index--; + return (it->has_value = loadlastvalue(it)); } -void roaring_bitmap_add(roaring_bitmap_t *r, uint32_t val) { - roaring_array_t *ra = &r->high_low_container; - - const uint16_t hb = val >> 16; - const int i = ra_get_index(ra, hb); - uint8_t typecode; - if (i >= 0) { - ra_unshare_container_at_index(ra, (uint16_t)i); - container_t *container = - ra_get_container_at_index(ra, (uint16_t)i, &typecode); - uint8_t newtypecode = typecode; - container_t *container2 = - container_add(container, val & 0xFFFF, typecode, &newtypecode); - if (container2 != container) { - container_free(container, typecode); - ra_set_container_at_index(&r->high_low_container, i, container2, - newtypecode); +uint32_t roaring_uint32_iterator_read(roaring_uint32_iterator_t *it, + uint32_t *buf, uint32_t count) { + uint32_t ret = 0; + while (it->has_value && ret < count) { + uint32_t consumed; + uint16_t low16 = (uint16_t)it->current_value; + bool has_value = container_iterator_read_into_uint32( + it->container, it->typecode, &it->container_it, it->highbits, buf, + count - ret, &consumed, &low16); + ret += consumed; + buf += consumed; + if (has_value) { + it->has_value = true; + it->current_value = it->highbits | low16; + // If the container still has values, we must have stopped because + // we read enough values. + assert(ret == count); + return ret; } - } else { - array_container_t *newac = array_container_create(); - container_t *container = - container_add(newac, val & 0xFFFF, ARRAY_CONTAINER_TYPE, &typecode); - // we could just assume that it stays an array container - ra_insert_new_key_value_at(&r->high_low_container, -i - 1, hb, - container, typecode); + it->container_index++; + it->has_value = loadfirstvalue(it); } + return ret; } -bool roaring_bitmap_add_checked(roaring_bitmap_t *r, uint32_t val) { - const uint16_t hb = val >> 16; - const int i = ra_get_index(&r->high_low_container, hb); - uint8_t typecode; - bool result = false; - if (i >= 0) { - ra_unshare_container_at_index(&r->high_low_container, (uint16_t)i); - container_t *container = ra_get_container_at_index( - &r->high_low_container, (uint16_t)i, &typecode); - - const int oldCardinality = - container_get_cardinality(container, typecode); - - uint8_t newtypecode = typecode; - container_t *container2 = - container_add(container, val & 0xFFFF, typecode, &newtypecode); - if (container2 != container) { - container_free(container, typecode); - ra_set_container_at_index(&r->high_low_container, i, container2, - newtypecode); - result = true; - } else { - const int newCardinality = - container_get_cardinality(container, newtypecode); - - result = oldCardinality != newCardinality; +uint32_t roaring_uint32_iterator_read_backward(roaring_uint32_iterator_t *it, + uint32_t *buf, uint32_t count) { + uint32_t ret = 0; + while (it->has_value && ret < count) { + uint32_t consumed; + uint16_t low16 = (uint16_t)it->current_value; + bool has_value = container_iterator_read_backward_into_uint32( + it->container, it->typecode, &it->container_it, it->highbits, buf, + count - ret, &consumed, &low16); + ret += consumed; + buf += consumed; + if (has_value) { + it->has_value = true; + it->current_value = it->highbits | low16; + // If the container still has values, we must have stopped because + // we read enough values. + assert(ret == count); + return ret; } - } else { - array_container_t *newac = array_container_create(); - container_t *container = - container_add(newac, val & 0xFFFF, ARRAY_CONTAINER_TYPE, &typecode); - // we could just assume that it stays an array container - ra_insert_new_key_value_at(&r->high_low_container, -i - 1, hb, - container, typecode); - result = true; + it->container_index--; + it->has_value = loadlastvalue(it); } - - return result; + return ret; } -void roaring_bitmap_remove(roaring_bitmap_t *r, uint32_t val) { - const uint16_t hb = val >> 16; - const int i = ra_get_index(&r->high_low_container, hb); - uint8_t typecode; - if (i >= 0) { - ra_unshare_container_at_index(&r->high_low_container, (uint16_t)i); - container_t *container = ra_get_container_at_index( - &r->high_low_container, (uint16_t)i, &typecode); - uint8_t newtypecode = typecode; - container_t *container2 = - container_remove(container, val & 0xFFFF, typecode, &newtypecode); - if (container2 != container) { - container_free(container, typecode); - ra_set_container_at_index(&r->high_low_container, i, container2, - newtypecode); - } - if (container_nonzero_cardinality(container2, newtypecode)) { - ra_set_container_at_index(&r->high_low_container, i, container2, - newtypecode); - } else { - ra_remove_at_index_and_free(&r->high_low_container, i); +uint32_t roaring_uint32_iterator_skip(roaring_uint32_iterator_t *it, + uint32_t count) { + uint32_t ret = 0; + while (it->has_value && ret < count) { + uint32_t consumed; + uint16_t low16 = (uint16_t)it->current_value; + bool has_value = container_iterator_skip(it->container, it->typecode, + &it->container_it, count - ret, + &consumed, &low16); + ret += consumed; + if (has_value) { + it->has_value = true; + it->current_value = it->highbits | low16; + // If the container still has values, we must have stopped because + // we skipped enough values. + assert(ret == count); + return ret; } + // We have skipped over all items in the current container, so set + // ourselves at the first item of the next container. + // We do NOT need to count another item skipped here. + it->container_index++; + it->has_value = loadfirstvalue(it); } + return ret; } -bool roaring_bitmap_remove_checked(roaring_bitmap_t *r, uint32_t val) { - const uint16_t hb = val >> 16; - const int i = ra_get_index(&r->high_low_container, hb); - uint8_t typecode; - bool result = false; - if (i >= 0) { - ra_unshare_container_at_index(&r->high_low_container, (uint16_t)i); - container_t *container = ra_get_container_at_index( - &r->high_low_container, (uint16_t)i, &typecode); - - const int oldCardinality = - container_get_cardinality(container, typecode); - - uint8_t newtypecode = typecode; - container_t *container2 = - container_remove(container, val & 0xFFFF, typecode, &newtypecode); - if (container2 != container) { - container_free(container, typecode); - ra_set_container_at_index(&r->high_low_container, i, container2, - newtypecode); +uint32_t roaring_uint32_iterator_skip_backward(roaring_uint32_iterator_t *it, + uint32_t count) { + uint32_t ret = 0; + while (it->has_value && ret < count) { + uint32_t consumed; + uint16_t low16 = (uint16_t)it->current_value; + bool has_value = container_iterator_skip_backward( + it->container, it->typecode, &it->container_it, count - ret, + &consumed, &low16); + ret += consumed; + if (has_value) { + it->has_value = true; + it->current_value = it->highbits | low16; + return ret; } + // We have skipped over all items in the current container backwards. + // Moving to the previous container counts as consuming one more skip. + it->container_index--; + it->has_value = loadlastvalue(it); + } + return ret; +} - const int newCardinality = - container_get_cardinality(container2, newtypecode); - - if (newCardinality != 0) { - ra_set_container_at_index(&r->high_low_container, i, container2, - newtypecode); - } else { - ra_remove_at_index_and_free(&r->high_low_container, i); +size_t roaring_uint32_iterator_read_ranges(roaring_uint32_iterator_t *it, + roaring_uint32_range_closed_t *buf, + size_t count) { + size_t ret = 0; + while (it->has_value && ret < count) { + buf[ret].min = it->current_value; + for (;;) { + uint16_t low16 = (uint16_t)it->current_value; + bool container_has_more; + uint16_t run_end_low16 = container_iterator_find_run_end( + it->container, it->typecode, &it->container_it, &low16, + &container_has_more); + buf[ret].max = it->highbits | run_end_low16; + + if (container_has_more) { + it->current_value = it->highbits | low16; + break; + } + // Move to next container + it->container_index++; + it->has_value = loadfirstvalue(it); + // Continue merging only if the run reached the container + // boundary and the next container starts exactly at max+1. + if (run_end_low16 != UINT16_MAX || !it->has_value || + it->current_value != buf[ret].max + 1) { + break; + } } - - result = oldCardinality != newCardinality; + ret++; } - return result; + return ret; } -void roaring_bitmap_remove_many(roaring_bitmap_t *r, size_t n_args, - const uint32_t *vals) { - if (n_args == 0 || r->high_low_container.size == 0) { - return; - } - int32_t pos = - -1; // position of the container used in the previous iteration - for (size_t i = 0; i < n_args; i++) { - uint16_t key = (uint16_t)(vals[i] >> 16); - if (pos < 0 || key != r->high_low_container.keys[pos]) { - pos = ra_get_index(&r->high_low_container, key); - } - if (pos >= 0) { - uint8_t new_typecode; - container_t *new_container; - new_container = container_remove( - r->high_low_container.containers[pos], vals[i] & 0xffff, - r->high_low_container.typecodes[pos], &new_typecode); - if (new_container != r->high_low_container.containers[pos]) { - container_free(r->high_low_container.containers[pos], - r->high_low_container.typecodes[pos]); - ra_replace_key_and_container_at_index(&r->high_low_container, - pos, key, new_container, - new_typecode); +size_t roaring_uint32_iterator_read_prev_ranges( + roaring_uint32_iterator_t *it, roaring_uint32_range_closed_t *buf, + size_t count) { + size_t ret = 0; + while (it->has_value && ret < count) { + buf[ret].max = it->current_value; + for (;;) { + uint16_t low16 = (uint16_t)it->current_value; + bool container_has_more; + uint16_t run_start_low16 = container_iterator_find_run_start( + it->container, it->typecode, &it->container_it, &low16, + &container_has_more); + buf[ret].min = it->highbits | run_start_low16; + + if (container_has_more) { + it->current_value = it->highbits | low16; + break; } - if (!container_nonzero_cardinality(new_container, new_typecode)) { - container_free(new_container, new_typecode); - ra_remove_at_index(&r->high_low_container, pos); - pos = -1; + // Move to previous container + it->container_index--; + it->has_value = loadlastvalue(it); + // Continue merging only if the run reached the container + // boundary and the previous container ends exactly at min-1. + if (run_start_low16 != 0 || !it->has_value || + it->current_value != buf[ret].min - 1) { + break; } } + ret++; + } + return ret; +} + +void roaring_uint32_iterator_free(roaring_uint32_iterator_t *it) { + roaring_free(it); +} + +/**** + * end of roaring_uint32_iterator_t + *****/ + +bool roaring_bitmap_equals(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2) { + const roaring_array_t *ra1 = &r1->high_low_container; + const roaring_array_t *ra2 = &r2->high_low_container; + + if (ra1->size != ra2->size) { + return false; + } + for (int i = 0; i < ra1->size; ++i) { + if (ra1->keys[i] != ra2->keys[i]) { + return false; + } + } + for (int i = 0; i < ra1->size; ++i) { + bool areequal = container_equals(ra1->containers[i], ra1->typecodes[i], + ra2->containers[i], ra2->typecodes[i]); + if (!areequal) { + return false; + } } + return true; } -// there should be some SIMD optimizations possible here -roaring_bitmap_t *roaring_bitmap_and(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - uint8_t result_type = 0; - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - uint32_t neededcap = length1 > length2 ? length2 : length1; - roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(neededcap); - roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); +bool roaring_bitmap_is_subset(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2) { + const roaring_array_t *ra1 = &r1->high_low_container; + const roaring_array_t *ra2 = &r2->high_low_container; + + const int length1 = ra1->size, length2 = ra2->size; int pos1 = 0, pos2 = 0; while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = - ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - const uint16_t s2 = - ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + const uint16_t s1 = ra_get_key_at_index(ra1, (uint16_t)pos1); + const uint16_t s2 = ra_get_key_at_index(ra2, (uint16_t)pos2); if (s1 == s2) { uint8_t type1, type2; - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - container_t *c = container_and(c1, type1, c2, type2, &result_type); - - if (container_nonzero_cardinality(c, result_type)) { - ra_append(&answer->high_low_container, s1, c, result_type); - } else { - container_free(c, result_type); // otherwise: memory leak! - } + container_t *c1 = + ra_get_container_at_index(ra1, (uint16_t)pos1, &type1); + container_t *c2 = + ra_get_container_at_index(ra2, (uint16_t)pos2, &type2); + if (!container_is_subset(c1, type1, c2, type2)) return false; ++pos1; ++pos2; } else if (s1 < s2) { // s1 < s2 - pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); + return false; } else { // s1 > s2 - pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + pos2 = ra_advance_until(ra2, s1, pos2); } } - return answer; + if (pos1 == length1) + return true; + else + return false; } -/** - * Compute the union of 'number' bitmaps. - */ -roaring_bitmap_t *roaring_bitmap_or_many(size_t number, - const roaring_bitmap_t **x) { - if (number == 0) { - return roaring_bitmap_create(); - } - if (number == 1) { - return roaring_bitmap_copy(x[0]); - } - roaring_bitmap_t *answer = - roaring_bitmap_lazy_or(x[0], x[1], LAZY_OR_BITSET_CONVERSION); - for (size_t i = 2; i < number; i++) { - roaring_bitmap_lazy_or_inplace(answer, x[i], LAZY_OR_BITSET_CONVERSION); - } - roaring_bitmap_repair_after_lazy(answer); - return answer; -} +static void insert_flipped_container(roaring_array_t *ans_arr, + const roaring_array_t *x1_arr, uint16_t hb, + uint16_t lb_start, uint16_t lb_end) { + const int i = ra_get_index(x1_arr, hb); + const int j = ra_get_index(ans_arr, hb); + uint8_t ctype_in, ctype_out; + container_t *flipped_container = NULL; + if (i >= 0) { + container_t *container_to_flip = + ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); + flipped_container = + container_not_range(container_to_flip, ctype_in, (uint32_t)lb_start, + (uint32_t)(lb_end + 1), &ctype_out); -/** - * Compute the xor of 'number' bitmaps. - */ -roaring_bitmap_t *roaring_bitmap_xor_many(size_t number, - const roaring_bitmap_t **x) { - if (number == 0) { - return roaring_bitmap_create(); - } - if (number == 1) { - return roaring_bitmap_copy(x[0]); - } - roaring_bitmap_t *answer = roaring_bitmap_lazy_xor(x[0], x[1]); - for (size_t i = 2; i < number; i++) { - roaring_bitmap_lazy_xor_inplace(answer, x[i]); + if (container_nonzero_cardinality(flipped_container, ctype_out)) + ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, + ctype_out); + else { + container_free(flipped_container, ctype_out); + } + } else { + flipped_container = container_range_of_ones( + (uint32_t)lb_start, (uint32_t)(lb_end + 1), &ctype_out); + ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, + ctype_out); } - roaring_bitmap_repair_after_lazy(answer); - return answer; } -// inplace and (modifies its first argument). -void roaring_bitmap_and_inplace(roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - if (x1 == x2) return; - int pos1 = 0, pos2 = 0, intersection_size = 0; - const int length1 = ra_get_size(&x1->high_low_container); - const int length2 = ra_get_size(&x2->high_low_container); - - // any skipped-over or newly emptied containers in x1 - // have to be freed. - while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = - ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - const uint16_t s2 = - ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - - if (s1 == s2) { - uint8_t type1, type2, result_type; - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - - // We do the computation "in place" only when c1 is not a shared - // container. Rationale: using a shared container safely with in - // place computation would require making a copy and then doing the - // computation in place which is likely less efficient than avoiding - // in place entirely and always generating a new container. - container_t *c = - (type1 == SHARED_CONTAINER_TYPE) - ? container_and(c1, type1, c2, type2, &result_type) - : container_iand(c1, type1, c2, type2, &result_type); - - if (c != c1) { // in this instance a new container was created, and - // we need to free the old one - container_free(c1, type1); - } - if (container_nonzero_cardinality(c, result_type)) { - ra_replace_key_and_container_at_index(&x1->high_low_container, - intersection_size, s1, c, - result_type); - intersection_size++; - } else { - container_free(c, result_type); - } - ++pos1; - ++pos2; - } else if (s1 < s2) { - pos1 = ra_advance_until_freeing(&x1->high_low_container, s2, pos1); - } else { // s1 > s2 - pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); +static void inplace_flip_container(roaring_array_t *x1_arr, uint16_t hb, + uint16_t lb_start, uint16_t lb_end) { + const int i = ra_get_index(x1_arr, hb); + uint8_t ctype_in, ctype_out; + container_t *flipped_container = NULL; + if (i >= 0) { + container_t *container_to_flip = + ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); + flipped_container = container_inot_range( + container_to_flip, ctype_in, (uint32_t)lb_start, + (uint32_t)(lb_end + 1), &ctype_out); + // if a new container was created, the old one was already freed + if (container_nonzero_cardinality(flipped_container, ctype_out)) { + ra_set_container_at_index(x1_arr, i, flipped_container, ctype_out); + } else { + container_free(flipped_container, ctype_out); + ra_remove_at_index(x1_arr, i); } - } - // if we ended early because x2 ran out, then all remaining in x1 should be - // freed - while (pos1 < length1) { - container_free(x1->high_low_container.containers[pos1], - x1->high_low_container.typecodes[pos1]); - ++pos1; + } else { + flipped_container = container_range_of_ones( + (uint32_t)lb_start, (uint32_t)(lb_end + 1), &ctype_out); + ra_insert_new_key_value_at(x1_arr, -i - 1, hb, flipped_container, + ctype_out); } - - // all containers after this have either been copied or freed - ra_downsize(&x1->high_low_container, intersection_size); } -roaring_bitmap_t *roaring_bitmap_or(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - uint8_t result_type = 0; - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - if (0 == length1) { - return roaring_bitmap_copy(x2); - } - if (0 == length2) { - return roaring_bitmap_copy(x1); +static void insert_fully_flipped_container(roaring_array_t *ans_arr, + const roaring_array_t *x1_arr, + uint16_t hb) { + const int i = ra_get_index(x1_arr, hb); + const int j = ra_get_index(ans_arr, hb); + uint8_t ctype_in, ctype_out; + container_t *flipped_container = NULL; + if (i >= 0) { + container_t *container_to_flip = + ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); + flipped_container = + container_not(container_to_flip, ctype_in, &ctype_out); + if (container_nonzero_cardinality(flipped_container, ctype_out)) + ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, + ctype_out); + else { + container_free(flipped_container, ctype_out); + } + } else { + flipped_container = container_range_of_ones(0U, 0x10000U, &ctype_out); + ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, + ctype_out); } - roaring_bitmap_t *answer = - roaring_bitmap_create_with_capacity(length1 + length2); - roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - container_t *c = container_or(c1, type1, c2, type2, &result_type); - - // since we assume that the initial containers are non-empty, the - // result here - // can only be non-empty - ra_append(&answer->high_low_container, s1, c, result_type); - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); +} - } else if (s1 < s2) { // s1 < s2 - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - // c1 = container_clone(c1, type1); - c1 = get_copy_of_container(c1, &type1, is_cow(x1)); - if (is_cow(x1)) { - ra_set_container_at_index(&x1->high_low_container, pos1, c1, - type1); - } - ra_append(&answer->high_low_container, s1, c1, type1); - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); +static void inplace_fully_flip_container(roaring_array_t *x1_arr, uint16_t hb) { + const int i = ra_get_index(x1_arr, hb); + uint8_t ctype_in, ctype_out; + container_t *flipped_container = NULL; + if (i >= 0) { + container_t *container_to_flip = + ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); + flipped_container = + container_inot(container_to_flip, ctype_in, &ctype_out); - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - // c2 = container_clone(c2, type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - ra_append(&answer->high_low_container, s2, c2, type2); - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + if (container_nonzero_cardinality(flipped_container, ctype_out)) { + ra_set_container_at_index(x1_arr, i, flipped_container, ctype_out); + } else { + container_free(flipped_container, ctype_out); + ra_remove_at_index(x1_arr, i); } + + } else { + flipped_container = container_range_of_ones(0U, 0x10000U, &ctype_out); + ra_insert_new_key_value_at(x1_arr, -i - 1, hb, flipped_container, + ctype_out); } - if (pos1 == length1) { - ra_append_copy_range(&answer->high_low_container, - &x2->high_low_container, pos2, length2, - is_cow(x2)); - } else if (pos2 == length2) { - ra_append_copy_range(&answer->high_low_container, - &x1->high_low_container, pos1, length1, - is_cow(x1)); +} + +roaring_bitmap_t *roaring_bitmap_flip(const roaring_bitmap_t *x1, + uint64_t range_start, + uint64_t range_end) { + if (range_start >= range_end || range_start > (uint64_t)UINT32_MAX + 1) { + return roaring_bitmap_copy(x1); } - return answer; + return roaring_bitmap_flip_closed(x1, (uint32_t)range_start, + (uint32_t)(range_end - 1)); } -// inplace or (modifies its first argument). -void roaring_bitmap_or_inplace(roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - uint8_t result_type = 0; - int length1 = x1->high_low_container.size; - const int length2 = x2->high_low_container.size; +roaring_bitmap_t *roaring_bitmap_flip_closed(const roaring_bitmap_t *x1, + uint32_t range_start, + uint32_t range_end) { + if (range_start > range_end) { + return roaring_bitmap_copy(x1); + } - if (0 == length2) return; + roaring_bitmap_t *ans = roaring_bitmap_create(); + roaring_bitmap_set_copy_on_write(ans, is_cow(x1)); - if (0 == length1) { - roaring_bitmap_overwrite(x1, x2); - return; - } - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - if (!container_is_full(c1, type1)) { - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, (uint16_t)pos2, &type2); - container_t *c = - (type1 == SHARED_CONTAINER_TYPE) - ? container_or(c1, type1, c2, type2, &result_type) - : container_ior(c1, type1, c2, type2, &result_type); + uint16_t hb_start = (uint16_t)(range_start >> 16); + const uint16_t lb_start = (uint16_t)range_start; // & 0xFFFF; + uint16_t hb_end = (uint16_t)(range_end >> 16); + const uint16_t lb_end = (uint16_t)range_end; // & 0xFFFF; - if (c != c1) { // in this instance a new container was created, - // and we need to free the old one - container_free(c1, type1); - } - ra_set_container_at_index(&x1->high_low_container, pos1, c, - result_type); - } - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + ra_append_copies_until(&ans->high_low_container, &x1->high_low_container, + hb_start, is_cow(x1)); + if (hb_start == hb_end) { + insert_flipped_container(&ans->high_low_container, + &x1->high_low_container, hb_start, lb_start, + lb_end); + } else { + // start and end containers are distinct + if (lb_start > 0) { + // handle first (partial) container + insert_flipped_container(&ans->high_low_container, + &x1->high_low_container, hb_start, + lb_start, 0xFFFF); + ++hb_start; // for the full containers. Can't wrap. + } - } else if (s1 < s2) { // s1 < s2 - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + if (lb_end != 0xFFFF) --hb_end; // later we'll handle the partial block - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } + for (uint32_t hb = hb_start; hb <= hb_end; ++hb) { + insert_fully_flipped_container(&ans->high_low_container, + &x1->high_low_container, + (uint16_t)hb); + } - // container_t *c2_clone = container_clone(c2, type2); - ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, - type2); - pos1++; - length1++; - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + // handle a partial final container + if (lb_end != 0xFFFF) { + insert_flipped_container(&ans->high_low_container, + &x1->high_low_container, hb_end + 1, 0, + lb_end); + ++hb_end; } } - if (pos1 == length1) { - ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, - pos2, length2, is_cow(x2)); - } + ra_append_copies_after(&ans->high_low_container, &x1->high_low_container, + hb_end, is_cow(x1)); + return ans; } -roaring_bitmap_t *roaring_bitmap_xor(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - uint8_t result_type = 0; - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - if (0 == length1) { - return roaring_bitmap_copy(x2); +void roaring_bitmap_flip_inplace(roaring_bitmap_t *x1, uint64_t range_start, + uint64_t range_end) { + if (range_start >= range_end || range_start > (uint64_t)UINT32_MAX + 1) { + return; } - if (0 == length2) { - return roaring_bitmap_copy(x1); + roaring_bitmap_flip_inplace_closed(x1, (uint32_t)range_start, + (uint32_t)(range_end - 1)); +} + +void roaring_bitmap_flip_inplace_closed(roaring_bitmap_t *x1, + uint32_t range_start, + uint32_t range_end) { + if (range_start > range_end) { + return; // empty range } - roaring_bitmap_t *answer = - roaring_bitmap_create_with_capacity(length1 + length2); - roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - container_t *c = container_xor(c1, type1, c2, type2, &result_type); - if (container_nonzero_cardinality(c, result_type)) { - ra_append(&answer->high_low_container, s1, c, result_type); - } else { - container_free(c, result_type); - } - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + uint16_t hb_start = (uint16_t)(range_start >> 16); + const uint16_t lb_start = (uint16_t)range_start; + uint16_t hb_end = (uint16_t)(range_end >> 16); + const uint16_t lb_end = (uint16_t)range_end; - } else if (s1 < s2) { // s1 < s2 - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - c1 = get_copy_of_container(c1, &type1, is_cow(x1)); - if (is_cow(x1)) { - ra_set_container_at_index(&x1->high_low_container, pos1, c1, - type1); - } - ra_append(&answer->high_low_container, s1, c1, type1); - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + if (hb_start == hb_end) { + inplace_flip_container(&x1->high_low_container, hb_start, lb_start, + lb_end); + } else { + // start and end containers are distinct + if (lb_start > 0) { + // handle first (partial) container + inplace_flip_container(&x1->high_low_container, hb_start, lb_start, + 0xFFFF); + ++hb_start; // for the full containers. Can't wrap. + } - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - ra_append(&answer->high_low_container, s2, c2, type2); - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + if (lb_end != 0xFFFF) --hb_end; + + for (uint32_t hb = hb_start; hb <= hb_end; ++hb) { + inplace_fully_flip_container(&x1->high_low_container, (uint16_t)hb); + } + // handle a partial final container + if (lb_end != 0xFFFF) { + inplace_flip_container(&x1->high_low_container, hb_end + 1, 0, + lb_end); + ++hb_end; } } - if (pos1 == length1) { - ra_append_copy_range(&answer->high_low_container, - &x2->high_low_container, pos2, length2, - is_cow(x2)); - } else if (pos2 == length2) { - ra_append_copy_range(&answer->high_low_container, - &x1->high_low_container, pos1, length1, - is_cow(x1)); - } - return answer; } -// inplace xor (modifies its first argument). +static void offset_append_with_merge(roaring_array_t *ra, int k, container_t *c, + uint8_t t) { + int size = ra_get_size(ra); + if (size == 0 || ra_get_key_at_index(ra, (uint16_t)(size - 1)) != k) { + // No merge. + ra_append(ra, (uint16_t)k, c, t); + return; + } -void roaring_bitmap_xor_inplace(roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - assert(x1 != x2); - uint8_t result_type = 0; - int length1 = x1->high_low_container.size; - const int length2 = x2->high_low_container.size; + uint8_t last_t, new_t; + container_t *last_c, *new_c; - if (0 == length2) return; + // NOTE: we don't need to unwrap here, since we added last_c ourselves + // we have the certainty it's not a shared container. + // The same applies to c, as it's the result of calling container_offset. + last_c = ra_get_container_at_index(ra, (uint16_t)(size - 1), &last_t); + new_c = container_ior(last_c, last_t, c, t, &new_t); - if (0 == length1) { - roaring_bitmap_overwrite(x1, x2); - return; + ra_set_container_at_index(ra, size - 1, new_c, new_t); + + // Comparison of pointers of different origin is UB (or so claim some + // compiler makers), so we compare their bit representation only. + if ((uintptr_t)last_c != (uintptr_t)new_c) { + container_free(last_c, last_t); } + container_free(c, t); +} - // XOR can have new containers inserted from x2, but can also - // lose containers when x1 and x2 are nonempty and identical. +// roaring_bitmap_add_offset adds the value 'offset' to each and every value in +// a bitmap, generating a new bitmap in the process. If offset + element is +// outside of the range [0,2^32), that the element will be dropped. +// We need "offset" to be 64 bits because we want to support values +// between -0xFFFFFFFF up to +0xFFFFFFFF. +roaring_bitmap_t *roaring_bitmap_add_offset(const roaring_bitmap_t *bm, + int64_t offset) { + roaring_bitmap_t *answer; + roaring_array_t *ans_ra; + int64_t container_offset; + uint16_t in_offset; - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); + const roaring_array_t *bm_ra = &bm->high_low_container; + int length = bm_ra->size; - // We do the computation "in place" only when c1 is not a shared - // container. Rationale: using a shared container safely with in - // place computation would require making a copy and then doing the - // computation in place which is likely less efficient than avoiding - // in place entirely and always generating a new container. + if (offset == 0) { + return roaring_bitmap_copy(bm); + } - container_t *c; - if (type1 == SHARED_CONTAINER_TYPE) { - c = container_xor(c1, type1, c2, type2, &result_type); - shared_container_free(CAST_shared(c1)); // so release - } else { - c = container_ixor(c1, type1, c2, type2, &result_type); - } + container_offset = offset >> 16; + in_offset = (uint16_t)(offset - container_offset * (1 << 16)); - if (container_nonzero_cardinality(c, result_type)) { - ra_set_container_at_index(&x1->high_low_container, pos1, c, - result_type); - ++pos1; - } else { - container_free(c, result_type); - ra_remove_at_index(&x1->high_low_container, pos1); - --length1; - } + answer = roaring_bitmap_create(); + bool cow = is_cow(bm); + roaring_bitmap_set_copy_on_write(answer, cow); - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + ans_ra = &answer->high_low_container; - } else if (s1 < s2) { // s1 < s2 - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + if (in_offset == 0) { + ans_ra = &answer->high_low_container; - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } + for (int i = 0, j = 0; i < length; ++i) { + int64_t key = ra_get_key_at_index(bm_ra, (uint16_t)i); + key += container_offset; - ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, - type2); - pos1++; - length1++; - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + if (key < 0 || key >= (1 << 16)) { + continue; + } + ra_append_copy(ans_ra, bm_ra, (uint16_t)i, cow); + ans_ra->keys[j++] = (uint16_t)key; } + return answer; } - if (pos1 == length1) { - ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, - pos2, length2, is_cow(x2)); - } -} -roaring_bitmap_t *roaring_bitmap_andnot(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - uint8_t result_type = 0; - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - if (0 == length1) { - roaring_bitmap_t *empty_bitmap = roaring_bitmap_create(); - roaring_bitmap_set_copy_on_write(empty_bitmap, - is_cow(x1) || is_cow(x2)); - return empty_bitmap; - } - if (0 == length2) { - return roaring_bitmap_copy(x1); - } - roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(length1); - roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + uint8_t t; + const container_t *c; + container_t *lo, *hi, **lo_ptr, **hi_ptr; + int64_t k; - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = 0; - uint16_t s2 = 0; - while (true) { - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + for (int i = 0; i < length; ++i) { + lo = hi = NULL; + lo_ptr = hi_ptr = NULL; - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - container_t *c = - container_andnot(c1, type1, c2, type2, &result_type); + k = ra_get_key_at_index(bm_ra, (uint16_t)i) + container_offset; + if (k >= 0 && k < (1 << 16)) { + lo_ptr = &lo; + } + if (k + 1 >= 0 && k + 1 < (1 << 16)) { + hi_ptr = &hi; + } + if (lo_ptr == NULL && hi_ptr == NULL) { + continue; + } + c = ra_get_container_at_index(bm_ra, (uint16_t)i, &t); + c = container_unwrap_shared(c, &t); - if (container_nonzero_cardinality(c, result_type)) { - ra_append(&answer->high_low_container, s1, c, result_type); - } else { - container_free(c, result_type); - } - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - } else if (s1 < s2) { // s1 < s2 - const int next_pos1 = - ra_advance_until(&x1->high_low_container, s2, pos1); - ra_append_copy_range(&answer->high_low_container, - &x1->high_low_container, pos1, next_pos1, - is_cow(x1)); - // TODO : perhaps some of the copy_on_write should be based on - // answer rather than x1 (more stringent?). Many similar cases - pos1 = next_pos1; - if (pos1 == length1) break; - } else { // s1 > s2 - pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); - if (pos2 == length2) break; + container_add_offset(c, t, lo_ptr, hi_ptr, in_offset); + if (lo != NULL) { + offset_append_with_merge(ans_ra, (int)k, lo, t); } + if (hi != NULL) { + ra_append(ans_ra, (uint16_t)(k + 1), hi, t); + } + // the `lo` and `hi` container type always keep same as container `c`. + // in the case of `container_add_offset` on bitset container, `lo` and + // `hi` may has small cardinality, they must be repaired to array + // container. } - if (pos2 == length2) { - ra_append_copy_range(&answer->high_low_container, - &x1->high_low_container, pos1, length1, - is_cow(x1)); - } + + roaring_bitmap_repair_after_lazy(answer); // do required type conversions. return answer; } -// inplace andnot (modifies its first argument). - -void roaring_bitmap_andnot_inplace(roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - assert(x1 != x2); - +roaring_bitmap_t *roaring_bitmap_lazy_or(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2, + const bool bitsetconversion) { uint8_t result_type = 0; - int length1 = x1->high_low_container.size; - const int length2 = x2->high_low_container.size; - int intersection_size = 0; - - if (0 == length2) return; - + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; if (0 == length1) { - roaring_bitmap_clear(x1); - return; + return roaring_bitmap_copy(x2); } - + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = + roaring_bitmap_create_with_capacity(length1 + length2); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); int pos1 = 0, pos2 = 0; uint8_t type1, type2; uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); @@ -15738,29 +16495,26 @@ void roaring_bitmap_andnot_inplace(roaring_bitmap_t *x1, (uint16_t)pos1, &type1); container_t *c2 = ra_get_container_at_index(&x2->high_low_container, (uint16_t)pos2, &type2); - - // We do the computation "in place" only when c1 is not a shared - // container. Rationale: using a shared container safely with in - // place computation would require making a copy and then doing the - // computation in place which is likely less efficient than avoiding - // in place entirely and always generating a new container. - container_t *c; - if (type1 == SHARED_CONTAINER_TYPE) { - c = container_andnot(c1, type1, c2, type2, &result_type); - shared_container_free(CAST_shared(c1)); // release - } else { - c = container_iandnot(c1, type1, c2, type2, &result_type); - } - - if (container_nonzero_cardinality(c, result_type)) { - ra_replace_key_and_container_at_index(&x1->high_low_container, - intersection_size++, s1, - c, result_type); + if (bitsetconversion && + (get_container_type(c1, type1) != BITSET_CONTAINER_TYPE) && + (get_container_type(c2, type2) != BITSET_CONTAINER_TYPE)) { + container_t *newc1 = + container_mutable_unwrap_shared(c1, &type1); + newc1 = container_to_bitset(newc1, type1); + type1 = BITSET_CONTAINER_TYPE; + c = container_lazy_ior(newc1, type1, c2, type2, &result_type); + if (c != newc1) { // should not happen + container_free(newc1, type1); + } } else { - container_free(c, result_type); + c = container_lazy_or(c1, type1, c2, type2, &result_type); } - + // since we assume that the initial containers are non-empty, + // the + // result here + // can only be non-empty + ra_append(&answer->high_low_container, s1, c, result_type); ++pos1; ++pos2; if (pos1 == length1) break; @@ -15769,5059 +16523,5750 @@ void roaring_bitmap_andnot_inplace(roaring_bitmap_t *x1, s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } else if (s1 < s2) { // s1 < s2 - if (pos1 != intersection_size) { - container_t *c1 = ra_get_container_at_index( - &x1->high_low_container, (uint16_t)pos1, &type1); - - ra_replace_key_and_container_at_index( - &x1->high_low_container, intersection_size, s1, c1, type1); + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + c1 = get_copy_of_container(c1, &type1, is_cow(x1)); + if (is_cow(x1)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c1, + type1); } - intersection_size++; + ra_append(&answer->high_low_container, s1, c1, type1); pos1++; if (pos1 == length1) break; s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); } else { // s1 > s2 - pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; if (pos2 == length2) break; s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } } - - if (pos1 < length1) { - // all containers between intersection_size and - // pos1 are junk. However, they have either been moved - // (thus still referenced) or involved in an iandnot - // that will clean up all containers that could not be reused. - // Thus we should not free the junk containers between - // intersection_size and pos1. - if (pos1 > intersection_size) { - // left slide of remaining items - ra_copy_range(&x1->high_low_container, pos1, length1, - intersection_size); - } - // else current placement is fine - intersection_size += (length1 - pos1); - } - ra_downsize(&x1->high_low_container, intersection_size); -} - -uint64_t roaring_bitmap_get_cardinality(const roaring_bitmap_t *r) { - const roaring_array_t *ra = &r->high_low_container; - - uint64_t card = 0; - for (int i = 0; i < ra->size; ++i) - card += container_get_cardinality(ra->containers[i], ra->typecodes[i]); - return card; -} - -uint64_t roaring_bitmap_range_cardinality(const roaring_bitmap_t *r, - uint64_t range_start, - uint64_t range_end) { - if (range_start >= range_end || range_start > (uint64_t)UINT32_MAX + 1) { - return 0; - } - return roaring_bitmap_range_cardinality_closed(r, (uint32_t)range_start, - (uint32_t)(range_end - 1)); -} - -uint64_t roaring_bitmap_range_cardinality_closed(const roaring_bitmap_t *r, - uint32_t range_start, - uint32_t range_end) { - const roaring_array_t *ra = &r->high_low_container; - - if (range_start > range_end) { - return 0; - } - - // now we have: 0 <= range_start <= range_end <= UINT32_MAX - - uint16_t minhb = (uint16_t)(range_start >> 16); - uint16_t maxhb = (uint16_t)(range_end >> 16); - - uint64_t card = 0; - - int i = ra_get_index(ra, minhb); - if (i >= 0) { - if (minhb == maxhb) { - card += container_rank(ra->containers[i], ra->typecodes[i], - range_end & 0xffff); - } else { - card += - container_get_cardinality(ra->containers[i], ra->typecodes[i]); - } - if ((range_start & 0xffff) != 0) { - card -= container_rank(ra->containers[i], ra->typecodes[i], - (range_start & 0xffff) - 1); - } - i++; - } else { - i = -i - 1; - } - - for (; i < ra->size; i++) { - uint16_t key = ra->keys[i]; - if (key < maxhb) { - card += - container_get_cardinality(ra->containers[i], ra->typecodes[i]); - } else if (key == maxhb) { - card += container_rank(ra->containers[i], ra->typecodes[i], - range_end & 0xffff); - break; - } else { - break; - } - } - - return card; -} - -bool roaring_bitmap_is_empty(const roaring_bitmap_t *r) { - return r->high_low_container.size == 0; -} - -void roaring_bitmap_to_uint32_array(const roaring_bitmap_t *r, uint32_t *ans) { - ra_to_uint32_array(&r->high_low_container, ans); -} - -bool roaring_bitmap_range_uint32_array(const roaring_bitmap_t *r, size_t offset, - size_t limit, uint32_t *ans) { - roaring_uint32_iterator_t it; - roaring_iterator_init(r, &it); - roaring_uint32_iterator_skip(&it, offset); - roaring_uint32_iterator_read(&it, ans, limit); - - // This function always succeeds - return true; -} - -/** convert array and bitmap containers to run containers when it is more - * efficient; - * also convert from run containers when more space efficient. Returns - * true if the result has at least one run container. - */ -bool roaring_bitmap_run_optimize(roaring_bitmap_t *r) { - bool answer = false; - for (int i = 0; i < r->high_low_container.size; i++) { - uint8_t type_original, type_after; - ra_unshare_container_at_index( - &r->high_low_container, - (uint16_t)i); // TODO: this introduces extra cloning! - container_t *c = ra_get_container_at_index(&r->high_low_container, - (uint16_t)i, &type_original); - container_t *c1 = convert_run_optimize(c, type_original, &type_after); - if (type_after == RUN_CONTAINER_TYPE) { - answer = true; - } - ra_set_container_at_index(&r->high_low_container, i, c1, type_after); - } - return answer; -} - -size_t roaring_bitmap_shrink_to_fit(roaring_bitmap_t *r) { - size_t answer = 0; - for (int i = 0; i < r->high_low_container.size; i++) { - uint8_t type_original; - container_t *c = ra_get_container_at_index(&r->high_low_container, - (uint16_t)i, &type_original); - answer += container_shrink_to_fit(c, type_original); - } - answer += ra_shrink_to_fit(&r->high_low_container); - return answer; -} - -/** - * Remove run-length encoding even when it is more space efficient - * return whether a change was applied - */ -bool roaring_bitmap_remove_run_compression(roaring_bitmap_t *r) { - bool answer = false; - for (int i = 0; i < r->high_low_container.size; i++) { - uint8_t type_original, type_after; - container_t *c = ra_get_container_at_index(&r->high_low_container, - (uint16_t)i, &type_original); - if (get_container_type(c, type_original) == RUN_CONTAINER_TYPE) { - answer = true; - if (type_original == SHARED_CONTAINER_TYPE) { - run_container_t *truec = CAST_run(CAST_shared(c)->container); - int32_t card = run_container_cardinality(truec); - container_t *c1 = convert_to_bitset_or_array_container( - truec, card, &type_after); - shared_container_free(CAST_shared(c)); // frees run as needed - ra_set_container_at_index(&r->high_low_container, i, c1, - type_after); - - } else { - int32_t card = run_container_cardinality(CAST_run(c)); - container_t *c1 = convert_to_bitset_or_array_container( - CAST_run(c), card, &type_after); - run_container_free(CAST_run(c)); - ra_set_container_at_index(&r->high_low_container, i, c1, - type_after); - } - } - } - return answer; -} - -size_t roaring_bitmap_serialize(const roaring_bitmap_t *r, char *buf) { - size_t portablesize = roaring_bitmap_portable_size_in_bytes(r); - uint64_t cardinality = roaring_bitmap_get_cardinality(r); - uint64_t sizeasarray = cardinality * sizeof(uint32_t) + sizeof(uint32_t); - if (portablesize < sizeasarray) { - buf[0] = CROARING_SERIALIZATION_CONTAINER; - return roaring_bitmap_portable_serialize(r, buf + 1) + 1; - } else { - buf[0] = CROARING_SERIALIZATION_ARRAY_UINT32; - memcpy(buf + 1, &cardinality, sizeof(uint32_t)); - roaring_bitmap_to_uint32_array( - r, (uint32_t *)(buf + 1 + sizeof(uint32_t))); - return 1 + (size_t)sizeasarray; + if (pos1 == length1) { + ra_append_copy_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2, + is_cow(x2)); + } else if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); } + return answer; } -size_t roaring_bitmap_size_in_bytes(const roaring_bitmap_t *r) { - size_t portablesize = roaring_bitmap_portable_size_in_bytes(r); - uint64_t sizeasarray = - roaring_bitmap_get_cardinality(r) * sizeof(uint32_t) + sizeof(uint32_t); - return portablesize < sizeasarray ? portablesize + 1 - : (size_t)sizeasarray + 1; -} +void roaring_bitmap_lazy_or_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2, + const bool bitsetconversion) { + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; -size_t roaring_bitmap_portable_size_in_bytes(const roaring_bitmap_t *r) { - return ra_portable_size_in_bytes(&r->high_low_container); -} + if (0 == length2) return; -roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, - size_t maxbytes) { - roaring_bitmap_t *ans = - (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); - if (ans == NULL) { - return NULL; - } - size_t bytesread; - bool is_ok = ra_portable_deserialize(&ans->high_low_container, buf, - maxbytes, &bytesread); - if (!is_ok) { - roaring_free(ans); - return NULL; - } - roaring_bitmap_set_copy_on_write(ans, false); - if (!is_ok) { - roaring_free(ans); - return NULL; + if (0 == length1) { + roaring_bitmap_overwrite(x1, x2); + return; } - return ans; -} - -roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf) { - return roaring_bitmap_portable_deserialize_safe(buf, SIZE_MAX); -} - -size_t roaring_bitmap_portable_deserialize_size(const char *buf, - size_t maxbytes) { - return ra_portable_deserialize_size(buf, maxbytes); -} + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + if (!container_is_full(c1, type1)) { + if ((bitsetconversion == false) || + (get_container_type(c1, type1) == BITSET_CONTAINER_TYPE)) { + c1 = get_writable_copy_if_shared(c1, &type1); + } else { + // convert to bitset + container_t *old_c1 = c1; + uint8_t old_type1 = type1; + c1 = container_mutable_unwrap_shared(c1, &type1); + c1 = container_to_bitset(c1, type1); + container_free(old_c1, old_type1); + type1 = BITSET_CONTAINER_TYPE; + } -size_t roaring_bitmap_portable_serialize(const roaring_bitmap_t *r, char *buf) { - return ra_portable_serialize(&r->high_low_container, buf); -} + container_t *c2 = ra_get_container_at_index( + &x2->high_low_container, (uint16_t)pos2, &type2); + container_t *c = + container_lazy_ior(c1, type1, c2, type2, &result_type); -roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf) { - const char *bufaschar = (const char *)buf; - if (bufaschar[0] == CROARING_SERIALIZATION_ARRAY_UINT32) { - /* This looks like a compressed set of uint32_t elements */ - uint32_t card; + if (c != c1) { // in this instance a new container was created, + // and we need to free the old one + container_free(c1, type1); + } - memcpy(&card, bufaschar + 1, sizeof(uint32_t)); + ra_set_container_at_index(&x1->high_low_container, pos1, c, + result_type); + } + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - const uint32_t *elems = - (const uint32_t *)(bufaschar + 1 + sizeof(uint32_t)); + } else if (s1 < s2) { // s1 < s2 + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - roaring_bitmap_t *bitmap = roaring_bitmap_create(); - if (bitmap == NULL) { - return NULL; - } - roaring_bulk_context_t context = CROARING_ZERO_INITIALIZER; - for (uint32_t i = 0; i < card; i++) { - // elems may not be aligned, read with memcpy - uint32_t elem; - memcpy(&elem, elems + i, sizeof(elem)); - roaring_bitmap_add_bulk(bitmap, &context, elem); + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + // container_t *c2_clone = container_clone(c2, type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, + type2); + pos1++; + length1++; + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } - return bitmap; - - } else if (bufaschar[0] == CROARING_SERIALIZATION_CONTAINER) { - return roaring_bitmap_portable_deserialize(bufaschar + 1); - } else - return (NULL); + } + if (pos1 == length1) { + ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, + pos2, length2, is_cow(x2)); + } } -roaring_bitmap_t *roaring_bitmap_deserialize_safe(const void *buf, - size_t maxbytes) { - if (maxbytes < 1) { - return NULL; +roaring_bitmap_t *roaring_bitmap_lazy_xor(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + if (0 == length1) { + return roaring_bitmap_copy(x2); + } + if (0 == length2) { + return roaring_bitmap_copy(x1); } + roaring_bitmap_t *answer = + roaring_bitmap_create_with_capacity(length1 + length2); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = + container_lazy_xor(c1, type1, c2, type2, &result_type); - const char *bufaschar = (const char *)buf; - if (bufaschar[0] == CROARING_SERIALIZATION_ARRAY_UINT32) { - if (maxbytes < 1 + sizeof(uint32_t)) { - return NULL; - } + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&answer->high_low_container, s1, c, result_type); + } else { + container_free(c, result_type); + } - /* This looks like a compressed set of uint32_t elements */ - uint32_t card; - memcpy(&card, bufaschar + 1, sizeof(uint32_t)); + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - // Check the buffer is big enough to contain card uint32_t elements - if (maxbytes < 1 + sizeof(uint32_t) + card * sizeof(uint32_t)) { - return NULL; + } else if (s1 < s2) { // s1 < s2 + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + c1 = get_copy_of_container(c1, &type1, is_cow(x1)); + if (is_cow(x1)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c1, + type1); + } + ra_append(&answer->high_low_container, s1, c1, type1); + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } + } + if (pos1 == length1) { + ra_append_copy_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2, + is_cow(x2)); + } else if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); + } + return answer; +} - const uint32_t *elems = - (const uint32_t *)(bufaschar + 1 + sizeof(uint32_t)); +void roaring_bitmap_lazy_xor_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + assert(x1 != x2); + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; - roaring_bitmap_t *bitmap = roaring_bitmap_create(); - if (bitmap == NULL) { - return NULL; - } - roaring_bulk_context_t context = CROARING_ZERO_INITIALIZER; - for (uint32_t i = 0; i < card; i++) { - // elems may not be aligned, read with memcpy - uint32_t elem; - memcpy((char *)&elem, (char *)(elems + i), sizeof(elem)); - roaring_bitmap_add_bulk(bitmap, &context, elem); - } - return bitmap; + if (0 == length2) return; - } else if (bufaschar[0] == CROARING_SERIALIZATION_CONTAINER) { - return roaring_bitmap_portable_deserialize_safe(bufaschar + 1, - maxbytes - 1); - } else - return (NULL); -} + if (0 == length1) { + roaring_bitmap_overwrite(x1, x2); + return; + } + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); -bool roaring_iterate(const roaring_bitmap_t *r, roaring_iterator iterator, - void *ptr) { - const roaring_array_t *ra = &r->high_low_container; + // We do the computation "in place" only when c1 is not a shared + // container. Rationale: using a shared container safely with in + // place computation would require making a copy and then doing the + // computation in place which is likely less efficient than avoiding + // in place entirely and always generating a new container. - for (int i = 0; i < ra->size; ++i) - if (!container_iterate(ra->containers[i], ra->typecodes[i], - ((uint32_t)ra->keys[i]) << 16, iterator, ptr)) { - return false; - } - return true; -} + container_t *c; + if (type1 == SHARED_CONTAINER_TYPE) { + c = container_lazy_xor(c1, type1, c2, type2, &result_type); + shared_container_free(CAST_shared(c1)); // release + } else { + c = container_lazy_ixor(c1, type1, c2, type2, &result_type); + } -bool roaring_iterate64(const roaring_bitmap_t *r, roaring_iterator64 iterator, - uint64_t high_bits, void *ptr) { - const roaring_array_t *ra = &r->high_low_container; + if (container_nonzero_cardinality(c, result_type)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c, + result_type); + ++pos1; + } else { + container_free(c, result_type); + ra_remove_at_index(&x1->high_low_container, pos1); + --length1; + } + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - for (int i = 0; i < ra->size; ++i) - if (!container_iterate64(ra->containers[i], ra->typecodes[i], - ((uint32_t)ra->keys[i]) << 16, iterator, - high_bits, ptr)) { - return false; + } else if (s1 < s2) { // s1 < s2 + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + // container_t *c2_clone = container_clone(c2, type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, + type2); + pos1++; + length1++; + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } - return true; + } + if (pos1 == length1) { + ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, + pos2, length2, is_cow(x2)); + } } -/**** - * begin roaring_uint32_iterator_t - *****/ +void roaring_bitmap_repair_after_lazy(roaring_bitmap_t *r) { + roaring_array_t *ra = &r->high_low_container; -/** - * Partially initializes the iterator. Leaves it in either state: - * 1. Invalid due to `has_value = false`, or - * 2. At a container, with the high bits set, `has_value = true`. - */ -CROARING_WARN_UNUSED static bool iter_new_container_partial_init( - roaring_uint32_iterator_t *newit) { - newit->current_value = 0; - if (newit->container_index >= newit->parent->high_low_container.size || - newit->container_index < 0) { - newit->current_value = UINT32_MAX; - return (newit->has_value = false); + for (int i = 0; i < ra->size; ++i) { + const uint8_t old_type = ra->typecodes[i]; + container_t *old_c = ra->containers[i]; + uint8_t new_type = old_type; + container_t *new_c = container_repair_after_lazy(old_c, &new_type); + ra->containers[i] = new_c; + ra->typecodes[i] = new_type; } - newit->has_value = true; - // we precompute container, typecode and highbits so that successive - // iterators do not have to grab them from odd memory locations - // and have to worry about the (easily predicted) container_unwrap_shared - // call. - newit->container = - newit->parent->high_low_container.containers[newit->container_index]; - newit->typecode = - newit->parent->high_low_container.typecodes[newit->container_index]; - newit->highbits = - ((uint32_t) - newit->parent->high_low_container.keys[newit->container_index]) - << 16; - newit->container = - container_unwrap_shared(newit->container, &(newit->typecode)); - return true; } /** - * Positions the iterator at the first value of the current container that the - * iterator points at, if available. + * roaring_bitmap_rank returns the number of integers that are smaller or equal + * to x. */ -CROARING_WARN_UNUSED static bool loadfirstvalue( - roaring_uint32_iterator_t *newit) { - if (iter_new_container_partial_init(newit)) { - uint16_t value = 0; - newit->container_it = - container_init_iterator(newit->container, newit->typecode, &value); - newit->current_value = newit->highbits | value; +uint64_t roaring_bitmap_rank(const roaring_bitmap_t *bm, uint32_t x) { + uint64_t size = 0; + uint32_t xhigh = x >> 16; + for (int i = 0; i < bm->high_low_container.size; i++) { + uint32_t key = bm->high_low_container.keys[i]; + if (xhigh > key) { + size += + container_get_cardinality(bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i]); + } else if (xhigh == key) { + return size + container_rank(bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i], + x & 0xFFFF); + } else { + return size; + } } - return newit->has_value; + return size; } +void roaring_bitmap_rank_many(const roaring_bitmap_t *bm, const uint32_t *begin, + const uint32_t *end, uint64_t *ans) { + uint64_t size = 0; -/** - * Positions the iterator at the last value of the current container that the - * iterator points at, if available. - */ -CROARING_WARN_UNUSED static bool loadlastvalue( - roaring_uint32_iterator_t *newit) { - if (iter_new_container_partial_init(newit)) { - uint16_t value = 0; - newit->container_it = container_init_iterator_last( - newit->container, newit->typecode, &value); - newit->current_value = newit->highbits | value; + int i = 0; + const uint32_t *iter = begin; + while (i < bm->high_low_container.size && iter != end) { + uint32_t x = *iter; + uint32_t xhigh = x >> 16; + uint32_t key = bm->high_low_container.keys[i]; + if (xhigh > key) { + size += + container_get_cardinality(bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i]); + i++; + } else if (xhigh == key) { + uint32_t consumed = container_rank_many( + bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i], size, iter, end, ans); + iter += consumed; + ans += consumed; + } else { + *(ans++) = size; + iter++; + } + } + while (iter != end) { // must have N outputs for N inputs... + *(ans++) = size; // ...everything left is beyond all containers + iter++; } - return newit->has_value; } /** - * Positions the iterator at the smallest value that is larger than or equal to - * `val` within the current container that the iterator points at. Assumes such - * a value exists within the current container. + * roaring_bitmap_get_index returns the index of x, if not exsist return -1. */ -CROARING_WARN_UNUSED static bool loadfirstvalue_largeorequal( - roaring_uint32_iterator_t *newit, uint32_t val) { - bool partial_init = iter_new_container_partial_init(newit); - assert(partial_init); - if (!partial_init) { - return false; - } - uint16_t value = 0; - newit->container_it = - container_init_iterator(newit->container, newit->typecode, &value); - bool found = container_iterator_lower_bound( - newit->container, newit->typecode, &newit->container_it, &value, - val & 0xFFFF); - assert(found); - if (!found) { - return false; - } - newit->current_value = newit->highbits | value; - return true; -} - -void roaring_iterator_init(const roaring_bitmap_t *r, - roaring_uint32_iterator_t *newit) { - newit->parent = r; - newit->container_index = 0; - newit->has_value = loadfirstvalue(newit); -} - -void roaring_iterator_init_last(const roaring_bitmap_t *r, - roaring_uint32_iterator_t *newit) { - newit->parent = r; - newit->container_index = newit->parent->high_low_container.size - 1; - newit->has_value = loadlastvalue(newit); -} - -roaring_uint32_iterator_t *roaring_iterator_create(const roaring_bitmap_t *r) { - roaring_uint32_iterator_t *newit = - (roaring_uint32_iterator_t *)roaring_malloc( - sizeof(roaring_uint32_iterator_t)); - if (newit == NULL) return NULL; - roaring_iterator_init(r, newit); - return newit; -} - -roaring_uint32_iterator_t *roaring_uint32_iterator_copy( - const roaring_uint32_iterator_t *it) { - roaring_uint32_iterator_t *newit = - (roaring_uint32_iterator_t *)roaring_malloc( - sizeof(roaring_uint32_iterator_t)); - memcpy(newit, it, sizeof(roaring_uint32_iterator_t)); - return newit; -} +int64_t roaring_bitmap_get_index(const roaring_bitmap_t *bm, uint32_t x) { + int64_t index = 0; + const uint16_t xhigh = x >> 16; + int32_t high_idx = ra_get_index(&bm->high_low_container, xhigh); + if (high_idx < 0) return -1; -bool roaring_uint32_iterator_move_equalorlarger(roaring_uint32_iterator_t *it, - uint32_t val) { - uint16_t hb = val >> 16; - const int i = ra_get_index(&it->parent->high_low_container, hb); - if (i >= 0) { - uint32_t lowvalue = - container_maximum(it->parent->high_low_container.containers[i], - it->parent->high_low_container.typecodes[i]); - uint16_t lb = val & 0xFFFF; - if (lowvalue < lb) { - // will have to load first value of next container - it->container_index = i + 1; + for (int i = 0; i < bm->high_low_container.size; i++) { + uint32_t key = bm->high_low_container.keys[i]; + if (xhigh > key) { + index += + container_get_cardinality(bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i]); + } else if (xhigh == key) { + int32_t low_idx = container_get_index( + bm->high_low_container.containers[high_idx], + bm->high_low_container.typecodes[high_idx], x & 0xFFFF); + if (low_idx < 0) return -1; + return index + low_idx; } else { - // the value is necessarily within the range of the container - it->container_index = i; - it->has_value = loadfirstvalue_largeorequal(it, val); - return it->has_value; + return -1; } - } else { - // there is no matching, so we are going for the next container - it->container_index = -i - 1; } - it->has_value = loadfirstvalue(it); - return it->has_value; + return index; } -bool roaring_uint32_iterator_advance(roaring_uint32_iterator_t *it) { - if (it->container_index >= it->parent->high_low_container.size) { - return (it->has_value = false); - } - if (it->container_index < 0) { - it->container_index = 0; - return (it->has_value = loadfirstvalue(it)); - } - uint16_t low16 = (uint16_t)it->current_value; - if (container_iterator_next(it->container, it->typecode, &it->container_it, - &low16)) { - it->current_value = it->highbits | low16; - return (it->has_value = true); +/** + * roaring_bitmap_smallest returns the smallest value in the set. + * Returns UINT32_MAX if the set is empty. + */ +uint32_t roaring_bitmap_minimum(const roaring_bitmap_t *bm) { + if (bm->high_low_container.size > 0) { + container_t *c = bm->high_low_container.containers[0]; + uint8_t type = bm->high_low_container.typecodes[0]; + uint32_t key = bm->high_low_container.keys[0]; + uint32_t lowvalue = container_minimum(c, type); + return lowvalue | (key << 16); } - it->container_index++; - return (it->has_value = loadfirstvalue(it)); + return UINT32_MAX; } -bool roaring_uint32_iterator_previous(roaring_uint32_iterator_t *it) { - if (it->container_index < 0) { - return (it->has_value = false); - } - if (it->container_index >= it->parent->high_low_container.size) { - it->container_index = it->parent->high_low_container.size - 1; - return (it->has_value = loadlastvalue(it)); +/** + * roaring_bitmap_smallest returns the greatest value in the set. + * Returns 0 if the set is empty. + */ +uint32_t roaring_bitmap_maximum(const roaring_bitmap_t *bm) { + if (bm->high_low_container.size > 0) { + container_t *container = + bm->high_low_container.containers[bm->high_low_container.size - 1]; + uint8_t typecode = + bm->high_low_container.typecodes[bm->high_low_container.size - 1]; + uint32_t key = + bm->high_low_container.keys[bm->high_low_container.size - 1]; + uint32_t lowvalue = container_maximum(container, typecode); + return lowvalue | (key << 16); } - uint16_t low16 = (uint16_t)it->current_value; - if (container_iterator_prev(it->container, it->typecode, &it->container_it, - &low16)) { - it->current_value = it->highbits | low16; - return (it->has_value = true); + return 0; +} + +bool roaring_bitmap_select(const roaring_bitmap_t *bm, uint32_t rank, + uint32_t *element) { + container_t *container; + uint8_t typecode; + uint16_t key; + uint32_t start_rank = 0; + int i = 0; + bool valid = false; + while (!valid && i < bm->high_low_container.size) { + container = bm->high_low_container.containers[i]; + typecode = bm->high_low_container.typecodes[i]; + valid = + container_select(container, typecode, &start_rank, rank, element); + i++; } - it->container_index--; - return (it->has_value = loadlastvalue(it)); + + if (valid) { + key = bm->high_low_container.keys[i - 1]; + *element |= (((uint32_t)key) << 16); // w/o cast, key promotes signed + return true; + } else + return false; } -uint32_t roaring_uint32_iterator_read(roaring_uint32_iterator_t *it, - uint32_t *buf, uint32_t count) { - uint32_t ret = 0; - while (it->has_value && ret < count) { - uint32_t consumed; - uint16_t low16 = (uint16_t)it->current_value; - bool has_value = container_iterator_read_into_uint32( - it->container, it->typecode, &it->container_it, it->highbits, buf, - count - ret, &consumed, &low16); - ret += consumed; - buf += consumed; - if (has_value) { - it->has_value = true; - it->current_value = it->highbits | low16; - // If the container still has values, we must have stopped because - // we skipped enough values. - assert(ret == count); - return ret; +bool roaring_bitmap_intersect(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + uint64_t answer = 0; + int pos1 = 0, pos2 = 0; + + while (pos1 < length1 && pos2 < length2) { + const uint16_t s1 = + ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + const uint16_t s2 = + ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + if (s1 == s2) { + uint8_t type1, type2; + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + if (container_intersect(c1, type1, c2, type2)) return true; + ++pos1; + ++pos2; + } else if (s1 < s2) { // s1 < s2 + pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); } - it->container_index++; - it->has_value = loadfirstvalue(it); } - return ret; + return answer != 0; } -uint32_t roaring_uint32_iterator_skip(roaring_uint32_iterator_t *it, - uint32_t count) { - uint32_t ret = 0; - while (it->has_value && ret < count) { - uint32_t consumed; - uint16_t low16 = (uint16_t)it->current_value; - bool has_value = container_iterator_skip(it->container, it->typecode, - &it->container_it, count - ret, - &consumed, &low16); - ret += consumed; - if (has_value) { - it->has_value = true; - it->current_value = it->highbits | low16; - // If the container still has values, we must have stopped because - // we skipped enough values. - assert(ret == count); - return ret; - } - // We have skipped over all items in the current container, so set - // ourselves at the first item of the next container. - // We do NOT need to count another item skipped here. - it->container_index++; - it->has_value = loadfirstvalue(it); +bool roaring_bitmap_intersect_with_range(const roaring_bitmap_t *bm, uint64_t x, + uint64_t y) { + if (x >= y) { + // Empty range. + return false; } - return ret; + roaring_uint32_iterator_t it; + roaring_iterator_init(bm, &it); + if (!roaring_uint32_iterator_move_equalorlarger(&it, (uint32_t)x)) { + // No values above x. + return false; + } + if (it.current_value >= y) { + // No values below y. + return false; + } + return true; } -uint32_t roaring_uint32_iterator_skip_backward(roaring_uint32_iterator_t *it, - uint32_t count) { - uint32_t ret = 0; - while (it->has_value && ret < count) { - uint32_t consumed; - uint16_t low16 = (uint16_t)it->current_value; - bool has_value = container_iterator_skip_backward( - it->container, it->typecode, &it->container_it, count - ret, - &consumed, &low16); - ret += consumed; - if (has_value) { - it->has_value = true; - it->current_value = it->highbits | low16; - return ret; +uint64_t roaring_bitmap_and_cardinality(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + uint64_t answer = 0; + int pos1 = 0, pos2 = 0; + while (pos1 < length1 && pos2 < length2) { + const uint16_t s1 = + ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + const uint16_t s2 = + ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + if (s1 == s2) { + uint8_t type1, type2; + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + answer += container_and_cardinality(c1, type1, c2, type2); + ++pos1; + ++pos2; + } else if (s1 < s2) { // s1 < s2 + pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); } - // We have skipped over all items in the current container backwards. - // Moving to the previous container counts as consuming one more skip. - it->container_index--; - it->has_value = loadlastvalue(it); } - return ret; + return answer; +} + +double roaring_bitmap_jaccard_index(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const uint64_t c1 = roaring_bitmap_get_cardinality(x1); + const uint64_t c2 = roaring_bitmap_get_cardinality(x2); + const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); + return (double)inter / (double)(c1 + c2 - inter); } -void roaring_uint32_iterator_free(roaring_uint32_iterator_t *it) { - roaring_free(it); +uint64_t roaring_bitmap_or_cardinality(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const uint64_t c1 = roaring_bitmap_get_cardinality(x1); + const uint64_t c2 = roaring_bitmap_get_cardinality(x2); + const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); + return c1 + c2 - inter; } -/**** - * end of roaring_uint32_iterator_t - *****/ +uint64_t roaring_bitmap_andnot_cardinality(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const uint64_t c1 = roaring_bitmap_get_cardinality(x1); + const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); + return c1 - inter; +} -bool roaring_bitmap_equals(const roaring_bitmap_t *r1, - const roaring_bitmap_t *r2) { - const roaring_array_t *ra1 = &r1->high_low_container; - const roaring_array_t *ra2 = &r2->high_low_container; +uint64_t roaring_bitmap_xor_cardinality(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const uint64_t c1 = roaring_bitmap_get_cardinality(x1); + const uint64_t c2 = roaring_bitmap_get_cardinality(x2); + const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); + return c1 + c2 - 2 * inter; +} - if (ra1->size != ra2->size) { +/** + * Check whether a range of values from range_start (included) to range_end + * (excluded) is present + */ +bool roaring_bitmap_contains_range(const roaring_bitmap_t *r, + uint64_t range_start, uint64_t range_end) { + if (range_start >= range_end || range_start > (uint64_t)UINT32_MAX + 1) { + return true; + } + return roaring_bitmap_contains_range_closed(r, (uint32_t)range_start, + (uint32_t)(range_end - 1)); +} + +/** + * Check whether a range of values from range_start (included) to range_end + * (included) is present + */ +bool roaring_bitmap_contains_range_closed(const roaring_bitmap_t *r, + uint32_t range_start, + uint32_t range_end) { + if (range_start > range_end) { + return true; + } // empty range are always contained! + if (range_end == range_start) { + return roaring_bitmap_contains(r, (uint32_t)range_start); + } + uint16_t hb_rs = (uint16_t)(range_start >> 16); + uint16_t hb_re = (uint16_t)(range_end >> 16); + const int32_t span = hb_re - hb_rs; + const int32_t hlc_sz = ra_get_size(&r->high_low_container); + if (hlc_sz < span + 1) { return false; } - for (int i = 0; i < ra1->size; ++i) { - if (ra1->keys[i] != ra2->keys[i]) { - return false; - } + int32_t is = ra_get_index(&r->high_low_container, hb_rs); + int32_t ie = ra_get_index(&r->high_low_container, hb_re); + if ((ie < 0) || (is < 0) || ((ie - is) != span) || ie >= hlc_sz) { + return false; } - for (int i = 0; i < ra1->size; ++i) { - bool areequal = container_equals(ra1->containers[i], ra1->typecodes[i], - ra2->containers[i], ra2->typecodes[i]); - if (!areequal) { + const uint32_t lb_rs = range_start & 0xFFFF; + const uint32_t lb_re = (range_end & 0xFFFF) + 1; + uint8_t type; + container_t *c = + ra_get_container_at_index(&r->high_low_container, (uint16_t)is, &type); + if (hb_rs == hb_re) { + return container_contains_range(c, lb_rs, lb_re, type); + } + if (!container_contains_range(c, lb_rs, 1 << 16, type)) { + return false; + } + c = ra_get_container_at_index(&r->high_low_container, (uint16_t)ie, &type); + if (!container_contains_range(c, 0, lb_re, type)) { + return false; + } + for (int32_t i = is + 1; i < ie; ++i) { + c = ra_get_container_at_index(&r->high_low_container, (uint16_t)i, + &type); + if (!container_is_full(c, type)) { return false; } } return true; } -bool roaring_bitmap_is_subset(const roaring_bitmap_t *r1, - const roaring_bitmap_t *r2) { - const roaring_array_t *ra1 = &r1->high_low_container; - const roaring_array_t *ra2 = &r2->high_low_container; +bool roaring_bitmap_is_strict_subset(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2) { + return (roaring_bitmap_get_cardinality(r2) > + roaring_bitmap_get_cardinality(r1) && + roaring_bitmap_is_subset(r1, r2)); +} - const int length1 = ra1->size, length2 = ra2->size; +/* + * FROZEN SERIALIZATION FORMAT DESCRIPTION + * + * -- (beginning must be aligned by 32 bytes) -- + * uint64_t[BITSET_CONTAINER_SIZE_IN_WORDS * + * num_bitset_containers] rle16_t[total number of rle elements in + * all run containers] uint16_t[total number of array elements in + * all array containers] uint16_t[num_containers] + * uint16_t[num_containers] uint8_t[num_containers]
+ * uint32_t + * + *
is a 4-byte value which is a bit union of FROZEN_COOKIE (15 bits) + * and the number of containers (17 bits). + * + * stores number of elements for every container. + * Its meaning depends on container type. + * For array and bitset containers, this value is the container cardinality + * minus one. For run container, it is the number of rle_t elements (n_runs). + * + * ,, are flat arrays of elements of + * all containers of respective type. + * + * <*_data> and are kept close together because they are not accessed + * during deserilization. This may reduce IO in case of large mmaped bitmaps. + * All members have their native alignments during deserilization except + *
, which is not guaranteed to be aligned by 4 bytes. + */ - int pos1 = 0, pos2 = 0; +size_t roaring_bitmap_frozen_size_in_bytes(const roaring_bitmap_t *rb) { + const roaring_array_t *ra = &rb->high_low_container; + size_t num_bytes = 0; + for (int32_t i = 0; i < ra->size; i++) { + switch (ra->typecodes[i]) { + case BITSET_CONTAINER_TYPE: { + num_bytes += BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + break; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(ra->containers[i]); + num_bytes += rc->n_runs * sizeof(rle16_t); + break; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = + const_CAST_array(ra->containers[i]); + num_bytes += ac->cardinality * sizeof(uint16_t); + break; + } + default: + roaring_unreachable; + } + } + num_bytes += (2 + 2 + 1) * ra->size; // keys, counts, typecodes + num_bytes += 4; // header + return num_bytes; +} - while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = ra_get_key_at_index(ra1, (uint16_t)pos1); - const uint16_t s2 = ra_get_key_at_index(ra2, (uint16_t)pos2); +inline static void *arena_alloc(char **arena, size_t num_bytes) { + char *res = *arena; + *arena += num_bytes; + return res; +} - if (s1 == s2) { - uint8_t type1, type2; - container_t *c1 = - ra_get_container_at_index(ra1, (uint16_t)pos1, &type1); - container_t *c2 = - ra_get_container_at_index(ra2, (uint16_t)pos2, &type2); - if (!container_is_subset(c1, type1, c2, type2)) return false; - ++pos1; - ++pos2; - } else if (s1 < s2) { // s1 < s2 - return false; - } else { // s1 > s2 - pos2 = ra_advance_until(ra2, s1, pos2); +void roaring_bitmap_frozen_serialize(const roaring_bitmap_t *rb, char *buf) { + /* + * Note: we do not require user to supply a specifically aligned buffer. + * Thus we have to use memcpy() everywhere. + */ + + const roaring_array_t *ra = &rb->high_low_container; + + size_t bitset_zone_size = 0; + size_t run_zone_size = 0; + size_t array_zone_size = 0; + for (int32_t i = 0; i < ra->size; i++) { + switch (ra->typecodes[i]) { + case BITSET_CONTAINER_TYPE: { + bitset_zone_size += + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + break; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(ra->containers[i]); + run_zone_size += rc->n_runs * sizeof(rle16_t); + break; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = + const_CAST_array(ra->containers[i]); + array_zone_size += ac->cardinality * sizeof(uint16_t); + break; + } + default: + roaring_unreachable; + } + } + + uint64_t *bitset_zone = (uint64_t *)arena_alloc(&buf, bitset_zone_size); + rle16_t *run_zone = (rle16_t *)arena_alloc(&buf, run_zone_size); + uint16_t *array_zone = (uint16_t *)arena_alloc(&buf, array_zone_size); + uint16_t *key_zone = (uint16_t *)arena_alloc(&buf, 2 * ra->size); + uint16_t *count_zone = (uint16_t *)arena_alloc(&buf, 2 * ra->size); + uint8_t *typecode_zone = (uint8_t *)arena_alloc(&buf, ra->size); + char *header_zone = (char *)arena_alloc(&buf, 4); + + for (int32_t i = 0; i < ra->size; i++) { + uint16_t count; + switch (ra->typecodes[i]) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = + const_CAST_bitset(ra->containers[i]); + memcpy(bitset_zone, bc->words, + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t)); + bitset_zone += BITSET_CONTAINER_SIZE_IN_WORDS; + if (bc->cardinality != BITSET_UNKNOWN_CARDINALITY) { + count = (uint16_t)(bc->cardinality - 1); + } else { + count = + (uint16_t)(bitset_container_compute_cardinality(bc) - + 1); + } + break; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(ra->containers[i]); + size_t num_bytes = rc->n_runs * sizeof(rle16_t); + memcpy(run_zone, rc->runs, num_bytes); + run_zone += rc->n_runs; + count = (uint16_t)rc->n_runs; + break; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = + const_CAST_array(ra->containers[i]); + size_t num_bytes = ac->cardinality * sizeof(uint16_t); + memcpy(array_zone, ac->array, num_bytes); + array_zone += ac->cardinality; + count = (uint16_t)(ac->cardinality - 1); + break; + } + default: + roaring_unreachable; } + memcpy(&count_zone[i], &count, 2); } - if (pos1 == length1) - return true; - else - return false; + memcpy(key_zone, ra->keys, ra->size * sizeof(uint16_t)); + memcpy(typecode_zone, ra->typecodes, ra->size * sizeof(uint8_t)); + uint32_t header = ((uint32_t)ra->size << 15) | FROZEN_COOKIE; + memcpy(header_zone, &header, 4); } -static void insert_flipped_container(roaring_array_t *ans_arr, - const roaring_array_t *x1_arr, uint16_t hb, - uint16_t lb_start, uint16_t lb_end) { - const int i = ra_get_index(x1_arr, hb); - const int j = ra_get_index(ans_arr, hb); - uint8_t ctype_in, ctype_out; - container_t *flipped_container = NULL; - if (i >= 0) { - container_t *container_to_flip = - ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); - flipped_container = - container_not_range(container_to_flip, ctype_in, (uint32_t)lb_start, - (uint32_t)(lb_end + 1), &ctype_out); +const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, + size_t length) { + if ((uintptr_t)buf % 32 != 0) { + return NULL; + } - if (container_nonzero_cardinality(flipped_container, ctype_out)) - ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, - ctype_out); - else { - container_free(flipped_container, ctype_out); - } - } else { - flipped_container = container_range_of_ones( - (uint32_t)lb_start, (uint32_t)(lb_end + 1), &ctype_out); - ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, - ctype_out); + // cookie and num_containers + if (length < 4) { + return NULL; } -} + uint32_t header; + memcpy(&header, buf + length - 4, 4); // header may be misaligned + if ((header & 0x7FFF) != FROZEN_COOKIE) { + return NULL; + } + int32_t num_containers = (header >> 15); -static void inplace_flip_container(roaring_array_t *x1_arr, uint16_t hb, - uint16_t lb_start, uint16_t lb_end) { - const int i = ra_get_index(x1_arr, hb); - uint8_t ctype_in, ctype_out; - container_t *flipped_container = NULL; - if (i >= 0) { - container_t *container_to_flip = - ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); - flipped_container = container_inot_range( - container_to_flip, ctype_in, (uint32_t)lb_start, - (uint32_t)(lb_end + 1), &ctype_out); - // if a new container was created, the old one was already freed - if (container_nonzero_cardinality(flipped_container, ctype_out)) { - ra_set_container_at_index(x1_arr, i, flipped_container, ctype_out); - } else { - container_free(flipped_container, ctype_out); - ra_remove_at_index(x1_arr, i); + // typecodes, counts and keys + if (length < 4 + (size_t)num_containers * (1 + 2 + 2)) { + return NULL; + } + uint16_t *keys = (uint16_t *)(buf + length - 4 - num_containers * 5); + uint16_t *counts = (uint16_t *)(buf + length - 4 - num_containers * 3); + uint8_t *typecodes = (uint8_t *)(buf + length - 4 - num_containers * 1); + + // {bitset,array,run}_zone + int32_t num_bitset_containers = 0; + int32_t num_run_containers = 0; + int32_t num_array_containers = 0; + size_t bitset_zone_size = 0; + size_t run_zone_size = 0; + size_t array_zone_size = 0; + for (int32_t i = 0; i < num_containers; i++) { + switch (typecodes[i]) { + case BITSET_CONTAINER_TYPE: + num_bitset_containers++; + bitset_zone_size += + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + break; + case RUN_CONTAINER_TYPE: + num_run_containers++; + run_zone_size += counts[i] * sizeof(rle16_t); + break; + case ARRAY_CONTAINER_TYPE: + num_array_containers++; + array_zone_size += (counts[i] + UINT32_C(1)) * sizeof(uint16_t); + break; + default: + return NULL; } + } + if (length != bitset_zone_size + run_zone_size + array_zone_size + + 5 * num_containers + 4) { + return NULL; + } + uint64_t *bitset_zone = (uint64_t *)(buf); + rle16_t *run_zone = (rle16_t *)(buf + bitset_zone_size); + uint16_t *array_zone = (uint16_t *)(buf + bitset_zone_size + run_zone_size); - } else { - flipped_container = container_range_of_ones( - (uint32_t)lb_start, (uint32_t)(lb_end + 1), &ctype_out); - ra_insert_new_key_value_at(x1_arr, -i - 1, hb, flipped_container, - ctype_out); + size_t alloc_size = 0; + alloc_size += sizeof(roaring_bitmap_t); + alloc_size += num_containers * sizeof(container_t *); + alloc_size += num_bitset_containers * sizeof(bitset_container_t); + alloc_size += num_run_containers * sizeof(run_container_t); + alloc_size += num_array_containers * sizeof(array_container_t); + + char *arena = (char *)roaring_malloc(alloc_size); + if (arena == NULL) { + return NULL; + } + + roaring_bitmap_t *rb = + (roaring_bitmap_t *)arena_alloc(&arena, sizeof(roaring_bitmap_t)); + rb->high_low_container.flags = ROARING_FLAG_FROZEN; + rb->high_low_container.allocation_size = num_containers; + rb->high_low_container.size = num_containers; + rb->high_low_container.keys = (uint16_t *)keys; + rb->high_low_container.typecodes = (uint8_t *)typecodes; + rb->high_low_container.containers = (container_t **)arena_alloc( + &arena, sizeof(container_t *) * num_containers); + // Ensure offset of high_low_container.containers is known distance used in + // C++ wrapper. sizeof(roaring_bitmap_t) is used as it is the size of the + // only allocation that precedes high_low_container.containers. If this is + // changed (new allocation or changed order), this offset will also need to + // be changed in the C++ wrapper. + assert(rb == + (roaring_bitmap_t *)((char *)rb->high_low_container.containers - + sizeof(roaring_bitmap_t))); + for (int32_t i = 0; i < num_containers; i++) { + switch (typecodes[i]) { + case BITSET_CONTAINER_TYPE: { + bitset_container_t *bitset = (bitset_container_t *)arena_alloc( + &arena, sizeof(bitset_container_t)); + bitset->words = bitset_zone; + bitset->cardinality = counts[i] + UINT32_C(1); + rb->high_low_container.containers[i] = bitset; + bitset_zone += BITSET_CONTAINER_SIZE_IN_WORDS; + break; + } + case RUN_CONTAINER_TYPE: { + run_container_t *run = (run_container_t *)arena_alloc( + &arena, sizeof(run_container_t)); + run->capacity = counts[i]; + run->n_runs = counts[i]; + run->runs = run_zone; + rb->high_low_container.containers[i] = run; + run_zone += run->n_runs; + break; + } + case ARRAY_CONTAINER_TYPE: { + array_container_t *array = (array_container_t *)arena_alloc( + &arena, sizeof(array_container_t)); + array->capacity = counts[i] + UINT32_C(1); + array->cardinality = counts[i] + UINT32_C(1); + array->array = array_zone; + rb->high_low_container.containers[i] = array; + array_zone += counts[i] + UINT32_C(1); + break; + } + default: + roaring_free(arena); + return NULL; + } } + + return rb; } -static void insert_fully_flipped_container(roaring_array_t *ans_arr, - const roaring_array_t *x1_arr, - uint16_t hb) { - const int i = ra_get_index(x1_arr, hb); - const int j = ra_get_index(ans_arr, hb); - uint8_t ctype_in, ctype_out; - container_t *flipped_container = NULL; - if (i >= 0) { - container_t *container_to_flip = - ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); - flipped_container = - container_not(container_to_flip, ctype_in, &ctype_out); - if (container_nonzero_cardinality(flipped_container, ctype_out)) - ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, - ctype_out); - else { - container_free(flipped_container, ctype_out); +CROARING_ALLOW_UNALIGNED +roaring_bitmap_t *roaring_bitmap_portable_deserialize_frozen(const char *buf) { +#if CROARING_IS_BIG_ENDIAN + // The portable format is little-endian on every host, and this function + // uses the container payloads where they sit rather than converting them. + // There is therefore no correct in-place view of them here: refuse rather + // than hand back a bitmap that silently reads byte-swapped values. Use + // roaring_bitmap_portable_deserialize_safe(), which converts as it copies. + (void)buf; + return NULL; +#else + char *start_of_buf = (char *)buf; + uint32_t cookie; + int32_t num_containers; + uint16_t *descriptive_headers; + uint32_t *offset_headers = NULL; + const char *run_flag_bitset = NULL; + bool hasrun = false; + + // deserialize cookie + memcpy(&cookie, buf, sizeof(uint32_t)); + buf += sizeof(uint32_t); + if (cookie == SERIAL_COOKIE_NO_RUNCONTAINER) { + memcpy(&num_containers, buf, sizeof(int32_t)); + buf += sizeof(int32_t); + descriptive_headers = (uint16_t *)buf; + buf += num_containers * 2 * sizeof(uint16_t); + offset_headers = (uint32_t *)buf; + buf += num_containers * sizeof(uint32_t); + } else if ((cookie & 0xFFFF) == SERIAL_COOKIE) { + num_containers = (cookie >> 16) + 1; + hasrun = true; + int32_t run_flag_bitset_size = (num_containers + 7) / 8; + run_flag_bitset = buf; + buf += run_flag_bitset_size; + descriptive_headers = (uint16_t *)buf; + buf += num_containers * 2 * sizeof(uint16_t); + if (num_containers >= NO_OFFSET_THRESHOLD) { + offset_headers = (uint32_t *)buf; + buf += num_containers * sizeof(uint32_t); } } else { - flipped_container = container_range_of_ones(0U, 0x10000U, &ctype_out); - ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, - ctype_out); + return NULL; } -} -static void inplace_fully_flip_container(roaring_array_t *x1_arr, uint16_t hb) { - const int i = ra_get_index(x1_arr, hb); - uint8_t ctype_in, ctype_out; - container_t *flipped_container = NULL; - if (i >= 0) { - container_t *container_to_flip = - ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); - flipped_container = - container_inot(container_to_flip, ctype_in, &ctype_out); + // calculate total size for allocation + int32_t num_bitset_containers = 0; + int32_t num_run_containers = 0; + int32_t num_array_containers = 0; + + for (int32_t i = 0; i < num_containers; i++) { + uint16_t tmp; + memcpy(&tmp, descriptive_headers + 2 * i + 1, sizeof(tmp)); + uint32_t cardinality = tmp + 1; + bool isbitmap = (cardinality > DEFAULT_MAX_SIZE); + bool isrun = false; + if (hasrun) { + if ((run_flag_bitset[i / 8] & (1 << (i % 8))) != 0) { + isbitmap = false; + isrun = true; + } + } - if (container_nonzero_cardinality(flipped_container, ctype_out)) { - ra_set_container_at_index(x1_arr, i, flipped_container, ctype_out); + if (isbitmap) { + num_bitset_containers++; + } else if (isrun) { + num_run_containers++; } else { - container_free(flipped_container, ctype_out); - ra_remove_at_index(x1_arr, i); + num_array_containers++; } - - } else { - flipped_container = container_range_of_ones(0U, 0x10000U, &ctype_out); - ra_insert_new_key_value_at(x1_arr, -i - 1, hb, flipped_container, - ctype_out); } -} -roaring_bitmap_t *roaring_bitmap_flip(const roaring_bitmap_t *x1, - uint64_t range_start, - uint64_t range_end) { - if (range_start >= range_end || range_start > (uint64_t)UINT32_MAX + 1) { - return roaring_bitmap_copy(x1); - } - return roaring_bitmap_flip_closed(x1, (uint32_t)range_start, - (uint32_t)(range_end - 1)); -} + size_t alloc_size = 0; + alloc_size += sizeof(roaring_bitmap_t); + alloc_size += num_containers * sizeof(container_t *); + alloc_size += num_bitset_containers * sizeof(bitset_container_t); + alloc_size += num_run_containers * sizeof(run_container_t); + alloc_size += num_array_containers * sizeof(array_container_t); + alloc_size += num_containers * sizeof(uint16_t); // keys + alloc_size += num_containers * sizeof(uint8_t); // typecodes -roaring_bitmap_t *roaring_bitmap_flip_closed(const roaring_bitmap_t *x1, - uint32_t range_start, - uint32_t range_end) { - if (range_start > range_end) { - return roaring_bitmap_copy(x1); + // allocate bitmap and construct containers + char *arena = (char *)roaring_malloc(alloc_size); + if (arena == NULL) { + return NULL; } - roaring_bitmap_t *ans = roaring_bitmap_create(); - roaring_bitmap_set_copy_on_write(ans, is_cow(x1)); - - uint16_t hb_start = (uint16_t)(range_start >> 16); - const uint16_t lb_start = (uint16_t)range_start; // & 0xFFFF; - uint16_t hb_end = (uint16_t)(range_end >> 16); - const uint16_t lb_end = (uint16_t)range_end; // & 0xFFFF; + roaring_bitmap_t *rb = + (roaring_bitmap_t *)arena_alloc(&arena, sizeof(roaring_bitmap_t)); + rb->high_low_container.flags = ROARING_FLAG_FROZEN; + rb->high_low_container.allocation_size = num_containers; + rb->high_low_container.size = num_containers; + rb->high_low_container.containers = (container_t **)arena_alloc( + &arena, sizeof(container_t *) * num_containers); - ra_append_copies_until(&ans->high_low_container, &x1->high_low_container, - hb_start, is_cow(x1)); - if (hb_start == hb_end) { - insert_flipped_container(&ans->high_low_container, - &x1->high_low_container, hb_start, lb_start, - lb_end); - } else { - // start and end containers are distinct - if (lb_start > 0) { - // handle first (partial) container - insert_flipped_container(&ans->high_low_container, - &x1->high_low_container, hb_start, - lb_start, 0xFFFF); - ++hb_start; // for the full containers. Can't wrap. - } + uint16_t *keys = + (uint16_t *)arena_alloc(&arena, num_containers * sizeof(uint16_t)); + uint8_t *typecodes = + (uint8_t *)arena_alloc(&arena, num_containers * sizeof(uint8_t)); - if (lb_end != 0xFFFF) --hb_end; // later we'll handle the partial block + rb->high_low_container.keys = keys; + rb->high_low_container.typecodes = typecodes; - for (uint32_t hb = hb_start; hb <= hb_end; ++hb) { - insert_fully_flipped_container(&ans->high_low_container, - &x1->high_low_container, - (uint16_t)hb); + for (int32_t i = 0; i < num_containers; i++) { + uint16_t tmp; + memcpy(&tmp, descriptive_headers + 2 * i + 1, sizeof(tmp)); + int32_t cardinality = tmp + 1; + bool isbitmap = (cardinality > DEFAULT_MAX_SIZE); + bool isrun = false; + if (hasrun) { + if ((run_flag_bitset[i / 8] & (1 << (i % 8))) != 0) { + isbitmap = false; + isrun = true; + } } - // handle a partial final container - if (lb_end != 0xFFFF) { - insert_flipped_container(&ans->high_low_container, - &x1->high_low_container, hb_end + 1, 0, - lb_end); - ++hb_end; + keys[i] = descriptive_headers[2 * i]; + + if (isbitmap) { + typecodes[i] = BITSET_CONTAINER_TYPE; + bitset_container_t *c = (bitset_container_t *)arena_alloc( + &arena, sizeof(bitset_container_t)); + c->cardinality = cardinality; + if (offset_headers != NULL) { + c->words = (uint64_t *)(start_of_buf + offset_headers[i]); + } else { + c->words = (uint64_t *)buf; + buf += BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + } + rb->high_low_container.containers[i] = c; + } else if (isrun) { + typecodes[i] = RUN_CONTAINER_TYPE; + run_container_t *c = + (run_container_t *)arena_alloc(&arena, sizeof(run_container_t)); + c->capacity = cardinality; + uint16_t n_runs; + if (offset_headers != NULL) { + memcpy(&n_runs, start_of_buf + offset_headers[i], + sizeof(uint16_t)); + c->n_runs = n_runs; + c->runs = (rle16_t *)(start_of_buf + offset_headers[i] + + sizeof(uint16_t)); + } else { + memcpy(&n_runs, buf, sizeof(uint16_t)); + c->n_runs = n_runs; + buf += sizeof(uint16_t); + c->runs = (rle16_t *)buf; + buf += c->n_runs * sizeof(rle16_t); + } + rb->high_low_container.containers[i] = c; + } else { + typecodes[i] = ARRAY_CONTAINER_TYPE; + array_container_t *c = (array_container_t *)arena_alloc( + &arena, sizeof(array_container_t)); + c->cardinality = cardinality; + c->capacity = cardinality; + if (offset_headers != NULL) { + c->array = (uint16_t *)(start_of_buf + offset_headers[i]); + } else { + c->array = (uint16_t *)buf; + buf += cardinality * sizeof(uint16_t); + } + rb->high_low_container.containers[i] = c; } } - ra_append_copies_after(&ans->high_low_container, &x1->high_low_container, - hb_end, is_cow(x1)); - return ans; -} -void roaring_bitmap_flip_inplace(roaring_bitmap_t *x1, uint64_t range_start, - uint64_t range_end) { - if (range_start >= range_end || range_start > (uint64_t)UINT32_MAX + 1) { - return; - } - roaring_bitmap_flip_inplace_closed(x1, (uint32_t)range_start, - (uint32_t)(range_end - 1)); + return rb; +#endif } -void roaring_bitmap_flip_inplace_closed(roaring_bitmap_t *x1, - uint32_t range_start, - uint32_t range_end) { - if (range_start > range_end) { - return; // empty range +bool roaring_bitmap_to_bitset(const roaring_bitmap_t *r, bitset_t *bitset) { + uint32_t max_value = roaring_bitmap_maximum(r); + size_t new_array_size = (size_t)(max_value / 64 + 1); + bool resize_ok = bitset_resize(bitset, new_array_size, true); + if (!resize_ok) { + return false; } - - uint16_t hb_start = (uint16_t)(range_start >> 16); - const uint16_t lb_start = (uint16_t)range_start; - uint16_t hb_end = (uint16_t)(range_end >> 16); - const uint16_t lb_end = (uint16_t)range_end; - - if (hb_start == hb_end) { - inplace_flip_container(&x1->high_low_container, hb_start, lb_start, - lb_end); - } else { - // start and end containers are distinct - if (lb_start > 0) { - // handle first (partial) container - inplace_flip_container(&x1->high_low_container, hb_start, lb_start, - 0xFFFF); - ++hb_start; // for the full containers. Can't wrap. - } - - if (lb_end != 0xFFFF) --hb_end; - - for (uint32_t hb = hb_start; hb <= hb_end; ++hb) { - inplace_fully_flip_container(&x1->high_low_container, (uint16_t)hb); + const roaring_array_t *ra = &r->high_low_container; + for (int i = 0; i < ra->size; ++i) { + uint64_t *words = bitset->array + (ra->keys[i] << 10); + uint8_t type = ra->typecodes[i]; + const container_t *c = ra->containers[i]; + if (type == SHARED_CONTAINER_TYPE) { + c = container_unwrap_shared(c, &type); } - // handle a partial final container - if (lb_end != 0xFFFF) { - inplace_flip_container(&x1->high_low_container, hb_end + 1, 0, - lb_end); - ++hb_end; + switch (type) { + case BITSET_CONTAINER_TYPE: { + size_t max_word_index = new_array_size - (ra->keys[i] << 10); + if (max_word_index > 1024) { + max_word_index = 1024; + } + const bitset_container_t *src = const_CAST_bitset(c); + memcpy(words, src->words, max_word_index * sizeof(uint64_t)); + } break; + case ARRAY_CONTAINER_TYPE: { + const array_container_t *src = const_CAST_array(c); + bitset_set_list(words, src->array, src->cardinality); + } break; + case RUN_CONTAINER_TYPE: { + const run_container_t *src = const_CAST_run(c); + for (int32_t rlepos = 0; rlepos < src->n_runs; ++rlepos) { + rle16_t rle = src->runs[rlepos]; + bitset_set_lenrange(words, rle.value, rle.length); + } + } break; + default: + roaring_unreachable; } } + return true; } -static void offset_append_with_merge(roaring_array_t *ra, int k, container_t *c, - uint8_t t) { - int size = ra_get_size(ra); - if (size == 0 || ra_get_key_at_index(ra, (uint16_t)(size - 1)) != k) { - // No merge. - ra_append(ra, (uint16_t)k, c, t); - return; - } - - uint8_t last_t, new_t; - container_t *last_c, *new_c; - - // NOTE: we don't need to unwrap here, since we added last_c ourselves - // we have the certainty it's not a shared container. - // The same applies to c, as it's the result of calling container_offset. - last_c = ra_get_container_at_index(ra, (uint16_t)(size - 1), &last_t); - new_c = container_ior(last_c, last_t, c, t, &new_t); - - ra_set_container_at_index(ra, size - 1, new_c, new_t); - - // Comparison of pointers of different origin is UB (or so claim some - // compiler makers), so we compare their bit representation only. - if ((uintptr_t)last_c != (uintptr_t)new_c) { - container_free(last_c, last_t); - } - container_free(c, t); -} - -// roaring_bitmap_add_offset adds the value 'offset' to each and every value in -// a bitmap, generating a new bitmap in the process. If offset + element is -// outside of the range [0,2^32), that the element will be dropped. -// We need "offset" to be 64 bits because we want to support values -// between -0xFFFFFFFF up to +0xFFFFFFFF. -roaring_bitmap_t *roaring_bitmap_add_offset(const roaring_bitmap_t *bm, - int64_t offset) { - roaring_bitmap_t *answer; - roaring_array_t *ans_ra; - int64_t container_offset; - uint16_t in_offset; +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { +#endif +/* end file src/roaring.c */ +/* begin file src/roaring64.c */ +#include +#include +#include +#include +#include - const roaring_array_t *bm_ra = &bm->high_low_container; - int length = bm_ra->size; - if (offset == 0) { - return roaring_bitmap_copy(bm); - } +// For serialization / deserialization +// containers.h last to avoid conflict with ROARING_CONTAINER_T. - container_offset = offset >> 16; - in_offset = (uint16_t)(offset - container_offset * (1 << 16)); +#define CROARING_ALIGN_BUF(buf, alignment) \ + (char *)(((uintptr_t)(buf) + ((alignment)-1)) & \ + (ptrdiff_t)(~((alignment)-1))) - answer = roaring_bitmap_create(); - bool cow = is_cow(bm); - roaring_bitmap_set_copy_on_write(answer, cow); +#define CROARING_BITSET_ALIGNMENT 64 - ans_ra = &answer->high_low_container; +#ifdef __cplusplus +using namespace ::roaring::internal; - if (in_offset == 0) { - ans_ra = &answer->high_low_container; +extern "C" { +namespace roaring { +namespace api { +#endif - for (int i = 0, j = 0; i < length; ++i) { - int64_t key = ra_get_key_at_index(bm_ra, (uint16_t)i); - key += container_offset; +// TODO: Copy on write. +// TODO: Error on failed allocation. - if (key < 0 || key >= (1 << 16)) { - continue; - } - ra_append_copy(ans_ra, bm_ra, (uint16_t)i, cow); - ans_ra->keys[j++] = (uint16_t)key; - } - return answer; - } +typedef struct roaring64_bitmap_s { + art_t art; + uint8_t flags; + uint64_t first_free; + uint64_t capacity; + container_t **containers; + // Parallel to containers[]. Live slots (non-NULL pointers) have the + // matching typecode; NULL slots are skipped and their typecodes ignored. + uint8_t *typecodes; +} roaring64_bitmap_t; - uint8_t t; - const container_t *c; - container_t *lo, *hi, **lo_ptr, **hi_ptr; - int64_t k; +// Leaf type of the ART used to keep the high 48 bits of each entry. +// Low 8 bits: typecode +// High 56 bits: container index +typedef roaring64_leaf_t leaf_t; - for (int i = 0; i < length; ++i) { - lo = hi = NULL; - lo_ptr = hi_ptr = NULL; +// Iterator struct to hold iteration state. +typedef struct roaring64_iterator_s { + // The order here is deliberate: everything `roaring64_iterator_advance` + // touches per value is packed into the first 64 bytes, and `art_it` -- + // 136 bytes, of which only `art_it.value` is read per value -- is last. + // Putting `art_it` earlier pushes the rest past the first cache line and + // costs ~18% on a scalar iteration loop. + + // Must stay first, and must stay a `roaring64_iterator_public_t`: the + // inline `roaring64_iterator_value` / `roaring64_iterator_has_value` in + // the public header reach these two members by converting a + // `roaring64_iterator_t *` to a pointer to its initial member. + roaring64_iterator_public_t pub; - k = ra_get_key_at_index(bm_ra, (uint16_t)i) + container_offset; - if (k >= 0 && k < (1 << 16)) { - lo_ptr = &lo; - } - if (k + 1 >= 0 && k + 1 < (1 << 16)) { - hi_ptr = &hi; - } - if (lo_ptr == NULL && hi_ptr == NULL) { - continue; - } - c = ra_get_container_at_index(bm_ra, (uint16_t)i, &t); - c = container_unwrap_shared(c, &t); + uint64_t high48; // Key that art_it points to. + roaring_container_iterator_t container_it; - container_add_offset(c, t, lo_ptr, hi_ptr, in_offset); - if (lo != NULL) { - offset_append_with_merge(ans_ra, (int)k, lo, t); - } - if (hi != NULL) { - ra_append(ans_ra, (uint16_t)(k + 1), hi, t); - } - // the `lo` and `hi` container type always keep same as container `c`. - // in the case of `container_add_offset` on bitset container, `lo` and - // `hi` may has small cardinality, they must be repaired to array - // container. - } + // Forward-iteration cache for bitset containers. `container_iterator_next` + // recomputes the word index from `container_it.index`, reloads the word + // and re-masks it for every value; keeping the remaining bits of the + // current word here turns that into a `tzcnt` and a `blsr`. Array and run + // containers already advance by a single increment and gain nothing from + // a cache, so they stay on the ordinary path. + // + // Every field is a pure function of the ART position and + // `container_it.index`, so the cache describes where the iterator is + // exactly when both still match what it was built from. `fast_type` is + // BITSET_CONTAINER_TYPE, or 0 when there is no usable cache. + uint32_t fast_wordindex; + const art_val_t *fast_art_value; + const uint64_t *fast_words; + uint64_t fast_word; + int32_t fast_index; + uint8_t fast_type; - roaring_bitmap_repair_after_lazy(answer); // do required type conversions. - return answer; + // If has_value is false, then the iterator is saturated. This field + // indicates the direction of saturation. If true, there are no more values + // in the forward direction. If false, there are no more values in the + // backward direction. + bool saturated_forward; + + const roaring64_bitmap_t *r; + art_iterator_t art_it; +} roaring64_iterator_t; + +static inline bool is_frozen64(const roaring64_bitmap_t *r) { + return r->flags & ROARING_FLAG_FROZEN; } -roaring_bitmap_t *roaring_bitmap_lazy_or(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2, - const bool bitsetconversion) { - uint8_t result_type = 0; - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - if (0 == length1) { - return roaring_bitmap_copy(x2); - } - if (0 == length2) { - return roaring_bitmap_copy(x1); - } - roaring_bitmap_t *answer = - roaring_bitmap_create_with_capacity(length1 + length2); - roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - container_t *c; - if (bitsetconversion && - (get_container_type(c1, type1) != BITSET_CONTAINER_TYPE) && - (get_container_type(c2, type2) != BITSET_CONTAINER_TYPE)) { - container_t *newc1 = - container_mutable_unwrap_shared(c1, &type1); - newc1 = container_to_bitset(newc1, type1); - type1 = BITSET_CONTAINER_TYPE; - c = container_lazy_ior(newc1, type1, c2, type2, &result_type); - if (c != newc1) { // should not happen - container_free(newc1, type1); - } - } else { - c = container_lazy_or(c1, type1, c2, type2, &result_type); - } - // since we assume that the initial containers are non-empty, - // the - // result here - // can only be non-empty - ra_append(&answer->high_low_container, s1, c, result_type); - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); +static inline bool is_frozen_art64(const roaring64_bitmap_t *r) { + return r->flags & ROARING_FLAG_FROZEN_ART; +} - } else if (s1 < s2) { // s1 < s2 - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - c1 = get_copy_of_container(c1, &type1, is_cow(x1)); - if (is_cow(x1)) { - ra_set_container_at_index(&x1->high_low_container, pos1, c1, - type1); - } - ra_append(&answer->high_low_container, s1, c1, type1); - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); +typedef union { + bitset_container_t bitset; + array_container_t array; + run_container_t run; +} frozen_container_header_t; - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - ra_append(&answer->high_low_container, s2, c2, type2); - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - } - } - if (pos1 == length1) { - ra_append_copy_range(&answer->high_low_container, - &x2->high_low_container, pos2, length2, - is_cow(x2)); - } else if (pos2 == length2) { - ra_append_copy_range(&answer->high_low_container, - &x1->high_low_container, pos1, length1, - is_cow(x1)); - } - return answer; +static void *roaring64_arena_alloc(char **arena, size_t num_bytes) { + char *res = *arena; + *arena += num_bytes; + return res; } -void roaring_bitmap_lazy_or_inplace(roaring_bitmap_t *x1, - const roaring_bitmap_t *x2, - const bool bitsetconversion) { - uint8_t result_type = 0; - int length1 = x1->high_low_container.size; - const int length2 = x2->high_low_container.size; +static char *roaring64_arena_pad(char *cursor, const char *base, + size_t alignment) { + uint64_t off = (uint64_t)(cursor - base); + uint64_t aligned = (off + alignment - 1) & ~(uint64_t)(alignment - 1); + return (char *)base + aligned; +} - if (0 == length2) return; +// Splits the given uint64 key into high 48 bit and low 16 bit components. +// Expects high48_out to be of length ART_KEY_BYTES. +static inline uint16_t split_key(uint64_t key, uint8_t high48_out[]) { + uint64_t tmp = croaring_htobe64(key); + memcpy(high48_out, (uint8_t *)(&tmp), ART_KEY_BYTES); + return (uint16_t)key; +} - if (0 == length1) { - roaring_bitmap_overwrite(x1, x2); - return; - } - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - if (!container_is_full(c1, type1)) { - if ((bitsetconversion == false) || - (get_container_type(c1, type1) == BITSET_CONTAINER_TYPE)) { - c1 = get_writable_copy_if_shared(c1, &type1); - } else { - // convert to bitset - container_t *old_c1 = c1; - uint8_t old_type1 = type1; - c1 = container_mutable_unwrap_shared(c1, &type1); - c1 = container_to_bitset(c1, type1); - container_free(old_c1, old_type1); - type1 = BITSET_CONTAINER_TYPE; - } +// Recombines the high 48 bit and low 16 bit components into a uint64 key. +// Expects high48_out to be of length ART_KEY_BYTES. +static inline uint64_t combine_key(const uint8_t high48[], uint16_t low16) { + uint64_t result = 0; + memcpy((uint8_t *)(&result), high48, ART_KEY_BYTES); + return croaring_be64toh(result) | low16; +} - container_t *c2 = ra_get_container_at_index( - &x2->high_low_container, (uint16_t)pos2, &type2); - container_t *c = - container_lazy_ior(c1, type1, c2, type2, &result_type); +static inline uint64_t minimum(uint64_t a, uint64_t b) { + return (a < b) ? a : b; +} - if (c != c1) { // in this instance a new container was created, - // and we need to free the old one - container_free(c1, type1); - } +static inline leaf_t create_leaf(uint64_t container_index, uint8_t typecode) { + return (container_index << 8) | typecode; +} - ra_set_container_at_index(&x1->high_low_container, pos1, c, - result_type); - } - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); +static inline uint8_t get_typecode(leaf_t leaf) { return (uint8_t)leaf; } - } else if (s1 < s2) { // s1 < s2 - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); +static inline uint64_t get_index(leaf_t leaf) { return leaf >> 8; } - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - // container_t *c2_clone = container_clone(c2, type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, - type2); - pos1++; - length1++; - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - } - } - if (pos1 == length1) { - ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, - pos2, length2, is_cow(x2)); - } +static inline container_t *get_container(const roaring64_bitmap_t *r, + leaf_t leaf) { + return r->containers[get_index(leaf)]; } -roaring_bitmap_t *roaring_bitmap_lazy_xor(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - uint8_t result_type = 0; - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - if (0 == length1) { - return roaring_bitmap_copy(x2); - } - if (0 == length2) { - return roaring_bitmap_copy(x1); - } - roaring_bitmap_t *answer = - roaring_bitmap_create_with_capacity(length1 + length2); - roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - container_t *c = - container_lazy_xor(c1, type1, c2, type2, &result_type); - - if (container_nonzero_cardinality(c, result_type)) { - ra_append(&answer->high_low_container, s1, c, result_type); - } else { - container_free(c, result_type); - } +// Writes the pointer and its typecode together so they cannot drift. +static inline void set_container_at(roaring64_bitmap_t *r, uint64_t index, + container_t *container, uint8_t typecode) { + r->containers[index] = container; + r->typecodes[index] = typecode; +} - ++pos1; - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); +// Replaces the container of `leaf` with the given container. Returns the +// modified leaf for convenience. +static inline leaf_t replace_container(roaring64_bitmap_t *r, leaf_t *leaf, + container_t *container, + uint8_t typecode) { + uint64_t index = get_index(*leaf); + set_container_at(r, index, container, typecode); + *leaf = create_leaf(index, typecode); + return *leaf; +} - } else if (s1 < s2) { // s1 < s2 - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - c1 = get_copy_of_container(c1, &type1, is_cow(x1)); - if (is_cow(x1)) { - ra_set_container_at_index(&x1->high_low_container, pos1, c1, - type1); - } - ra_append(&answer->high_low_container, s1, c1, type1); - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); +/** + * Extends the array of container pointers (and the parallel typecode array). + */ +static void extend_containers(roaring64_bitmap_t *r) { + uint64_t size = r->first_free; + if (size < r->capacity) { + return; + } + uint64_t new_capacity; + if (r->capacity == 0) { + new_capacity = 2; + } else if (r->capacity < 1024) { + new_capacity = 2 * r->capacity; + } else { + new_capacity = 5 * r->capacity / 4; + } + uint64_t increase = new_capacity - r->capacity; + r->containers = (container_t **)roaring_realloc( + r->containers, new_capacity * sizeof(container_t *)); + memset(r->containers + r->capacity, 0, increase * sizeof(container_t *)); + r->typecodes = (uint8_t *)roaring_realloc(r->typecodes, + new_capacity * sizeof(uint8_t)); + r->capacity = new_capacity; +} - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - ra_append(&answer->high_low_container, s2, c2, type2); - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); +static uint64_t next_free_container_idx(const roaring64_bitmap_t *r) { + for (uint64_t i = r->first_free + 1; i < r->capacity; ++i) { + if (r->containers[i] == NULL) { + return i; } } - if (pos1 == length1) { - ra_append_copy_range(&answer->high_low_container, - &x2->high_low_container, pos2, length2, - is_cow(x2)); - } else if (pos2 == length2) { - ra_append_copy_range(&answer->high_low_container, - &x1->high_low_container, pos1, length1, - is_cow(x1)); - } - return answer; + return r->capacity; } -void roaring_bitmap_lazy_xor_inplace(roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - assert(x1 != x2); - uint8_t result_type = 0; - int length1 = x1->high_low_container.size; - const int length2 = x2->high_low_container.size; +static uint64_t allocate_index(roaring64_bitmap_t *r) { + uint64_t first_free = r->first_free; + if (first_free == r->capacity) { + extend_containers(r); + } + r->first_free = next_free_container_idx(r); + return first_free; +} - if (0 == length2) return; +static leaf_t add_container(roaring64_bitmap_t *r, container_t *container, + uint8_t typecode) { + uint64_t index = allocate_index(r); + set_container_at(r, index, container, typecode); + return create_leaf(index, typecode); +} - if (0 == length1) { - roaring_bitmap_overwrite(x1, x2); +static void ensure_container_capacity(roaring64_bitmap_t *r, uint64_t extra) { + uint64_t needed = r->first_free + extra; + if (needed <= r->capacity) { return; } - int pos1 = 0, pos2 = 0; - uint8_t type1, type2; - uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - while (true) { - if (s1 == s2) { - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - - // We do the computation "in place" only when c1 is not a shared - // container. Rationale: using a shared container safely with in - // place computation would require making a copy and then doing the - // computation in place which is likely less efficient than avoiding - // in place entirely and always generating a new container. + uint64_t new_capacity = r->capacity; + if (new_capacity == 0) { + new_capacity = 2; + } + while (new_capacity < needed) { + uint64_t grown; + if (new_capacity < 1024) { + grown = 2 * new_capacity; + } else { + grown = new_capacity + new_capacity / 4; + } + if (grown <= new_capacity) { + new_capacity = needed; + break; + } + new_capacity = grown; + } + uint64_t increase = new_capacity - r->capacity; + r->containers = (container_t **)roaring_realloc( + r->containers, new_capacity * sizeof(container_t *)); + memset(r->containers + r->capacity, 0, increase * sizeof(container_t *)); + r->typecodes = (uint8_t *)roaring_realloc(r->typecodes, + new_capacity * sizeof(uint8_t)); + r->capacity = new_capacity; +} - container_t *c; - if (type1 == SHARED_CONTAINER_TYPE) { - c = container_lazy_xor(c1, type1, c2, type2, &result_type); - shared_container_free(CAST_shared(c1)); // release - } else { - c = container_lazy_ixor(c1, type1, c2, type2, &result_type); - } +static void remove_container(roaring64_bitmap_t *r, leaf_t leaf) { + uint64_t index = get_index(leaf); + r->containers[index] = NULL; + if (index < r->first_free) { + r->first_free = index; + } +} - if (container_nonzero_cardinality(c, result_type)) { - ra_set_container_at_index(&x1->high_low_container, pos1, c, - result_type); - ++pos1; - } else { - container_free(c, result_type); - ra_remove_at_index(&x1->high_low_container, pos1); - --length1; - } - ++pos2; - if (pos1 == length1) break; - if (pos2 == length2) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); +// Copies the container referenced by `leaf` from `r1` to `r2`. +static inline leaf_t copy_leaf_container(const roaring64_bitmap_t *r1, + roaring64_bitmap_t *r2, leaf_t leaf) { + uint8_t typecode = get_typecode(leaf); + // get_copy_of_container modifies the typecode passed in. + container_t *container = get_copy_of_container( + get_container(r1, leaf), &typecode, /*copy_on_write=*/false); + return add_container(r2, container, typecode); +} - } else if (s1 < s2) { // s1 < s2 - pos1++; - if (pos1 == length1) break; - s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); +static inline int compare_high48(art_key_chunk_t key1[], + art_key_chunk_t key2[]) { + return art_compare_keys(key1, key2); +} - } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - // container_t *c2_clone = container_clone(c2, type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, - type2); - pos1++; - length1++; - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); - } +// Cache the current container so advance() need not re-read the ART leaf, +// reload the container pointer, or (for bitsets) re-mask the current word. +// Called after any positioning that sets container_it.index. +// fast_type is 0 when there is no usable cache (run containers, exhausted). +static inline void roaring64_iterator_prime(roaring64_iterator_t *it) { + it->fast_type = 0; + if (it->art_it.value == NULL) { + return; } - if (pos1 == length1) { - ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, - pos2, length2, is_cow(x2)); + leaf_t leaf = (leaf_t)*it->art_it.value; + if (get_typecode(leaf) != BITSET_CONTAINER_TYPE) { + return; } + const bitset_container_t *bc = + const_CAST_bitset(get_container(it->r, leaf)); + int32_t index = it->container_it.index; + uint32_t wordindex = (uint32_t)index >> 6; + uint32_t bit = (uint32_t)index & 63u; + uint64_t word = bc->words[wordindex]; + // Bits strictly after the current value in this word. A shift of 64 is + // undefined, so the last bit of a word is a special case. + it->fast_word = + (bit == 63u) ? UINT64_C(0) : (word & (UINT64_MAX << (bit + 1))); + it->fast_words = bc->words; + it->fast_wordindex = wordindex; + it->fast_index = index; + it->fast_art_value = it->art_it.value; + it->fast_type = BITSET_CONTAINER_TYPE; } -void roaring_bitmap_repair_after_lazy(roaring_bitmap_t *r) { - roaring_array_t *ra = &r->high_low_container; +static inline bool roaring64_iterator_init_at_leaf_first( + roaring64_iterator_t *it) { + it->high48 = combine_key(it->art_it.key, 0); + leaf_t leaf = (leaf_t)*it->art_it.value; + uint16_t low16 = 0; + it->container_it = container_init_iterator(get_container(it->r, leaf), + get_typecode(leaf), &low16); + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + roaring64_iterator_prime(it); + return true; +} - for (int i = 0; i < ra->size; ++i) { - const uint8_t old_type = ra->typecodes[i]; - container_t *old_c = ra->containers[i]; - uint8_t new_type = old_type; - container_t *new_c = container_repair_after_lazy(old_c, &new_type); - ra->containers[i] = new_c; - ra->typecodes[i] = new_type; - } +static inline bool roaring64_iterator_init_at_leaf_last( + roaring64_iterator_t *it) { + it->high48 = combine_key(it->art_it.key, 0); + leaf_t leaf = (leaf_t)*it->art_it.value; + uint16_t low16 = 0; + it->container_it = container_init_iterator_last(get_container(it->r, leaf), + get_typecode(leaf), &low16); + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + roaring64_iterator_prime(it); + return true; } -/** - * roaring_bitmap_rank returns the number of integers that are smaller or equal - * to x. - */ -uint64_t roaring_bitmap_rank(const roaring_bitmap_t *bm, uint32_t x) { - uint64_t size = 0; - uint32_t xhigh = x >> 16; - for (int i = 0; i < bm->high_low_container.size; i++) { - uint32_t key = bm->high_low_container.keys[i]; - if (xhigh > key) { - size += - container_get_cardinality(bm->high_low_container.containers[i], - bm->high_low_container.typecodes[i]); - } else if (xhigh == key) { - return size + container_rank(bm->high_low_container.containers[i], - bm->high_low_container.typecodes[i], - x & 0xFFFF); +static inline roaring64_iterator_t *roaring64_iterator_init_at( + const roaring64_bitmap_t *r, roaring64_iterator_t *it, bool first) { + it->r = r; + it->art_it = art_init_iterator((art_t *)&r->art, first); + it->pub.has_value = it->art_it.value != NULL; + if (it->pub.has_value) { + if (first) { + roaring64_iterator_init_at_leaf_first(it); } else { - return size; + roaring64_iterator_init_at_leaf_last(it); } + } else { + it->saturated_forward = first; + it->fast_type = 0; } - return size; + return it; } -void roaring_bitmap_rank_many(const roaring_bitmap_t *bm, const uint32_t *begin, - const uint32_t *end, uint64_t *ans) { - uint64_t size = 0; - int i = 0; - const uint32_t *iter = begin; - while (i < bm->high_low_container.size && iter != end) { - uint32_t x = *iter; - uint32_t xhigh = x >> 16; - uint32_t key = bm->high_low_container.keys[i]; - if (xhigh > key) { - size += - container_get_cardinality(bm->high_low_container.containers[i], - bm->high_low_container.typecodes[i]); - i++; - } else if (xhigh == key) { - uint32_t consumed = container_rank_many( - bm->high_low_container.containers[i], - bm->high_low_container.typecodes[i], size, iter, end, ans); - iter += consumed; - ans += consumed; - } else { - *(ans++) = size; - iter++; +roaring64_bitmap_t *roaring64_bitmap_create(void) { + roaring64_bitmap_t *r = + (roaring64_bitmap_t *)roaring_malloc(sizeof(roaring64_bitmap_t)); + art_init_cleared(&r->art); + r->flags = 0; + r->capacity = 0; + r->first_free = 0; + r->containers = NULL; + r->typecodes = NULL; + return r; +} + +void roaring64_bitmap_free(roaring64_bitmap_t *r) { + if (!r) { + return; + } + if (is_frozen64(r)) { + // Headers, containers[], and typecodes[] live in the same allocation + // as `r`. Payloads alias a caller buffer. + if (!is_frozen_art64(r)) { + art_free(&r->art); } + roaring_free(r); + return; + } + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + container_free(get_container(r, leaf), get_typecode(leaf)); + art_iterator_next(&it); } + art_free(&r->art); + roaring_free(r->containers); + roaring_free(r->typecodes); + roaring_free(r); } -/** - * roaring_bitmap_get_index returns the index of x, if not exsist return -1. - */ -int64_t roaring_bitmap_get_index(const roaring_bitmap_t *bm, uint32_t x) { - int64_t index = 0; - const uint16_t xhigh = x >> 16; - int32_t high_idx = ra_get_index(&bm->high_low_container, xhigh); - if (high_idx < 0) return -1; +roaring64_bitmap_t *roaring64_bitmap_copy(const roaring64_bitmap_t *r) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); - for (int i = 0; i < bm->high_low_container.size; i++) { - uint32_t key = bm->high_low_container.keys[i]; - if (xhigh > key) { - index += - container_get_cardinality(bm->high_low_container.containers[i], - bm->high_low_container.typecodes[i]); - } else if (xhigh == key) { - int32_t low_idx = container_get_index( - bm->high_low_container.containers[high_idx], - bm->high_low_container.typecodes[high_idx], x & 0xFFFF); - if (low_idx < 0) return -1; - return index + low_idx; - } else { - return -1; - } + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + uint8_t result_typecode = get_typecode(leaf); + container_t *result_container = get_copy_of_container( + get_container(r, leaf), &result_typecode, /*copy_on_write=*/false); + leaf_t result_leaf = + add_container(result, result_container, result_typecode); + art_insert(&result->art, it.key, (art_val_t)result_leaf); + art_iterator_next(&it); } - return index; + return result; } -/** - * roaring_bitmap_smallest returns the smallest value in the set. - * Returns UINT32_MAX if the set is empty. - */ -uint32_t roaring_bitmap_minimum(const roaring_bitmap_t *bm) { - if (bm->high_low_container.size > 0) { - container_t *c = bm->high_low_container.containers[0]; - uint8_t type = bm->high_low_container.typecodes[0]; - uint32_t key = bm->high_low_container.keys[0]; - uint32_t lowvalue = container_minimum(c, type); - return lowvalue | (key << 16); +void roaring64_bitmap_overwrite(roaring64_bitmap_t *dest, + const roaring64_bitmap_t *src) { + if (dest == src) { + return; } - return UINT32_MAX; -} -/** - * roaring_bitmap_smallest returns the greatest value in the set. - * Returns 0 if the set is empty. - */ -uint32_t roaring_bitmap_maximum(const roaring_bitmap_t *bm) { - if (bm->high_low_container.size > 0) { - container_t *container = - bm->high_low_container.containers[bm->high_low_container.size - 1]; - uint8_t typecode = - bm->high_low_container.typecodes[bm->high_low_container.size - 1]; - uint32_t key = - bm->high_low_container.keys[bm->high_low_container.size - 1]; - uint32_t lowvalue = container_maximum(container, typecode); - return lowvalue | (key << 16); + // Free dest's containers. + art_iterator_t it = art_init_iterator(&dest->art, /*first=*/true); + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + container_free(get_container(dest, leaf), get_typecode(leaf)); + art_iterator_next(&it); } - return 0; -} + art_free(&dest->art); -bool roaring_bitmap_select(const roaring_bitmap_t *bm, uint32_t rank, - uint32_t *element) { - container_t *container; - uint8_t typecode; - uint16_t key; - uint32_t start_rank = 0; - int i = 0; - bool valid = false; - while (!valid && i < bm->high_low_container.size) { - container = bm->high_low_container.containers[i]; - typecode = bm->high_low_container.typecodes[i]; - valid = - container_select(container, typecode, &start_rank, rank, element); - i++; + // Reinitialize dest. + art_init_cleared(&dest->art); + dest->flags = 0; + dest->first_free = 0; + if (dest->capacity > 0) { + memset(dest->containers, 0, + sizeof(dest->containers[0]) * dest->capacity); + } + + // Copy src's containers into dest. + it = art_init_iterator((art_t *)&src->art, /*first=*/true); + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + uint8_t typecode = get_typecode(leaf); + container_t *container = get_copy_of_container( + get_container(src, leaf), &typecode, /*copy_on_write=*/false); + leaf_t dest_leaf = add_container(dest, container, typecode); + art_insert(&dest->art, it.key, (art_val_t)dest_leaf); + art_iterator_next(&it); } +} + +/** + * Steal the containers from a 32-bit bitmap and insert them into a 64-bit + * bitmap (with an offset) + * + * After calling this function, the original bitmap will be empty, and the + * returned bitmap will contain all the values from the original bitmap. + */ +static void move_from_roaring32_offset(roaring64_bitmap_t *dst, + roaring_bitmap_t *src, + uint32_t high_bits) { + uint64_t key_base = ((uint64_t)high_bits) << 32; + uint32_t r32_size = ra_get_size(&src->high_low_container); + ensure_container_capacity(dst, r32_size); + for (uint32_t i = 0; i < r32_size; ++i) { + uint16_t key = ra_get_key_at_index(&src->high_low_container, i); + uint8_t typecode; + container_t *container = ra_get_container_at_index( + &src->high_low_container, (uint16_t)i, &typecode); - if (valid) { - key = bm->high_low_container.keys[i - 1]; - *element |= (((uint32_t)key) << 16); // w/o cast, key promotes signed - return true; - } else - return false; + uint8_t high48[ART_KEY_BYTES]; + uint64_t high48_bits = key_base | ((uint64_t)key << 16); + split_key(high48_bits, high48); + leaf_t leaf = add_container(dst, container, typecode); + art_insert(&dst->art, high48, (art_val_t)leaf); + } + // We stole all the containers, so leave behind a size of zero + src->high_low_container.size = 0; } -bool roaring_bitmap_intersect(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - uint64_t answer = 0; - int pos1 = 0, pos2 = 0; +roaring64_bitmap_t *roaring64_bitmap_move_from_roaring32( + roaring_bitmap_t *bitmap32) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); - while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = - ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - const uint16_t s2 = - ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + move_from_roaring32_offset(result, bitmap32, 0); - if (s1 == s2) { - uint8_t type1, type2; - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - if (container_intersect(c1, type1, c2, type2)) return true; - ++pos1; - ++pos2; - } else if (s1 < s2) { // s1 < s2 - pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); - } else { // s1 > s2 - pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); - } - } - return answer != 0; + return result; } -bool roaring_bitmap_intersect_with_range(const roaring_bitmap_t *bm, uint64_t x, - uint64_t y) { - if (x >= y) { - // Empty range. - return false; - } - roaring_uint32_iterator_t it; - roaring_iterator_init(bm, &it); - if (!roaring_uint32_iterator_move_equalorlarger(&it, (uint32_t)x)) { - // No values above x. - return false; +roaring64_bitmap_t *roaring64_bitmap_from_range(uint64_t min, uint64_t max, + uint64_t step) { + if (step == 0 || max <= min) { + return NULL; } - if (it.current_value >= y) { - // No values below y. - return false; + roaring64_bitmap_t *r = roaring64_bitmap_create(); + if (step >= (1 << 16)) { + // Only one value per container. + for (uint64_t value = min; value < max; value += step) { + roaring64_bitmap_add(r, value); + if (value > UINT64_MAX - step) { + break; + } + } + return r; } - return true; -} + do { + uint64_t high_bits = min & 0xFFFFFFFFFFFF0000; + uint16_t container_min = min & 0xFFFF; + uint32_t container_max = (uint32_t)minimum(max - high_bits, 1 << 16); -uint64_t roaring_bitmap_and_cardinality(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - const int length1 = x1->high_low_container.size, - length2 = x2->high_low_container.size; - uint64_t answer = 0; - int pos1 = 0, pos2 = 0; - while (pos1 < length1 && pos2 < length2) { - const uint16_t s1 = - ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); - const uint16_t s2 = - ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + uint8_t typecode; + container_t *container = container_from_range( + &typecode, container_min, container_max, (uint16_t)step); - if (s1 == s2) { - uint8_t type1, type2; - container_t *c1 = ra_get_container_at_index(&x1->high_low_container, - (uint16_t)pos1, &type1); - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - answer += container_and_cardinality(c1, type1, c2, type2); - ++pos1; - ++pos2; - } else if (s1 < s2) { // s1 < s2 - pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); - } else { // s1 > s2 - pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + uint8_t high48[ART_KEY_BYTES]; + split_key(min, high48); + leaf_t leaf = add_container(r, container, typecode); + art_insert(&r->art, high48, (art_val_t)leaf); + + uint64_t gap = container_max - container_min + step - 1; + uint64_t increment = gap - (gap % step); + if (min > UINT64_MAX - increment) { + break; } - } - return answer; + min += increment; + } while (min < max); + return r; } -double roaring_bitmap_jaccard_index(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - const uint64_t c1 = roaring_bitmap_get_cardinality(x1); - const uint64_t c2 = roaring_bitmap_get_cardinality(x2); - const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); - return (double)inter / (double)(c1 + c2 - inter); +roaring64_bitmap_t *roaring64_bitmap_of_ptr(size_t n_args, + const uint64_t *vals) { + roaring64_bitmap_t *r = roaring64_bitmap_create(); + roaring64_bitmap_add_many(r, n_args, vals); + return r; } -uint64_t roaring_bitmap_or_cardinality(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - const uint64_t c1 = roaring_bitmap_get_cardinality(x1); - const uint64_t c2 = roaring_bitmap_get_cardinality(x2); - const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); - return c1 + c2 - inter; +static inline leaf_t *containerptr_roaring64_bitmap_add(roaring64_bitmap_t *r, + uint8_t *high48, + uint16_t low16, + leaf_t *leaf) { + if (leaf != NULL) { + uint8_t typecode = get_typecode(*leaf); + container_t *container = get_container(r, *leaf); + uint8_t typecode2; + container_t *container2 = + container_add(container, low16, typecode, &typecode2); + if (container2 != container) { + container_free(container, typecode); + replace_container(r, leaf, container2, typecode2); + } + return leaf; + } else { + array_container_t *ac = array_container_create(); + uint8_t typecode; + container_t *container = + container_add(ac, low16, ARRAY_CONTAINER_TYPE, &typecode); + assert(ac == container); + leaf_t new_leaf = add_container(r, container, typecode); + return (leaf_t *)art_insert(&r->art, high48, (art_val_t)new_leaf); + } } -uint64_t roaring_bitmap_andnot_cardinality(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - const uint64_t c1 = roaring_bitmap_get_cardinality(x1); - const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); - return c1 - inter; +void roaring64_bitmap_add(roaring64_bitmap_t *r, uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + containerptr_roaring64_bitmap_add(r, high48, low16, leaf); } -uint64_t roaring_bitmap_xor_cardinality(const roaring_bitmap_t *x1, - const roaring_bitmap_t *x2) { - const uint64_t c1 = roaring_bitmap_get_cardinality(x1); - const uint64_t c2 = roaring_bitmap_get_cardinality(x2); - const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); - return c1 + c2 - 2 * inter; -} +bool roaring64_bitmap_add_checked(roaring64_bitmap_t *r, uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); -/** - * Check whether a range of values from range_start (included) to range_end - * (excluded) is present - */ -bool roaring_bitmap_contains_range(const roaring_bitmap_t *r, - uint64_t range_start, uint64_t range_end) { - if (range_start >= range_end || range_start > (uint64_t)UINT32_MAX + 1) { - return true; + int old_cardinality = 0; + if (leaf != NULL) { + old_cardinality = container_get_cardinality(get_container(r, *leaf), + get_typecode(*leaf)); } - return roaring_bitmap_contains_range_closed(r, (uint32_t)range_start, - (uint32_t)(range_end - 1)); + leaf = containerptr_roaring64_bitmap_add(r, high48, low16, leaf); + int new_cardinality = + container_get_cardinality(get_container(r, *leaf), get_typecode(*leaf)); + return old_cardinality != new_cardinality; } -/** - * Check whether a range of values from range_start (included) to range_end - * (included) is present - */ -bool roaring_bitmap_contains_range_closed(const roaring_bitmap_t *r, - uint32_t range_start, - uint32_t range_end) { - if (range_start > range_end) { - return true; - } // empty range are always contained! - if (range_end == range_start) { - return roaring_bitmap_contains(r, (uint32_t)range_start); - } - uint16_t hb_rs = (uint16_t)(range_start >> 16); - uint16_t hb_re = (uint16_t)(range_end >> 16); - const int32_t span = hb_re - hb_rs; - const int32_t hlc_sz = ra_get_size(&r->high_low_container); - if (hlc_sz < span + 1) { - return false; - } - int32_t is = ra_get_index(&r->high_low_container, hb_rs); - int32_t ie = ra_get_index(&r->high_low_container, hb_re); - if ((ie < 0) || (is < 0) || ((ie - is) != span) || ie >= hlc_sz) { - return false; - } - const uint32_t lb_rs = range_start & 0xFFFF; - const uint32_t lb_re = (range_end & 0xFFFF) + 1; - uint8_t type; - container_t *c = - ra_get_container_at_index(&r->high_low_container, (uint16_t)is, &type); - if (hb_rs == hb_re) { - return container_contains_range(c, lb_rs, lb_re, type); - } - if (!container_contains_range(c, lb_rs, 1 << 16, type)) { - return false; - } - c = ra_get_container_at_index(&r->high_low_container, (uint16_t)ie, &type); - if (!container_contains_range(c, 0, lb_re, type)) { - return false; - } - for (int32_t i = is + 1; i < ie; ++i) { - c = ra_get_container_at_index(&r->high_low_container, (uint16_t)i, - &type); - if (!container_is_full(c, type)) { - return false; +void roaring64_bitmap_add_bulk(roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, + uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + leaf_t *leaf = context->leaf; + if (leaf != NULL && compare_high48(context->high_bytes, high48) == 0) { + // We're at a container with the correct high bits. + uint8_t typecode1 = get_typecode(*leaf); + container_t *container1 = get_container(r, *leaf); + uint8_t typecode2; + container_t *container2 = + container_add(container1, low16, typecode1, &typecode2); + if (container2 != container1) { + container_free(container1, typecode1); + replace_container(r, leaf, container2, typecode2); } + } else { + // We're not positioned anywhere yet or the high bits of the key + // differ. + leaf = (leaf_t *)art_find(&r->art, high48); + context->leaf = + containerptr_roaring64_bitmap_add(r, high48, low16, leaf); + memcpy(context->high_bytes, high48, ART_KEY_BYTES); } - return true; } -bool roaring_bitmap_is_strict_subset(const roaring_bitmap_t *r1, - const roaring_bitmap_t *r2) { - return (roaring_bitmap_get_cardinality(r2) > - roaring_bitmap_get_cardinality(r1) && - roaring_bitmap_is_subset(r1, r2)); +void roaring64_bitmap_add_many(roaring64_bitmap_t *r, size_t n_args, + const uint64_t *vals) { + if (n_args == 0) { + return; + } + const uint64_t *end = vals + n_args; + roaring64_bulk_context_t context = CROARING_ZERO_INITIALIZER; + for (const uint64_t *current_val = vals; current_val != end; + current_val++) { + roaring64_bitmap_add_bulk(r, &context, *current_val); + } } -/* - * FROZEN SERIALIZATION FORMAT DESCRIPTION - * - * -- (beginning must be aligned by 32 bytes) -- - * uint64_t[BITSET_CONTAINER_SIZE_IN_WORDS * - * num_bitset_containers] rle16_t[total number of rle elements in - * all run containers] uint16_t[total number of array elements in - * all array containers] uint16_t[num_containers] - * uint16_t[num_containers] uint8_t[num_containers]
- * uint32_t - * - *
is a 4-byte value which is a bit union of FROZEN_COOKIE (15 bits) - * and the number of containers (17 bits). - * - * stores number of elements for every container. - * Its meaning depends on container type. - * For array and bitset containers, this value is the container cardinality - * minus one. For run container, it is the number of rle_t elements (n_runs). - * - * ,, are flat arrays of elements of - * all containers of respective type. - * - * <*_data> and are kept close together because they are not accessed - * during deserilization. This may reduce IO in case of large mmaped bitmaps. - * All members have their native alignments during deserilization except - *
, which is not guaranteed to be aligned by 4 bytes. - */ - -size_t roaring_bitmap_frozen_size_in_bytes(const roaring_bitmap_t *rb) { - const roaring_array_t *ra = &rb->high_low_container; - size_t num_bytes = 0; - for (int32_t i = 0; i < ra->size; i++) { - switch (ra->typecodes[i]) { - case BITSET_CONTAINER_TYPE: { - num_bytes += BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); - break; - } - case RUN_CONTAINER_TYPE: { - const run_container_t *rc = const_CAST_run(ra->containers[i]); - num_bytes += rc->n_runs * sizeof(rle16_t); - break; - } - case ARRAY_CONTAINER_TYPE: { - const array_container_t *ac = - const_CAST_array(ra->containers[i]); - num_bytes += ac->cardinality * sizeof(uint16_t); - break; - } - default: - roaring_unreachable; +static inline void add_range_closed_at(roaring64_bitmap_t *r, art_t *art, + uint8_t *high48, uint16_t min, + uint16_t max) { + leaf_t *leaf = (leaf_t *)art_find(art, high48); + if (leaf != NULL) { + uint8_t typecode1 = get_typecode(*leaf); + container_t *container1 = get_container(r, *leaf); + uint8_t typecode2; + container_t *container2 = + container_add_range(container1, typecode1, min, max, &typecode2); + if (container2 != container1) { + container_free(container1, typecode1); + replace_container(r, leaf, container2, typecode2); } + return; } - num_bytes += (2 + 2 + 1) * ra->size; // keys, counts, typecodes - num_bytes += 4; // header - return num_bytes; + uint8_t typecode; + // container_add_range is inclusive, but `container_range_of_ones` is + // exclusive. + container_t *container = container_range_of_ones(min, max + 1, &typecode); + leaf_t new_leaf = add_container(r, container, typecode); + art_insert(art, high48, (art_val_t)new_leaf); } -inline static void *arena_alloc(char **arena, size_t num_bytes) { - char *res = *arena; - *arena += num_bytes; - return res; +void roaring64_bitmap_add_range(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min >= max) { + return; + } + roaring64_bitmap_add_range_closed(r, min, max - 1); } -void roaring_bitmap_frozen_serialize(const roaring_bitmap_t *rb, char *buf) { - /* - * Note: we do not require user to supply a specifically aligned buffer. - * Thus we have to use memcpy() everywhere. - */ - - const roaring_array_t *ra = &rb->high_low_container; - - size_t bitset_zone_size = 0; - size_t run_zone_size = 0; - size_t array_zone_size = 0; - for (int32_t i = 0; i < ra->size; i++) { - switch (ra->typecodes[i]) { - case BITSET_CONTAINER_TYPE: { - bitset_zone_size += - BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); - break; - } - case RUN_CONTAINER_TYPE: { - const run_container_t *rc = const_CAST_run(ra->containers[i]); - run_zone_size += rc->n_runs * sizeof(rle16_t); - break; - } - case ARRAY_CONTAINER_TYPE: { - const array_container_t *ac = - const_CAST_array(ra->containers[i]); - array_zone_size += ac->cardinality * sizeof(uint16_t); - break; - } - default: - roaring_unreachable; - } +void roaring64_bitmap_add_range_closed(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min > max) { + return; } - uint64_t *bitset_zone = (uint64_t *)arena_alloc(&buf, bitset_zone_size); - rle16_t *run_zone = (rle16_t *)arena_alloc(&buf, run_zone_size); - uint16_t *array_zone = (uint16_t *)arena_alloc(&buf, array_zone_size); - uint16_t *key_zone = (uint16_t *)arena_alloc(&buf, 2 * ra->size); - uint16_t *count_zone = (uint16_t *)arena_alloc(&buf, 2 * ra->size); - uint8_t *typecode_zone = (uint8_t *)arena_alloc(&buf, ra->size); - char *header_zone = (char *)arena_alloc(&buf, 4); + art_t *art = &r->art; + uint8_t min_high48[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48); + uint8_t max_high48[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48); + if (compare_high48(min_high48, max_high48) == 0) { + // Only populate range within one container. + add_range_closed_at(r, art, min_high48, min_low16, max_low16); + return; + } - for (int32_t i = 0; i < ra->size; i++) { - uint16_t count; - switch (ra->typecodes[i]) { - case BITSET_CONTAINER_TYPE: { - const bitset_container_t *bc = - const_CAST_bitset(ra->containers[i]); - memcpy(bitset_zone, bc->words, - BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t)); - bitset_zone += BITSET_CONTAINER_SIZE_IN_WORDS; - if (bc->cardinality != BITSET_UNKNOWN_CARDINALITY) { - count = (uint16_t)(bc->cardinality - 1); - } else { - count = - (uint16_t)(bitset_container_compute_cardinality(bc) - - 1); - } - break; - } - case RUN_CONTAINER_TYPE: { - const run_container_t *rc = const_CAST_run(ra->containers[i]); - size_t num_bytes = rc->n_runs * sizeof(rle16_t); - memcpy(run_zone, rc->runs, num_bytes); - run_zone += rc->n_runs; - count = (uint16_t)rc->n_runs; - break; - } - case ARRAY_CONTAINER_TYPE: { - const array_container_t *ac = - const_CAST_array(ra->containers[i]); - size_t num_bytes = ac->cardinality * sizeof(uint16_t); - memcpy(array_zone, ac->array, num_bytes); - array_zone += ac->cardinality; - count = (uint16_t)(ac->cardinality - 1); - break; - } - default: - roaring_unreachable; - } - memcpy(&count_zone[i], &count, 2); + // Populate a range across containers. Fill intermediate containers + // entirely. + add_range_closed_at(r, art, min_high48, min_low16, 0xffff); + uint64_t min_high_bits = min >> 16; + uint64_t max_high_bits = max >> 16; + for (uint64_t current = min_high_bits + 1; current < max_high_bits; + ++current) { + uint8_t current_high48[ART_KEY_BYTES]; + split_key(current << 16, current_high48); + add_range_closed_at(r, art, current_high48, 0, 0xffff); } - memcpy(key_zone, ra->keys, ra->size * sizeof(uint16_t)); - memcpy(typecode_zone, ra->typecodes, ra->size * sizeof(uint8_t)); - uint32_t header = ((uint32_t)ra->size << 15) | FROZEN_COOKIE; - memcpy(header_zone, &header, 4); + add_range_closed_at(r, art, max_high48, 0, max_low16); } -const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, - size_t length) { - if ((uintptr_t)buf % 32 != 0) { - return NULL; +bool roaring64_bitmap_contains(const roaring64_bitmap_t *r, uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + if (leaf != NULL) { + return container_contains(get_container(r, *leaf), low16, + get_typecode(*leaf)); } + return false; +} - // cookie and num_containers - if (length < 4) { - return NULL; +bool roaring64_bitmap_contains_range(const roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min >= max) { + return true; } - uint32_t header; - memcpy(&header, buf + length - 4, 4); // header may be misaligned - if ((header & 0x7FFF) != FROZEN_COOKIE) { - return NULL; + return roaring64_bitmap_contains_range_closed(r, min, max - 1); +} + +bool roaring64_bitmap_contains_range_closed(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max) { + if (min > max) { + return true; } - int32_t num_containers = (header >> 15); - // typecodes, counts and keys - if (length < 4 + (size_t)num_containers * (1 + 2 + 2)) { - return NULL; + uint8_t min_high48[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48); + uint8_t max_high48[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48); + uint64_t max_high48_bits = max & 0xFFFFFFFFFFFF0000; + + art_iterator_t it = art_lower_bound((art_t *)&r->art, min_high48); + if (it.value == NULL || combine_key(it.key, 0) > min) { + return false; } - uint16_t *keys = (uint16_t *)(buf + length - 4 - num_containers * 5); - uint16_t *counts = (uint16_t *)(buf + length - 4 - num_containers * 3); - uint8_t *typecodes = (uint8_t *)(buf + length - 4 - num_containers * 1); + uint64_t prev_high48_bits = min & 0xFFFFFFFFFFFF0000; + while (it.value != NULL) { + uint64_t current_high48_bits = combine_key(it.key, 0); + if (current_high48_bits > max_high48_bits) { + // We've passed the end of the range with all containers containing + // the range. + return true; + } + if (current_high48_bits - prev_high48_bits > 0x10000) { + // There is a gap in the iterator that falls in the range. + return false; + } - // {bitset,array,run}_zone - int32_t num_bitset_containers = 0; - int32_t num_run_containers = 0; - int32_t num_array_containers = 0; - size_t bitset_zone_size = 0; - size_t run_zone_size = 0; - size_t array_zone_size = 0; - for (int32_t i = 0; i < num_containers; i++) { - switch (typecodes[i]) { - case BITSET_CONTAINER_TYPE: - num_bitset_containers++; - bitset_zone_size += - BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); - break; - case RUN_CONTAINER_TYPE: - num_run_containers++; - run_zone_size += counts[i] * sizeof(rle16_t); - break; - case ARRAY_CONTAINER_TYPE: - num_array_containers++; - array_zone_size += (counts[i] + UINT32_C(1)) * sizeof(uint16_t); - break; - default: - return NULL; + leaf_t leaf = (leaf_t)*it.value; + uint32_t container_min = 0; + if (compare_high48(it.key, min_high48) == 0) { + container_min = min_low16; } + uint32_t container_max = 0xFFFF + 1; // Exclusive + if (compare_high48(it.key, max_high48) == 0) { + container_max = (uint32_t)max_low16 + 1; + } + + // For the first and last containers we use container_contains_range, + // for the intermediate containers we can use container_is_full. + if (container_min == 0 && container_max == 0xFFFF + 1) { + if (!container_is_full(get_container(r, leaf), + get_typecode(leaf))) { + return false; + } + } else if (!container_contains_range(get_container(r, leaf), + container_min, container_max, + get_typecode(leaf))) { + return false; + } + prev_high48_bits = current_high48_bits; + art_iterator_next(&it); } - if (length != bitset_zone_size + run_zone_size + array_zone_size + - 5 * num_containers + 4) { - return NULL; - } - uint64_t *bitset_zone = (uint64_t *)(buf); - rle16_t *run_zone = (rle16_t *)(buf + bitset_zone_size); - uint16_t *array_zone = (uint16_t *)(buf + bitset_zone_size + run_zone_size); + return prev_high48_bits == max_high48_bits; +} - size_t alloc_size = 0; - alloc_size += sizeof(roaring_bitmap_t); - alloc_size += num_containers * sizeof(container_t *); - alloc_size += num_bitset_containers * sizeof(bitset_container_t); - alloc_size += num_run_containers * sizeof(run_container_t); - alloc_size += num_array_containers * sizeof(array_container_t); +bool roaring64_bitmap_contains_bulk(const roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, + uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); - char *arena = (char *)roaring_malloc(alloc_size); - if (arena == NULL) { - return NULL; + if (context->leaf == NULL || + art_compare_keys(context->high_bytes, high48) != 0) { + // We're not positioned anywhere yet or the high bits of the key + // differ. + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + if (leaf == NULL) { + return false; + } + context->leaf = leaf; + memcpy(context->high_bytes, high48, ART_KEY_BYTES); } + return container_contains(get_container(r, *context->leaf), low16, + get_typecode(*context->leaf)); +} - roaring_bitmap_t *rb = - (roaring_bitmap_t *)arena_alloc(&arena, sizeof(roaring_bitmap_t)); - rb->high_low_container.flags = ROARING_FLAG_FROZEN; - rb->high_low_container.allocation_size = num_containers; - rb->high_low_container.size = num_containers; - rb->high_low_container.keys = (uint16_t *)keys; - rb->high_low_container.typecodes = (uint8_t *)typecodes; - rb->high_low_container.containers = (container_t **)arena_alloc( - &arena, sizeof(container_t *) * num_containers); - // Ensure offset of high_low_container.containers is known distance used in - // C++ wrapper. sizeof(roaring_bitmap_t) is used as it is the size of the - // only allocation that precedes high_low_container.containers. If this is - // changed (new allocation or changed order), this offset will also need to - // be changed in the C++ wrapper. - assert(rb == - (roaring_bitmap_t *)((char *)rb->high_low_container.containers - - sizeof(roaring_bitmap_t))); - for (int32_t i = 0; i < num_containers; i++) { - switch (typecodes[i]) { - case BITSET_CONTAINER_TYPE: { - bitset_container_t *bitset = (bitset_container_t *)arena_alloc( - &arena, sizeof(bitset_container_t)); - bitset->words = bitset_zone; - bitset->cardinality = counts[i] + UINT32_C(1); - rb->high_low_container.containers[i] = bitset; - bitset_zone += BITSET_CONTAINER_SIZE_IN_WORDS; - break; - } - case RUN_CONTAINER_TYPE: { - run_container_t *run = (run_container_t *)arena_alloc( - &arena, sizeof(run_container_t)); - run->capacity = counts[i]; - run->n_runs = counts[i]; - run->runs = run_zone; - rb->high_low_container.containers[i] = run; - run_zone += run->n_runs; - break; - } - case ARRAY_CONTAINER_TYPE: { - array_container_t *array = (array_container_t *)arena_alloc( - &arena, sizeof(array_container_t)); - array->capacity = counts[i] + UINT32_C(1); - array->cardinality = counts[i] + UINT32_C(1); - array->array = array_zone; - rb->high_low_container.containers[i] = array; - array_zone += counts[i] + UINT32_C(1); - break; +bool roaring64_bitmap_select(const roaring64_bitmap_t *r, uint64_t rank, + uint64_t *element) { + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + uint64_t start_rank = 0; + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + uint64_t cardinality = container_get_cardinality(get_container(r, leaf), + get_typecode(leaf)); + if (start_rank + cardinality > rank) { + uint32_t uint32_start = 0; + uint32_t uint32_rank = rank - start_rank; + uint32_t uint32_element = 0; + if (container_select(get_container(r, leaf), get_typecode(leaf), + &uint32_start, uint32_rank, &uint32_element)) { + *element = combine_key(it.key, (uint16_t)uint32_element); + return true; } - default: - roaring_free(arena); - return NULL; + return false; } + start_rank += cardinality; + art_iterator_next(&it); } - - return rb; + return false; } -CROARING_ALLOW_UNALIGNED -roaring_bitmap_t *roaring_bitmap_portable_deserialize_frozen(const char *buf) { - char *start_of_buf = (char *)buf; - uint32_t cookie; - int32_t num_containers; - uint16_t *descriptive_headers; - uint32_t *offset_headers = NULL; - const char *run_flag_bitset = NULL; - bool hasrun = false; +uint64_t roaring64_bitmap_rank(const roaring64_bitmap_t *r, uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); - // deserialize cookie - memcpy(&cookie, buf, sizeof(uint32_t)); - buf += sizeof(uint32_t); - if (cookie == SERIAL_COOKIE_NO_RUNCONTAINER) { - memcpy(&num_containers, buf, sizeof(int32_t)); - buf += sizeof(int32_t); - descriptive_headers = (uint16_t *)buf; - buf += num_containers * 2 * sizeof(uint16_t); - offset_headers = (uint32_t *)buf; - buf += num_containers * sizeof(uint32_t); - } else if ((cookie & 0xFFFF) == SERIAL_COOKIE) { - num_containers = (cookie >> 16) + 1; - hasrun = true; - int32_t run_flag_bitset_size = (num_containers + 7) / 8; - run_flag_bitset = buf; - buf += run_flag_bitset_size; - descriptive_headers = (uint16_t *)buf; - buf += num_containers * 2 * sizeof(uint16_t); - if (num_containers >= NO_OFFSET_THRESHOLD) { - offset_headers = (uint32_t *)buf; - buf += num_containers * sizeof(uint32_t); + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + uint64_t rank = 0; + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + int compare_result = compare_high48(it.key, high48); + if (compare_result < 0) { + rank += container_get_cardinality(get_container(r, leaf), + get_typecode(leaf)); + } else if (compare_result == 0) { + return rank + container_rank(get_container(r, leaf), + get_typecode(leaf), low16); + } else { + return rank; } - } else { - return NULL; + art_iterator_next(&it); } + return rank; +} - // calculate total size for allocation - int32_t num_bitset_containers = 0; - int32_t num_run_containers = 0; - int32_t num_array_containers = 0; - - for (int32_t i = 0; i < num_containers; i++) { - uint16_t tmp; - memcpy(&tmp, descriptive_headers + 2 * i + 1, sizeof(tmp)); - uint32_t cardinality = tmp + 1; - bool isbitmap = (cardinality > DEFAULT_MAX_SIZE); - bool isrun = false; - if (hasrun) { - if ((run_flag_bitset[i / 8] & (1 << (i % 8))) != 0) { - isbitmap = false; - isrun = true; - } - } +bool roaring64_bitmap_get_index(const roaring64_bitmap_t *r, uint64_t val, + uint64_t *out_index) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); - if (isbitmap) { - num_bitset_containers++; - } else if (isrun) { - num_run_containers++; + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + uint64_t index = 0; + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + int compare_result = compare_high48(it.key, high48); + if (compare_result < 0) { + index += container_get_cardinality(get_container(r, leaf), + get_typecode(leaf)); + } else if (compare_result == 0) { + int index16 = container_get_index(get_container(r, leaf), + get_typecode(leaf), low16); + if (index16 < 0) { + return false; + } + *out_index = index + index16; + return true; } else { - num_array_containers++; + return false; } + art_iterator_next(&it); } + return false; +} - size_t alloc_size = 0; - alloc_size += sizeof(roaring_bitmap_t); - alloc_size += num_containers * sizeof(container_t *); - alloc_size += num_bitset_containers * sizeof(bitset_container_t); - alloc_size += num_run_containers * sizeof(run_container_t); - alloc_size += num_array_containers * sizeof(array_container_t); - alloc_size += num_containers * sizeof(uint16_t); // keys - alloc_size += num_containers * sizeof(uint8_t); // typecodes - - // allocate bitmap and construct containers - char *arena = (char *)roaring_malloc(alloc_size); - if (arena == NULL) { - return NULL; +// Returns true if a container was removed. +static inline bool containerptr_roaring64_bitmap_remove(roaring64_bitmap_t *r, + uint8_t *high48, + uint16_t low16, + leaf_t *leaf) { + if (leaf == NULL) { + return false; } - roaring_bitmap_t *rb = - (roaring_bitmap_t *)arena_alloc(&arena, sizeof(roaring_bitmap_t)); - rb->high_low_container.flags = ROARING_FLAG_FROZEN; - rb->high_low_container.allocation_size = num_containers; - rb->high_low_container.size = num_containers; - rb->high_low_container.containers = (container_t **)arena_alloc( - &arena, sizeof(container_t *) * num_containers); + uint8_t typecode = get_typecode(*leaf); + container_t *container = get_container(r, *leaf); + uint8_t typecode2; + container_t *container2 = + container_remove(container, low16, typecode, &typecode2); + if (container2 != container) { + container_free(container, typecode); + replace_container(r, leaf, container2, typecode2); + } + if (!container_nonzero_cardinality(container2, typecode2)) { + container_free(container2, typecode2); + bool erased = art_erase(&r->art, high48, (art_val_t *)leaf); + assert(erased); + (void)erased; + remove_container(r, *leaf); + return true; + } + return false; +} - uint16_t *keys = - (uint16_t *)arena_alloc(&arena, num_containers * sizeof(uint16_t)); - uint8_t *typecodes = - (uint8_t *)arena_alloc(&arena, num_containers * sizeof(uint8_t)); +void roaring64_bitmap_remove(roaring64_bitmap_t *r, uint64_t val) { + art_t *art = &r->art; + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); - rb->high_low_container.keys = keys; - rb->high_low_container.typecodes = typecodes; + leaf_t *leaf = (leaf_t *)art_find(art, high48); + containerptr_roaring64_bitmap_remove(r, high48, low16, leaf); +} - for (int32_t i = 0; i < num_containers; i++) { - uint16_t tmp; - memcpy(&tmp, descriptive_headers + 2 * i + 1, sizeof(tmp)); - int32_t cardinality = tmp + 1; - bool isbitmap = (cardinality > DEFAULT_MAX_SIZE); - bool isrun = false; - if (hasrun) { - if ((run_flag_bitset[i / 8] & (1 << (i % 8))) != 0) { - isbitmap = false; - isrun = true; - } - } +bool roaring64_bitmap_remove_checked(roaring64_bitmap_t *r, uint64_t val) { + art_t *art = &r->art; + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + leaf_t *leaf = (leaf_t *)art_find(art, high48); - keys[i] = descriptive_headers[2 * i]; + if (leaf == NULL) { + return false; + } + int old_cardinality = + container_get_cardinality(get_container(r, *leaf), get_typecode(*leaf)); + if (containerptr_roaring64_bitmap_remove(r, high48, low16, leaf)) { + return true; + } + int new_cardinality = + container_get_cardinality(get_container(r, *leaf), get_typecode(*leaf)); + return new_cardinality != old_cardinality; +} - if (isbitmap) { - typecodes[i] = BITSET_CONTAINER_TYPE; - bitset_container_t *c = (bitset_container_t *)arena_alloc( - &arena, sizeof(bitset_container_t)); - c->cardinality = cardinality; - if (offset_headers != NULL) { - c->words = (uint64_t *)(start_of_buf + offset_headers[i]); - } else { - c->words = (uint64_t *)buf; - buf += BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); - } - rb->high_low_container.containers[i] = c; - } else if (isrun) { - typecodes[i] = RUN_CONTAINER_TYPE; - run_container_t *c = - (run_container_t *)arena_alloc(&arena, sizeof(run_container_t)); - c->capacity = cardinality; - uint16_t n_runs; - if (offset_headers != NULL) { - memcpy(&n_runs, start_of_buf + offset_headers[i], - sizeof(uint16_t)); - c->n_runs = n_runs; - c->runs = (rle16_t *)(start_of_buf + offset_headers[i] + - sizeof(uint16_t)); - } else { - memcpy(&n_runs, buf, sizeof(uint16_t)); - c->n_runs = n_runs; - buf += sizeof(uint16_t); - c->runs = (rle16_t *)buf; - buf += c->n_runs * sizeof(rle16_t); - } - rb->high_low_container.containers[i] = c; - } else { - typecodes[i] = ARRAY_CONTAINER_TYPE; - array_container_t *c = (array_container_t *)arena_alloc( - &arena, sizeof(array_container_t)); - c->cardinality = cardinality; - c->capacity = cardinality; - if (offset_headers != NULL) { - c->array = (uint16_t *)(start_of_buf + offset_headers[i]); - } else { - c->array = (uint16_t *)buf; - buf += cardinality * sizeof(uint16_t); - } - rb->high_low_container.containers[i] = c; +void roaring64_bitmap_remove_bulk(roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, + uint64_t val) { + art_t *art = &r->art; + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + if (context->leaf != NULL && + compare_high48(context->high_bytes, high48) == 0) { + // We're at a container with the correct high bits. + uint8_t typecode = get_typecode(*context->leaf); + container_t *container = get_container(r, *context->leaf); + uint8_t typecode2; + container_t *container2 = + container_remove(container, low16, typecode, &typecode2); + if (container2 != container) { + container_free(container, typecode); + replace_container(r, context->leaf, container2, typecode2); } + if (!container_nonzero_cardinality(container2, typecode2)) { + container_free(container2, typecode2); + leaf_t leaf = 0; + bool erased = art_erase(art, high48, (art_val_t *)&leaf); + assert(erased); + (void)erased; + remove_container(r, leaf); + context->leaf = NULL; + } + } else { + // We're not positioned anywhere yet or the high bits of the key + // differ. + leaf_t *leaf = (leaf_t *)art_find(art, high48); + containerptr_roaring64_bitmap_remove(r, high48, low16, leaf); + context->leaf = leaf; + memcpy(context->high_bytes, high48, ART_KEY_BYTES); } +} - return rb; +void roaring64_bitmap_remove_many(roaring64_bitmap_t *r, size_t n_args, + const uint64_t *vals) { + if (n_args == 0) { + return; + } + const uint64_t *end = vals + n_args; + roaring64_bulk_context_t context = CROARING_ZERO_INITIALIZER; + for (const uint64_t *current_val = vals; current_val != end; + current_val++) { + roaring64_bitmap_remove_bulk(r, &context, *current_val); + } } -bool roaring_bitmap_to_bitset(const roaring_bitmap_t *r, bitset_t *bitset) { - uint32_t max_value = roaring_bitmap_maximum(r); - size_t new_array_size = (size_t)(max_value / 64 + 1); - bool resize_ok = bitset_resize(bitset, new_array_size, true); - if (!resize_ok) { - return false; +static inline void remove_range_closed_at(roaring64_bitmap_t *r, art_t *art, + uint8_t *high48, uint16_t min, + uint16_t max) { + leaf_t *leaf = (leaf_t *)art_find(art, high48); + if (leaf == NULL) { + return; } - const roaring_array_t *ra = &r->high_low_container; - for (int i = 0; i < ra->size; ++i) { - uint64_t *words = bitset->array + (ra->keys[i] << 10); - uint8_t type = ra->typecodes[i]; - const container_t *c = ra->containers[i]; - if (type == SHARED_CONTAINER_TYPE) { - c = container_unwrap_shared(c, &type); - } - switch (type) { - case BITSET_CONTAINER_TYPE: { - size_t max_word_index = new_array_size - (ra->keys[i] << 10); - if (max_word_index > 1024) { - max_word_index = 1024; - } - const bitset_container_t *src = const_CAST_bitset(c); - memcpy(words, src->words, max_word_index * sizeof(uint64_t)); - } break; - case ARRAY_CONTAINER_TYPE: { - const array_container_t *src = const_CAST_array(c); - bitset_set_list(words, src->array, src->cardinality); - } break; - case RUN_CONTAINER_TYPE: { - const run_container_t *src = const_CAST_run(c); - for (int32_t rlepos = 0; rlepos < src->n_runs; ++rlepos) { - rle16_t rle = src->runs[rlepos]; - bitset_set_lenrange(words, rle.value, rle.length); - } - } break; - default: - roaring_unreachable; + uint8_t typecode = get_typecode(*leaf); + container_t *container = get_container(r, *leaf); + uint8_t typecode2; + container_t *container2 = + container_remove_range(container, typecode, min, max, &typecode2); + if (container2 != container) { + container_free(container, typecode); + if (container2 != NULL) { + replace_container(r, leaf, container2, typecode2); + } else { + bool erased = art_erase(art, high48, NULL); + assert(erased); + (void)erased; + remove_container(r, *leaf); } } - return true; } -#ifdef __cplusplus -} +void roaring64_bitmap_remove_range(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min >= max) { + return; + } + roaring64_bitmap_remove_range_closed(r, min, max - 1); } -} // extern "C" { namespace roaring { -#endif -/* end file src/roaring.c */ -/* begin file src/roaring64.c */ -#include -#include -#include -#include -#include - - -// For serialization / deserialization -// containers.h last to avoid conflict with ROARING_CONTAINER_T. - -#define CROARING_ALIGN_BUF(buf, alignment) \ - (char *)(((uintptr_t)(buf) + ((alignment)-1)) & \ - (ptrdiff_t)(~((alignment)-1))) - -#define CROARING_BITSET_ALIGNMENT 64 - -#ifdef __cplusplus -using namespace ::roaring::internal; - -extern "C" { -namespace roaring { -namespace api { -#endif - -// TODO: Copy on write. -// TODO: Error on failed allocation. - -typedef struct roaring64_bitmap_s { - art_t art; - uint8_t flags; - uint64_t first_free; - uint64_t capacity; - container_t **containers; -} roaring64_bitmap_t; - -// Leaf type of the ART used to keep the high 48 bits of each entry. -// Low 8 bits: typecode -// High 56 bits: container index -typedef roaring64_leaf_t leaf_t; -// Iterator struct to hold iteration state. -typedef struct roaring64_iterator_s { - const roaring64_bitmap_t *r; - art_iterator_t art_it; - roaring_container_iterator_t container_it; - uint64_t high48; // Key that art_it points to. +void roaring64_bitmap_remove_range_closed(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min > max) { + return; + } - uint64_t value; - bool has_value; + art_t *art = &r->art; + uint8_t min_high48[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48); + uint8_t max_high48[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48); + if (compare_high48(min_high48, max_high48) == 0) { + // Only remove a range within one container. + remove_range_closed_at(r, art, min_high48, min_low16, max_low16); + return; + } - // If has_value is false, then the iterator is saturated. This field - // indicates the direction of saturation. If true, there are no more values - // in the forward direction. If false, there are no more values in the - // backward direction. - bool saturated_forward; -} roaring64_iterator_t; + // Remove a range across containers. Remove intermediate containers + // entirely. + remove_range_closed_at(r, art, min_high48, min_low16, 0xffff); -static inline bool is_frozen64(const roaring64_bitmap_t *r) { - return r->flags & ROARING_FLAG_FROZEN; + art_iterator_t it = art_upper_bound(art, min_high48); + while (it.value != NULL && art_compare_keys(it.key, max_high48) < 0) { + leaf_t leaf = 0; + bool erased = art_iterator_erase(&it, (art_val_t *)&leaf); + assert(erased); + (void)erased; + container_free(get_container(r, leaf), get_typecode(leaf)); + remove_container(r, leaf); + } + remove_range_closed_at(r, art, max_high48, 0, max_low16); } -// Splits the given uint64 key into high 48 bit and low 16 bit components. -// Expects high48_out to be of length ART_KEY_BYTES. -static inline uint16_t split_key(uint64_t key, uint8_t high48_out[]) { - uint64_t tmp = croaring_htobe64(key); - memcpy(high48_out, (uint8_t *)(&tmp), ART_KEY_BYTES); - return (uint16_t)key; +void roaring64_bitmap_clear(roaring64_bitmap_t *r) { + roaring64_bitmap_remove_range_closed(r, 0, UINT64_MAX); } -// Recombines the high 48 bit and low 16 bit components into a uint64 key. -// Expects high48_out to be of length ART_KEY_BYTES. -static inline uint64_t combine_key(const uint8_t high48[], uint16_t low16) { - uint64_t result = 0; - memcpy((uint8_t *)(&result), high48, ART_KEY_BYTES); - return croaring_be64toh(result) | low16; +uint64_t roaring64_bitmap_get_cardinality(const roaring64_bitmap_t *r) { + // Scan the pointer array rather than the ART: the arrays are sequential + // and the ART is not. first_free is a free-list head, not a size, so + // after deletes live containers can sit above it; skip NULL slots. + uint64_t cardinality = 0; + for (uint64_t i = 0; i < r->capacity; ++i) { + if (r->containers[i] != NULL) { + cardinality += + container_get_cardinality(r->containers[i], r->typecodes[i]); + } + } + return cardinality; } -static inline uint64_t minimum(uint64_t a, uint64_t b) { - return (a < b) ? a : b; +uint64_t roaring64_bitmap_range_cardinality(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max) { + if (min >= max) { + return 0; + } + // Convert to a closed range + // No underflow here: passing the above condition implies min < max, so + // there is a number less than max + return roaring64_bitmap_range_closed_cardinality(r, min, max - 1); } -static inline leaf_t create_leaf(uint64_t container_index, uint8_t typecode) { - return (container_index << 8) | typecode; -} +uint64_t roaring64_bitmap_range_closed_cardinality(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max) { + if (min > max) { + return 0; + } -static inline uint8_t get_typecode(leaf_t leaf) { return (uint8_t)leaf; } + uint64_t cardinality = 0; + uint8_t min_high48[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48); + uint8_t max_high48[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48); -static inline uint64_t get_index(leaf_t leaf) { return leaf >> 8; } + art_iterator_t it = art_lower_bound((art_t *)&r->art, min_high48); + while (it.value != NULL) { + int max_compare_result = compare_high48(it.key, max_high48); + if (max_compare_result > 0) { + // We're outside the range. + break; + } -static inline container_t *get_container(const roaring64_bitmap_t *r, - leaf_t leaf) { - return r->containers[get_index(leaf)]; + leaf_t leaf = (leaf_t)*it.value; + uint8_t typecode = get_typecode(leaf); + container_t *container = get_container(r, leaf); + if (max_compare_result == 0) { + // We're at the max high key, add only the range up to the low + // 16 bits of max. + cardinality += container_rank(container, typecode, max_low16); + } else { + // We're not yet at the max high key, add the full container + // range. + cardinality += container_get_cardinality(container, typecode); + } + if (compare_high48(it.key, min_high48) == 0 && min_low16 > 0) { + // We're at the min high key, remove the range up to the low 16 + // bits of min. + cardinality -= container_rank(container, typecode, min_low16 - 1); + } + art_iterator_next(&it); + } + return cardinality; } -// Replaces the container of `leaf` with the given container. Returns the -// modified leaf for convenience. -static inline leaf_t replace_container(roaring64_bitmap_t *r, leaf_t *leaf, - container_t *container, - uint8_t typecode) { - uint64_t index = get_index(*leaf); - r->containers[index] = container; - *leaf = create_leaf(index, typecode); - return *leaf; +bool roaring64_bitmap_is_empty(const roaring64_bitmap_t *r) { + return art_is_empty(&r->art); } -/** - * Extends the array of container pointers. - */ -static void extend_containers(roaring64_bitmap_t *r) { - uint64_t size = r->first_free; - if (size < r->capacity) { - return; +uint64_t roaring64_bitmap_minimum(const roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + if (it.value == NULL) { + return UINT64_MAX; } - uint64_t new_capacity; - if (r->capacity == 0) { - new_capacity = 2; - } else if (r->capacity < 1024) { - new_capacity = 2 * r->capacity; - } else { - new_capacity = 5 * r->capacity / 4; + leaf_t leaf = (leaf_t)*it.value; + return combine_key( + it.key, container_minimum(get_container(r, leaf), get_typecode(leaf))); +} + +uint64_t roaring64_bitmap_maximum(const roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/false); + if (it.value == NULL) { + return 0; } - uint64_t increase = new_capacity - r->capacity; - r->containers = (container_t **)roaring_realloc( - r->containers, new_capacity * sizeof(container_t *)); - memset(r->containers + r->capacity, 0, increase * sizeof(container_t *)); - r->capacity = new_capacity; + leaf_t leaf = (leaf_t)*it.value; + return combine_key( + it.key, container_maximum(get_container(r, leaf), get_typecode(leaf))); } -static uint64_t next_free_container_idx(const roaring64_bitmap_t *r) { - for (uint64_t i = r->first_free + 1; i < r->capacity; ++i) { - if (r->containers[i] == NULL) { - return i; +bool roaring64_bitmap_remove_run_compression(roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + bool removed = false; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + if (get_typecode(*leaf) == RUN_CONTAINER_TYPE) { + run_container_t *run = CAST_run(get_container(r, *leaf)); + int32_t card = run_container_cardinality(run); + uint8_t new_typecode; + container_t *new_container = + convert_to_bitset_or_array_container(run, card, &new_typecode); + run_container_free(run); + replace_container(r, leaf, new_container, new_typecode); + removed = true; } + art_iterator_next(&it); } - return r->capacity; + return removed; } -static uint64_t allocate_index(roaring64_bitmap_t *r) { - uint64_t first_free = r->first_free; - if (first_free == r->capacity) { - extend_containers(r); +bool roaring64_bitmap_run_optimize(roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + bool has_run_container = false; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + uint8_t new_typecode; + // We don't need to free the existing container if a new one was + // created, convert_run_optimize does that internally. + container_t *new_container = convert_run_optimize( + get_container(r, *leaf), get_typecode(*leaf), &new_typecode); + replace_container(r, leaf, new_container, new_typecode); + has_run_container |= new_typecode == RUN_CONTAINER_TYPE; + art_iterator_next(&it); + } + return has_run_container; +} + +static void move_to_shrink(roaring64_bitmap_t *r, leaf_t *leaf) { + uint64_t idx = get_index(*leaf); + if (idx < r->first_free) { + return; } + uint8_t typecode = get_typecode(*leaf); + set_container_at(r, r->first_free, get_container(r, *leaf), typecode); + r->containers[idx] = NULL; + *leaf = create_leaf(r->first_free, typecode); r->first_free = next_free_container_idx(r); - return first_free; } -static leaf_t add_container(roaring64_bitmap_t *r, container_t *container, - uint8_t typecode) { - uint64_t index = allocate_index(r); - r->containers[index] = container; - return create_leaf(index, typecode); +static inline bool is_shrunken(const roaring64_bitmap_t *r) { + return art_is_shrunken(&r->art) && r->first_free == r->capacity; } -static void remove_container(roaring64_bitmap_t *r, leaf_t leaf) { - uint64_t index = get_index(leaf); - r->containers[index] = NULL; - if (index < r->first_free) { - r->first_free = index; +size_t roaring64_bitmap_shrink_to_fit(roaring64_bitmap_t *r) { + size_t freed = art_shrink_to_fit(&r->art); + art_iterator_t it = art_init_iterator(&r->art, true); + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + freed += container_shrink_to_fit(get_container(r, *leaf), + get_typecode(*leaf)); + move_to_shrink(r, leaf); + art_iterator_next(&it); + } + if (is_shrunken(r)) { + return freed; + } + uint64_t new_capacity = r->first_free; + if (new_capacity < r->capacity) { + r->containers = (container_t **)roaring_realloc( + r->containers, new_capacity * sizeof(container_t *)); + r->typecodes = (uint8_t *)roaring_realloc( + r->typecodes, new_capacity * sizeof(uint8_t)); + freed += (r->capacity - new_capacity) * + (sizeof(container_t *) + sizeof(uint8_t)); + r->capacity = new_capacity; } + return freed; } -// Copies the container referenced by `leaf` from `r1` to `r2`. -static inline leaf_t copy_leaf_container(const roaring64_bitmap_t *r1, - roaring64_bitmap_t *r2, leaf_t leaf) { - uint8_t typecode = get_typecode(leaf); - // get_copy_of_container modifies the typecode passed in. - container_t *container = get_copy_of_container( - get_container(r1, leaf), &typecode, /*copy_on_write=*/false); - return add_container(r2, container, typecode); -} +/** + * (For advanced users.) + * Collect statistics about the bitmap + */ +void roaring64_bitmap_statistics(const roaring64_bitmap_t *r, + roaring64_statistics_t *stat) { + memset(stat, 0, sizeof(*stat)); + stat->min_value = roaring64_bitmap_minimum(r); + stat->max_value = roaring64_bitmap_maximum(r); -static inline int compare_high48(art_key_chunk_t key1[], - art_key_chunk_t key2[]) { - return art_compare_keys(key1, key2); + art_iterator_t it = art_init_iterator((art_t *)&r->art, true); + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + stat->n_containers++; + uint8_t truetype = + get_container_type(get_container(r, leaf), get_typecode(leaf)); + uint32_t card = container_get_cardinality(get_container(r, leaf), + get_typecode(leaf)); + uint32_t sbytes = + container_size_in_bytes(get_container(r, leaf), get_typecode(leaf)); + stat->cardinality += card; + switch (truetype) { + case BITSET_CONTAINER_TYPE: + stat->n_bitset_containers++; + stat->n_values_bitset_containers += card; + stat->n_bytes_bitset_containers += sbytes; + break; + case ARRAY_CONTAINER_TYPE: + stat->n_array_containers++; + stat->n_values_array_containers += card; + stat->n_bytes_array_containers += sbytes; + break; + case RUN_CONTAINER_TYPE: + stat->n_run_containers++; + stat->n_values_run_containers += card; + stat->n_bytes_run_containers += sbytes; + break; + default: + assert(false); + roaring_unreachable; + } + art_iterator_next(&it); + } } -static inline bool roaring64_iterator_init_at_leaf_first( - roaring64_iterator_t *it) { - it->high48 = combine_key(it->art_it.key, 0); - leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = 0; - it->container_it = container_init_iterator(get_container(it->r, leaf), - get_typecode(leaf), &low16); - it->value = it->high48 | low16; - return (it->has_value = true); +static bool roaring64_leaf_internal_validate(const art_val_t val, + const char **reason, + void *context) { + leaf_t leaf = (leaf_t)val; + roaring64_bitmap_t *r = (roaring64_bitmap_t *)context; + uint64_t index = get_index(leaf); + uint8_t typecode = get_typecode(leaf); + if (index >= r->capacity || r->containers[index] == NULL) { + *reason = "ART leaf points at an empty container slot"; + return false; + } + if (r->typecodes[index] != typecode) { + *reason = "typecode array does not match ART leaf"; + return false; + } + return container_internal_validate(r->containers[index], typecode, reason); } -static inline bool roaring64_iterator_init_at_leaf_last( - roaring64_iterator_t *it) { - it->high48 = combine_key(it->art_it.key, 0); - leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = 0; - it->container_it = container_init_iterator_last(get_container(it->r, leaf), - get_typecode(leaf), &low16); - it->value = it->high48 | low16; - return (it->has_value = true); +bool roaring64_bitmap_internal_validate(const roaring64_bitmap_t *r, + const char **reason) { + return art_internal_validate(&r->art, reason, + roaring64_leaf_internal_validate, (void *)r); } -static inline roaring64_iterator_t *roaring64_iterator_init_at( - const roaring64_bitmap_t *r, roaring64_iterator_t *it, bool first) { - it->r = r; - it->art_it = art_init_iterator((art_t *)&r->art, first); - it->has_value = it->art_it.value != NULL; - if (it->has_value) { - if (first) { - roaring64_iterator_init_at_leaf_first(it); - } else { - roaring64_iterator_init_at_leaf_last(it); +bool roaring64_bitmap_equals(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); + + while (it1.value != NULL && it2.value != NULL) { + if (compare_high48(it1.key, it2.key) != 0) { + return false; } - } else { - it->saturated_forward = first; + leaf_t leaf1 = (leaf_t)*it1.value; + leaf_t leaf2 = (leaf_t)*it2.value; + if (!container_equals(get_container(r1, leaf1), get_typecode(leaf1), + get_container(r2, leaf2), get_typecode(leaf2))) { + return false; + } + art_iterator_next(&it1); + art_iterator_next(&it2); } - return it; + return it1.value == NULL && it2.value == NULL; } -roaring64_bitmap_t *roaring64_bitmap_create(void) { - roaring64_bitmap_t *r = - (roaring64_bitmap_t *)roaring_malloc(sizeof(roaring64_bitmap_t)); - art_init_cleared(&r->art); - r->flags = 0; - r->capacity = 0; - r->first_free = 0; - r->containers = NULL; - return r; -} +bool roaring64_bitmap_is_subset(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); -void roaring64_bitmap_free(roaring64_bitmap_t *r) { - if (!r) { - return; - } - art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - if (is_frozen64(r)) { - // Only free the container itself, not the buffer-backed contents - // within. - roaring_free(get_container(r, leaf)); - } else { - container_free(get_container(r, leaf), get_typecode(leaf)); + while (it1.value != NULL) { + bool it2_present = it2.value != NULL; + + int compare_result = 0; + if (it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + leaf_t leaf1 = (leaf_t)*it1.value; + leaf_t leaf2 = (leaf_t)*it2.value; + if (!container_is_subset( + get_container(r1, leaf1), get_typecode(leaf1), + get_container(r2, leaf2), get_typecode(leaf2))) { + return false; + } + art_iterator_next(&it1); + art_iterator_next(&it2); + } + } + if (!it2_present || compare_result < 0) { + return false; + } else if (compare_result > 0) { + art_iterator_lower_bound(&it2, it1.key); } - art_iterator_next(&it); - } - if (!is_frozen64(r)) { - art_free(&r->art); } - roaring_free(r->containers); - roaring_free(r); + return true; } -roaring64_bitmap_t *roaring64_bitmap_copy(const roaring64_bitmap_t *r) { +bool roaring64_bitmap_is_strict_subset(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + return roaring64_bitmap_get_cardinality(r1) < + roaring64_bitmap_get_cardinality(r2) && + roaring64_bitmap_is_subset(r1, r2); +} + +roaring64_bitmap_t *roaring64_bitmap_and(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { roaring64_bitmap_t *result = roaring64_bitmap_create(); - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - uint8_t result_typecode = get_typecode(leaf); - container_t *result_container = get_copy_of_container( - get_container(r, leaf), &result_typecode, /*copy_on_write=*/false); - leaf_t result_leaf = - add_container(result, result_container, result_typecode); - art_insert(&result->art, it.key, (art_val_t)result_leaf); - art_iterator_next(&it); + art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); + + while (it1.value != NULL && it2.value != NULL) { + // Cases: + // 1. it1 < it2 -> it1++ + // 2. it1 == it1 -> output it1 & it2, it1++, it2++ + // 3. it1 > it2 -> it2++ + int compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2: iterators at the same high key position. + leaf_t leaf1 = (leaf_t)*it1.value; + leaf_t leaf2 = (leaf_t)*it2.value; + uint8_t result_typecode; + container_t *result_container = + container_and(get_container(r1, leaf1), get_typecode(leaf1), + get_container(r2, leaf2), get_typecode(leaf2), + &result_typecode); + if (container_nonzero_cardinality(result_container, + result_typecode)) { + leaf_t result_leaf = + add_container(result, result_container, result_typecode); + art_insert(&result->art, it1.key, (art_val_t)result_leaf); + } else { + container_free(result_container, result_typecode); + } + art_iterator_next(&it1); + art_iterator_next(&it2); + } else if (compare_result < 0) { + // Case 1: it1 is before it2. + art_iterator_lower_bound(&it1, it2.key); + } else { + // Case 3: it2 is before it1. + art_iterator_lower_bound(&it2, it1.key); + } } return result; } -void roaring64_bitmap_overwrite(roaring64_bitmap_t *dest, - const roaring64_bitmap_t *src) { - if (dest == src) { - return; - } - - // Free dest's containers. - art_iterator_t it = art_init_iterator(&dest->art, /*first=*/true); - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - container_free(get_container(dest, leaf), get_typecode(leaf)); - art_iterator_next(&it); - } - art_free(&dest->art); +uint64_t roaring64_bitmap_and_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t result = 0; - // Reinitialize dest. - art_init_cleared(&dest->art); - dest->flags = 0; - dest->first_free = 0; - if (dest->capacity > 0) { - memset(dest->containers, 0, - sizeof(dest->containers[0]) * dest->capacity); - } + art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); - // Copy src's containers into dest. - it = art_init_iterator((art_t *)&src->art, /*first=*/true); - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - uint8_t typecode = get_typecode(leaf); - container_t *container = get_copy_of_container( - get_container(src, leaf), &typecode, /*copy_on_write=*/false); - leaf_t dest_leaf = add_container(dest, container, typecode); - art_insert(&dest->art, it.key, (art_val_t)dest_leaf); - art_iterator_next(&it); + while (it1.value != NULL && it2.value != NULL) { + // Cases: + // 1. it1 < it2 -> it1++ + // 2. it1 == it1 -> output cardinaltiy it1 & it2, it1++, it2++ + // 3. it1 > it2 -> it2++ + int compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2: iterators at the same high key position. + leaf_t leaf1 = (leaf_t)*it1.value; + leaf_t leaf2 = (leaf_t)*it2.value; + result += container_and_cardinality( + get_container(r1, leaf1), get_typecode(leaf1), + get_container(r2, leaf2), get_typecode(leaf2)); + art_iterator_next(&it1); + art_iterator_next(&it2); + } else if (compare_result < 0) { + // Case 1: it1 is before it2. + art_iterator_lower_bound(&it1, it2.key); + } else { + // Case 3: it2 is before it1. + art_iterator_lower_bound(&it2, it1.key); + } } + return result; } -/** - * Steal the containers from a 32-bit bitmap and insert them into a 64-bit - * bitmap (with an offset) - * - * After calling this function, the original bitmap will be empty, and the - * returned bitmap will contain all the values from the original bitmap. - */ -static void move_from_roaring32_offset(roaring64_bitmap_t *dst, - roaring_bitmap_t *src, - uint32_t high_bits) { - uint64_t key_base = ((uint64_t)high_bits) << 32; - uint32_t r32_size = ra_get_size(&src->high_low_container); - for (uint32_t i = 0; i < r32_size; ++i) { - uint16_t key = ra_get_key_at_index(&src->high_low_container, i); - uint8_t typecode; - container_t *container = ra_get_container_at_index( - &src->high_low_container, (uint16_t)i, &typecode); - - uint8_t high48[ART_KEY_BYTES]; - uint64_t high48_bits = key_base | ((uint64_t)key << 16); - split_key(high48_bits, high48); - leaf_t leaf = add_container(dst, container, typecode); - art_insert(&dst->art, high48, (art_val_t)leaf); +// Inplace and (modifies its first argument). +void roaring64_bitmap_and_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + if (r1 == r2) { + return; } - // We stole all the containers, so leave behind a size of zero - src->high_low_container.size = 0; -} - -roaring64_bitmap_t *roaring64_bitmap_move_from_roaring32( - roaring_bitmap_t *bitmap32) { - roaring64_bitmap_t *result = roaring64_bitmap_create(); + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); - move_from_roaring32_offset(result, bitmap32, 0); + while (it1.value != NULL) { + // Cases: + // 1. !it2_present -> erase it1 + // 2. it2_present + // a. it1 < it2 -> erase it1 + // b. it1 == it2 -> output it1 & it2, it1++, it2++ + // c. it1 > it2 -> it2++ + bool it2_present = it2.value != NULL; + int compare_result = 0; + if (it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2a: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t leaf2 = (leaf_t)*it2.value; - return result; -} + // We do the computation "in place" only when c1 is not a + // shared container. Rationale: using a shared container + // safely with in place computation would require making a + // copy and then doing the computation in place which is + // likely less efficient than avoiding in place entirely and + // always generating a new container. + uint8_t typecode = get_typecode(*leaf1); + container_t *container = get_container(r1, *leaf1); + uint8_t typecode2; + container_t *container2; + if (typecode == SHARED_CONTAINER_TYPE) { + container2 = container_and(container, typecode, + get_container(r2, leaf2), + get_typecode(leaf2), &typecode2); + } else { + container2 = container_iand( + container, typecode, get_container(r2, leaf2), + get_typecode(leaf2), &typecode2); + } -roaring64_bitmap_t *roaring64_bitmap_from_range(uint64_t min, uint64_t max, - uint64_t step) { - if (step == 0 || max <= min) { - return NULL; - } - roaring64_bitmap_t *r = roaring64_bitmap_create(); - if (step >= (1 << 16)) { - // Only one value per container. - for (uint64_t value = min; value < max; value += step) { - roaring64_bitmap_add(r, value); - if (value > UINT64_MAX - step) { - break; + if (container2 != container) { + container_free(container, typecode); + } + if (!container_nonzero_cardinality(container2, typecode2)) { + container_free(container2, typecode2); + art_iterator_erase(&it1, NULL); + remove_container(r1, *leaf1); + } else { + if (container2 != container) { + replace_container(r1, leaf1, container2, typecode2); + } + // Only advance the iterator if we didn't delete the + // leaf, as erasing advances by itself. + art_iterator_next(&it1); + } + art_iterator_next(&it2); } } - return r; - } - do { - uint64_t high_bits = min & 0xFFFFFFFFFFFF0000; - uint16_t container_min = min & 0xFFFF; - uint32_t container_max = (uint32_t)minimum(max - high_bits, 1 << 16); - - uint8_t typecode; - container_t *container = container_from_range( - &typecode, container_min, container_max, (uint16_t)step); - - uint8_t high48[ART_KEY_BYTES]; - split_key(min, high48); - leaf_t leaf = add_container(r, container, typecode); - art_insert(&r->art, high48, (art_val_t)leaf); - uint64_t gap = container_max - container_min + step - 1; - uint64_t increment = gap - (gap % step); - if (min > UINT64_MAX - increment) { - break; + if (!it2_present || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + leaf_t leaf = 0; + bool erased = art_iterator_erase(&it1, (art_val_t *)&leaf); + assert(erased); + (void)erased; + container_free(get_container(r1, leaf), get_typecode(leaf)); + remove_container(r1, leaf); + } else if (compare_result > 0) { + // Case 2c: it1 is after it2. + art_iterator_lower_bound(&it2, it1.key); } - min += increment; - } while (min < max); - return r; + } } -roaring64_bitmap_t *roaring64_bitmap_of_ptr(size_t n_args, - const uint64_t *vals) { - roaring64_bitmap_t *r = roaring64_bitmap_create(); - roaring64_bitmap_add_many(r, n_args, vals); - return r; -} +bool roaring64_bitmap_intersect(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + bool intersect = false; + art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); -static inline leaf_t *containerptr_roaring64_bitmap_add(roaring64_bitmap_t *r, - uint8_t *high48, - uint16_t low16, - leaf_t *leaf) { - if (leaf != NULL) { - uint8_t typecode = get_typecode(*leaf); - container_t *container = get_container(r, *leaf); - uint8_t typecode2; - container_t *container2 = - container_add(container, low16, typecode, &typecode2); - if (container2 != container) { - container_free(container, typecode); - replace_container(r, leaf, container2, typecode2); + while (it1.value != NULL && it2.value != NULL) { + // Cases: + // 1. it1 < it2 -> it1++ + // 2. it1 == it1 -> intersect |= it1 & it2, it1++, it2++ + // 3. it1 > it2 -> it2++ + int compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2: iterators at the same high key position. + leaf_t leaf1 = (leaf_t)*it1.value; + leaf_t leaf2 = (leaf_t)*it2.value; + intersect |= container_intersect( + get_container(r1, leaf1), get_typecode(leaf1), + get_container(r2, leaf2), get_typecode(leaf2)); + art_iterator_next(&it1); + art_iterator_next(&it2); + } else if (compare_result < 0) { + // Case 1: it1 is before it2. + art_iterator_lower_bound(&it1, it2.key); + } else { + // Case 3: it2 is before it1. + art_iterator_lower_bound(&it2, it1.key); } - return leaf; - } else { - array_container_t *ac = array_container_create(); - uint8_t typecode; - container_t *container = - container_add(ac, low16, ARRAY_CONTAINER_TYPE, &typecode); - assert(ac == container); - leaf_t new_leaf = add_container(r, container, typecode); - return (leaf_t *)art_insert(&r->art, high48, (art_val_t)new_leaf); } + return intersect; +} + +bool roaring64_bitmap_intersect_with_range(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max) { + if (min >= max) { + return false; + } + roaring64_iterator_t it; + roaring64_iterator_init_at(r, &it, /*first=*/true); + if (!roaring64_iterator_move_equalorlarger(&it, min)) { + return false; + } + return roaring64_iterator_has_value(&it) && + roaring64_iterator_value(&it) < max; } -void roaring64_bitmap_add(roaring64_bitmap_t *r, uint64_t val) { - uint8_t high48[ART_KEY_BYTES]; - uint16_t low16 = split_key(val, high48); - leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); - containerptr_roaring64_bitmap_add(r, high48, low16, leaf); +double roaring64_bitmap_jaccard_index(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t c1 = roaring64_bitmap_get_cardinality(r1); + uint64_t c2 = roaring64_bitmap_get_cardinality(r2); + uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); + return (double)inter / (double)(c1 + c2 - inter); } -bool roaring64_bitmap_add_checked(roaring64_bitmap_t *r, uint64_t val) { - uint8_t high48[ART_KEY_BYTES]; - uint16_t low16 = split_key(val, high48); - leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); +roaring64_bitmap_t *roaring64_bitmap_or(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); - int old_cardinality = 0; - if (leaf != NULL) { - old_cardinality = container_get_cardinality(get_container(r, *leaf), - get_typecode(*leaf)); - } - leaf = containerptr_roaring64_bitmap_add(r, high48, low16, leaf); - int new_cardinality = - container_get_cardinality(get_container(r, *leaf), get_typecode(*leaf)); - return old_cardinality != new_cardinality; -} + art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); -void roaring64_bitmap_add_bulk(roaring64_bitmap_t *r, - roaring64_bulk_context_t *context, - uint64_t val) { - uint8_t high48[ART_KEY_BYTES]; - uint16_t low16 = split_key(val, high48); - leaf_t *leaf = context->leaf; - if (leaf != NULL && compare_high48(context->high_bytes, high48) == 0) { - // We're at a container with the correct high bits. - uint8_t typecode1 = get_typecode(*leaf); - container_t *container1 = get_container(r, *leaf); - uint8_t typecode2; - container_t *container2 = - container_add(container1, low16, typecode1, &typecode2); - if (container2 != container1) { - container_free(container1, typecode1); - replace_container(r, leaf, container2, typecode2); + while (it1.value != NULL || it2.value != NULL) { + bool it1_present = it1.value != NULL; + bool it2_present = it2.value != NULL; + + // Cases: + // 1. it1_present && !it2_present -> output it1, it1++ + // 2. !it1_present && it2_present -> output it2, it2++ + // 3. it1_present && it2_present + // a. it1 < it2 -> output it1, it1++ + // b. it1 == it2 -> output it1 | it2, it1++, it2++ + // c. it1 > it2 -> output it2, it2++ + int compare_result = 0; + if (it1_present && it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 3b: iterators at the same high key position. + leaf_t leaf1 = (leaf_t)*it1.value; + leaf_t leaf2 = (leaf_t)*it2.value; + uint8_t result_typecode; + container_t *result_container = + container_or(get_container(r1, leaf1), get_typecode(leaf1), + get_container(r2, leaf2), get_typecode(leaf2), + &result_typecode); + leaf_t result_leaf = + add_container(result, result_container, result_typecode); + art_insert(&result->art, it1.key, (art_val_t)result_leaf); + art_iterator_next(&it1); + art_iterator_next(&it2); + } + } + if ((it1_present && !it2_present) || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + leaf_t result_leaf = + copy_leaf_container(r1, result, (leaf_t)*it1.value); + art_insert(&result->art, it1.key, (art_val_t)result_leaf); + art_iterator_next(&it1); + } else if ((!it1_present && it2_present) || compare_result > 0) { + // Cases 2 and 3c: it2 is the only iterator or is before it1. + leaf_t result_leaf = + copy_leaf_container(r2, result, (leaf_t)*it2.value); + art_insert(&result->art, it2.key, (art_val_t)result_leaf); + art_iterator_next(&it2); } - } else { - // We're not positioned anywhere yet or the high bits of the key - // differ. - leaf = (leaf_t *)art_find(&r->art, high48); - context->leaf = - containerptr_roaring64_bitmap_add(r, high48, low16, leaf); - memcpy(context->high_bytes, high48, ART_KEY_BYTES); } + return result; } -void roaring64_bitmap_add_many(roaring64_bitmap_t *r, size_t n_args, - const uint64_t *vals) { - if (n_args == 0) { - return; - } - const uint64_t *end = vals + n_args; - roaring64_bulk_context_t context = CROARING_ZERO_INITIALIZER; - for (const uint64_t *current_val = vals; current_val != end; - current_val++) { - roaring64_bitmap_add_bulk(r, &context, *current_val); - } +uint64_t roaring64_bitmap_or_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t c1 = roaring64_bitmap_get_cardinality(r1); + uint64_t c2 = roaring64_bitmap_get_cardinality(r2); + uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); + return c1 + c2 - inter; } -static inline void add_range_closed_at(roaring64_bitmap_t *r, art_t *art, - uint8_t *high48, uint16_t min, - uint16_t max) { - leaf_t *leaf = (leaf_t *)art_find(art, high48); - if (leaf != NULL) { - uint8_t typecode1 = get_typecode(*leaf); - container_t *container1 = get_container(r, *leaf); - uint8_t typecode2; - container_t *container2 = - container_add_range(container1, typecode1, min, max, &typecode2); - if (container2 != container1) { - container_free(container1, typecode1); - replace_container(r, leaf, container2, typecode2); - } +void roaring64_bitmap_or_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + if (r1 == r2) { return; } - uint8_t typecode; - // container_add_range is inclusive, but `container_range_of_ones` is - // exclusive. - container_t *container = container_range_of_ones(min, max + 1, &typecode); - leaf_t new_leaf = add_container(r, container, typecode); - art_insert(art, high48, (art_val_t)new_leaf); -} + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); -void roaring64_bitmap_add_range(roaring64_bitmap_t *r, uint64_t min, - uint64_t max) { - if (min >= max) { - return; + while (it1.value != NULL || it2.value != NULL) { + bool it1_present = it1.value != NULL; + bool it2_present = it2.value != NULL; + + // Cases: + // 1. it1_present && !it2_present -> it1++ + // 2. !it1_present && it2_present -> add it2, it2++ + // 3. it1_present && it2_present + // a. it1 < it2 -> it1++ + // b. it1 == it2 -> it1 | it2, it1++, it2++ + // c. it1 > it2 -> add it2, it2++ + int compare_result = 0; + if (it1_present && it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 3b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t leaf2 = (leaf_t)*it2.value; + uint8_t typecode1 = get_typecode(*leaf1); + container_t *container1 = get_container(r1, *leaf1); + uint8_t typecode2; + container_t *container2; + if (get_typecode(*leaf1) == SHARED_CONTAINER_TYPE) { + container2 = container_or(container1, typecode1, + get_container(r2, leaf2), + get_typecode(leaf2), &typecode2); + } else { + container2 = container_ior(container1, typecode1, + get_container(r2, leaf2), + get_typecode(leaf2), &typecode2); + } + if (container2 != container1) { + container_free(container1, typecode1); + replace_container(r1, leaf1, container2, typecode2); + } + art_iterator_next(&it1); + art_iterator_next(&it2); + } + } + if ((it1_present && !it2_present) || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + art_iterator_next(&it1); + } else if ((!it1_present && it2_present) || compare_result > 0) { + // Cases 2 and 3c: it2 is the only iterator or is before it1. + leaf_t result_leaf = + copy_leaf_container(r2, r1, (leaf_t)*it2.value); + art_iterator_insert(&it1, it2.key, (art_val_t)result_leaf); + art_iterator_next(&it2); + } } - roaring64_bitmap_add_range_closed(r, min, max - 1); } -void roaring64_bitmap_add_range_closed(roaring64_bitmap_t *r, uint64_t min, - uint64_t max) { - if (min > max) { - return; - } +roaring64_bitmap_t *roaring64_bitmap_xor(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); - art_t *art = &r->art; - uint8_t min_high48[ART_KEY_BYTES]; - uint16_t min_low16 = split_key(min, min_high48); - uint8_t max_high48[ART_KEY_BYTES]; - uint16_t max_low16 = split_key(max, max_high48); - if (compare_high48(min_high48, max_high48) == 0) { - // Only populate range within one container. - add_range_closed_at(r, art, min_high48, min_low16, max_low16); - return; - } + art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); - // Populate a range across containers. Fill intermediate containers - // entirely. - add_range_closed_at(r, art, min_high48, min_low16, 0xffff); - uint64_t min_high_bits = min >> 16; - uint64_t max_high_bits = max >> 16; - for (uint64_t current = min_high_bits + 1; current < max_high_bits; - ++current) { - uint8_t current_high48[ART_KEY_BYTES]; - split_key(current << 16, current_high48); - add_range_closed_at(r, art, current_high48, 0, 0xffff); + while (it1.value != NULL || it2.value != NULL) { + bool it1_present = it1.value != NULL; + bool it2_present = it2.value != NULL; + + // Cases: + // 1. it1_present && !it2_present -> output it1, it1++ + // 2. !it1_present && it2_present -> output it2, it2++ + // 3. it1_present && it2_present + // a. it1 < it2 -> output it1, it1++ + // b. it1 == it2 -> output it1 ^ it2, it1++, it2++ + // c. it1 > it2 -> output it2, it2++ + int compare_result = 0; + if (it1_present && it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 3b: iterators at the same high key position. + leaf_t leaf1 = (leaf_t)*it1.value; + leaf_t leaf2 = (leaf_t)*it2.value; + uint8_t result_typecode; + container_t *result_container = + container_xor(get_container(r1, leaf1), get_typecode(leaf1), + get_container(r2, leaf2), get_typecode(leaf2), + &result_typecode); + if (container_nonzero_cardinality(result_container, + result_typecode)) { + leaf_t result_leaf = add_container(result, result_container, + result_typecode); + art_insert(&result->art, it1.key, (art_val_t)result_leaf); + } else { + container_free(result_container, result_typecode); + } + art_iterator_next(&it1); + art_iterator_next(&it2); + } + } + if ((it1_present && !it2_present) || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + leaf_t result_leaf = + copy_leaf_container(r1, result, (leaf_t)*it1.value); + art_insert(&result->art, it1.key, (art_val_t)result_leaf); + art_iterator_next(&it1); + } else if ((!it1_present && it2_present) || compare_result > 0) { + // Cases 2 and 3c: it2 is the only iterator or is before it1. + leaf_t result_leaf = + copy_leaf_container(r2, result, (leaf_t)*it2.value); + art_insert(&result->art, it2.key, (art_val_t)result_leaf); + art_iterator_next(&it2); + } } - add_range_closed_at(r, art, max_high48, 0, max_low16); + return result; } -bool roaring64_bitmap_contains(const roaring64_bitmap_t *r, uint64_t val) { - uint8_t high48[ART_KEY_BYTES]; - uint16_t low16 = split_key(val, high48); - leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); - if (leaf != NULL) { - return container_contains(get_container(r, *leaf), low16, - get_typecode(*leaf)); - } - return false; +uint64_t roaring64_bitmap_xor_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t c1 = roaring64_bitmap_get_cardinality(r1); + uint64_t c2 = roaring64_bitmap_get_cardinality(r2); + uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); + return c1 + c2 - 2 * inter; } -bool roaring64_bitmap_contains_range(const roaring64_bitmap_t *r, uint64_t min, - uint64_t max) { - if (min >= max) { - return true; - } +void roaring64_bitmap_xor_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + assert(r1 != r2); + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); - uint8_t min_high48[ART_KEY_BYTES]; - uint16_t min_low16 = split_key(min, min_high48); - uint8_t max_high48[ART_KEY_BYTES]; - uint16_t max_low16 = split_key(max, max_high48); - uint64_t max_high48_bits = (max - 1) & 0xFFFFFFFFFFFF0000; // Inclusive + while (it1.value != NULL || it2.value != NULL) { + bool it1_present = it1.value != NULL; + bool it2_present = it2.value != NULL; - art_iterator_t it = art_lower_bound((art_t *)&r->art, min_high48); - if (it.value == NULL || combine_key(it.key, 0) > min) { - return false; - } - uint64_t prev_high48_bits = min & 0xFFFFFFFFFFFF0000; - while (it.value != NULL) { - uint64_t current_high48_bits = combine_key(it.key, 0); - if (current_high48_bits > max_high48_bits) { - // We've passed the end of the range with all containers containing - // the range. - return true; - } - if (current_high48_bits - prev_high48_bits > 0x10000) { - // There is a gap in the iterator that falls in the range. - return false; - } + // Cases: + // 1. it1_present && !it2_present -> it1++ + // 2. !it1_present && it2_present -> add it2, it2++ + // 3. it1_present && it2_present + // a. it1 < it2 -> it1++ + // b. it1 == it2 -> it1 ^ it2, it1++, it2++ + // c. it1 > it2 -> add it2, it2++ + int compare_result = 0; + if (it1_present && it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 3b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t leaf2 = (leaf_t)*it2.value; + uint8_t typecode1 = get_typecode(*leaf1); + container_t *container1 = get_container(r1, *leaf1); + uint8_t typecode2; + container_t *container2; + if (typecode1 == SHARED_CONTAINER_TYPE) { + container2 = container_xor(container1, typecode1, + get_container(r2, leaf2), + get_typecode(leaf2), &typecode2); + if (container2 != container1) { + // We only free when doing container_xor, not + // container_ixor, as ixor frees the original + // internally. + container_free(container1, typecode1); + } + } else { + container2 = container_ixor( + container1, typecode1, get_container(r2, leaf2), + get_typecode(leaf2), &typecode2); + } - leaf_t leaf = (leaf_t)*it.value; - uint32_t container_min = 0; - if (compare_high48(it.key, min_high48) == 0) { - container_min = min_low16; - } - uint32_t container_max = 0xFFFF + 1; // Exclusive - if (compare_high48(it.key, max_high48) == 0) { - container_max = max_low16; + if (!container_nonzero_cardinality(container2, typecode2)) { + container_free(container2, typecode2); + bool erased = art_iterator_erase(&it1, NULL); + assert(erased); + (void)erased; + remove_container(r1, *leaf1); + } else { + if (container2 != container1) { + replace_container(r1, leaf1, container2, typecode2); + } + // Only advance the iterator if we didn't delete the + // leaf, as erasing advances by itself. + art_iterator_next(&it1); + } + art_iterator_next(&it2); + } } - - // For the first and last containers we use container_contains_range, - // for the intermediate containers we can use container_is_full. - if (container_min == 0 && container_max == 0xFFFF + 1) { - if (!container_is_full(get_container(r, leaf), - get_typecode(leaf))) { - return false; + if ((it1_present && !it2_present) || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + art_iterator_next(&it1); + } else if ((!it1_present && it2_present) || compare_result > 0) { + // Cases 2 and 3c: it2 is the only iterator or is before it1. + leaf_t result_leaf = + copy_leaf_container(r2, r1, (leaf_t)*it2.value); + if (it1_present) { + art_iterator_insert(&it1, it2.key, (art_val_t)result_leaf); + art_iterator_next(&it1); + } else { + art_insert(&r1->art, it2.key, (art_val_t)result_leaf); } - } else if (!container_contains_range(get_container(r, leaf), - container_min, container_max, - get_typecode(leaf))) { - return false; + art_iterator_next(&it2); } - prev_high48_bits = current_high48_bits; - art_iterator_next(&it); } - return prev_high48_bits == max_high48_bits; } -bool roaring64_bitmap_contains_bulk(const roaring64_bitmap_t *r, - roaring64_bulk_context_t *context, - uint64_t val) { - uint8_t high48[ART_KEY_BYTES]; - uint16_t low16 = split_key(val, high48); +roaring64_bitmap_t *roaring64_bitmap_andnot(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); - if (context->leaf == NULL || - art_compare_keys(context->high_bytes, high48) != 0) { - // We're not positioned anywhere yet or the high bits of the key - // differ. - leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); - if (leaf == NULL) { - return false; - } - context->leaf = leaf; - memcpy(context->high_bytes, high48, ART_KEY_BYTES); - } - return container_contains(get_container(r, *context->leaf), low16, - get_typecode(*context->leaf)); -} + art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); -bool roaring64_bitmap_select(const roaring64_bitmap_t *r, uint64_t rank, - uint64_t *element) { - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); - uint64_t start_rank = 0; - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - uint64_t cardinality = container_get_cardinality(get_container(r, leaf), - get_typecode(leaf)); - if (start_rank + cardinality > rank) { - uint32_t uint32_start = 0; - uint32_t uint32_rank = rank - start_rank; - uint32_t uint32_element = 0; - if (container_select(get_container(r, leaf), get_typecode(leaf), - &uint32_start, uint32_rank, &uint32_element)) { - *element = combine_key(it.key, (uint16_t)uint32_element); - return true; + while (it1.value != NULL) { + // Cases: + // 1. it1_present && !it2_present -> output it1, it1++ + // 2. it1_present && it2_present + // a. it1 < it2 -> output it1, it1++ + // b. it1 == it2 -> output it1 - it2, it1++, it2++ + // c. it1 > it2 -> it2++ + bool it2_present = it2.value != NULL; + int compare_result = 0; + if (it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t leaf2 = (leaf_t)*it2.value; + uint8_t result_typecode; + container_t *result_container = container_andnot( + get_container(r1, *leaf1), get_typecode(*leaf1), + get_container(r2, leaf2), get_typecode(leaf2), + &result_typecode); + + if (container_nonzero_cardinality(result_container, + result_typecode)) { + leaf_t result_leaf = add_container(result, result_container, + result_typecode); + art_insert(&result->art, it1.key, (art_val_t)result_leaf); + } else { + container_free(result_container, result_typecode); + } + art_iterator_next(&it1); + art_iterator_next(&it2); } - return false; } - start_rank += cardinality; - art_iterator_next(&it); - } - return false; -} - -uint64_t roaring64_bitmap_rank(const roaring64_bitmap_t *r, uint64_t val) { - uint8_t high48[ART_KEY_BYTES]; - uint16_t low16 = split_key(val, high48); - - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); - uint64_t rank = 0; - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - int compare_result = compare_high48(it.key, high48); - if (compare_result < 0) { - rank += container_get_cardinality(get_container(r, leaf), - get_typecode(leaf)); - } else if (compare_result == 0) { - return rank + container_rank(get_container(r, leaf), - get_typecode(leaf), low16); - } else { - return rank; + if (!it2_present || compare_result < 0) { + // Cases 1 and 2a: it1 is the only iterator or is before it2. + leaf_t result_leaf = + copy_leaf_container(r1, result, (leaf_t)*it1.value); + art_insert(&result->art, it1.key, (art_val_t)result_leaf); + art_iterator_next(&it1); + } else if (compare_result > 0) { + // Case 2c: it1 is after it2. + art_iterator_next(&it2); } - art_iterator_next(&it); } - return rank; + return result; } -bool roaring64_bitmap_get_index(const roaring64_bitmap_t *r, uint64_t val, - uint64_t *out_index) { - uint8_t high48[ART_KEY_BYTES]; - uint16_t low16 = split_key(val, high48); +uint64_t roaring64_bitmap_andnot_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t c1 = roaring64_bitmap_get_cardinality(r1); + uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); + return c1 - inter; +} - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); - uint64_t index = 0; - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - int compare_result = compare_high48(it.key, high48); - if (compare_result < 0) { - index += container_get_cardinality(get_container(r, leaf), - get_typecode(leaf)); - } else if (compare_result == 0) { - int index16 = container_get_index(get_container(r, leaf), - get_typecode(leaf), low16); - if (index16 < 0) { - return false; +void roaring64_bitmap_andnot_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); + + while (it1.value != NULL) { + // Cases: + // 1. it1_present && !it2_present -> it1++ + // 2. it1_present && it2_present + // a. it1 < it2 -> it1++ + // b. it1 == it2 -> it1 - it2, it1++, it2++ + // c. it1 > it2 -> it2++ + bool it2_present = it2.value != NULL; + int compare_result = 0; + if (it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t leaf2 = (leaf_t)*it2.value; + uint8_t typecode1 = get_typecode(*leaf1); + container_t *container1 = get_container(r1, *leaf1); + uint8_t typecode2; + container_t *container2; + if (typecode1 == SHARED_CONTAINER_TYPE) { + container2 = container_andnot( + container1, typecode1, get_container(r2, leaf2), + get_typecode(leaf2), &typecode2); + if (container2 != container1) { + // We only free when doing container_andnot, not + // container_iandnot, as iandnot frees the original + // internally. + container_free(container1, typecode1); + } + } else { + container2 = container_iandnot( + container1, typecode1, get_container(r2, leaf2), + get_typecode(leaf2), &typecode2); + } + + if (!container_nonzero_cardinality(container2, typecode2)) { + container_free(container2, typecode2); + bool erased = art_iterator_erase(&it1, NULL); + assert(erased); + (void)erased; + remove_container(r1, *leaf1); + } else { + if (container2 != container1) { + replace_container(r1, leaf1, container2, typecode2); + } + // Only advance the iterator if we didn't delete the + // leaf, as erasing advances by itself. + art_iterator_next(&it1); + } + art_iterator_next(&it2); } - *out_index = index + index16; - return true; - } else { - return false; } - art_iterator_next(&it); + if (!it2_present || compare_result < 0) { + // Cases 1 and 2a: it1 is the only iterator or is before it2. + art_iterator_next(&it1); + } else if (compare_result > 0) { + // Case 2c: it1 is after it2. + art_iterator_next(&it2); + } } - return false; } -// Returns true if a container was removed. -static inline bool containerptr_roaring64_bitmap_remove(roaring64_bitmap_t *r, - uint8_t *high48, - uint16_t low16, - leaf_t *leaf) { - if (leaf == NULL) { - return false; +/** + * Flips the leaf at high48 in the range [min, max), adding the result to + * `r2`. If the high48 key is not found in `r1`, a new container is created. + */ +static void roaring64_flip_leaf(const roaring64_bitmap_t *r1, + roaring64_bitmap_t *r2, uint8_t high48[], + uint32_t min, uint32_t max) { + leaf_t *leaf1 = (leaf_t *)art_find(&r1->art, high48); + uint8_t typecode2; + container_t *container2; + if (leaf1 == NULL) { + // No container at this key, create a full container. + container2 = container_range_of_ones(min, max, &typecode2); + } else if (min == 0 && max > 0xFFFF) { + // Flip whole container. + container2 = container_not(get_container(r1, *leaf1), + get_typecode(*leaf1), &typecode2); + } else { + // Partially flip a container. + container2 = + container_not_range(get_container(r1, *leaf1), get_typecode(*leaf1), + min, max, &typecode2); + } + if (container_nonzero_cardinality(container2, typecode2)) { + leaf_t leaf2 = add_container(r2, container2, typecode2); + art_insert(&r2->art, high48, (art_val_t)leaf2); + } else { + container_free(container2, typecode2); } +} - uint8_t typecode = get_typecode(*leaf); - container_t *container = get_container(r, *leaf); +/** + * Flips the leaf at high48 in the range [min, max). If the high48 key is + * not found in the bitmap, a new container is created. Deletes the leaf and + * associated container if the negation results in an empty range. + */ +static void roaring64_flip_leaf_inplace(roaring64_bitmap_t *r, uint8_t high48[], + uint32_t min, uint32_t max) { + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + container_t *container2; uint8_t typecode2; - container_t *container2 = - container_remove(container, low16, typecode, &typecode2); - if (container2 != container) { - container_free(container, typecode); - replace_container(r, leaf, container2, typecode2); + if (leaf == NULL) { + // No container at this key, insert a full container. + container2 = container_range_of_ones(min, max, &typecode2); + leaf_t new_leaf = add_container(r, container2, typecode2); + art_insert(&r->art, high48, (art_val_t)new_leaf); + return; } - if (!container_nonzero_cardinality(container2, typecode2)) { - container_free(container2, typecode2); - bool erased = art_erase(&r->art, high48, (art_val_t *)leaf); + + if (min == 0 && max > 0xFFFF) { + // Flip whole container. + container2 = container_inot(get_container(r, *leaf), + get_typecode(*leaf), &typecode2); + } else { + // Partially flip a container. + container2 = container_inot_range( + get_container(r, *leaf), get_typecode(*leaf), min, max, &typecode2); + } + + if (container_nonzero_cardinality(container2, typecode2)) { + replace_container(r, leaf, container2, typecode2); + } else { + bool erased = art_erase(&r->art, high48, NULL); assert(erased); (void)erased; + container_free(container2, typecode2); remove_container(r, *leaf); - return true; } - return false; } -void roaring64_bitmap_remove(roaring64_bitmap_t *r, uint64_t val) { - art_t *art = &r->art; - uint8_t high48[ART_KEY_BYTES]; - uint16_t low16 = split_key(val, high48); - - leaf_t *leaf = (leaf_t *)art_find(art, high48); - containerptr_roaring64_bitmap_remove(r, high48, low16, leaf); +roaring64_bitmap_t *roaring64_bitmap_flip(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max) { + if (min >= max) { + return roaring64_bitmap_copy(r); + } + return roaring64_bitmap_flip_closed(r, min, max - 1); } -bool roaring64_bitmap_remove_checked(roaring64_bitmap_t *r, uint64_t val) { - art_t *art = &r->art; - uint8_t high48[ART_KEY_BYTES]; - uint16_t low16 = split_key(val, high48); - leaf_t *leaf = (leaf_t *)art_find(art, high48); - - if (leaf == NULL) { - return false; +roaring64_bitmap_t *roaring64_bitmap_flip_closed(const roaring64_bitmap_t *r1, + uint64_t min, uint64_t max) { + if (min > max) { + return roaring64_bitmap_copy(r1); } - int old_cardinality = - container_get_cardinality(get_container(r, *leaf), get_typecode(*leaf)); - if (containerptr_roaring64_bitmap_remove(r, high48, low16, leaf)) { - return true; + uint8_t min_high48_key[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48_key); + uint8_t max_high48_key[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48_key); + uint64_t min_high48_bits = (min & 0xFFFFFFFFFFFF0000ULL) >> 16; + uint64_t max_high48_bits = (max & 0xFFFFFFFFFFFF0000ULL) >> 16; + + roaring64_bitmap_t *r2 = roaring64_bitmap_create(); + art_iterator_t it = art_init_iterator((art_t *)&r1->art, /*first=*/true); + + // Copy the containers before min unchanged. + while (it.value != NULL && compare_high48(it.key, min_high48_key) < 0) { + leaf_t leaf1 = (leaf_t)*it.value; + uint8_t typecode2 = get_typecode(leaf1); + container_t *container2 = get_copy_of_container( + get_container(r1, leaf1), &typecode2, /*copy_on_write=*/false); + leaf_t leaf2 = add_container(r2, container2, typecode2); + art_insert(&r2->art, it.key, (art_val_t)leaf2); + art_iterator_next(&it); } - int new_cardinality = - container_get_cardinality(get_container(r, *leaf), get_typecode(*leaf)); - return new_cardinality != old_cardinality; -} -void roaring64_bitmap_remove_bulk(roaring64_bitmap_t *r, - roaring64_bulk_context_t *context, - uint64_t val) { - art_t *art = &r->art; - uint8_t high48[ART_KEY_BYTES]; - uint16_t low16 = split_key(val, high48); - if (context->leaf != NULL && - compare_high48(context->high_bytes, high48) == 0) { - // We're at a container with the correct high bits. - uint8_t typecode = get_typecode(*context->leaf); - container_t *container = get_container(r, *context->leaf); - uint8_t typecode2; - container_t *container2 = - container_remove(container, low16, typecode, &typecode2); - if (container2 != container) { - container_free(container, typecode); - replace_container(r, context->leaf, container2, typecode2); + // Flip the range (including non-existent containers!) between min and + // max. + for (uint64_t high48_bits = min_high48_bits; high48_bits <= max_high48_bits; + high48_bits++) { + uint8_t current_high48_key[ART_KEY_BYTES]; + split_key(high48_bits << 16, current_high48_key); + + uint32_t min_container = 0; + if (high48_bits == min_high48_bits) { + min_container = min_low16; } - if (!container_nonzero_cardinality(container2, typecode2)) { - container_free(container2, typecode2); - leaf_t leaf; - bool erased = art_erase(art, high48, (art_val_t *)&leaf); - assert(erased); - (void)erased; - remove_container(r, leaf); - context->leaf = NULL; + uint32_t max_container = 0xFFFF + 1; // Exclusive range. + if (high48_bits == max_high48_bits) { + max_container = max_low16 + 1; // Exclusive. } - } else { - // We're not positioned anywhere yet or the high bits of the key - // differ. - leaf_t *leaf = (leaf_t *)art_find(art, high48); - containerptr_roaring64_bitmap_remove(r, high48, low16, leaf); - context->leaf = leaf; - memcpy(context->high_bytes, high48, ART_KEY_BYTES); - } -} -void roaring64_bitmap_remove_many(roaring64_bitmap_t *r, size_t n_args, - const uint64_t *vals) { - if (n_args == 0) { - return; - } - const uint64_t *end = vals + n_args; - roaring64_bulk_context_t context = CROARING_ZERO_INITIALIZER; - for (const uint64_t *current_val = vals; current_val != end; - current_val++) { - roaring64_bitmap_remove_bulk(r, &context, *current_val); + roaring64_flip_leaf(r1, r2, current_high48_key, min_container, + max_container); } -} -static inline void remove_range_closed_at(roaring64_bitmap_t *r, art_t *art, - uint8_t *high48, uint16_t min, - uint16_t max) { - leaf_t *leaf = (leaf_t *)art_find(art, high48); - if (leaf == NULL) { - return; - } - uint8_t typecode = get_typecode(*leaf); - container_t *container = get_container(r, *leaf); - uint8_t typecode2; - container_t *container2 = - container_remove_range(container, typecode, min, max, &typecode2); - if (container2 != container) { - container_free(container, typecode); - if (container2 != NULL) { - replace_container(r, leaf, container2, typecode2); - } else { - bool erased = art_erase(art, high48, NULL); - assert(erased); - (void)erased; - remove_container(r, *leaf); - } + // Copy the containers after max unchanged. + it = art_upper_bound((art_t *)&r1->art, max_high48_key); + while (it.value != NULL) { + leaf_t leaf1 = (leaf_t)*it.value; + uint8_t typecode2 = get_typecode(leaf1); + container_t *container2 = get_copy_of_container( + get_container(r1, leaf1), &typecode2, /*copy_on_write=*/false); + leaf_t leaf2 = add_container(r2, container2, typecode2); + art_insert(&r2->art, it.key, (art_val_t)leaf2); + art_iterator_next(&it); } + + return r2; } -void roaring64_bitmap_remove_range(roaring64_bitmap_t *r, uint64_t min, +void roaring64_bitmap_flip_inplace(roaring64_bitmap_t *r, uint64_t min, uint64_t max) { if (min >= max) { return; } - roaring64_bitmap_remove_range_closed(r, min, max - 1); + roaring64_bitmap_flip_closed_inplace(r, min, max - 1); } -void roaring64_bitmap_remove_range_closed(roaring64_bitmap_t *r, uint64_t min, +void roaring64_bitmap_flip_closed_inplace(roaring64_bitmap_t *r, uint64_t min, uint64_t max) { if (min > max) { return; } + uint16_t min_low16 = (uint16_t)min; + uint16_t max_low16 = (uint16_t)max; + uint64_t min_high48_bits = (min & 0xFFFFFFFFFFFF0000ULL) >> 16; + uint64_t max_high48_bits = (max & 0xFFFFFFFFFFFF0000ULL) >> 16; - art_t *art = &r->art; - uint8_t min_high48[ART_KEY_BYTES]; - uint16_t min_low16 = split_key(min, min_high48); - uint8_t max_high48[ART_KEY_BYTES]; - uint16_t max_low16 = split_key(max, max_high48); - if (compare_high48(min_high48, max_high48) == 0) { - // Only remove a range within one container. - remove_range_closed_at(r, art, min_high48, min_low16, max_low16); - return; - } + // Flip the range (including non-existent containers!) between min and + // max. + for (uint64_t high48_bits = min_high48_bits; high48_bits <= max_high48_bits; + high48_bits++) { + uint8_t current_high48_key[ART_KEY_BYTES]; + split_key(high48_bits << 16, current_high48_key); - // Remove a range across containers. Remove intermediate containers - // entirely. - remove_range_closed_at(r, art, min_high48, min_low16, 0xffff); + uint32_t min_container = 0; + if (high48_bits == min_high48_bits) { + min_container = min_low16; + } + uint32_t max_container = 0xFFFF + 1; // Exclusive range. + if (high48_bits == max_high48_bits) { + max_container = max_low16 + 1; // Exclusive. + } - art_iterator_t it = art_upper_bound(art, min_high48); - while (it.value != NULL && art_compare_keys(it.key, max_high48) < 0) { - leaf_t leaf; - bool erased = art_iterator_erase(&it, (art_val_t *)&leaf); - assert(erased); - (void)erased; - container_free(get_container(r, leaf), get_typecode(leaf)); - remove_container(r, leaf); + roaring64_flip_leaf_inplace(r, current_high48_key, min_container, + max_container); } - remove_range_closed_at(r, art, max_high48, 0, max_low16); -} - -void roaring64_bitmap_clear(roaring64_bitmap_t *r) { - roaring64_bitmap_remove_range_closed(r, 0, UINT64_MAX); } -uint64_t roaring64_bitmap_get_cardinality(const roaring64_bitmap_t *r) { - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); - uint64_t cardinality = 0; - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - cardinality += container_get_cardinality(get_container(r, leaf), - get_typecode(leaf)); - art_iterator_next(&it); +roaring64_bitmap_t *roaring64_bitmap_add_offset_signed( + const roaring64_bitmap_t *r, bool positive, uint64_t offset) { + if (offset == 0) { + return roaring64_bitmap_copy(r); } - return cardinality; -} -uint64_t roaring64_bitmap_range_cardinality(const roaring64_bitmap_t *r, - uint64_t min, uint64_t max) { - if (min >= max) { - return 0; + roaring64_bitmap_t *answer = roaring64_bitmap_create(); + + // Decompose the offset into a signed container-level shift and an + // intra-container shift. For negative offsets the low 16 bits wrap: e.g. + // -1 = container_offset(-1) + in_offset(0xffff), because shifting by -1 + // container is a shift of -0x1_0000, so we need to shift up within + // containers to get back to -1 + uint16_t low16 = (uint16_t)offset; + int64_t container_offset; + uint16_t in_offset; + if (positive) { + container_offset = (int64_t)(offset >> 16); + in_offset = low16; + } else if (low16 == 0) { + container_offset = -(int64_t)(offset >> 16); + in_offset = 0; + } else { + container_offset = -(int64_t)(offset >> 16) - 1; + in_offset = (uint16_t)-low16; } - // Convert to a closed range - // No underflow here: passing the above condition implies min < max, so - // there is a number less than max - return roaring64_bitmap_range_closed_cardinality(r, min, max - 1); -} -uint64_t roaring64_bitmap_range_closed_cardinality(const roaring64_bitmap_t *r, - uint64_t min, uint64_t max) { - if (min > max) { - return 0; + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + + if (in_offset == 0) { + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + int64_t k = + (int64_t)(combine_key(it.key, 0) >> 16) + container_offset; + if ((uint64_t)k < (uint64_t)1 << 48) { + uint8_t new_high48[ART_KEY_BYTES]; + split_key((uint64_t)k << 16, new_high48); + uint8_t typecode = get_typecode(leaf); + container_t *container = + get_copy_of_container(get_container(r, leaf), &typecode, + /*copy_on_write=*/false); + leaf_t new_leaf = add_container(answer, container, typecode); + art_insert(&answer->art, new_high48, (art_val_t)new_leaf); + } + art_iterator_next(&it); + } + return answer; } - uint64_t cardinality = 0; - uint8_t min_high48[ART_KEY_BYTES]; - uint16_t min_low16 = split_key(min, min_high48); - uint8_t max_high48[ART_KEY_BYTES]; - uint16_t max_low16 = split_key(max, max_high48); + // Track the most recently inserted hi container so that the next + // iteration's lo can merge with it without re-searching the ART. + leaf_t *prev_hi_leaf = NULL; + int64_t prev_hi_k = -1; - art_iterator_t it = art_lower_bound((art_t *)&r->art, min_high48); while (it.value != NULL) { - int max_compare_result = compare_high48(it.key, max_high48); - if (max_compare_result > 0) { - // We're outside the range. - break; + leaf_t leaf = (leaf_t)*it.value; + int64_t k = (int64_t)(combine_key(it.key, 0) >> 16) + container_offset; + + container_t *lo = NULL, *hi = NULL; + container_t **lo_ptr = NULL, **hi_ptr = NULL; + + if ((uint64_t)k < (uint64_t)1 << 48) { + lo_ptr = &lo; + } + if ((uint64_t)(k + 1) < (uint64_t)1 << 48) { + hi_ptr = &hi; + } + if (lo_ptr == NULL && hi_ptr == NULL) { + art_iterator_next(&it); + continue; } - leaf_t leaf = (leaf_t)*it.value; uint8_t typecode = get_typecode(leaf); - container_t *container = get_container(r, leaf); - if (max_compare_result == 0) { - // We're at the max high key, add only the range up to the low - // 16 bits of max. - cardinality += container_rank(container, typecode, max_low16); - } else { - // We're not yet at the max high key, add the full container - // range. - cardinality += container_get_cardinality(container, typecode); + const container_t *c = + container_unwrap_shared(get_container(r, leaf), &typecode); + container_add_offset(c, typecode, lo_ptr, hi_ptr, in_offset); + + if (lo != NULL) { + if (prev_hi_leaf != NULL && prev_hi_k == k) { + uint8_t existing_type = get_typecode(*prev_hi_leaf); + container_t *existing_c = get_container(answer, *prev_hi_leaf); + uint8_t merged_type; + container_t *merged_c = container_ior( + existing_c, existing_type, lo, typecode, &merged_type); + if (merged_c != existing_c) { + container_free(existing_c, existing_type); + } + replace_container(answer, prev_hi_leaf, merged_c, merged_type); + container_free(lo, typecode); + } else { + uint8_t lo_high48[ART_KEY_BYTES]; + split_key((uint64_t)k << 16, lo_high48); + leaf_t new_leaf = add_container(answer, lo, typecode); + art_insert(&answer->art, lo_high48, (art_val_t)new_leaf); + } } - if (compare_high48(it.key, min_high48) == 0 && min_low16 > 0) { - // We're at the min high key, remove the range up to the low 16 - // bits of min. - cardinality -= container_rank(container, typecode, min_low16 - 1); + + prev_hi_leaf = NULL; + if (hi != NULL) { + uint8_t hi_high48[ART_KEY_BYTES]; + split_key((uint64_t)(k + 1) << 16, hi_high48); + leaf_t new_leaf = add_container(answer, hi, typecode); + prev_hi_leaf = (leaf_t *)art_insert(&answer->art, hi_high48, + (art_val_t)new_leaf); + prev_hi_k = k + 1; } + art_iterator_next(&it); } - return cardinality; -} - -bool roaring64_bitmap_is_empty(const roaring64_bitmap_t *r) { - return art_is_empty(&r->art); -} -uint64_t roaring64_bitmap_minimum(const roaring64_bitmap_t *r) { - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); - if (it.value == NULL) { - return UINT64_MAX; + // Repair containers (e.g., convert low-cardinality bitset containers to + // array containers after lazy union operations). + art_iterator_t repair_it = art_init_iterator(&answer->art, /*first=*/true); + while (repair_it.value != NULL) { + leaf_t *leaf_ptr = (leaf_t *)repair_it.value; + uint8_t typecode = get_typecode(*leaf_ptr); + container_t *repaired = container_repair_after_lazy( + get_container(answer, *leaf_ptr), &typecode); + replace_container(answer, leaf_ptr, repaired, typecode); + art_iterator_next(&repair_it); } - leaf_t leaf = (leaf_t)*it.value; - return combine_key( - it.key, container_minimum(get_container(r, leaf), get_typecode(leaf))); -} -uint64_t roaring64_bitmap_maximum(const roaring64_bitmap_t *r) { - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/false); - if (it.value == NULL) { - return 0; - } - leaf_t leaf = (leaf_t)*it.value; - return combine_key( - it.key, container_maximum(get_container(r, leaf), get_typecode(leaf))); + return answer; } -bool roaring64_bitmap_remove_run_compression(roaring64_bitmap_t *r) { - art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); - bool removed = false; +// Returns the number of distinct high 32-bit entries in the bitmap. +static inline uint64_t count_high32(const roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + uint64_t high32_count = 0; + uint32_t prev_high32 = 0; while (it.value != NULL) { - leaf_t *leaf = (leaf_t *)it.value; - if (get_typecode(*leaf) == RUN_CONTAINER_TYPE) { - run_container_t *run = CAST_run(get_container(r, *leaf)); - int32_t card = run_container_cardinality(run); - uint8_t new_typecode; - container_t *new_container = - convert_to_bitset_or_array_container(run, card, &new_typecode); - run_container_free(run); - replace_container(r, leaf, new_container, new_typecode); - removed = true; + uint32_t current_high32 = (uint32_t)(combine_key(it.key, 0) >> 32); + if (high32_count == 0 || prev_high32 != current_high32) { + high32_count++; + prev_high32 = current_high32; } art_iterator_next(&it); } - return removed; + return high32_count; } -bool roaring64_bitmap_run_optimize(roaring64_bitmap_t *r) { - art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); - bool has_run_container = false; - while (it.value != NULL) { - leaf_t *leaf = (leaf_t *)it.value; - uint8_t new_typecode; - // We don't need to free the existing container if a new one was - // created, convert_run_optimize does that internally. - container_t *new_container = convert_run_optimize( - get_container(r, *leaf), get_typecode(*leaf), &new_typecode); - replace_container(r, leaf, new_container, new_typecode); - has_run_container |= new_typecode == RUN_CONTAINER_TYPE; - art_iterator_next(&it); - } - return has_run_container; +// Frees the (32-bit!) bitmap without freeing the containers. +static inline void roaring_bitmap_free_without_containers(roaring_bitmap_t *r) { + ra_clear_without_containers(&r->high_low_container); + roaring_free(r); } -static void move_to_shrink(roaring64_bitmap_t *r, leaf_t *leaf) { - uint64_t idx = get_index(*leaf); - if (idx < r->first_free) { - return; - } - r->containers[r->first_free] = get_container(r, *leaf); - r->containers[idx] = NULL; - *leaf = create_leaf(r->first_free, get_typecode(*leaf)); - r->first_free = next_free_container_idx(r); -} +size_t roaring64_bitmap_portable_size_in_bytes(const roaring64_bitmap_t *r) { + // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + size_t size = 0; -static inline bool is_shrunken(const roaring64_bitmap_t *r) { - return art_is_shrunken(&r->art) && r->first_free == r->capacity; -} + // Write as uint64 the distinct number of "buckets", where a bucket is + // defined as the most significant 32 bits of an element. + uint64_t high32_count; + size += sizeof(high32_count); -size_t roaring64_bitmap_shrink_to_fit(roaring64_bitmap_t *r) { - size_t freed = art_shrink_to_fit(&r->art); - art_iterator_t it = art_init_iterator(&r->art, true); + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + uint32_t prev_high32 = 0; + roaring_bitmap_t *bitmap32 = NULL; + + // Iterate through buckets ordered by increasing keys. while (it.value != NULL) { - leaf_t *leaf = (leaf_t *)it.value; - freed += container_shrink_to_fit(get_container(r, *leaf), - get_typecode(*leaf)); - move_to_shrink(r, leaf); - art_iterator_next(&it); - } - if (is_shrunken(r)) { - return freed; - } - uint64_t new_capacity = r->first_free; - if (new_capacity < r->capacity) { - r->containers = (container_t **)roaring_realloc( - r->containers, new_capacity * sizeof(container_t *)); - freed += (r->capacity - new_capacity) * sizeof(container_t *); - r->capacity = new_capacity; - } - return freed; -} + uint32_t current_high32 = (uint32_t)(combine_key(it.key, 0) >> 32); + if (bitmap32 == NULL || prev_high32 != current_high32) { + if (bitmap32 != NULL) { + // Write as uint32 the most significant 32 bits of the + // bucket. + size += sizeof(prev_high32); -/** - * (For advanced users.) - * Collect statistics about the bitmap - */ -void roaring64_bitmap_statistics(const roaring64_bitmap_t *r, - roaring64_statistics_t *stat) { - memset(stat, 0, sizeof(*stat)); - stat->min_value = roaring64_bitmap_minimum(r); - stat->max_value = roaring64_bitmap_maximum(r); + // Write the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. + size += roaring_bitmap_portable_size_in_bytes(bitmap32); + roaring_bitmap_free_without_containers(bitmap32); + } - art_iterator_t it = art_init_iterator((art_t *)&r->art, true); - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - stat->n_containers++; - uint8_t truetype = - get_container_type(get_container(r, leaf), get_typecode(leaf)); - uint32_t card = container_get_cardinality(get_container(r, leaf), - get_typecode(leaf)); - uint32_t sbytes = - container_size_in_bytes(get_container(r, leaf), get_typecode(leaf)); - stat->cardinality += card; - switch (truetype) { - case BITSET_CONTAINER_TYPE: - stat->n_bitset_containers++; - stat->n_values_bitset_containers += card; - stat->n_bytes_bitset_containers += sbytes; - break; - case ARRAY_CONTAINER_TYPE: - stat->n_array_containers++; - stat->n_values_array_containers += card; - stat->n_bytes_array_containers += sbytes; - break; - case RUN_CONTAINER_TYPE: - stat->n_run_containers++; - stat->n_values_run_containers += card; - stat->n_bytes_run_containers += sbytes; - break; - default: - assert(false); - roaring_unreachable; + // Start a new 32-bit bitmap with the current high 32 bits. + art_iterator_t it2 = it; + uint32_t containers_with_high32 = 0; + while (it2.value != NULL && (uint32_t)(combine_key(it2.key, 0) >> + 32) == current_high32) { + containers_with_high32++; + art_iterator_next(&it2); + } + bitmap32 = + roaring_bitmap_create_with_capacity(containers_with_high32); + + prev_high32 = current_high32; } + leaf_t leaf = (leaf_t)*it.value; + ra_append(&bitmap32->high_low_container, + (uint16_t)(current_high32 >> 16), get_container(r, leaf), + get_typecode(leaf)); art_iterator_next(&it); } -} -static bool roaring64_leaf_internal_validate(const art_val_t val, - const char **reason, - void *context) { - leaf_t leaf = (leaf_t)val; - roaring64_bitmap_t *r = (roaring64_bitmap_t *)context; - return container_internal_validate(get_container(r, leaf), - get_typecode(leaf), reason); -} + if (bitmap32 != NULL) { + // Write as uint32 the most significant 32 bits of the bucket. + size += sizeof(prev_high32); -bool roaring64_bitmap_internal_validate(const roaring64_bitmap_t *r, - const char **reason) { - return art_internal_validate(&r->art, reason, - roaring64_leaf_internal_validate, (void *)r); -} + // Write the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. + size += roaring_bitmap_portable_size_in_bytes(bitmap32); + roaring_bitmap_free_without_containers(bitmap32); + } -bool roaring64_bitmap_equals(const roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); - art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); + return size; +} - while (it1.value != NULL && it2.value != NULL) { - if (compare_high48(it1.key, it2.key) != 0) { - return false; - } - leaf_t leaf1 = (leaf_t)*it1.value; - leaf_t leaf2 = (leaf_t)*it2.value; - if (!container_equals(get_container(r1, leaf1), get_typecode(leaf1), - get_container(r2, leaf2), get_typecode(leaf2))) { - return false; - } - art_iterator_next(&it1); - art_iterator_next(&it2); +size_t roaring64_bitmap_portable_serialize(const roaring64_bitmap_t *r, + char *buf) { + // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + if (buf == NULL) { + return 0; } - return it1.value == NULL && it2.value == NULL; -} + const char *initial_buf = buf; -bool roaring64_bitmap_is_subset(const roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); - art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); + // Write as uint64 the distinct number of "buckets", where a bucket is + // defined as the most significant 32 bits of an element. + uint64_t high32_count = count_high32(r); + uint64_t high32_count_le = croaring_htole64(high32_count); + memcpy(buf, &high32_count_le, sizeof(high32_count_le)); + buf += sizeof(high32_count); - while (it1.value != NULL) { - bool it2_present = it2.value != NULL; + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + uint32_t prev_high32 = 0; + roaring_bitmap_t *bitmap32 = NULL; + + // Iterate through buckets ordered by increasing keys. + while (it.value != NULL) { + uint64_t current_high48 = combine_key(it.key, 0); + uint32_t current_high32 = (uint32_t)(current_high48 >> 32); + if (bitmap32 == NULL || prev_high32 != current_high32) { + if (bitmap32 != NULL) { + // Write as uint32 the most significant 32 bits of the + // bucket. + uint32_t prev_high32_le = croaring_htole32(prev_high32); + memcpy(buf, &prev_high32_le, sizeof(prev_high32_le)); + buf += sizeof(prev_high32); + + // Write the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. + buf += roaring_bitmap_portable_serialize(bitmap32, buf); + roaring_bitmap_free_without_containers(bitmap32); + } - int compare_result = 0; - if (it2_present) { - compare_result = compare_high48(it1.key, it2.key); - if (compare_result == 0) { - leaf_t leaf1 = (leaf_t)*it1.value; - leaf_t leaf2 = (leaf_t)*it2.value; - if (!container_is_subset( - get_container(r1, leaf1), get_typecode(leaf1), - get_container(r2, leaf2), get_typecode(leaf2))) { - return false; - } - art_iterator_next(&it1); + // Start a new 32-bit bitmap with the current high 32 bits. + art_iterator_t it2 = it; + uint32_t containers_with_high32 = 0; + while (it2.value != NULL && + (uint32_t)combine_key(it2.key, 0) == current_high32) { + containers_with_high32++; art_iterator_next(&it2); } + bitmap32 = + roaring_bitmap_create_with_capacity(containers_with_high32); + + prev_high32 = current_high32; } - if (!it2_present || compare_result < 0) { - return false; - } else if (compare_result > 0) { - art_iterator_lower_bound(&it2, it1.key); - } + leaf_t leaf = (leaf_t)*it.value; + ra_append(&bitmap32->high_low_container, + (uint16_t)(current_high48 >> 16), get_container(r, leaf), + get_typecode(leaf)); + art_iterator_next(&it); } - return true; -} -bool roaring64_bitmap_is_strict_subset(const roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - return roaring64_bitmap_get_cardinality(r1) < - roaring64_bitmap_get_cardinality(r2) && - roaring64_bitmap_is_subset(r1, r2); -} + if (bitmap32 != NULL) { + // Write as uint32 the most significant 32 bits of the bucket. + uint32_t prev_high32_le = croaring_htole32(prev_high32); + memcpy(buf, &prev_high32_le, sizeof(prev_high32_le)); + buf += sizeof(prev_high32); -roaring64_bitmap_t *roaring64_bitmap_and(const roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - roaring64_bitmap_t *result = roaring64_bitmap_create(); + // Write the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. + buf += roaring_bitmap_portable_serialize(bitmap32, buf); + roaring_bitmap_free_without_containers(bitmap32); + } - art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); - art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); + return buf - initial_buf; +} - while (it1.value != NULL && it2.value != NULL) { - // Cases: - // 1. it1 < it2 -> it1++ - // 2. it1 == it1 -> output it1 & it2, it1++, it2++ - // 3. it1 > it2 -> it2++ - int compare_result = compare_high48(it1.key, it2.key); - if (compare_result == 0) { - // Case 2: iterators at the same high key position. - leaf_t leaf1 = (leaf_t)*it1.value; - leaf_t leaf2 = (leaf_t)*it2.value; - uint8_t result_typecode; - container_t *result_container = - container_and(get_container(r1, leaf1), get_typecode(leaf1), - get_container(r2, leaf2), get_typecode(leaf2), - &result_typecode); - if (container_nonzero_cardinality(result_container, - result_typecode)) { - leaf_t result_leaf = - add_container(result, result_container, result_typecode); - art_insert(&result->art, it1.key, (art_val_t)result_leaf); - } else { - container_free(result_container, result_typecode); - } - art_iterator_next(&it1); - art_iterator_next(&it2); - } else if (compare_result < 0) { - // Case 1: it1 is before it2. - art_iterator_lower_bound(&it1, it2.key); - } else { - // Case 3: it2 is before it1. - art_iterator_lower_bound(&it2, it1.key); - } +size_t roaring64_bitmap_portable_deserialize_size(const char *buf, + size_t maxbytes) { + // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + if (buf == NULL) { + return 0; } - return result; -} + size_t read_bytes = 0; -uint64_t roaring64_bitmap_and_cardinality(const roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - uint64_t result = 0; + // Read as uint64 the distinct number of "buckets", where a bucket is + // defined as the most significant 32 bits of an element. + uint64_t buckets; + if (read_bytes + sizeof(buckets) > maxbytes) { + return 0; + } + memcpy(&buckets, buf, sizeof(buckets)); + buckets = croaring_letoh64(buckets); + buf += sizeof(buckets); + read_bytes += sizeof(buckets); - art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); - art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); + // Buckets should be 32 bits with 4 bits of zero padding. + if (buckets > UINT32_MAX) { + return 0; + } - while (it1.value != NULL && it2.value != NULL) { - // Cases: - // 1. it1 < it2 -> it1++ - // 2. it1 == it1 -> output cardinaltiy it1 & it2, it1++, it2++ - // 3. it1 > it2 -> it2++ - int compare_result = compare_high48(it1.key, it2.key); - if (compare_result == 0) { - // Case 2: iterators at the same high key position. - leaf_t leaf1 = (leaf_t)*it1.value; - leaf_t leaf2 = (leaf_t)*it2.value; - result += container_and_cardinality( - get_container(r1, leaf1), get_typecode(leaf1), - get_container(r2, leaf2), get_typecode(leaf2)); - art_iterator_next(&it1); - art_iterator_next(&it2); - } else if (compare_result < 0) { - // Case 1: it1 is before it2. - art_iterator_lower_bound(&it1, it2.key); - } else { - // Case 3: it2 is before it1. - art_iterator_lower_bound(&it2, it1.key); + // Iterate through buckets ordered by increasing keys. + for (uint64_t bucket = 0; bucket < buckets; ++bucket) { + // Read as uint32 the most significant 32 bits of the bucket. + uint32_t high32; + if (read_bytes + sizeof(high32) > maxbytes) { + return 0; } + buf += sizeof(high32); + read_bytes += sizeof(high32); + + // Read the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. + size_t bitmap32_size = roaring_bitmap_portable_deserialize_size( + buf, maxbytes - read_bytes); + if (bitmap32_size == 0) { + return 0; + } + buf += bitmap32_size; + read_bytes += bitmap32_size; } - return result; + return read_bytes; } -// Inplace and (modifies its first argument). -void roaring64_bitmap_and_inplace(roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - if (r1 == r2) { - return; +roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe( + const char *buf, size_t maxbytes) { + // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + if (buf == NULL) { + return NULL; } - art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); - art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); - - while (it1.value != NULL) { - // Cases: - // 1. !it2_present -> erase it1 - // 2. it2_present - // a. it1 < it2 -> erase it1 - // b. it1 == it2 -> output it1 & it2, it1++, it2++ - // c. it1 > it2 -> it2++ - bool it2_present = it2.value != NULL; - int compare_result = 0; - if (it2_present) { - compare_result = compare_high48(it1.key, it2.key); - if (compare_result == 0) { - // Case 2a: iterators at the same high key position. - leaf_t *leaf1 = (leaf_t *)it1.value; - leaf_t leaf2 = (leaf_t)*it2.value; + size_t read_bytes = 0; - // We do the computation "in place" only when c1 is not a - // shared container. Rationale: using a shared container - // safely with in place computation would require making a - // copy and then doing the computation in place which is - // likely less efficient than avoiding in place entirely and - // always generating a new container. - uint8_t typecode = get_typecode(*leaf1); - container_t *container = get_container(r1, *leaf1); - uint8_t typecode2; - container_t *container2; - if (typecode == SHARED_CONTAINER_TYPE) { - container2 = container_and(container, typecode, - get_container(r2, leaf2), - get_typecode(leaf2), &typecode2); - } else { - container2 = container_iand( - container, typecode, get_container(r2, leaf2), - get_typecode(leaf2), &typecode2); - } + // Read as uint64 the distinct number of "buckets", where a bucket is + // defined as the most significant 32 bits of an element. + uint64_t buckets; + if (read_bytes + sizeof(buckets) > maxbytes) { + return NULL; + } + memcpy(&buckets, buf, sizeof(buckets)); + buckets = croaring_letoh64(buckets); + buf += sizeof(buckets); + read_bytes += sizeof(buckets); - if (container2 != container) { - container_free(container, typecode); - } - if (!container_nonzero_cardinality(container2, typecode2)) { - container_free(container2, typecode2); - art_iterator_erase(&it1, NULL); - remove_container(r1, *leaf1); - } else { - if (container2 != container) { - replace_container(r1, leaf1, container2, typecode2); - } - // Only advance the iterator if we didn't delete the - // leaf, as erasing advances by itself. - art_iterator_next(&it1); - } - art_iterator_next(&it2); - } - } + // Buckets should be 32 bits with 4 bits of zero padding. + if (buckets > UINT32_MAX) { + return NULL; + } - if (!it2_present || compare_result < 0) { - // Cases 1 and 3a: it1 is the only iterator or is before it2. - leaf_t leaf; - bool erased = art_iterator_erase(&it1, (art_val_t *)&leaf); - assert(erased); - (void)erased; - container_free(get_container(r1, leaf), get_typecode(leaf)); - remove_container(r1, leaf); - } else if (compare_result > 0) { - // Case 2c: it1 is after it2. - art_iterator_lower_bound(&it2, it1.key); + roaring64_bitmap_t *r = roaring64_bitmap_create(); + // Iterate through buckets ordered by increasing keys. + int64_t previous_high32 = -1; + for (uint64_t bucket = 0; bucket < buckets; ++bucket) { + // Read as uint32 the most significant 32 bits of the bucket. + uint32_t high32; + if (read_bytes + sizeof(high32) > maxbytes) { + roaring64_bitmap_free(r); + return NULL; } - } -} + memcpy(&high32, buf, sizeof(high32)); + high32 = croaring_letoh32(high32); + buf += sizeof(high32); + read_bytes += sizeof(high32); + // High 32 bits must be strictly increasing. + if (high32 <= previous_high32) { + roaring64_bitmap_free(r); + return NULL; + } + previous_high32 = high32; -bool roaring64_bitmap_intersect(const roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - bool intersect = false; - art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); - art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); + // Read the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. ra_portable_deserialize + // already reports bytes consumed, so we do not walk the 32-bit + // format a second time with deserialize_size. + roaring_bitmap_t *bitmap32 = + (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); + if (bitmap32 == NULL) { + roaring64_bitmap_free(r); + return NULL; + } + size_t bytesread = 0; + bool is_ok = ra_portable_deserialize(&bitmap32->high_low_container, buf, + maxbytes - read_bytes, &bytesread); + if (!is_ok) { + roaring_free(bitmap32); + roaring64_bitmap_free(r); + return NULL; + } + roaring_bitmap_set_copy_on_write(bitmap32, false); + buf += bytesread; + read_bytes += bytesread; - while (it1.value != NULL && it2.value != NULL) { - // Cases: - // 1. it1 < it2 -> it1++ - // 2. it1 == it1 -> intersect |= it1 & it2, it1++, it2++ - // 3. it1 > it2 -> it2++ - int compare_result = compare_high48(it1.key, it2.key); - if (compare_result == 0) { - // Case 2: iterators at the same high key position. - leaf_t leaf1 = (leaf_t)*it1.value; - leaf_t leaf2 = (leaf_t)*it2.value; - intersect |= container_intersect( - get_container(r1, leaf1), get_typecode(leaf1), - get_container(r2, leaf2), get_typecode(leaf2)); - art_iterator_next(&it1); - art_iterator_next(&it2); - } else if (compare_result < 0) { - // Case 1: it1 is before it2. - art_iterator_lower_bound(&it1, it2.key); - } else { - // Case 3: it2 is before it1. - art_iterator_lower_bound(&it2, it1.key); + // While we don't attempt to validate much, we must ensure that there + // is no duplication in the high 48 bits - inserting into the ART + // assumes (or UB) no duplicate keys. The top 32 bits must be unique + // because we check for strict increasing values of high32, but we + // must also ensure the top 16 bits within each 32-bit bitmap are also + // at least unique (we ensure they're strictly increasing as well, + // which they must be for a _valid_ bitmap, since it's cheaper to check) + int32_t last_bitmap_key = -1; + for (int i = 0; i < bitmap32->high_low_container.size; i++) { + uint16_t key = bitmap32->high_low_container.keys[i]; + if (key <= last_bitmap_key) { + roaring_bitmap_free(bitmap32); + roaring64_bitmap_free(r); + return NULL; + } + last_bitmap_key = key; } - } - return intersect; -} -bool roaring64_bitmap_intersect_with_range(const roaring64_bitmap_t *r, - uint64_t min, uint64_t max) { - if (min >= max) { - return false; - } - roaring64_iterator_t it; - roaring64_iterator_init_at(r, &it, /*first=*/true); - if (!roaring64_iterator_move_equalorlarger(&it, min)) { - return false; + // Insert all containers of the 32-bit bitmap into the 64-bit bitmap. + move_from_roaring32_offset(r, bitmap32, high32); + roaring_bitmap_free(bitmap32); } - return roaring64_iterator_has_value(&it) && - roaring64_iterator_value(&it) < max; + return r; } -double roaring64_bitmap_jaccard_index(const roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - uint64_t c1 = roaring64_bitmap_get_cardinality(r1); - uint64_t c2 = roaring64_bitmap_get_cardinality(r2); - uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); - return (double)inter / (double)(c1 + c2 - inter); +// Returns an "element count" for the given container. This has a different +// meaning for each container type, but the purpose is the minimal information +// required to serialize the container metadata. +static inline uint32_t container_get_element_count(const container_t *c, + uint8_t typecode) { + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + return ((bitset_container_t *)c)->cardinality; + } + case ARRAY_CONTAINER_TYPE: { + return ((array_container_t *)c)->cardinality; + } + case RUN_CONTAINER_TYPE: { + return ((run_container_t *)c)->n_runs; + } + default: { + assert(false); + roaring_unreachable; + return 0; + } + } } -roaring64_bitmap_t *roaring64_bitmap_or(const roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - roaring64_bitmap_t *result = roaring64_bitmap_create(); - - art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); - art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); - - while (it1.value != NULL || it2.value != NULL) { - bool it1_present = it1.value != NULL; - bool it2_present = it2.value != NULL; - - // Cases: - // 1. it1_present && !it2_present -> output it1, it1++ - // 2. !it1_present && it2_present -> output it2, it2++ - // 3. it1_present && it2_present - // a. it1 < it2 -> output it1, it1++ - // b. it1 == it2 -> output it1 | it2, it1++, it2++ - // c. it1 > it2 -> output it2, it2++ - int compare_result = 0; - if (it1_present && it2_present) { - compare_result = compare_high48(it1.key, it2.key); - if (compare_result == 0) { - // Case 3b: iterators at the same high key position. - leaf_t leaf1 = (leaf_t)*it1.value; - leaf_t leaf2 = (leaf_t)*it2.value; - uint8_t result_typecode; - container_t *result_container = - container_or(get_container(r1, leaf1), get_typecode(leaf1), - get_container(r2, leaf2), get_typecode(leaf2), - &result_typecode); - leaf_t result_leaf = - add_container(result, result_container, result_typecode); - art_insert(&result->art, it1.key, (art_val_t)result_leaf); - art_iterator_next(&it1); - art_iterator_next(&it2); - } +static inline size_t container_get_frozen_size(const container_t *c, + uint8_t typecode) { + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + return BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); } - if ((it1_present && !it2_present) || compare_result < 0) { - // Cases 1 and 3a: it1 is the only iterator or is before it2. - leaf_t result_leaf = - copy_leaf_container(r1, result, (leaf_t)*it1.value); - art_insert(&result->art, it1.key, (art_val_t)result_leaf); - art_iterator_next(&it1); - } else if ((!it1_present && it2_present) || compare_result > 0) { - // Cases 2 and 3c: it2 is the only iterator or is before it1. - leaf_t result_leaf = - copy_leaf_container(r2, result, (leaf_t)*it2.value); - art_insert(&result->art, it2.key, (art_val_t)result_leaf); - art_iterator_next(&it2); + case ARRAY_CONTAINER_TYPE: { + return container_get_element_count(c, typecode) * sizeof(uint16_t); + } + case RUN_CONTAINER_TYPE: { + return container_get_element_count(c, typecode) * sizeof(rle16_t); + } + default: { + assert(false); + roaring_unreachable; + return 0; } } - return result; } -uint64_t roaring64_bitmap_or_cardinality(const roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - uint64_t c1 = roaring64_bitmap_get_cardinality(r1); - uint64_t c2 = roaring64_bitmap_get_cardinality(r2); - uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); - return c1 + c2 - inter; +uint64_t align_size(uint64_t size, uint64_t alignment) { + return (size + alignment - 1) & ~(alignment - 1); } -void roaring64_bitmap_or_inplace(roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - if (r1 == r2) { - return; +size_t roaring64_bitmap_frozen_size_in_bytes(const roaring64_bitmap_t *r) { + if (!is_shrunken(r)) { + return 0; } - art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); - art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); + // Flags. + uint64_t size = sizeof(r->flags); + // Container count. + size += sizeof(r->capacity); + // Container element counts. + size += r->capacity * sizeof(uint16_t); + // Total container sizes. + size += 3 * sizeof(uint64_t); + // ART (8 byte aligned). + size = align_size(size, 8); + size += art_size_in_bytes(&r->art); - while (it1.value != NULL || it2.value != NULL) { - bool it1_present = it1.value != NULL; - bool it2_present = it2.value != NULL; + uint64_t total_sizes[4] = + CROARING_ZERO_INITIALIZER; // Indexed by typecode. + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + uint8_t typecode = get_typecode(leaf); + total_sizes[typecode] += + container_get_frozen_size(get_container(r, leaf), typecode); + art_iterator_next(&it); + } + // Containers (aligned). + size = align_size(size, CROARING_BITSET_ALIGNMENT); + size += total_sizes[BITSET_CONTAINER_TYPE]; + size = align_size(size, alignof(rle16_t)); + size += total_sizes[ARRAY_CONTAINER_TYPE]; + size = align_size(size, alignof(uint16_t)); + size += total_sizes[RUN_CONTAINER_TYPE]; + // Padding to make overall size a multiple of required alignment. + size = align_size(size, CROARING_BITSET_ALIGNMENT); + return size; +} - // Cases: - // 1. it1_present && !it2_present -> it1++ - // 2. !it1_present && it2_present -> add it2, it2++ - // 3. it1_present && it2_present - // a. it1 < it2 -> it1++ - // b. it1 == it2 -> it1 | it2, it1++, it2++ - // c. it1 > it2 -> add it2, it2++ - int compare_result = 0; - if (it1_present && it2_present) { - compare_result = compare_high48(it1.key, it2.key); - if (compare_result == 0) { - // Case 3b: iterators at the same high key position. - leaf_t *leaf1 = (leaf_t *)it1.value; - leaf_t leaf2 = (leaf_t)*it2.value; - uint8_t typecode1 = get_typecode(*leaf1); - container_t *container1 = get_container(r1, *leaf1); - uint8_t typecode2; - container_t *container2; - if (get_typecode(*leaf1) == SHARED_CONTAINER_TYPE) { - container2 = container_or(container1, typecode1, - get_container(r2, leaf2), - get_typecode(leaf2), &typecode2); - } else { - container2 = container_ior(container1, typecode1, - get_container(r2, leaf2), - get_typecode(leaf2), &typecode2); - } - if (container2 != container1) { - container_free(container1, typecode1); - replace_container(r1, leaf1, container2, typecode2); - } - art_iterator_next(&it1); - art_iterator_next(&it2); - } +static inline void container_frozen_serialize(const container_t *container, + uint8_t typecode, + uint64_t **bitsets, + uint16_t **arrays, + rle16_t **runs) { + size_t size = container_get_frozen_size(container, typecode); + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + bitset_container_t *bitset = (bitset_container_t *)container; + memcpy(*bitsets, bitset->words, size); + *bitsets += BITSET_CONTAINER_SIZE_IN_WORDS; + break; } - if ((it1_present && !it2_present) || compare_result < 0) { - // Cases 1 and 3a: it1 is the only iterator or is before it2. - art_iterator_next(&it1); - } else if ((!it1_present && it2_present) || compare_result > 0) { - // Cases 2 and 3c: it2 is the only iterator or is before it1. - leaf_t result_leaf = - copy_leaf_container(r2, r1, (leaf_t)*it2.value); - art_iterator_insert(&it1, it2.key, (art_val_t)result_leaf); - art_iterator_next(&it2); + case ARRAY_CONTAINER_TYPE: { + array_container_t *array = (array_container_t *)container; + memcpy(*arrays, array->array, size); + *arrays += container_get_element_count(container, typecode); + break; + } + case RUN_CONTAINER_TYPE: { + run_container_t *run = (run_container_t *)container; + memcpy(*runs, run->runs, size); + *runs += container_get_element_count(container, typecode); + break; + } + default: { + assert(false); + roaring_unreachable; } } } -roaring64_bitmap_t *roaring64_bitmap_xor(const roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - roaring64_bitmap_t *result = roaring64_bitmap_create(); +static inline char *pad_align(char *buf, const char *initial_buf, + size_t alignment) { + uint64_t buf_size = buf - initial_buf; + uint64_t pad = align_size(buf_size, alignment) - buf_size; + memset(buf, 0, pad); + return buf + pad; +} - art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); - art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); +size_t roaring64_bitmap_frozen_serialize(const roaring64_bitmap_t *r, + char *buf) { + if (buf == NULL) { + return 0; + } + if (!is_shrunken(r)) { + return 0; + } + const char *initial_buf = buf; - while (it1.value != NULL || it2.value != NULL) { - bool it1_present = it1.value != NULL; - bool it2_present = it2.value != NULL; + // Flags. + memcpy(buf, &r->flags, sizeof(r->flags)); + buf += sizeof(r->flags); - // Cases: - // 1. it1_present && !it2_present -> output it1, it1++ - // 2. !it1_present && it2_present -> output it2, it2++ - // 3. it1_present && it2_present - // a. it1 < it2 -> output it1, it1++ - // b. it1 == it2 -> output it1 ^ it2, it1++, it2++ - // c. it1 > it2 -> output it2, it2++ - int compare_result = 0; - if (it1_present && it2_present) { - compare_result = compare_high48(it1.key, it2.key); - if (compare_result == 0) { - // Case 3b: iterators at the same high key position. - leaf_t leaf1 = (leaf_t)*it1.value; - leaf_t leaf2 = (leaf_t)*it2.value; - uint8_t result_typecode; - container_t *result_container = - container_xor(get_container(r1, leaf1), get_typecode(leaf1), - get_container(r2, leaf2), get_typecode(leaf2), - &result_typecode); - if (container_nonzero_cardinality(result_container, - result_typecode)) { - leaf_t result_leaf = add_container(result, result_container, - result_typecode); - art_insert(&result->art, it1.key, (art_val_t)result_leaf); - } else { - container_free(result_container, result_typecode); - } - art_iterator_next(&it1); - art_iterator_next(&it2); - } - } - if ((it1_present && !it2_present) || compare_result < 0) { - // Cases 1 and 3a: it1 is the only iterator or is before it2. - leaf_t result_leaf = - copy_leaf_container(r1, result, (leaf_t)*it1.value); - art_insert(&result->art, it1.key, (art_val_t)result_leaf); - art_iterator_next(&it1); - } else if ((!it1_present && it2_present) || compare_result > 0) { - // Cases 2 and 3c: it2 is the only iterator or is before it1. - leaf_t result_leaf = - copy_leaf_container(r2, result, (leaf_t)*it2.value); - art_insert(&result->art, it2.key, (art_val_t)result_leaf); - art_iterator_next(&it2); - } + // Container count. + memcpy(buf, &r->capacity, sizeof(r->capacity)); + buf += sizeof(r->capacity); + + // Container element counts. + uint64_t total_sizes[4] = + CROARING_ZERO_INITIALIZER; // Indexed by typecode. + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + uint8_t typecode = get_typecode(leaf); + container_t *container = get_container(r, leaf); + + uint32_t elem_count = container_get_element_count(container, typecode); + uint16_t compressed_elem_count = (uint16_t)(elem_count - 1); + memcpy(buf, &compressed_elem_count, sizeof(compressed_elem_count)); + buf += sizeof(compressed_elem_count); + + total_sizes[typecode] += container_get_frozen_size(container, typecode); + art_iterator_next(&it); } - return result; -} -uint64_t roaring64_bitmap_xor_cardinality(const roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - uint64_t c1 = roaring64_bitmap_get_cardinality(r1); - uint64_t c2 = roaring64_bitmap_get_cardinality(r2); - uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); - return c1 + c2 - 2 * inter; -} + // Total container sizes. + memcpy(buf, &(total_sizes[BITSET_CONTAINER_TYPE]), sizeof(uint64_t)); + buf += sizeof(uint64_t); + memcpy(buf, &(total_sizes[RUN_CONTAINER_TYPE]), sizeof(uint64_t)); + buf += sizeof(uint64_t); + memcpy(buf, &(total_sizes[ARRAY_CONTAINER_TYPE]), sizeof(uint64_t)); + buf += sizeof(uint64_t); -void roaring64_bitmap_xor_inplace(roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - assert(r1 != r2); - art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); - art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); + // ART. + buf = pad_align(buf, initial_buf, 8); + buf += art_serialize(&r->art, buf); - while (it1.value != NULL || it2.value != NULL) { - bool it1_present = it1.value != NULL; - bool it2_present = it2.value != NULL; + // Containers (aligned). + // Runs before arrays as run elements are larger than array elements and + // smaller than bitset elements. + buf = pad_align(buf, initial_buf, CROARING_BITSET_ALIGNMENT); + uint64_t *bitsets = (uint64_t *)buf; + buf += total_sizes[BITSET_CONTAINER_TYPE]; + buf = pad_align(buf, initial_buf, alignof(rle16_t)); + rle16_t *runs = (rle16_t *)buf; + buf += total_sizes[RUN_CONTAINER_TYPE]; + buf = pad_align(buf, initial_buf, alignof(uint16_t)); + uint16_t *arrays = (uint16_t *)buf; + buf += total_sizes[ARRAY_CONTAINER_TYPE]; - // Cases: - // 1. it1_present && !it2_present -> it1++ - // 2. !it1_present && it2_present -> add it2, it2++ - // 3. it1_present && it2_present - // a. it1 < it2 -> it1++ - // b. it1 == it2 -> it1 ^ it2, it1++, it2++ - // c. it1 > it2 -> add it2, it2++ - int compare_result = 0; - if (it1_present && it2_present) { - compare_result = compare_high48(it1.key, it2.key); - if (compare_result == 0) { - // Case 3b: iterators at the same high key position. - leaf_t *leaf1 = (leaf_t *)it1.value; - leaf_t leaf2 = (leaf_t)*it2.value; - uint8_t typecode1 = get_typecode(*leaf1); - container_t *container1 = get_container(r1, *leaf1); - uint8_t typecode2; - container_t *container2; - if (typecode1 == SHARED_CONTAINER_TYPE) { - container2 = container_xor(container1, typecode1, - get_container(r2, leaf2), - get_typecode(leaf2), &typecode2); - if (container2 != container1) { - // We only free when doing container_xor, not - // container_ixor, as ixor frees the original - // internally. - container_free(container1, typecode1); - } - } else { - container2 = container_ixor( - container1, typecode1, get_container(r2, leaf2), - get_typecode(leaf2), &typecode2); - } + it = art_init_iterator((art_t *)&r->art, /*first=*/true); + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + uint8_t typecode = get_typecode(leaf); + container_t *container = get_container(r, leaf); + container_frozen_serialize(container, typecode, &bitsets, &arrays, + &runs); + art_iterator_next(&it); + } - if (!container_nonzero_cardinality(container2, typecode2)) { - container_free(container2, typecode2); - bool erased = art_iterator_erase(&it1, NULL); - assert(erased); - (void)erased; - remove_container(r1, *leaf1); - } else { - if (container2 != container1) { - replace_container(r1, leaf1, container2, typecode2); - } - // Only advance the iterator if we didn't delete the - // leaf, as erasing advances by itself. - art_iterator_next(&it1); - } - art_iterator_next(&it2); - } + // Padding to make overall size a multiple of required alignment. + buf = pad_align(buf, initial_buf, CROARING_BITSET_ALIGNMENT); + + return buf - initial_buf; +} + +static roaring64_bitmap_t *alloc_frozen_bitmap( + uint64_t capacity, frozen_container_header_t **headers_out) { + if (capacity > SIZE_MAX / sizeof(frozen_container_header_t)) { + return NULL; + } + size_t ptrs = (size_t)capacity * sizeof(container_t *); + size_t codes = (size_t)capacity * sizeof(uint8_t); + size_t headers = (size_t)capacity * sizeof(frozen_container_header_t); + size_t pad = alignof(container_t *) + alignof(frozen_container_header_t); + if (sizeof(roaring64_bitmap_t) > SIZE_MAX - ptrs || + sizeof(roaring64_bitmap_t) + ptrs > SIZE_MAX - codes || + sizeof(roaring64_bitmap_t) + ptrs + codes > SIZE_MAX - headers || + sizeof(roaring64_bitmap_t) + ptrs + codes + headers > SIZE_MAX - pad) { + return NULL; + } + size_t sz = sizeof(roaring64_bitmap_t) + ptrs + codes + headers + pad; + char *base = (char *)roaring_malloc(sz); + if (base == NULL) { + return NULL; + } + char *cursor = base; + roaring64_bitmap_t *r = (roaring64_bitmap_t *)roaring64_arena_alloc( + &cursor, sizeof(roaring64_bitmap_t)); + art_init_cleared(&r->art); + r->flags = ROARING_FLAG_FROZEN; + r->capacity = capacity; + r->first_free = 0; + cursor = roaring64_arena_pad(cursor, base, alignof(container_t *)); + if (capacity == 0) { + r->containers = NULL; + r->typecodes = NULL; + if (headers_out != NULL) { + *headers_out = NULL; + } + return r; + } + r->containers = (container_t **)roaring64_arena_alloc(&cursor, ptrs); + memset(r->containers, 0, ptrs); + r->typecodes = (uint8_t *)roaring64_arena_alloc(&cursor, codes); + cursor = + roaring64_arena_pad(cursor, base, alignof(frozen_container_header_t)); + frozen_container_header_t *hdrs = + (frozen_container_header_t *)roaring64_arena_alloc(&cursor, headers); + if (headers_out != NULL) { + *headers_out = hdrs; + } + return r; +} + +static container_t *container_frozen_view_at( + frozen_container_header_t *header, uint8_t typecode, uint32_t elem_count, + const uint64_t **bitsets, const uint16_t **arrays, const rle16_t **runs) { + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + bitset_container_t *c = &header->bitset; + c->cardinality = elem_count; + c->words = (uint64_t *)*bitsets; + *bitsets += BITSET_CONTAINER_SIZE_IN_WORDS; + return (container_t *)c; + } + case ARRAY_CONTAINER_TYPE: { + array_container_t *c = &header->array; + c->cardinality = elem_count; + c->capacity = elem_count; + c->array = (uint16_t *)*arrays; + *arrays += elem_count; + return (container_t *)c; } - if ((it1_present && !it2_present) || compare_result < 0) { - // Cases 1 and 3a: it1 is the only iterator or is before it2. - art_iterator_next(&it1); - } else if ((!it1_present && it2_present) || compare_result > 0) { - // Cases 2 and 3c: it2 is the only iterator or is before it1. - leaf_t result_leaf = - copy_leaf_container(r2, r1, (leaf_t)*it2.value); - if (it1_present) { - art_iterator_insert(&it1, it2.key, (art_val_t)result_leaf); - art_iterator_next(&it1); - } else { - art_insert(&r1->art, it2.key, (art_val_t)result_leaf); - } - art_iterator_next(&it2); + case RUN_CONTAINER_TYPE: { + run_container_t *c = &header->run; + c->n_runs = elem_count; + c->capacity = elem_count; + c->runs = (rle16_t *)*runs; + *runs += elem_count; + return (container_t *)c; } + default: + return NULL; } } -roaring64_bitmap_t *roaring64_bitmap_andnot(const roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - roaring64_bitmap_t *result = roaring64_bitmap_create(); +roaring64_bitmap_t *roaring64_bitmap_frozen_view(const char *buf, + size_t maxbytes) { + if (buf == NULL) { + return NULL; + } + if ((uintptr_t)buf % CROARING_BITSET_ALIGNMENT != 0) { + return NULL; + } - art_iterator_t it1 = art_init_iterator((art_t *)&r1->art, /*first=*/true); - art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); + uint8_t flags; + uint64_t capacity; + if (maxbytes < sizeof(flags) + sizeof(capacity)) { + return NULL; + } + memcpy(&flags, buf, sizeof(flags)); + buf += sizeof(flags); + maxbytes -= sizeof(flags); + memcpy(&capacity, buf, sizeof(capacity)); + buf += sizeof(capacity); + maxbytes -= sizeof(capacity); + + // The element counts alone need two bytes per container, so a capacity + // larger than that cannot be satisfied by this buffer. Checked before + // allocating, so a short buffer claiming a huge count cannot make us + // reserve (and clear) an arena sized from attacker-controlled bytes. + if (capacity > maxbytes / sizeof(uint16_t)) { + return NULL; + } - while (it1.value != NULL) { - // Cases: - // 1. it1_present && !it2_present -> output it1, it1++ - // 2. it1_present && it2_present - // a. it1 < it2 -> output it1, it1++ - // b. it1 == it2 -> output it1 - it2, it1++, it2++ - // c. it1 > it2 -> it2++ - bool it2_present = it2.value != NULL; - int compare_result = 0; - if (it2_present) { - compare_result = compare_high48(it1.key, it2.key); - if (compare_result == 0) { - // Case 2b: iterators at the same high key position. - leaf_t *leaf1 = (leaf_t *)it1.value; - leaf_t leaf2 = (leaf_t)*it2.value; - uint8_t result_typecode; - container_t *result_container = container_andnot( - get_container(r1, *leaf1), get_typecode(*leaf1), - get_container(r2, leaf2), get_typecode(leaf2), - &result_typecode); + frozen_container_header_t *headers = NULL; + roaring64_bitmap_t *r = alloc_frozen_bitmap(capacity, &headers); + if (r == NULL) { + return NULL; + } + // Only flags the format actually defines; the byte comes from the buffer. + r->flags = (uint8_t)(flags & ROARING_FLAG_COW) | ROARING_FLAG_FROZEN | + ROARING_FLAG_FROZEN_ART; - if (container_nonzero_cardinality(result_container, - result_typecode)) { - leaf_t result_leaf = add_container(result, result_container, - result_typecode); - art_insert(&result->art, it1.key, (art_val_t)result_leaf); - } else { - container_free(result_container, result_typecode); - } - art_iterator_next(&it1); - art_iterator_next(&it2); - } - } - if (!it2_present || compare_result < 0) { - // Cases 1 and 2a: it1 is the only iterator or is before it2. - leaf_t result_leaf = - copy_leaf_container(r1, result, (leaf_t)*it1.value); - art_insert(&result->art, it1.key, (art_val_t)result_leaf); - art_iterator_next(&it1); - } else if (compare_result > 0) { - // Case 2c: it1 is after it2. - art_iterator_next(&it2); - } + // Container element counts. + if (maxbytes < capacity * sizeof(uint16_t)) { + roaring64_bitmap_free(r); + return NULL; } - return result; -} + const char *elem_counts = buf; + buf += capacity * sizeof(uint16_t); + maxbytes -= capacity * sizeof(uint16_t); -uint64_t roaring64_bitmap_andnot_cardinality(const roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - uint64_t c1 = roaring64_bitmap_get_cardinality(r1); - uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); - return c1 - inter; -} + // Total container sizes. + uint64_t total_sizes[4]; + if (maxbytes < sizeof(uint64_t) * 3) { + roaring64_bitmap_free(r); + return NULL; + } + memcpy(&(total_sizes[BITSET_CONTAINER_TYPE]), buf, sizeof(uint64_t)); + buf += sizeof(uint64_t); + maxbytes -= sizeof(uint64_t); + memcpy(&(total_sizes[RUN_CONTAINER_TYPE]), buf, sizeof(uint64_t)); + buf += sizeof(uint64_t); + maxbytes -= sizeof(uint64_t); + memcpy(&(total_sizes[ARRAY_CONTAINER_TYPE]), buf, sizeof(uint64_t)); + buf += sizeof(uint64_t); + maxbytes -= sizeof(uint64_t); -void roaring64_bitmap_andnot_inplace(roaring64_bitmap_t *r1, - const roaring64_bitmap_t *r2) { - art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); - art_iterator_t it2 = art_init_iterator((art_t *)&r2->art, /*first=*/true); + // ART (8 byte aligned). + buf = CROARING_ALIGN_BUF(buf, 8); + size_t art_size = art_frozen_view(buf, maxbytes, &r->art); + if (art_size == 0) { + roaring64_bitmap_free(r); + return NULL; + } + buf += art_size; + maxbytes -= art_size; - while (it1.value != NULL) { - // Cases: - // 1. it1_present && !it2_present -> it1++ - // 2. it1_present && it2_present - // a. it1 < it2 -> it1++ - // b. it1 == it2 -> it1 - it2, it1++, it2++ - // c. it1 > it2 -> it2++ - bool it2_present = it2.value != NULL; - int compare_result = 0; - if (it2_present) { - compare_result = compare_high48(it1.key, it2.key); - if (compare_result == 0) { - // Case 2b: iterators at the same high key position. - leaf_t *leaf1 = (leaf_t *)it1.value; - leaf_t leaf2 = (leaf_t)*it2.value; - uint8_t typecode1 = get_typecode(*leaf1); - container_t *container1 = get_container(r1, *leaf1); - uint8_t typecode2; - container_t *container2; - if (typecode1 == SHARED_CONTAINER_TYPE) { - container2 = container_andnot( - container1, typecode1, get_container(r2, leaf2), - get_typecode(leaf2), &typecode2); - if (container2 != container1) { - // We only free when doing container_andnot, not - // container_iandnot, as iandnot frees the original - // internally. - container_free(container1, typecode1); - } - } else { - container2 = container_iandnot( - container1, typecode1, get_container(r2, leaf2), - get_typecode(leaf2), &typecode2); - } + // Containers (aligned). + const char *before_containers = buf; + buf = CROARING_ALIGN_BUF(buf, CROARING_BITSET_ALIGNMENT); + const uint64_t *bitsets = (const uint64_t *)buf; + buf += total_sizes[BITSET_CONTAINER_TYPE]; + buf = CROARING_ALIGN_BUF(buf, alignof(rle16_t)); + const rle16_t *runs = (const rle16_t *)buf; + buf += total_sizes[RUN_CONTAINER_TYPE]; + buf = CROARING_ALIGN_BUF(buf, alignof(uint16_t)); + const uint16_t *arrays = (const uint16_t *)buf; + buf += total_sizes[ARRAY_CONTAINER_TYPE]; + if (maxbytes < (uint64_t)(buf - before_containers)) { + roaring64_bitmap_free(r); + return NULL; + } + maxbytes -= buf - before_containers; - if (!container_nonzero_cardinality(container2, typecode2)) { - container_free(container2, typecode2); - bool erased = art_iterator_erase(&it1, NULL); - assert(erased); - (void)erased; - remove_container(r1, *leaf1); - } else { - if (container2 != container1) { - replace_container(r1, leaf1, container2, typecode2); - } - // Only advance the iterator if we didn't delete the - // leaf, as erasing advances by itself. - art_iterator_next(&it1); - } - art_iterator_next(&it2); - } + // Deserialize in ART iteration order. + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + for (size_t i = 0; it.value != NULL; ++i) { + if (i >= capacity) { + roaring64_bitmap_free(r); + return NULL; } - if (!it2_present || compare_result < 0) { - // Cases 1 and 2a: it1 is the only iterator or is before it2. - art_iterator_next(&it1); - } else if (compare_result > 0) { - // Case 2c: it1 is after it2. - art_iterator_next(&it2); + leaf_t leaf = (leaf_t)*it.value; + uint8_t typecode = get_typecode(leaf); + + uint16_t compressed_elem_count; + memcpy(&compressed_elem_count, elem_counts + (i * sizeof(uint16_t)), + sizeof(compressed_elem_count)); + uint32_t elem_count = (uint32_t)(compressed_elem_count) + 1; + + // The container index is unrelated to the iteration order. + uint64_t index = get_index(leaf); + if (index >= capacity) { + roaring64_bitmap_free(r); + return NULL; } - } -} + container_t *c = container_frozen_view_at( + headers + index, typecode, elem_count, &bitsets, &arrays, &runs); + if (c == NULL) { + roaring64_bitmap_free(r); + return NULL; + } + set_container_at(r, index, c, typecode); -/** - * Flips the leaf at high48 in the range [min, max), adding the result to - * `r2`. If the high48 key is not found in `r1`, a new container is created. - */ -static void roaring64_flip_leaf(const roaring64_bitmap_t *r1, - roaring64_bitmap_t *r2, uint8_t high48[], - uint32_t min, uint32_t max) { - leaf_t *leaf1 = (leaf_t *)art_find(&r1->art, high48); - uint8_t typecode2; - container_t *container2; - if (leaf1 == NULL) { - // No container at this key, create a full container. - container2 = container_range_of_ones(min, max, &typecode2); - } else if (min == 0 && max > 0xFFFF) { - // Flip whole container. - container2 = container_not(get_container(r1, *leaf1), - get_typecode(*leaf1), &typecode2); - } else { - // Partially flip a container. - container2 = - container_not_range(get_container(r1, *leaf1), get_typecode(*leaf1), - min, max, &typecode2); - } - if (container_nonzero_cardinality(container2, typecode2)) { - leaf_t leaf2 = add_container(r2, container2, typecode2); - art_insert(&r2->art, high48, (art_val_t)leaf2); - } else { - container_free(container2, typecode2); + art_iterator_next(&it); } + + // Padding to make overall size a multiple of required alignment. + buf = CROARING_ALIGN_BUF(buf, CROARING_BITSET_ALIGNMENT); + + r->first_free = r->capacity; + return r; } -/** - * Flips the leaf at high48 in the range [min, max). If the high48 key is - * not found in the bitmap, a new container is created. Deletes the leaf and - * associated container if the negation results in an empty range. - */ -static void roaring64_flip_leaf_inplace(roaring64_bitmap_t *r, uint8_t high48[], - uint32_t min, uint32_t max) { - leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); - container_t *container2; - uint8_t typecode2; - if (leaf == NULL) { - // No container at this key, insert a full container. - container2 = container_range_of_ones(min, max, &typecode2); - leaf_t new_leaf = add_container(r, container2, typecode2); - art_insert(&r->art, high48, (art_val_t)new_leaf); - return; +static bool view_one_portable32(roaring64_bitmap_t *r, + frozen_container_header_t *headers, + uint32_t high32, const char *buf, + size_t maxbytes, size_t *consumed) { + *consumed = roaring_bitmap_portable_deserialize_size(buf, maxbytes); + if (*consumed == 0 || *consumed > maxbytes) { + return false; } + const char *start = buf; + size_t remaining = *consumed; - if (min == 0 && max > 0xFFFF) { - // Flip whole container. - container2 = container_inot(get_container(r, *leaf), - get_typecode(*leaf), &typecode2); + uint32_t cookie; + memcpy(&cookie, buf, sizeof(cookie)); + cookie = croaring_letoh32(cookie); + buf += sizeof(cookie); + remaining -= sizeof(cookie); + + int32_t num_containers; + const char *run_flag_bitset = NULL; + bool hasrun = false; + bool has_offsets; + + if (cookie == SERIAL_COOKIE_NO_RUNCONTAINER) { + if (remaining < sizeof(uint32_t)) { + return false; + } + uint32_t n_le; + memcpy(&n_le, buf, sizeof(n_le)); + num_containers = (int32_t)croaring_letoh32(n_le); + buf += sizeof(uint32_t); + remaining -= sizeof(uint32_t); + has_offsets = true; + } else if ((cookie & 0xFFFF) == SERIAL_COOKIE) { + num_containers = (int32_t)(cookie >> 16) + 1; + hasrun = true; + int32_t run_flag_bitset_size = (num_containers + 7) / 8; + if (num_containers < 0 || remaining < (size_t)run_flag_bitset_size) { + return false; + } + run_flag_bitset = buf; + buf += run_flag_bitset_size; + remaining -= (size_t)run_flag_bitset_size; + has_offsets = num_containers >= NO_OFFSET_THRESHOLD; } else { - // Partially flip a container. - container2 = container_inot_range( - get_container(r, *leaf), get_typecode(*leaf), min, max, &typecode2); + return false; + } + if (num_containers < 0 || num_containers > (1 << 16)) { + return false; } - if (container_nonzero_cardinality(container2, typecode2)) { - replace_container(r, leaf, container2, typecode2); - } else { - bool erased = art_erase(&r->art, high48, NULL); - assert(erased); - (void)erased; - container_free(container2, typecode2); - remove_container(r, *leaf); + size_t desc_bytes = (size_t)num_containers * 2 * sizeof(uint16_t); + if (remaining < desc_bytes) { + return false; } -} + const char *keyscards = buf; + buf += desc_bytes; + remaining -= desc_bytes; -roaring64_bitmap_t *roaring64_bitmap_flip(const roaring64_bitmap_t *r, - uint64_t min, uint64_t max) { - if (min >= max) { - return roaring64_bitmap_copy(r); + const char *offset_bytes = NULL; + if (has_offsets) { + size_t off_bytes = (size_t)num_containers * sizeof(uint32_t); + if (remaining < off_bytes) { + return false; + } + offset_bytes = buf; + buf += off_bytes; + remaining -= off_bytes; } - return roaring64_bitmap_flip_closed(r, min, max - 1); + + int32_t last_key = -1; + uint64_t key_base = ((uint64_t)high32) << 32; + + for (int32_t i = 0; i < num_containers; ++i) { + uint16_t key, card_m1; + memcpy(&key, keyscards + 4 * (size_t)i, sizeof(key)); + key = croaring_letoh16(key); + memcpy(&card_m1, keyscards + 4 * (size_t)i + 2, sizeof(card_m1)); + card_m1 = croaring_letoh16(card_m1); + if ((int32_t)key <= last_key) { + return false; + } + last_key = (int32_t)key; + + uint32_t cardinality = (uint32_t)card_m1 + 1; + bool isbitmap = cardinality > DEFAULT_MAX_SIZE; + bool isrun = false; + if (hasrun && (run_flag_bitset[i / 8] & (1 << (i % 8))) != 0) { + isbitmap = false; + isrun = true; + } + + const char *payload; + if (offset_bytes != NULL) { + uint32_t off; + memcpy(&off, offset_bytes + (size_t)i * sizeof(uint32_t), + sizeof(off)); + off = croaring_letoh32(off); + if ((size_t)off >= *consumed) { + return false; + } + payload = start + off; + } else { + payload = buf; + } + + uint8_t typecode; + size_t payload_size; + if (isbitmap) { + typecode = BITSET_CONTAINER_TYPE; + payload_size = BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + } else if (isrun) { + typecode = RUN_CONTAINER_TYPE; + if ((size_t)(payload - start) + sizeof(uint16_t) > *consumed) { + return false; + } + uint16_t n_runs; + memcpy(&n_runs, payload, sizeof(n_runs)); + n_runs = croaring_letoh16(n_runs); + payload_size = sizeof(uint16_t) + (size_t)n_runs * sizeof(rle16_t); + } else { + typecode = ARRAY_CONTAINER_TYPE; + payload_size = (size_t)cardinality * sizeof(uint16_t); + } + if ((size_t)(payload - start) + payload_size > *consumed) { + return false; + } + + if (r->first_free >= r->capacity) { + return false; + } + uint64_t index = allocate_index(r); + frozen_container_header_t *header = headers + index; + container_t *c; + if (isbitmap) { + header->bitset.cardinality = (int32_t)cardinality; + header->bitset.words = (uint64_t *)payload; + c = (container_t *)&header->bitset; + } else if (isrun) { + uint16_t n_runs; + memcpy(&n_runs, payload, sizeof(n_runs)); + n_runs = croaring_letoh16(n_runs); + header->run.n_runs = n_runs; + header->run.capacity = n_runs; + header->run.runs = (rle16_t *)(payload + sizeof(uint16_t)); + c = (container_t *)&header->run; + } else { + header->array.cardinality = (int32_t)cardinality; + header->array.capacity = (int32_t)cardinality; + header->array.array = (uint16_t *)payload; + c = (container_t *)&header->array; + } + set_container_at(r, index, c, typecode); + + uint8_t high48[ART_KEY_BYTES]; + uint64_t high48_bits = key_base | ((uint64_t)key << 16); + split_key(high48_bits, high48); + art_insert(&r->art, high48, (art_val_t)create_leaf(index, typecode)); + + if (offset_bytes == NULL) { + buf += payload_size; + remaining -= payload_size; + } + } + return true; } -roaring64_bitmap_t *roaring64_bitmap_flip_closed(const roaring64_bitmap_t *r1, - uint64_t min, uint64_t max) { - if (min > max) { - return roaring64_bitmap_copy(r1); +CROARING_ALLOW_UNALIGNED +roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_frozen( + const char *buf, size_t maxbytes) { + if (buf == NULL) { + return NULL; } - uint8_t min_high48_key[ART_KEY_BYTES]; - uint16_t min_low16 = split_key(min, min_high48_key); - uint8_t max_high48_key[ART_KEY_BYTES]; - uint16_t max_low16 = split_key(max, max_high48_key); - uint64_t min_high48_bits = (min & 0xFFFFFFFFFFFF0000ULL) >> 16; - uint64_t max_high48_bits = (max & 0xFFFFFFFFFFFF0000ULL) >> 16; +#if CROARING_IS_BIG_ENDIAN + // The portable format is little-endian and this function uses the payload + // bytes where they sit, so there is no correct view of them here. Refuse + // rather than hand back a bitmap that silently reads byte-swapped values. + (void)maxbytes; + return NULL; +#else + size_t remaining = maxbytes; - roaring64_bitmap_t *r2 = roaring64_bitmap_create(); - art_iterator_t it = art_init_iterator((art_t *)&r1->art, /*first=*/true); + if (remaining < sizeof(uint64_t)) { + return NULL; + } + uint64_t buckets; + memcpy(&buckets, buf, sizeof(buckets)); + buckets = croaring_letoh64(buckets); + buf += sizeof(buckets); + remaining -= sizeof(buckets); + if (buckets > UINT32_MAX) { + return NULL; + } - // Copy the containers before min unchanged. - while (it.value != NULL && compare_high48(it.key, min_high48_key) < 0) { - leaf_t leaf1 = (leaf_t)*it.value; - uint8_t typecode2 = get_typecode(leaf1); - container_t *container2 = get_copy_of_container( - get_container(r1, leaf1), &typecode2, /*copy_on_write=*/false); - leaf_t leaf2 = add_container(r2, container2, typecode2); - art_insert(&r2->art, it.key, (art_val_t)leaf2); - art_iterator_next(&it); + uint64_t ncontainers = 0; + const char *count_buf = buf; + size_t count_remaining = remaining; + int64_t previous_high32 = -1; + for (uint64_t bucket = 0; bucket < buckets; ++bucket) { + if (count_remaining < sizeof(uint32_t)) { + return NULL; + } + uint32_t high32; + memcpy(&high32, count_buf, sizeof(high32)); + high32 = croaring_letoh32(high32); + count_buf += sizeof(high32); + count_remaining -= sizeof(high32); + if (high32 <= previous_high32) { + return NULL; + } + previous_high32 = high32; + size_t bitmap32_size = roaring_bitmap_portable_deserialize_size( + count_buf, count_remaining); + if (bitmap32_size == 0) { + return NULL; + } + uint32_t cookie; + if (count_remaining < sizeof(cookie)) { + return NULL; + } + memcpy(&cookie, count_buf, sizeof(cookie)); + cookie = croaring_letoh32(cookie); + int32_t size; + if ((cookie & 0xFFFF) == SERIAL_COOKIE) { + size = (int32_t)(cookie >> 16) + 1; + } else if (cookie == SERIAL_COOKIE_NO_RUNCONTAINER) { + if (count_remaining < 2 * sizeof(uint32_t)) { + return NULL; + } + uint32_t size_le; + memcpy(&size_le, count_buf + sizeof(uint32_t), sizeof(size_le)); + size = (int32_t)croaring_letoh32(size_le); + } else { + return NULL; + } + if (size < 0) { + return NULL; + } + ncontainers += (uint64_t)size; + count_buf += bitmap32_size; + count_remaining -= bitmap32_size; } - // Flip the range (including non-existent containers!) between min and - // max. - for (uint64_t high48_bits = min_high48_bits; high48_bits <= max_high48_bits; - high48_bits++) { - uint8_t current_high48_key[ART_KEY_BYTES]; - split_key(high48_bits << 16, current_high48_key); + frozen_container_header_t *headers = NULL; + roaring64_bitmap_t *r = alloc_frozen_bitmap(ncontainers, &headers); + if (r == NULL) { + return NULL; + } + // ART is owned; payloads alias buf. + r->flags = ROARING_FLAG_FROZEN; - uint32_t min_container = 0; - if (high48_bits == min_high48_bits) { - min_container = min_low16; + previous_high32 = -1; + for (uint64_t bucket = 0; bucket < buckets; ++bucket) { + if (remaining < sizeof(uint32_t)) { + roaring64_bitmap_free(r); + return NULL; } - uint32_t max_container = 0xFFFF + 1; // Exclusive range. - if (high48_bits == max_high48_bits) { - max_container = max_low16 + 1; // Exclusive. + uint32_t high32; + memcpy(&high32, buf, sizeof(high32)); + high32 = croaring_letoh32(high32); + buf += sizeof(high32); + remaining -= sizeof(high32); + if (high32 <= previous_high32) { + roaring64_bitmap_free(r); + return NULL; } + previous_high32 = high32; - roaring64_flip_leaf(r1, r2, current_high48_key, min_container, - max_container); + size_t consumed = 0; + if (!view_one_portable32(r, headers, high32, buf, remaining, + &consumed)) { + roaring64_bitmap_free(r); + return NULL; + } + buf += consumed; + remaining -= consumed; } + return r; +#endif +} - // Copy the containers after max unchanged. - it = art_upper_bound((art_t *)&r1->art, max_high48_key); +bool roaring64_bitmap_iterate(const roaring64_bitmap_t *r, + roaring_iterator64 iterator, void *ptr) { + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); while (it.value != NULL) { - leaf_t leaf1 = (leaf_t)*it.value; - uint8_t typecode2 = get_typecode(leaf1); - container_t *container2 = get_copy_of_container( - get_container(r1, leaf1), &typecode2, /*copy_on_write=*/false); - leaf_t leaf2 = add_container(r2, container2, typecode2); - art_insert(&r2->art, it.key, (art_val_t)leaf2); + uint64_t high48 = combine_key(it.key, 0); + uint64_t high32 = high48 & 0xFFFFFFFF00000000ULL; + uint32_t low32 = high48; + leaf_t leaf = (leaf_t)*it.value; + if (!container_iterate64(get_container(r, leaf), get_typecode(leaf), + low32, iterator, high32, ptr)) { + return false; + } art_iterator_next(&it); } - - return r2; + return true; +} + +void roaring64_bitmap_to_uint64_array(const roaring64_bitmap_t *r, + uint64_t *out) { + roaring64_iterator_t it; // gets initialized in the next line + roaring64_iterator_init_at(r, &it, /*first=*/true); + roaring64_iterator_read(&it, out, UINT64_MAX); +} + +roaring64_iterator_t *roaring64_iterator_create(const roaring64_bitmap_t *r) { + roaring64_iterator_t *it = + (roaring64_iterator_t *)roaring_malloc(sizeof(roaring64_iterator_t)); + return roaring64_iterator_init_at(r, it, /*first=*/true); +} + +roaring64_iterator_t *roaring64_iterator_create_last( + const roaring64_bitmap_t *r) { + roaring64_iterator_t *it = + (roaring64_iterator_t *)roaring_malloc(sizeof(roaring64_iterator_t)); + return roaring64_iterator_init_at(r, it, /*first=*/false); +} + +void roaring64_iterator_reinit(const roaring64_bitmap_t *r, + roaring64_iterator_t *it) { + roaring64_iterator_init_at(r, it, /*first=*/true); +} + +void roaring64_iterator_reinit_last(const roaring64_bitmap_t *r, + roaring64_iterator_t *it) { + roaring64_iterator_init_at(r, it, /*first=*/false); +} + +roaring64_iterator_t *roaring64_iterator_copy(const roaring64_iterator_t *it) { + roaring64_iterator_t *new_it = + (roaring64_iterator_t *)roaring_malloc(sizeof(roaring64_iterator_t)); + memcpy(new_it, it, sizeof(*it)); + return new_it; +} + +void roaring64_iterator_free(roaring64_iterator_t *it) { roaring_free(it); } + +CROARING_STATIC_ASSERT(offsetof(roaring64_iterator_t, pub) == 0, + "the public members must be first in the iterator"); + +extern inline bool roaring64_iterator_has_value(const roaring64_iterator_t *it); +extern inline uint64_t roaring64_iterator_value(const roaring64_iterator_t *it); + +// The current container is exhausted: step the ART to the next leaf. +static inline bool roaring64_iterator_next_leaf(roaring64_iterator_t *it) { + if (art_iterator_next(&it->art_it)) { + return roaring64_iterator_init_at_leaf_first(it); + } + it->saturated_forward = true; + it->fast_type = 0; + return (it->pub.has_value = false); } -void roaring64_bitmap_flip_inplace(roaring64_bitmap_t *r, uint64_t min, - uint64_t max) { - if (min >= max) { - return; +// Everything `advance` needs when the bitset cache does not describe where the +// iterator is: a restart, or a step through the container iterator. +static inline bool roaring64_iterator_advance_slow(roaring64_iterator_t *it) { + if (it->art_it.value == NULL) { + if (it->saturated_forward) { + return (it->pub.has_value = false); + } + roaring64_iterator_init_at(it->r, it, /*first=*/true); + return it->pub.has_value; } - roaring64_bitmap_flip_closed_inplace(r, min, max - 1); + leaf_t leaf = (leaf_t)*it->art_it.value; + uint8_t typecode = get_typecode(leaf); + uint16_t low16 = (uint16_t)it->pub.value; + if (container_iterator_next(get_container(it->r, leaf), typecode, + &it->container_it, &low16)) { + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + if (typecode == BITSET_CONTAINER_TYPE) { + // Only reached when the cache was stale, i.e. something else moved + // the cursor. Array and run containers never have a cache, and + // priming them would cost a leaf load per value. + roaring64_iterator_prime(it); + } + return true; + } + return roaring64_iterator_next_leaf(it); } -void roaring64_bitmap_flip_closed_inplace(roaring64_bitmap_t *r, uint64_t min, - uint64_t max) { - if (min > max) { - return; +bool roaring64_iterator_advance(roaring64_iterator_t *it) { + // Matching both the leaf and `container_it.index` is enough to know the + // cache describes where the iterator actually is: within a container the + // index determines the position. Anything that moved the cursor -- + // `previous`, `move_equalorlarger`, any of the `read` variants -- moved one + // of them, and drops through to the slow path, which rebuilds the cache. + if (it->fast_type != 0 && it->fast_art_value == it->art_it.value && + it->fast_index == it->container_it.index) { + uint64_t word = it->fast_word; + uint32_t wi = it->fast_wordindex; + while (word == 0) { + if (++wi >= BITSET_CONTAINER_SIZE_IN_WORDS) { + return roaring64_iterator_next_leaf(it); + } + word = it->fast_words[wi]; + it->fast_wordindex = wi; + } + uint32_t index = wi * 64 + (uint32_t)roaring_trailing_zeroes(word); + it->fast_word = word & (word - 1); + it->fast_index = (int32_t)index; + it->container_it.index = (int32_t)index; + it->pub.value = it->high48 | index; + return (it->pub.has_value = true); } - uint16_t min_low16 = (uint16_t)min; - uint16_t max_low16 = (uint16_t)max; - uint64_t min_high48_bits = (min & 0xFFFFFFFFFFFF0000ULL) >> 16; - uint64_t max_high48_bits = (max & 0xFFFFFFFFFFFF0000ULL) >> 16; - - // Flip the range (including non-existent containers!) between min and - // max. - for (uint64_t high48_bits = min_high48_bits; high48_bits <= max_high48_bits; - high48_bits++) { - uint8_t current_high48_key[ART_KEY_BYTES]; - split_key(high48_bits << 16, current_high48_key); + return roaring64_iterator_advance_slow(it); +} - uint32_t min_container = 0; - if (high48_bits == min_high48_bits) { - min_container = min_low16; - } - uint32_t max_container = 0xFFFF + 1; // Exclusive range. - if (high48_bits == max_high48_bits) { - max_container = max_low16 + 1; // Exclusive. +bool roaring64_iterator_previous(roaring64_iterator_t *it) { + if (it->art_it.value == NULL) { + if (!it->saturated_forward) { + // Saturated backward. + return (it->pub.has_value = false); } - - roaring64_flip_leaf_inplace(r, current_high48_key, min_container, - max_container); + roaring64_iterator_init_at(it->r, it, /*first=*/false); + return it->pub.has_value; + } + leaf_t leaf = (leaf_t)*it->art_it.value; + uint16_t low16 = (uint16_t)it->pub.value; + if (container_iterator_prev(get_container(it->r, leaf), get_typecode(leaf), + &it->container_it, &low16)) { + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + it->fast_type = 0; + return true; + } + if (art_iterator_prev(&it->art_it)) { + return roaring64_iterator_init_at_leaf_last(it); } + it->saturated_forward = false; // Saturated backward. + it->fast_type = 0; + return (it->pub.has_value = false); } -roaring64_bitmap_t *roaring64_bitmap_add_offset_signed( - const roaring64_bitmap_t *r, bool positive, uint64_t offset) { - if (offset == 0) { - return roaring64_bitmap_copy(r); +bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, + uint64_t val) { + uint8_t val_high48[ART_KEY_BYTES]; + uint16_t val_low16 = split_key(val, val_high48); + if (!it->pub.has_value || it->high48 != (val & 0xFFFFFFFFFFFF0000)) { + // The ART iterator is before or after the high48 bits of `val` (or + // beyond the ART altogether), so we need to move to a leaf with a + // key equal or greater. + if (!art_iterator_lower_bound(&it->art_it, val_high48)) { + // Only smaller keys found. + it->saturated_forward = true; + it->fast_type = 0; + return (it->pub.has_value = false); + } + it->high48 = combine_key(it->art_it.key, 0); + // Fall through to the next if statement. } - roaring64_bitmap_t *answer = roaring64_bitmap_create(); - - // Decompose the offset into a signed container-level shift and an - // intra-container shift. For negative offsets the low 16 bits wrap: e.g. - // -1 = container_offset(-1) + in_offset(0xffff), because shifting by -1 - // container is a shift of -0x1_0000, so we need to shift up within - // containers to get back to -1 - uint16_t low16 = (uint16_t)offset; - int64_t container_offset; - uint16_t in_offset; - if (positive) { - container_offset = (int64_t)(offset >> 16); - in_offset = low16; - } else if (low16 == 0) { - container_offset = -(int64_t)(offset >> 16); - in_offset = 0; - } else { - container_offset = -(int64_t)(offset >> 16) - 1; - in_offset = (uint16_t)-low16; + if (it->high48 == (val & 0xFFFFFFFFFFFF0000)) { + // We're at equal high bits, check if a suitable value can be found + // in this container. + leaf_t leaf = (leaf_t)*it->art_it.value; + uint16_t low16 = (uint16_t)it->pub.value; + if (container_iterator_lower_bound( + get_container(it->r, leaf), get_typecode(leaf), + &it->container_it, &low16, val_low16)) { + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + it->fast_type = 0; + return true; + } + // Only smaller entries in this container, move to the next. + if (!art_iterator_next(&it->art_it)) { + it->saturated_forward = true; + it->fast_type = 0; + return (it->pub.has_value = false); + } } - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + // We're at a leaf with high bits greater than `val`, so the first entry + // in this container is our result. + return roaring64_iterator_init_at_leaf_first(it); +} - if (in_offset == 0) { - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - int64_t k = - (int64_t)(combine_key(it.key, 0) >> 16) + container_offset; - if ((uint64_t)k < (uint64_t)1 << 48) { - uint8_t new_high48[ART_KEY_BYTES]; - split_key((uint64_t)k << 16, new_high48); - uint8_t typecode = get_typecode(leaf); - container_t *container = - get_copy_of_container(get_container(r, leaf), &typecode, - /*copy_on_write=*/false); - leaf_t new_leaf = add_container(answer, container, typecode); - art_insert(&answer->art, new_high48, (art_val_t)new_leaf); - } - art_iterator_next(&it); +uint64_t roaring64_iterator_read(roaring64_iterator_t *it, uint64_t *buf, + uint64_t count) { + uint64_t consumed = 0; + while (it->pub.has_value && consumed < count) { + uint32_t container_consumed; + leaf_t leaf = (leaf_t)*it->art_it.value; + uint16_t low16 = (uint16_t)it->pub.value; + uint32_t container_count = UINT32_MAX; + if (count - consumed < (uint64_t)UINT32_MAX) { + container_count = count - consumed; + } + bool has_value = container_iterator_read_into_uint64( + get_container(it->r, leaf), get_typecode(leaf), &it->container_it, + it->high48, buf, container_count, &container_consumed, &low16); + consumed += container_consumed; + buf += container_consumed; + if (has_value) { + it->pub.has_value = true; + it->pub.value = it->high48 | low16; + it->fast_type = 0; + assert(consumed == count); + return consumed; + } + it->pub.has_value = art_iterator_next(&it->art_it); + if (it->pub.has_value) { + roaring64_iterator_init_at_leaf_first(it); + } else { + it->saturated_forward = true; + it->fast_type = 0; } - return answer; } + return consumed; +} - // Track the most recently inserted hi container so that the next - // iteration's lo can merge with it without re-searching the ART. - leaf_t *prev_hi_leaf = NULL; - int64_t prev_hi_k = -1; - - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - int64_t k = (int64_t)(combine_key(it.key, 0) >> 16) + container_offset; - - container_t *lo = NULL, *hi = NULL; - container_t **lo_ptr = NULL, **hi_ptr = NULL; - - if ((uint64_t)k < (uint64_t)1 << 48) { - lo_ptr = &lo; +uint64_t roaring64_iterator_read_backward(roaring64_iterator_t *it, + uint64_t *buf, uint64_t count) { + uint64_t consumed = 0; + while (it->pub.has_value && consumed < count) { + uint32_t container_consumed; + leaf_t leaf = *it->art_it.value; + uint16_t low16 = (uint16_t)it->pub.value; + uint32_t container_count = UINT32_MAX; + if (count - consumed < (uint64_t)UINT32_MAX) { + container_count = count - consumed; } - if ((uint64_t)(k + 1) < (uint64_t)1 << 48) { - hi_ptr = &hi; + bool has_value = container_iterator_read_backward_into_uint64( + get_container(it->r, leaf), get_typecode(leaf), &it->container_it, + it->high48, buf, container_count, &container_consumed, &low16); + consumed += container_consumed; + buf += container_consumed; + if (has_value) { + it->pub.has_value = true; + it->pub.value = it->high48 | low16; + assert(consumed == count); + return consumed; } - if (lo_ptr == NULL && hi_ptr == NULL) { - art_iterator_next(&it); - continue; + it->pub.has_value = art_iterator_prev(&it->art_it); + if (it->pub.has_value) { + roaring64_iterator_init_at_leaf_last(it); + } else { + it->saturated_forward = false; } + } + return consumed; +} - uint8_t typecode = get_typecode(leaf); - const container_t *c = - container_unwrap_shared(get_container(r, leaf), &typecode); - container_add_offset(c, typecode, lo_ptr, hi_ptr, in_offset); +size_t roaring64_iterator_read_ranges(roaring64_iterator_t *it, + roaring64_range_closed_t *buf, + size_t count) { + size_t ret = 0; + while (it->pub.has_value && ret < count) { + buf[ret].min = it->pub.value; + for (;;) { + uint16_t low16 = (uint16_t)it->pub.value; + leaf_t leaf = (leaf_t)*it->art_it.value; + bool container_has_more; + uint16_t run_end_low16 = container_iterator_find_run_end( + get_container(it->r, leaf), get_typecode(leaf), + &it->container_it, &low16, &container_has_more); + buf[ret].max = it->high48 | run_end_low16; - if (lo != NULL) { - if (prev_hi_leaf != NULL && prev_hi_k == k) { - uint8_t existing_type = get_typecode(*prev_hi_leaf); - container_t *existing_c = get_container(answer, *prev_hi_leaf); - uint8_t merged_type; - container_t *merged_c = container_ior( - existing_c, existing_type, lo, typecode, &merged_type); - if (merged_c != existing_c) { - container_free(existing_c, existing_type); - } - replace_container(answer, prev_hi_leaf, merged_c, merged_type); - container_free(lo, typecode); + if (container_has_more) { + it->pub.value = it->high48 | low16; + break; + } + // Move to next leaf + it->pub.has_value = art_iterator_next(&it->art_it); + if (it->pub.has_value) { + roaring64_iterator_init_at_leaf_first(it); } else { - uint8_t lo_high48[ART_KEY_BYTES]; - split_key((uint64_t)k << 16, lo_high48); - leaf_t new_leaf = add_container(answer, lo, typecode); - art_insert(&answer->art, lo_high48, (art_val_t)new_leaf); + it->saturated_forward = true; + break; + } + // Continue merging only if the run reached the container + // boundary and the next leaf starts exactly at max+1. + if (run_end_low16 != UINT16_MAX || + it->pub.value != buf[ret].max + 1) { + break; } } - - prev_hi_leaf = NULL; - if (hi != NULL) { - uint8_t hi_high48[ART_KEY_BYTES]; - split_key((uint64_t)(k + 1) << 16, hi_high48); - leaf_t new_leaf = add_container(answer, hi, typecode); - prev_hi_leaf = (leaf_t *)art_insert(&answer->art, hi_high48, - (art_val_t)new_leaf); - prev_hi_k = k + 1; - } - - art_iterator_next(&it); - } - - // Repair containers (e.g., convert low-cardinality bitset containers to - // array containers after lazy union operations). - art_iterator_t repair_it = art_init_iterator(&answer->art, /*first=*/true); - while (repair_it.value != NULL) { - leaf_t *leaf_ptr = (leaf_t *)repair_it.value; - uint8_t typecode = get_typecode(*leaf_ptr); - container_t *repaired = container_repair_after_lazy( - get_container(answer, *leaf_ptr), &typecode); - replace_container(answer, leaf_ptr, repaired, typecode); - art_iterator_next(&repair_it); + ret++; } - - return answer; + return ret; } - -// Returns the number of distinct high 32-bit entries in the bitmap. -static inline uint64_t count_high32(const roaring64_bitmap_t *r) { - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); - uint64_t high32_count = 0; - uint32_t prev_high32 = 0; - while (it.value != NULL) { - uint32_t current_high32 = (uint32_t)(combine_key(it.key, 0) >> 32); - if (high32_count == 0 || prev_high32 != current_high32) { - high32_count++; - prev_high32 = current_high32; + +size_t roaring64_iterator_read_prev_ranges(roaring64_iterator_t *it, + roaring64_range_closed_t *buf, + size_t count) { + size_t ret = 0; + while (it->pub.has_value && ret < count) { + buf[ret].max = it->pub.value; + for (;;) { + uint16_t low16 = (uint16_t)it->pub.value; + leaf_t leaf = (leaf_t)*it->art_it.value; + bool container_has_more; + uint16_t run_start_low16 = container_iterator_find_run_start( + get_container(it->r, leaf), get_typecode(leaf), + &it->container_it, &low16, &container_has_more); + buf[ret].min = it->high48 | run_start_low16; + + if (container_has_more) { + it->pub.value = it->high48 | low16; + break; + } + // Move to previous leaf + it->pub.has_value = art_iterator_prev(&it->art_it); + if (it->pub.has_value) { + roaring64_iterator_init_at_leaf_last(it); + } else { + it->saturated_forward = false; + break; + } + // Continue merging only if the run reached the container + // boundary and the previous leaf ends exactly at min-1. + if (run_start_low16 != 0 || it->pub.value != buf[ret].min - 1) { + break; + } } - art_iterator_next(&it); + ret++; } - return high32_count; + return ret; } -// Frees the (32-bit!) bitmap without freeing the containers. -static inline void roaring_bitmap_free_without_containers(roaring_bitmap_t *r) { - ra_clear_without_containers(&r->high_low_container); - roaring_free(r); -} +#ifdef __cplusplus +} // extern "C" +} // namespace roaring +} // namespace api +#endif +/* end file src/roaring64.c */ +/* begin file src/roaring_array.c */ +#include +#include +#include +#include +#include +#include -size_t roaring64_bitmap_portable_size_in_bytes(const roaring64_bitmap_t *r) { - // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations - size_t size = 0; - // Write as uint64 the distinct number of "buckets", where a bucket is - // defined as the most significant 32 bits of an element. - uint64_t high32_count; - size += sizeof(high32_count); +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); - uint32_t prev_high32 = 0; - roaring_bitmap_t *bitmap32 = NULL; +// Convention: [0,ra->size) all elements are initialized +// [ra->size, ra->allocation_size) is junk and contains nothing needing freeing - // Iterate through buckets ordered by increasing keys. - while (it.value != NULL) { - uint32_t current_high32 = (uint32_t)(combine_key(it.key, 0) >> 32); - if (bitmap32 == NULL || prev_high32 != current_high32) { - if (bitmap32 != NULL) { - // Write as uint32 the most significant 32 bits of the - // bucket. - size += sizeof(prev_high32); +extern inline int32_t ra_get_size(const roaring_array_t *ra); +extern inline int32_t ra_get_index(const roaring_array_t *ra, uint16_t x); - // Write the 32-bit Roaring bitmaps representing the least - // significant bits of a set of elements. - size += roaring_bitmap_portable_size_in_bytes(bitmap32); - roaring_bitmap_free_without_containers(bitmap32); - } +extern inline container_t *ra_get_container_at_index(const roaring_array_t *ra, + uint16_t i, + uint8_t *typecode); - // Start a new 32-bit bitmap with the current high 32 bits. - art_iterator_t it2 = it; - uint32_t containers_with_high32 = 0; - while (it2.value != NULL && (uint32_t)(combine_key(it2.key, 0) >> - 32) == current_high32) { - containers_with_high32++; - art_iterator_next(&it2); - } - bitmap32 = - roaring_bitmap_create_with_capacity(containers_with_high32); +extern inline void ra_unshare_container_at_index(roaring_array_t *ra, + uint16_t i); - prev_high32 = current_high32; - } - leaf_t leaf = (leaf_t)*it.value; - ra_append(&bitmap32->high_low_container, - (uint16_t)(current_high32 >> 16), get_container(r, leaf), - get_typecode(leaf)); - art_iterator_next(&it); +extern inline void ra_replace_key_and_container_at_index(roaring_array_t *ra, + int32_t i, + uint16_t key, + container_t *c, + uint8_t typecode); + +extern inline void ra_set_container_at_index(const roaring_array_t *ra, + int32_t i, container_t *c, + uint8_t typecode); + +static bool realloc_array(roaring_array_t *ra, int32_t new_capacity) { + // + // Note: not implemented using C's realloc(), because the memory layout is + // Struct-of-Arrays vs. Array-of-Structs: + // https://github.com/RoaringBitmap/CRoaring/issues/256 + + if (new_capacity == 0) { + roaring_free(ra->containers); + ra->containers = NULL; + ra->keys = NULL; + ra->typecodes = NULL; + ra->allocation_size = 0; + return true; + } + const size_t memoryneeded = + new_capacity * + (sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t)); + void *bigalloc = roaring_malloc(memoryneeded); + if (!bigalloc) return false; + void *oldbigalloc = ra->containers; + container_t **newcontainers = (container_t **)bigalloc; + uint16_t *newkeys = (uint16_t *)(newcontainers + new_capacity); + uint8_t *newtypecodes = (uint8_t *)(newkeys + new_capacity); + assert((char *)(newtypecodes + new_capacity) == + (char *)bigalloc + memoryneeded); + if (ra->size > 0) { + memcpy(newcontainers, ra->containers, sizeof(container_t *) * ra->size); + memcpy(newkeys, ra->keys, sizeof(uint16_t) * ra->size); + memcpy(newtypecodes, ra->typecodes, sizeof(uint8_t) * ra->size); } + ra->containers = newcontainers; + ra->keys = newkeys; + ra->typecodes = newtypecodes; + ra->allocation_size = new_capacity; + roaring_free(oldbigalloc); + return true; +} - if (bitmap32 != NULL) { - // Write as uint32 the most significant 32 bits of the bucket. - size += sizeof(prev_high32); +bool ra_init_with_capacity(roaring_array_t *new_ra, uint32_t cap) { + if (!new_ra) return false; + ra_init(new_ra); - // Write the 32-bit Roaring bitmaps representing the least - // significant bits of a set of elements. - size += roaring_bitmap_portable_size_in_bytes(bitmap32); - roaring_bitmap_free_without_containers(bitmap32); + // Containers hold 64Ki elements, so 64Ki containers is enough to hold + // `0x10000 * 0x10000` (all 2^32) elements + if (cap > 0x10000) { + cap = 0x10000; } - return size; + if (cap > 0) { + void *bigalloc = roaring_malloc( + cap * (sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t))); + if (bigalloc == NULL) return false; + new_ra->containers = (container_t **)bigalloc; + new_ra->keys = (uint16_t *)(new_ra->containers + cap); + new_ra->typecodes = (uint8_t *)(new_ra->keys + cap); + // Narrowing is safe because of above check + new_ra->allocation_size = (int32_t)cap; + } + return true; } -size_t roaring64_bitmap_portable_serialize(const roaring64_bitmap_t *r, - char *buf) { - // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations - if (buf == NULL) { +int ra_shrink_to_fit(roaring_array_t *ra) { + int savings = (ra->allocation_size - ra->size) * + (sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t)); + if (!realloc_array(ra, ra->size)) { return 0; } - const char *initial_buf = buf; - - // Write as uint64 the distinct number of "buckets", where a bucket is - // defined as the most significant 32 bits of an element. - uint64_t high32_count = count_high32(r); - memcpy(buf, &high32_count, sizeof(high32_count)); - buf += sizeof(high32_count); - - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); - uint32_t prev_high32 = 0; - roaring_bitmap_t *bitmap32 = NULL; - - // Iterate through buckets ordered by increasing keys. - while (it.value != NULL) { - uint64_t current_high48 = combine_key(it.key, 0); - uint32_t current_high32 = (uint32_t)(current_high48 >> 32); - if (bitmap32 == NULL || prev_high32 != current_high32) { - if (bitmap32 != NULL) { - // Write as uint32 the most significant 32 bits of the - // bucket. - memcpy(buf, &prev_high32, sizeof(prev_high32)); - buf += sizeof(prev_high32); + ra->allocation_size = ra->size; + return savings; +} - // Write the 32-bit Roaring bitmaps representing the least - // significant bits of a set of elements. - buf += roaring_bitmap_portable_serialize(bitmap32, buf); - roaring_bitmap_free_without_containers(bitmap32); - } +void ra_init(roaring_array_t *new_ra) { + if (!new_ra) { + return; + } + new_ra->keys = NULL; + new_ra->containers = NULL; + new_ra->typecodes = NULL; - // Start a new 32-bit bitmap with the current high 32 bits. - art_iterator_t it2 = it; - uint32_t containers_with_high32 = 0; - while (it2.value != NULL && - (uint32_t)combine_key(it2.key, 0) == current_high32) { - containers_with_high32++; - art_iterator_next(&it2); - } - bitmap32 = - roaring_bitmap_create_with_capacity(containers_with_high32); + new_ra->allocation_size = 0; + new_ra->size = 0; + new_ra->flags = 0; +} - prev_high32 = current_high32; +bool ra_overwrite(const roaring_array_t *source, roaring_array_t *dest, + bool copy_on_write) { + ra_clear_containers(dest); // we are going to overwrite them + if (source->size == 0) { // Note: can't call memcpy(NULL), even w/size + dest->size = 0; // <--- This is important. + return true; // output was just cleared, so they match + } + if (dest->allocation_size < source->size) { + if (!realloc_array(dest, source->size)) { + return false; + } + } + dest->size = source->size; + memcpy(dest->keys, source->keys, dest->size * sizeof(uint16_t)); + // we go through the containers, turning them into shared containers... + if (copy_on_write) { + for (int32_t i = 0; i < dest->size; ++i) { + source->containers[i] = get_copy_of_container( + source->containers[i], &source->typecodes[i], copy_on_write); + } + // we do a shallow copy to the other bitmap + memcpy(dest->containers, source->containers, + dest->size * sizeof(container_t *)); + memcpy(dest->typecodes, source->typecodes, + dest->size * sizeof(uint8_t)); + } else { + memcpy(dest->typecodes, source->typecodes, + dest->size * sizeof(uint8_t)); + for (int32_t i = 0; i < dest->size; i++) { + dest->containers[i] = + container_clone(source->containers[i], source->typecodes[i]); + if (dest->containers[i] == NULL) { + for (int32_t j = 0; j < i; j++) { + container_free(dest->containers[j], dest->typecodes[j]); + } + ra_clear_without_containers(dest); + return false; + } } - leaf_t leaf = (leaf_t)*it.value; - ra_append(&bitmap32->high_low_container, - (uint16_t)(current_high48 >> 16), get_container(r, leaf), - get_typecode(leaf)); - art_iterator_next(&it); } + return true; +} - if (bitmap32 != NULL) { - // Write as uint32 the most significant 32 bits of the bucket. - memcpy(buf, &prev_high32, sizeof(prev_high32)); - buf += sizeof(prev_high32); - - // Write the 32-bit Roaring bitmaps representing the least - // significant bits of a set of elements. - buf += roaring_bitmap_portable_serialize(bitmap32, buf); - roaring_bitmap_free_without_containers(bitmap32); +void ra_clear_containers(roaring_array_t *ra) { + for (int32_t i = 0; i < ra->size; ++i) { + container_free(ra->containers[i], ra->typecodes[i]); } - - return buf - initial_buf; } -size_t roaring64_bitmap_portable_deserialize_size(const char *buf, - size_t maxbytes) { - // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations - if (buf == NULL) { - return 0; - } - size_t read_bytes = 0; +void ra_reset(roaring_array_t *ra) { + ra_clear_containers(ra); + ra->size = 0; + ra_shrink_to_fit(ra); +} - // Read as uint64 the distinct number of "buckets", where a bucket is - // defined as the most significant 32 bits of an element. - uint64_t buckets; - if (read_bytes + sizeof(buckets) > maxbytes) { - return 0; - } - memcpy(&buckets, buf, sizeof(buckets)); - buf += sizeof(buckets); - read_bytes += sizeof(buckets); +void ra_clear_without_containers(roaring_array_t *ra) { + roaring_free( + ra->containers); // keys and typecodes are allocated with containers + ra->size = 0; + ra->allocation_size = 0; + ra->containers = NULL; + ra->keys = NULL; + ra->typecodes = NULL; +} - // Buckets should be 32 bits with 4 bits of zero padding. - if (buckets > UINT32_MAX) { - return 0; - } +void ra_clear(roaring_array_t *ra) { + ra_clear_containers(ra); + ra_clear_without_containers(ra); +} - // Iterate through buckets ordered by increasing keys. - for (uint64_t bucket = 0; bucket < buckets; ++bucket) { - // Read as uint32 the most significant 32 bits of the bucket. - uint32_t high32; - if (read_bytes + sizeof(high32) > maxbytes) { - return 0; +bool extend_array(roaring_array_t *ra, int32_t k) { + int32_t desired_size = ra->size + k; + const int32_t max_containers = 65536; + assert(desired_size <= max_containers); + if (desired_size > ra->allocation_size) { + int32_t new_capacity = + (ra->size < 1024) ? 2 * desired_size : 5 * desired_size / 4; + if (new_capacity > max_containers) { + new_capacity = max_containers; } - buf += sizeof(high32); - read_bytes += sizeof(high32); - // Read the 32-bit Roaring bitmaps representing the least - // significant bits of a set of elements. - size_t bitmap32_size = roaring_bitmap_portable_deserialize_size( - buf, maxbytes - read_bytes); - if (bitmap32_size == 0) { - return 0; - } - buf += bitmap32_size; - read_bytes += bitmap32_size; + return realloc_array(ra, new_capacity); } - return read_bytes; + return true; } -roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe( - const char *buf, size_t maxbytes) { - // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations - if (buf == NULL) { - return NULL; - } - size_t read_bytes = 0; +void ra_append(roaring_array_t *ra, uint16_t key, container_t *c, + uint8_t typecode) { + extend_array(ra, 1); + const int32_t pos = ra->size; - // Read as uint64 the distinct number of "buckets", where a bucket is - // defined as the most significant 32 bits of an element. - uint64_t buckets; - if (read_bytes + sizeof(buckets) > maxbytes) { - return NULL; + ra->keys[pos] = key; + ra->containers[pos] = c; + ra->typecodes[pos] = typecode; + ra->size++; +} + +void ra_append_copy(roaring_array_t *ra, const roaring_array_t *sa, + uint16_t index, bool copy_on_write) { + extend_array(ra, 1); + const int32_t pos = ra->size; + + // old contents is junk that does not need freeing + ra->keys[pos] = sa->keys[index]; + // the shared container will be in two bitmaps + if (copy_on_write) { + sa->containers[index] = get_copy_of_container( + sa->containers[index], &sa->typecodes[index], copy_on_write); + ra->containers[pos] = sa->containers[index]; + ra->typecodes[pos] = sa->typecodes[index]; + } else { + ra->containers[pos] = + container_clone(sa->containers[index], sa->typecodes[index]); + ra->typecodes[pos] = sa->typecodes[index]; } - memcpy(&buckets, buf, sizeof(buckets)); - buf += sizeof(buckets); - read_bytes += sizeof(buckets); + ra->size++; +} - // Buckets should be 32 bits with 4 bits of zero padding. - if (buckets > UINT32_MAX) { - return NULL; +void ra_append_copies_until(roaring_array_t *ra, const roaring_array_t *sa, + uint16_t stopping_key, bool copy_on_write) { + for (int32_t i = 0; i < sa->size; ++i) { + if (sa->keys[i] >= stopping_key) break; + ra_append_copy(ra, sa, (uint16_t)i, copy_on_write); } +} - roaring64_bitmap_t *r = roaring64_bitmap_create(); - // Iterate through buckets ordered by increasing keys. - int64_t previous_high32 = -1; - for (uint64_t bucket = 0; bucket < buckets; ++bucket) { - // Read as uint32 the most significant 32 bits of the bucket. - uint32_t high32; - if (read_bytes + sizeof(high32) > maxbytes) { - roaring64_bitmap_free(r); - return NULL; - } - memcpy(&high32, buf, sizeof(high32)); - buf += sizeof(high32); - read_bytes += sizeof(high32); - // High 32 bits must be strictly increasing. - if (high32 <= previous_high32) { - roaring64_bitmap_free(r); - return NULL; +void ra_append_copy_range(roaring_array_t *ra, const roaring_array_t *sa, + int32_t start_index, int32_t end_index, + bool copy_on_write) { + extend_array(ra, end_index - start_index); + for (int32_t i = start_index; i < end_index; ++i) { + const int32_t pos = ra->size; + ra->keys[pos] = sa->keys[i]; + if (copy_on_write) { + sa->containers[i] = get_copy_of_container( + sa->containers[i], &sa->typecodes[i], copy_on_write); + ra->containers[pos] = sa->containers[i]; + ra->typecodes[pos] = sa->typecodes[i]; + } else { + ra->containers[pos] = + container_clone(sa->containers[i], sa->typecodes[i]); + ra->typecodes[pos] = sa->typecodes[i]; } - previous_high32 = high32; + ra->size++; + } +} - // Read the 32-bit Roaring bitmaps representing the least - // significant bits of a set of elements. - size_t bitmap32_size = roaring_bitmap_portable_deserialize_size( - buf, maxbytes - read_bytes); - if (bitmap32_size == 0) { - roaring64_bitmap_free(r); - return NULL; - } +void ra_append_copies_after(roaring_array_t *ra, const roaring_array_t *sa, + uint16_t before_start, bool copy_on_write) { + int start_location = ra_get_index(sa, before_start); + if (start_location >= 0) + ++start_location; + else + start_location = -start_location - 1; + ra_append_copy_range(ra, sa, start_location, sa->size, copy_on_write); +} - roaring_bitmap_t *bitmap32 = roaring_bitmap_portable_deserialize_safe( - buf, maxbytes - read_bytes); - if (bitmap32 == NULL) { - roaring64_bitmap_free(r); - return NULL; - } - buf += bitmap32_size; - read_bytes += bitmap32_size; +void ra_append_move_range(roaring_array_t *ra, roaring_array_t *sa, + int32_t start_index, int32_t end_index) { + extend_array(ra, end_index - start_index); - // While we don't attempt to validate much, we must ensure that there - // is no duplication in the high 48 bits - inserting into the ART - // assumes (or UB) no duplicate keys. The top 32 bits must be unique - // because we check for strict increasing values of high32, but we - // must also ensure the top 16 bits within each 32-bit bitmap are also - // at least unique (we ensure they're strictly increasing as well, - // which they must be for a _valid_ bitmap, since it's cheaper to check) - int32_t last_bitmap_key = -1; - for (int i = 0; i < bitmap32->high_low_container.size; i++) { - uint16_t key = bitmap32->high_low_container.keys[i]; - if (key <= last_bitmap_key) { - roaring_bitmap_free(bitmap32); - roaring64_bitmap_free(r); - return NULL; - } - last_bitmap_key = key; - } + for (int32_t i = start_index; i < end_index; ++i) { + const int32_t pos = ra->size; - // Insert all containers of the 32-bit bitmap into the 64-bit bitmap. - move_from_roaring32_offset(r, bitmap32, high32); - roaring_bitmap_free(bitmap32); + ra->keys[pos] = sa->keys[i]; + ra->containers[pos] = sa->containers[i]; + ra->typecodes[pos] = sa->typecodes[i]; + ra->size++; } - return r; } -// Returns an "element count" for the given container. This has a different -// meaning for each container type, but the purpose is the minimal information -// required to serialize the container metadata. -static inline uint32_t container_get_element_count(const container_t *c, - uint8_t typecode) { - switch (typecode) { - case BITSET_CONTAINER_TYPE: { - return ((bitset_container_t *)c)->cardinality; - } - case ARRAY_CONTAINER_TYPE: { - return ((array_container_t *)c)->cardinality; - } - case RUN_CONTAINER_TYPE: { - return ((run_container_t *)c)->n_runs; - } - default: { - assert(false); - roaring_unreachable; - return 0; +void ra_append_range(roaring_array_t *ra, roaring_array_t *sa, + int32_t start_index, int32_t end_index, + bool copy_on_write) { + extend_array(ra, end_index - start_index); + + for (int32_t i = start_index; i < end_index; ++i) { + const int32_t pos = ra->size; + ra->keys[pos] = sa->keys[i]; + if (copy_on_write) { + sa->containers[i] = get_copy_of_container( + sa->containers[i], &sa->typecodes[i], copy_on_write); + ra->containers[pos] = sa->containers[i]; + ra->typecodes[pos] = sa->typecodes[i]; + } else { + ra->containers[pos] = + container_clone(sa->containers[i], sa->typecodes[i]); + ra->typecodes[pos] = sa->typecodes[i]; } + ra->size++; } } -static inline size_t container_get_frozen_size(const container_t *c, - uint8_t typecode) { - switch (typecode) { - case BITSET_CONTAINER_TYPE: { - return BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); - } - case ARRAY_CONTAINER_TYPE: { - return container_get_element_count(c, typecode) * sizeof(uint16_t); - } - case RUN_CONTAINER_TYPE: { - return container_get_element_count(c, typecode) * sizeof(rle16_t); - } - default: { - assert(false); - roaring_unreachable; - return 0; - } - } +container_t *ra_get_container(roaring_array_t *ra, uint16_t x, + uint8_t *typecode) { + int i = binarySearch(ra->keys, (int32_t)ra->size, x); + if (i < 0) return NULL; + *typecode = ra->typecodes[i]; + return ra->containers[i]; } -uint64_t align_size(uint64_t size, uint64_t alignment) { - return (size + alignment - 1) & ~(alignment - 1); -} +extern inline container_t *ra_get_container_at_index(const roaring_array_t *ra, + uint16_t i, + uint8_t *typecode); -size_t roaring64_bitmap_frozen_size_in_bytes(const roaring64_bitmap_t *r) { - if (!is_shrunken(r)) { - return 0; - } - // Flags. - uint64_t size = sizeof(r->flags); - // Container count. - size += sizeof(r->capacity); - // Container element counts. - size += r->capacity * sizeof(uint16_t); - // Total container sizes. - size += 3 * sizeof(uint64_t); - // ART (8 byte aligned). - size = align_size(size, 8); - size += art_size_in_bytes(&r->art); +extern inline uint16_t ra_get_key_at_index(const roaring_array_t *ra, + uint16_t i); - uint64_t total_sizes[4] = - CROARING_ZERO_INITIALIZER; // Indexed by typecode. - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - uint8_t typecode = get_typecode(leaf); - total_sizes[typecode] += - container_get_frozen_size(get_container(r, leaf), typecode); - art_iterator_next(&it); +extern inline int32_t ra_get_index(const roaring_array_t *ra, uint16_t x); + +extern inline int32_t ra_advance_until(const roaring_array_t *ra, uint16_t x, + int32_t pos); + +// everything skipped over is freed +int32_t ra_advance_until_freeing(roaring_array_t *ra, uint16_t x, int32_t pos) { + while (pos < ra->size && ra->keys[pos] < x) { + container_free(ra->containers[pos], ra->typecodes[pos]); + ++pos; } - // Containers (aligned). - size = align_size(size, CROARING_BITSET_ALIGNMENT); - size += total_sizes[BITSET_CONTAINER_TYPE]; - size = align_size(size, alignof(rle16_t)); - size += total_sizes[ARRAY_CONTAINER_TYPE]; - size = align_size(size, alignof(uint16_t)); - size += total_sizes[RUN_CONTAINER_TYPE]; - // Padding to make overall size a multiple of required alignment. - size = align_size(size, CROARING_BITSET_ALIGNMENT); - return size; + return pos; } -static inline void container_frozen_serialize(const container_t *container, - uint8_t typecode, - uint64_t **bitsets, - uint16_t **arrays, - rle16_t **runs) { - size_t size = container_get_frozen_size(container, typecode); - switch (typecode) { - case BITSET_CONTAINER_TYPE: { - bitset_container_t *bitset = (bitset_container_t *)container; - memcpy(*bitsets, bitset->words, size); - *bitsets += BITSET_CONTAINER_SIZE_IN_WORDS; - break; - } - case ARRAY_CONTAINER_TYPE: { - array_container_t *array = (array_container_t *)container; - memcpy(*arrays, array->array, size); - *arrays += container_get_element_count(container, typecode); - break; - } - case RUN_CONTAINER_TYPE: { - run_container_t *run = (run_container_t *)container; - memcpy(*runs, run->runs, size); - *runs += container_get_element_count(container, typecode); - break; - } - default: { - assert(false); - roaring_unreachable; - } - } +void ra_insert_new_key_value_at(roaring_array_t *ra, int32_t i, uint16_t key, + container_t *c, uint8_t typecode) { + extend_array(ra, 1); + // May be an optimization opportunity with DIY memmove + memmove(&(ra->keys[i + 1]), &(ra->keys[i]), + sizeof(uint16_t) * (ra->size - i)); + memmove(&(ra->containers[i + 1]), &(ra->containers[i]), + sizeof(container_t *) * (ra->size - i)); + memmove(&(ra->typecodes[i + 1]), &(ra->typecodes[i]), + sizeof(uint8_t) * (ra->size - i)); + ra->keys[i] = key; + ra->containers[i] = c; + ra->typecodes[i] = typecode; + ra->size++; } -static inline char *pad_align(char *buf, const char *initial_buf, - size_t alignment) { - uint64_t buf_size = buf - initial_buf; - uint64_t pad = align_size(buf_size, alignment) - buf_size; - memset(buf, 0, pad); - return buf + pad; +// note: Java routine set things to 0, enabling GC. +// Java called it "resize" but it was always used to downsize. +// Allowing upsize would break the conventions about +// valid containers below ra->size. + +void ra_downsize(roaring_array_t *ra, int32_t new_length) { + assert(new_length <= ra->size); + ra->size = new_length; } -size_t roaring64_bitmap_frozen_serialize(const roaring64_bitmap_t *r, - char *buf) { - if (buf == NULL) { - return 0; - } - if (!is_shrunken(r)) { - return 0; - } - const char *initial_buf = buf; +void ra_remove_at_index(roaring_array_t *ra, int32_t i) { + memmove(&(ra->containers[i]), &(ra->containers[i + 1]), + sizeof(container_t *) * (ra->size - i - 1)); + memmove(&(ra->keys[i]), &(ra->keys[i + 1]), + sizeof(uint16_t) * (ra->size - i - 1)); + memmove(&(ra->typecodes[i]), &(ra->typecodes[i + 1]), + sizeof(uint8_t) * (ra->size - i - 1)); + ra->size--; +} - // Flags. - memcpy(buf, &r->flags, sizeof(r->flags)); - buf += sizeof(r->flags); +void ra_remove_at_index_and_free(roaring_array_t *ra, int32_t i) { + container_free(ra->containers[i], ra->typecodes[i]); + ra_remove_at_index(ra, i); +} - // Container count. - memcpy(buf, &r->capacity, sizeof(r->capacity)); - buf += sizeof(r->capacity); +// used in inplace andNot only, to slide left the containers from +// the mutated RoaringBitmap that are after the largest container of +// the argument RoaringBitmap. In use it should be followed by a call to +// downsize. +// +void ra_copy_range(roaring_array_t *ra, uint32_t begin, uint32_t end, + uint32_t new_begin) { + assert(begin <= end); + assert(new_begin < begin); - // Container element counts. - uint64_t total_sizes[4] = - CROARING_ZERO_INITIALIZER; // Indexed by typecode. - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - uint8_t typecode = get_typecode(leaf); - container_t *container = get_container(r, leaf); + const int range = end - begin; - uint32_t elem_count = container_get_element_count(container, typecode); - uint16_t compressed_elem_count = (uint16_t)(elem_count - 1); - memcpy(buf, &compressed_elem_count, sizeof(compressed_elem_count)); - buf += sizeof(compressed_elem_count); + // We ensure to previously have freed overwritten containers + // that are not copied elsewhere - total_sizes[typecode] += container_get_frozen_size(container, typecode); - art_iterator_next(&it); - } + memmove(&(ra->containers[new_begin]), &(ra->containers[begin]), + sizeof(container_t *) * range); + memmove(&(ra->keys[new_begin]), &(ra->keys[begin]), + sizeof(uint16_t) * range); + memmove(&(ra->typecodes[new_begin]), &(ra->typecodes[begin]), + sizeof(uint8_t) * range); +} - // Total container sizes. - memcpy(buf, &(total_sizes[BITSET_CONTAINER_TYPE]), sizeof(uint64_t)); - buf += sizeof(uint64_t); - memcpy(buf, &(total_sizes[RUN_CONTAINER_TYPE]), sizeof(uint64_t)); - buf += sizeof(uint64_t); - memcpy(buf, &(total_sizes[ARRAY_CONTAINER_TYPE]), sizeof(uint64_t)); - buf += sizeof(uint64_t); +void ra_shift_tail(roaring_array_t *ra, int32_t count, int32_t distance) { + if (distance > 0) { + extend_array(ra, distance); + } + int32_t srcpos = ra->size - count; + int32_t dstpos = srcpos + distance; + memmove(&(ra->keys[dstpos]), &(ra->keys[srcpos]), sizeof(uint16_t) * count); + memmove(&(ra->containers[dstpos]), &(ra->containers[srcpos]), + sizeof(container_t *) * count); + memmove(&(ra->typecodes[dstpos]), &(ra->typecodes[srcpos]), + sizeof(uint8_t) * count); + ra->size += distance; +} - // ART. - buf = pad_align(buf, initial_buf, 8); - buf += art_serialize(&r->art, buf); +void ra_to_uint32_array(const roaring_array_t *ra, uint32_t *ans) { + size_t ctr = 0; + for (int32_t i = 0; i < ra->size; ++i) { + int num_added = container_to_uint32_array( + ans + ctr, ra->containers[i], ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16); + ctr += num_added; + } +} - // Containers (aligned). - // Runs before arrays as run elements are larger than array elements and - // smaller than bitset elements. - buf = pad_align(buf, initial_buf, CROARING_BITSET_ALIGNMENT); - uint64_t *bitsets = (uint64_t *)buf; - buf += total_sizes[BITSET_CONTAINER_TYPE]; - buf = pad_align(buf, initial_buf, alignof(rle16_t)); - rle16_t *runs = (rle16_t *)buf; - buf += total_sizes[RUN_CONTAINER_TYPE]; - buf = pad_align(buf, initial_buf, alignof(uint16_t)); - uint16_t *arrays = (uint16_t *)buf; - buf += total_sizes[ARRAY_CONTAINER_TYPE]; +bool ra_has_run_container(const roaring_array_t *ra) { + for (int32_t k = 0; k < ra->size; ++k) { + if (get_container_type(ra->containers[k], ra->typecodes[k]) == + RUN_CONTAINER_TYPE) + return true; + } + return false; +} - it = art_init_iterator((art_t *)&r->art, /*first=*/true); - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - uint8_t typecode = get_typecode(leaf); - container_t *container = get_container(r, leaf); - container_frozen_serialize(container, typecode, &bitsets, &arrays, - &runs); - art_iterator_next(&it); +uint32_t ra_portable_header_size(const roaring_array_t *ra) { + if (ra_has_run_container(ra)) { + if (ra->size < + NO_OFFSET_THRESHOLD) { // for small bitmaps, we omit the offsets + return 4 + (ra->size + 7) / 8 + 4 * ra->size; + } + return 4 + (ra->size + 7) / 8 + + 8 * ra->size; // - 4 because we pack the size with the cookie + } else { + return 4 + 4 + 8 * ra->size; } +} - // Padding to make overall size a multiple of required alignment. - buf = pad_align(buf, initial_buf, CROARING_BITSET_ALIGNMENT); +size_t ra_portable_size_in_bytes(const roaring_array_t *ra) { + size_t count = ra_portable_header_size(ra); - return buf - initial_buf; + for (int32_t k = 0; k < ra->size; ++k) { + count += container_size_in_bytes(ra->containers[k], ra->typecodes[k]); + } + return count; } -static container_t *container_frozen_view(uint8_t typecode, uint32_t elem_count, - const uint64_t **bitsets, - const uint16_t **arrays, - const rle16_t **runs) { - switch (typecode) { - case BITSET_CONTAINER_TYPE: { - bitset_container_t *c = (bitset_container_t *)roaring_malloc( - sizeof(bitset_container_t)); - c->cardinality = elem_count; - c->words = (uint64_t *)*bitsets; - *bitsets += BITSET_CONTAINER_SIZE_IN_WORDS; - return (container_t *)c; - } - case ARRAY_CONTAINER_TYPE: { - array_container_t *c = - (array_container_t *)roaring_malloc(sizeof(array_container_t)); - c->cardinality = elem_count; - c->capacity = elem_count; - c->array = (uint16_t *)*arrays; - *arrays += elem_count; - return (container_t *)c; +// The portable serialization format is little-endian. On big-endian hosts we +// byte-swap multi-byte fields before writing them to the buffer. +size_t ra_portable_serialize(const roaring_array_t *ra, char *buf) { + char *initbuf = buf; + uint32_t startOffset = 0; + bool hasrun = ra_has_run_container(ra); + if (hasrun) { + uint32_t cookie = SERIAL_COOKIE | ((uint32_t)(ra->size - 1) << 16); + uint32_t cookie_le = croaring_htole32(cookie); + memcpy(buf, &cookie_le, sizeof(cookie_le)); + buf += sizeof(cookie_le); + uint32_t s = (ra->size + 7) / 8; + memset(buf, 0, s); + for (int32_t i = 0; i < ra->size; ++i) { + if (get_container_type(ra->containers[i], ra->typecodes[i]) == + RUN_CONTAINER_TYPE) { + buf[i / 8] |= 1 << (i % 8); + } } - case RUN_CONTAINER_TYPE: { - run_container_t *c = - (run_container_t *)roaring_malloc(sizeof(run_container_t)); - c->n_runs = elem_count; - c->capacity = elem_count; - c->runs = (rle16_t *)*runs; - *runs += elem_count; - return (container_t *)c; + buf += s; + if (ra->size < NO_OFFSET_THRESHOLD) { + startOffset = 4 + 4 * ra->size + s; + } else { + startOffset = 4 + 8 * ra->size + s; } - default: { - assert(false); - roaring_unreachable; - return NULL; + } else { // backwards compatibility + uint32_t cookie = SERIAL_COOKIE_NO_RUNCONTAINER; + uint32_t cookie_le = croaring_htole32(cookie); + memcpy(buf, &cookie_le, sizeof(cookie_le)); + buf += sizeof(cookie_le); + uint32_t size_le = croaring_htole32((uint32_t)ra->size); + memcpy(buf, &size_le, sizeof(size_le)); + buf += sizeof(size_le); + + startOffset = 4 + 4 + 4 * ra->size + 4 * ra->size; + } + for (int32_t k = 0; k < ra->size; ++k) { + uint16_t key_le = croaring_htole16(ra->keys[k]); + memcpy(buf, &key_le, sizeof(key_le)); + buf += sizeof(key_le); + // get_cardinality returns a value in [1,1<<16], subtracting one + // we get [0,1<<16 - 1] which fits in 16 bits + uint16_t card = (uint16_t)(container_get_cardinality(ra->containers[k], + ra->typecodes[k]) - + 1); + uint16_t card_le = croaring_htole16(card); + memcpy(buf, &card_le, sizeof(card_le)); + buf += sizeof(card_le); + } + if ((!hasrun) || (ra->size >= NO_OFFSET_THRESHOLD)) { + // writing the containers offsets + for (int32_t k = 0; k < ra->size; k++) { + uint32_t off_le = croaring_htole32(startOffset); + memcpy(buf, &off_le, sizeof(off_le)); + buf += sizeof(off_le); + startOffset = + startOffset + + container_size_in_bytes(ra->containers[k], ra->typecodes[k]); } } + for (int32_t k = 0; k < ra->size; ++k) { + buf += container_write(ra->containers[k], ra->typecodes[k], buf); + } + return buf - initbuf; } -roaring64_bitmap_t *roaring64_bitmap_frozen_view(const char *buf, - size_t maxbytes) { - if (buf == NULL) { - return NULL; - } - if ((uintptr_t)buf % CROARING_BITSET_ALIGNMENT != 0) { - return NULL; +// Quickly checks whether there is a serialized bitmap at the pointer, +// not exceeding size "maxbytes" in bytes. This function does not allocate +// memory dynamically. +// +// This function returns 0 if and only if no valid bitmap is found. +// Otherwise, it returns how many bytes are occupied. +// +size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { + size_t bytestotal = sizeof(int32_t); // for cookie + if (bytestotal > maxbytes) return 0; + uint32_t cookie; + memcpy(&cookie, buf, sizeof(int32_t)); + cookie = croaring_letoh32(cookie); + buf += sizeof(uint32_t); + if ((cookie & 0xFFFF) != SERIAL_COOKIE && + cookie != SERIAL_COOKIE_NO_RUNCONTAINER) { + return 0; } + int32_t size; - roaring64_bitmap_t *r = roaring64_bitmap_create(); + if ((cookie & 0xFFFF) == SERIAL_COOKIE) + size = (cookie >> 16) + 1; + else { + bytestotal += sizeof(int32_t); + if (bytestotal > maxbytes) return 0; + uint32_t size_le; + memcpy(&size_le, buf, sizeof(int32_t)); + size = (int32_t)croaring_letoh32(size_le); + buf += sizeof(uint32_t); + } + if (size > (1 << 16) || size < 0) { + return 0; + } + char *bitmapOfRunContainers = NULL; + bool hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE; + if (hasrun) { + int32_t s = (size + 7) / 8; + bytestotal += s; + if (bytestotal > maxbytes) return 0; + bitmapOfRunContainers = (char *)buf; + buf += s; + } + bytestotal += size * 2 * sizeof(uint16_t); + if (bytestotal > maxbytes) return 0; + const char *keyscards = buf; + buf += size * 2 * sizeof(uint16_t); + if ((!hasrun) || (size >= NO_OFFSET_THRESHOLD)) { + // skipping the offsets + bytestotal += size * 4; + if (bytestotal > maxbytes) return 0; + buf += size * 4; + } + // Reading the containers + for (int32_t k = 0; k < size; ++k) { + uint16_t tmp; + memcpy(&tmp, keyscards + 4 * k + 2, sizeof(tmp)); + tmp = croaring_letoh16(tmp); + uint32_t thiscard = tmp + 1; + bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); + bool isrun = false; + if (hasrun) { + if ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) { + isbitmap = false; + isrun = true; + } + } + if (isbitmap) { + size_t containersize = + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + bytestotal += containersize; + if (bytestotal > maxbytes) return 0; + buf += containersize; + } else if (isrun) { + bytestotal += sizeof(uint16_t); + if (bytestotal > maxbytes) return 0; + uint16_t n_runs; + memcpy(&n_runs, buf, sizeof(uint16_t)); + n_runs = croaring_letoh16(n_runs); + buf += sizeof(uint16_t); + size_t containersize = n_runs * sizeof(rle16_t); + bytestotal += containersize; + if (bytestotal > maxbytes) return 0; + buf += containersize; + } else { + size_t containersize = thiscard * sizeof(uint16_t); + bytestotal += containersize; + if (bytestotal > maxbytes) return 0; + buf += containersize; + } + } + return bytestotal; +} - // Flags. - if (maxbytes < sizeof(r->flags)) { - roaring64_bitmap_free(r); - return NULL; +// This function populates answer from the content of buf (reading up to +// maxbytes bytes). The function returns false if a properly serialized bitmap +// cannot be found. If it returns true, readbytes is populated by how many bytes +// were read, we have that *readbytes <= maxbytes. +// +// The portable serialization format is little-endian. On big-endian hosts we +// byte-swap multi-byte fields after reading them from the buffer. +bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, + const size_t maxbytes, size_t *readbytes) { + *readbytes = sizeof(int32_t); // for cookie + if (*readbytes > maxbytes) { + // Ran out of bytes while reading first 4 bytes. + return false; } - memcpy(&r->flags, buf, sizeof(r->flags)); - buf += sizeof(r->flags); - maxbytes -= sizeof(r->flags); - r->flags |= ROARING_FLAG_FROZEN; + uint32_t cookie; + memcpy(&cookie, buf, sizeof(int32_t)); + cookie = croaring_letoh32(cookie); + buf += sizeof(uint32_t); + if ((cookie & 0xFFFF) != SERIAL_COOKIE && + cookie != SERIAL_COOKIE_NO_RUNCONTAINER) { + // "I failed to find one of the right cookies. + return false; + } + int32_t size; - // Container count. - if (maxbytes < sizeof(r->capacity)) { - roaring64_bitmap_free(r); - return NULL; + if ((cookie & 0xFFFF) == SERIAL_COOKIE) + size = (cookie >> 16) + 1; + else { + *readbytes += sizeof(int32_t); + if (*readbytes > maxbytes) { + // Ran out of bytes while reading second part of the cookie. + return false; + } + uint32_t size_le; + memcpy(&size_le, buf, sizeof(int32_t)); + size = (int32_t)croaring_letoh32(size_le); + buf += sizeof(uint32_t); } - memcpy(&r->capacity, buf, sizeof(r->capacity)); - buf += sizeof(r->capacity); - maxbytes -= sizeof(r->capacity); + if (size < 0) { + // You cannot have a negative number of containers, the data must be + // corrupted. + return false; + } + if (size > (1 << 16)) { + // You cannot have so many containers, the data must be corrupted. + return false; + } + const char *bitmapOfRunContainers = NULL; + bool hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE; + if (hasrun) { + int32_t s = (size + 7) / 8; + *readbytes += s; + if (*readbytes > maxbytes) { // data is corrupted? + // Ran out of bytes while reading run bitmap. + return false; + } + bitmapOfRunContainers = buf; + buf += s; + } + const char *keyscards = buf; - r->containers = - (container_t **)roaring_malloc(r->capacity * sizeof(container_t *)); + *readbytes += size * 2 * sizeof(uint16_t); + if (*readbytes > maxbytes) { + // Ran out of bytes while reading key-cardinality array. + return false; + } + buf += size * 2 * sizeof(uint16_t); - // Container element counts. - if (maxbytes < r->capacity * sizeof(uint16_t)) { - roaring64_bitmap_free(r); - return NULL; + bool is_ok = ra_init_with_capacity(answer, size); + if (!is_ok) { + // Failed to allocate memory for roaring array. Bailing out. + return false; } - const char *elem_counts = buf; - buf += r->capacity * sizeof(uint16_t); - maxbytes -= r->capacity * sizeof(uint16_t); - // Total container sizes. - uint64_t total_sizes[4]; - if (maxbytes < sizeof(uint64_t) * 3) { - roaring64_bitmap_free(r); - return NULL; + for (int32_t k = 0; k < size; ++k) { + uint16_t tmp; + memcpy(&tmp, keyscards + 4 * k, sizeof(tmp)); + answer->keys[k] = croaring_letoh16(tmp); } - memcpy(&(total_sizes[BITSET_CONTAINER_TYPE]), buf, sizeof(uint64_t)); - buf += sizeof(uint64_t); - maxbytes -= sizeof(uint64_t); - memcpy(&(total_sizes[RUN_CONTAINER_TYPE]), buf, sizeof(uint64_t)); - buf += sizeof(uint64_t); - maxbytes -= sizeof(uint64_t); - memcpy(&(total_sizes[ARRAY_CONTAINER_TYPE]), buf, sizeof(uint64_t)); - buf += sizeof(uint64_t); - maxbytes -= sizeof(uint64_t); + if ((!hasrun) || (size >= NO_OFFSET_THRESHOLD)) { + *readbytes += size * 4; + if (*readbytes > maxbytes) { // data is corrupted? + // Ran out of bytes while reading offsets. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } - // ART (8 byte aligned). - buf = CROARING_ALIGN_BUF(buf, 8); - size_t art_size = art_frozen_view(buf, maxbytes, &r->art); - if (art_size == 0) { - roaring64_bitmap_free(r); - return NULL; + // skipping the offsets + buf += size * 4; } - buf += art_size; - maxbytes -= art_size; + // Reading the containers + for (int32_t k = 0; k < size; ++k) { + uint16_t tmp; + memcpy(&tmp, keyscards + 4 * k + 2, sizeof(tmp)); + tmp = croaring_letoh16(tmp); + uint32_t thiscard = tmp + 1; + bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); + bool isrun = false; + if (hasrun) { + if ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) { + isbitmap = false; + isrun = true; + } + } + if (isbitmap) { + // we check that the read is allowed + size_t containersize = + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + *readbytes += containersize; + if (*readbytes > maxbytes) { + // Running out of bytes while reading a bitset container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + // it is now safe to read + bitset_container_t *c = bitset_container_create_uninitialized(); + if (c == NULL) { // memory allocation failure + // Failed to allocate memory for a bitset container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + answer->size++; + buf += bitset_container_read(thiscard, c, buf); + answer->containers[k] = c; + answer->typecodes[k] = BITSET_CONTAINER_TYPE; + } else if (isrun) { + // we check that the read is allowed + *readbytes += sizeof(uint16_t); + if (*readbytes > maxbytes) { + // Running out of bytes while reading a run container (header). + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + uint16_t n_runs; + memcpy(&n_runs, buf, sizeof(uint16_t)); + n_runs = croaring_letoh16(n_runs); + size_t containersize = n_runs * sizeof(rle16_t); + *readbytes += containersize; + if (*readbytes > maxbytes) { // data is corrupted? + // Running out of bytes while reading a run container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + // it is now safe to read - // Containers (aligned). - const char *before_containers = buf; - buf = CROARING_ALIGN_BUF(buf, CROARING_BITSET_ALIGNMENT); - const uint64_t *bitsets = (const uint64_t *)buf; - buf += total_sizes[BITSET_CONTAINER_TYPE]; - buf = CROARING_ALIGN_BUF(buf, alignof(rle16_t)); - const rle16_t *runs = (const rle16_t *)buf; - buf += total_sizes[RUN_CONTAINER_TYPE]; - buf = CROARING_ALIGN_BUF(buf, alignof(uint16_t)); - const uint16_t *arrays = (const uint16_t *)buf; - buf += total_sizes[ARRAY_CONTAINER_TYPE]; - if (maxbytes < (uint64_t)(buf - before_containers)) { - roaring64_bitmap_free(r); - return NULL; + run_container_t *c = run_container_create(); + if (c == NULL) { // memory allocation failure + // Failed to allocate memory for a run container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + answer->size++; + buf += run_container_read(thiscard, c, buf); + answer->containers[k] = c; + answer->typecodes[k] = RUN_CONTAINER_TYPE; + } else { + // we check that the read is allowed + size_t containersize = thiscard * sizeof(uint16_t); + *readbytes += containersize; + if (*readbytes > maxbytes) { // data is corrupted? + // Running out of bytes while reading an array container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + // it is now safe to read + array_container_t *c = + array_container_create_given_capacity(thiscard); + if (c == NULL) { // memory allocation failure + // Failed to allocate memory for an array container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + answer->size++; + buf += array_container_read(thiscard, c, buf); + answer->containers[k] = c; + answer->typecodes[k] = ARRAY_CONTAINER_TYPE; + } } - maxbytes -= buf - before_containers; + return true; +} - // Deserialize in ART iteration order. - art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); - for (size_t i = 0; it.value != NULL; ++i) { - leaf_t leaf = (leaf_t)*it.value; - uint8_t typecode = get_typecode(leaf); +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif +/* end file src/roaring_array.c */ +/* begin file src/roaring_priority_queue.c */ - uint16_t compressed_elem_count; - memcpy(&compressed_elem_count, elem_counts + (i * sizeof(uint16_t)), - sizeof(compressed_elem_count)); - uint32_t elem_count = (uint32_t)(compressed_elem_count) + 1; +#ifdef __cplusplus +using namespace ::roaring::internal; - // The container index is unrelated to the iteration order. - uint64_t index = get_index(leaf); - r->containers[index] = container_frozen_view(typecode, elem_count, - &bitsets, &arrays, &runs); +extern "C" { +namespace roaring { +namespace api { +#endif - art_iterator_next(&it); - } +struct roaring_pq_element_s { + uint64_t size; + bool is_temporary; + roaring_bitmap_t *bitmap; +}; - // Padding to make overall size a multiple of required alignment. - buf = CROARING_ALIGN_BUF(buf, CROARING_BITSET_ALIGNMENT); +typedef struct roaring_pq_element_s roaring_pq_element_t; - return r; +struct roaring_pq_s { + roaring_pq_element_t *elements; + uint64_t size; +}; + +typedef struct roaring_pq_s roaring_pq_t; + +static inline bool compare(roaring_pq_element_t *t1, roaring_pq_element_t *t2) { + return t1->size < t2->size; } -bool roaring64_bitmap_iterate(const roaring64_bitmap_t *r, - roaring_iterator64 iterator, void *ptr) { - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); - while (it.value != NULL) { - uint64_t high48 = combine_key(it.key, 0); - uint64_t high32 = high48 & 0xFFFFFFFF00000000ULL; - uint32_t low32 = high48; - leaf_t leaf = (leaf_t)*it.value; - if (!container_iterate64(get_container(r, leaf), get_typecode(leaf), - low32, iterator, high32, ptr)) { - return false; - } - art_iterator_next(&it); +static void pq_add(roaring_pq_t *pq, roaring_pq_element_t *t) { + uint64_t i = pq->size; + pq->elements[pq->size++] = *t; + while (i > 0) { + uint64_t p = (i - 1) >> 1; + roaring_pq_element_t ap = pq->elements[p]; + if (!compare(t, &ap)) break; + pq->elements[i] = ap; + i = p; } - return true; + pq->elements[i] = *t; } -void roaring64_bitmap_to_uint64_array(const roaring64_bitmap_t *r, - uint64_t *out) { - roaring64_iterator_t it; // gets initialized in the next line - roaring64_iterator_init_at(r, &it, /*first=*/true); - roaring64_iterator_read(&it, out, UINT64_MAX); -} +static void pq_free(roaring_pq_t *pq) { roaring_free(pq); } -roaring64_iterator_t *roaring64_iterator_create(const roaring64_bitmap_t *r) { - roaring64_iterator_t *it = - (roaring64_iterator_t *)roaring_malloc(sizeof(roaring64_iterator_t)); - return roaring64_iterator_init_at(r, it, /*first=*/true); +static void percolate_down(roaring_pq_t *pq, uint32_t i) { + uint32_t size = (uint32_t)pq->size; + uint32_t hsize = size >> 1; + roaring_pq_element_t ai = pq->elements[i]; + while (i < hsize) { + uint32_t l = (i << 1) + 1; + uint32_t r = l + 1; + roaring_pq_element_t bestc = pq->elements[l]; + if (r < size) { + if (compare(pq->elements + r, &bestc)) { + l = r; + bestc = pq->elements[r]; + } + } + if (!compare(&bestc, &ai)) { + break; + } + pq->elements[i] = bestc; + i = l; + } + pq->elements[i] = ai; } -roaring64_iterator_t *roaring64_iterator_create_last( - const roaring64_bitmap_t *r) { - roaring64_iterator_t *it = - (roaring64_iterator_t *)roaring_malloc(sizeof(roaring64_iterator_t)); - return roaring64_iterator_init_at(r, it, /*first=*/false); +static roaring_pq_t *create_pq(const roaring_bitmap_t **arr, uint32_t length) { + size_t alloc_size = + sizeof(roaring_pq_t) + sizeof(roaring_pq_element_t) * length; + roaring_pq_t *answer = (roaring_pq_t *)roaring_malloc(alloc_size); + answer->elements = (roaring_pq_element_t *)(answer + 1); + answer->size = length; + for (uint32_t i = 0; i < length; i++) { + answer->elements[i].bitmap = (roaring_bitmap_t *)arr[i]; + answer->elements[i].is_temporary = false; + answer->elements[i].size = + roaring_bitmap_portable_size_in_bytes(arr[i]); + } + for (int32_t i = (length >> 1); i >= 0; i--) { + percolate_down(answer, i); + } + return answer; } -void roaring64_iterator_reinit(const roaring64_bitmap_t *r, - roaring64_iterator_t *it) { - roaring64_iterator_init_at(r, it, /*first=*/true); +static roaring_pq_element_t pq_poll(roaring_pq_t *pq) { + roaring_pq_element_t ans = *pq->elements; + if (pq->size > 1) { + pq->elements[0] = pq->elements[--pq->size]; + percolate_down(pq, 0); + } else + --pq->size; + // memmove(pq->elements,pq->elements+1,(pq->size-1)*sizeof(roaring_pq_element_t));--pq->size; + return ans; } -void roaring64_iterator_reinit_last(const roaring64_bitmap_t *r, - roaring64_iterator_t *it) { - roaring64_iterator_init_at(r, it, /*first=*/false); -} +// this function consumes and frees the inputs +static roaring_bitmap_t *lazy_or_from_lazy_inputs(roaring_bitmap_t *x1, + roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = ra_get_size(&x1->high_low_container), + length2 = ra_get_size(&x2->high_low_container); + if (0 == length1) { + roaring_bitmap_free(x1); + return x2; + } + if (0 == length2) { + roaring_bitmap_free(x2); + return x1; + } + uint32_t neededcap = length1 > length2 ? length2 : length1; + roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(neededcap); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + // todo: unsharing can be inefficient as it may create a clone where + // none + // is needed, but it has the benefit of being easy to reason about. -roaring64_iterator_t *roaring64_iterator_copy(const roaring64_iterator_t *it) { - roaring64_iterator_t *new_it = - (roaring64_iterator_t *)roaring_malloc(sizeof(roaring64_iterator_t)); - memcpy(new_it, it, sizeof(*it)); - return new_it; -} + ra_unshare_container_at_index(&x1->high_low_container, + (uint16_t)pos1); + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + assert(type1 != SHARED_CONTAINER_TYPE); -void roaring64_iterator_free(roaring64_iterator_t *it) { roaring_free(it); } + ra_unshare_container_at_index(&x2->high_low_container, + (uint16_t)pos2); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + assert(type2 != SHARED_CONTAINER_TYPE); -bool roaring64_iterator_has_value(const roaring64_iterator_t *it) { - return it->has_value; -} + container_t *c; -uint64_t roaring64_iterator_value(const roaring64_iterator_t *it) { - return it->value; -} + if ((type2 == BITSET_CONTAINER_TYPE) && + (type1 != BITSET_CONTAINER_TYPE)) { + c = container_lazy_ior(c2, type2, c1, type1, &result_type); + container_free(c1, type1); + if (c != c2) { + container_free(c2, type2); + } + } else { + c = container_lazy_ior(c1, type1, c2, type2, &result_type); + container_free(c2, type2); + if (c != c1) { + container_free(c1, type1); + } + } + // since we assume that the initial containers are non-empty, the + // result here + // can only be non-empty + ra_append(&answer->high_low_container, s1, c, result_type); + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); -bool roaring64_iterator_advance(roaring64_iterator_t *it) { - if (it->art_it.value == NULL) { - if (it->saturated_forward) { - return (it->has_value = false); - } - roaring64_iterator_init_at(it->r, it, /*first=*/true); - return it->has_value; - } - leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = (uint16_t)it->value; - if (container_iterator_next(get_container(it->r, leaf), get_typecode(leaf), - &it->container_it, &low16)) { - it->value = it->high48 | low16; - return (it->has_value = true); - } - if (art_iterator_next(&it->art_it)) { - return roaring64_iterator_init_at_leaf_first(it); - } - it->saturated_forward = true; - return (it->has_value = false); -} + } else if (s1 < s2) { // s1 < s2 + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + ra_append(&answer->high_low_container, s1, c1, type1); + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); -bool roaring64_iterator_previous(roaring64_iterator_t *it) { - if (it->art_it.value == NULL) { - if (!it->saturated_forward) { - // Saturated backward. - return (it->has_value = false); + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); } - roaring64_iterator_init_at(it->r, it, /*first=*/false); - return it->has_value; - } - leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = (uint16_t)it->value; - if (container_iterator_prev(get_container(it->r, leaf), get_typecode(leaf), - &it->container_it, &low16)) { - it->value = it->high48 | low16; - return (it->has_value = true); } - if (art_iterator_prev(&it->art_it)) { - return roaring64_iterator_init_at_leaf_last(it); + if (pos1 == length1) { + ra_append_move_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2); + } else if (pos2 == length2) { + ra_append_move_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1); } - it->saturated_forward = false; // Saturated backward. - return (it->has_value = false); + ra_clear_without_containers(&x1->high_low_container); + ra_clear_without_containers(&x2->high_low_container); + roaring_free(x1); + roaring_free(x2); + return answer; } -bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, - uint64_t val) { - uint8_t val_high48[ART_KEY_BYTES]; - uint16_t val_low16 = split_key(val, val_high48); - if (!it->has_value || it->high48 != (val & 0xFFFFFFFFFFFF0000)) { - // The ART iterator is before or after the high48 bits of `val` (or - // beyond the ART altogether), so we need to move to a leaf with a - // key equal or greater. - if (!art_iterator_lower_bound(&it->art_it, val_high48)) { - // Only smaller keys found. - it->saturated_forward = true; - return (it->has_value = false); - } - it->high48 = combine_key(it->art_it.key, 0); - // Fall through to the next if statement. +/** + * Compute the union of 'number' bitmaps using a heap. This can + * sometimes be faster than roaring_bitmap_or_many which uses + * a naive algorithm. Caller is responsible for freeing the + * result. + */ +roaring_bitmap_t *roaring_bitmap_or_many_heap(uint32_t number, + const roaring_bitmap_t **x) { + if (number == 0) { + return roaring_bitmap_create(); } - - if (it->high48 == (val & 0xFFFFFFFFFFFF0000)) { - // We're at equal high bits, check if a suitable value can be found - // in this container. - leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = (uint16_t)it->value; - if (container_iterator_lower_bound( - get_container(it->r, leaf), get_typecode(leaf), - &it->container_it, &low16, val_low16)) { - it->value = it->high48 | low16; - return (it->has_value = true); - } - // Only smaller entries in this container, move to the next. - if (!art_iterator_next(&it->art_it)) { - it->saturated_forward = true; - return (it->has_value = false); - } + if (number == 1) { + return roaring_bitmap_copy(x[0]); } + roaring_pq_t *pq = create_pq(x, number); + while (pq->size > 1) { + roaring_pq_element_t x1 = pq_poll(pq); + roaring_pq_element_t x2 = pq_poll(pq); - // We're at a leaf with high bits greater than `val`, so the first entry - // in this container is our result. - return roaring64_iterator_init_at_leaf_first(it); -} + if (x1.is_temporary && x2.is_temporary) { + roaring_bitmap_t *newb = + lazy_or_from_lazy_inputs(x1.bitmap, x2.bitmap); + // should normally return a fresh new bitmap *except* that + // it can return x1.bitmap or x2.bitmap in degenerate cases + bool temporary = !((newb == x1.bitmap) && (newb == x2.bitmap)); + uint64_t bsize = roaring_bitmap_portable_size_in_bytes(newb); + roaring_pq_element_t newelement = { + .size = bsize, .is_temporary = temporary, .bitmap = newb}; + pq_add(pq, &newelement); + } else if (x2.is_temporary) { + roaring_bitmap_lazy_or_inplace(x2.bitmap, x1.bitmap, false); + x2.size = roaring_bitmap_portable_size_in_bytes(x2.bitmap); + pq_add(pq, &x2); + } else if (x1.is_temporary) { + roaring_bitmap_lazy_or_inplace(x1.bitmap, x2.bitmap, false); + x1.size = roaring_bitmap_portable_size_in_bytes(x1.bitmap); -uint64_t roaring64_iterator_read(roaring64_iterator_t *it, uint64_t *buf, - uint64_t count) { - uint64_t consumed = 0; - while (it->has_value && consumed < count) { - uint32_t container_consumed; - leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = (uint16_t)it->value; - uint32_t container_count = UINT32_MAX; - if (count - consumed < (uint64_t)UINT32_MAX) { - container_count = count - consumed; - } - bool has_value = container_iterator_read_into_uint64( - get_container(it->r, leaf), get_typecode(leaf), &it->container_it, - it->high48, buf, container_count, &container_consumed, &low16); - consumed += container_consumed; - buf += container_consumed; - if (has_value) { - it->has_value = true; - it->value = it->high48 | low16; - assert(consumed == count); - return consumed; - } - it->has_value = art_iterator_next(&it->art_it); - if (it->has_value) { - roaring64_iterator_init_at_leaf_first(it); + pq_add(pq, &x1); } else { - it->saturated_forward = true; + roaring_bitmap_t *newb = + roaring_bitmap_lazy_or(x1.bitmap, x2.bitmap, false); + uint64_t bsize = roaring_bitmap_portable_size_in_bytes(newb); + roaring_pq_element_t newelement = { + .size = bsize, .is_temporary = true, .bitmap = newb}; + + pq_add(pq, &newelement); } } - return consumed; + roaring_pq_element_t X = pq_poll(pq); + roaring_bitmap_t *answer = X.bitmap; + roaring_bitmap_repair_after_lazy(answer); + pq_free(pq); + return answer; } #ifdef __cplusplus -} // extern "C" -} // namespace roaring -} // namespace api +} +} +} // extern "C" { namespace roaring { namespace api { #endif -/* end file src/roaring64.c */ +/* end file src/roaring_priority_queue.c */ diff --git a/pyroaring/roaring.h b/pyroaring/roaring.h index c8e6d1c..6df01e4 100644 --- a/pyroaring/roaring.h +++ b/pyroaring/roaring.h @@ -1,5 +1,5 @@ // !!! DO NOT EDIT - THIS IS AN AUTO-GENERATED FILE !!! -// Created by amalgamation.sh on 2026-03-09T14:59:44Z +// Created by amalgamation.sh on 2026-09-17T17:49:11Z /* * The CRoaring project is under a dual license (Apache/MIT). @@ -59,11 +59,11 @@ // /include/roaring/roaring_version.h automatically generated by release.py, do not change by hand #ifndef ROARING_INCLUDE_ROARING_VERSION #define ROARING_INCLUDE_ROARING_VERSION -#define ROARING_VERSION "4.6.1" +#define ROARING_VERSION "5.2.2" enum { - ROARING_VERSION_MAJOR = 4, - ROARING_VERSION_MINOR = 6, - ROARING_VERSION_REVISION = 1 + ROARING_VERSION_MAJOR = 5, + ROARING_VERSION_MINOR = 2, + ROARING_VERSION_REVISION = 2 }; #endif // ROARING_INCLUDE_ROARING_VERSION // clang-format on/* end file include/roaring/roaring_version.h */ @@ -71,12 +71,21 @@ enum { /* * portability.h * + * This header centralizes compiler-, platform-, and architecture-specific + * portability definitions used throughout CRoaring. It provides feature + * detection, calling-convention and attribute macros, intrinsic and inline + * assembly enablement, endianness helpers, alignment annotations, atomic + * reference-count support, and other low-level compatibility glue. + * + * The goal is to keep these conditional definitions in one place so the rest + * of the codebase can rely on a more uniform interface across GCC, Clang, + * MSVC, x86/x64, ARM/NEON, and other supported environments. */ /** * All macros should be prefixed with either CROARING or ROARING. * The library uses both ROARING_... - * as well as CROAIRING_ as prefixes. The ROARING_ prefix is for + * as well as CROARING_ as prefixes. The ROARING_ prefix is for * macros that are provided by the build system or that are closely * related to the format. The header macros may also use ROARING_. * The CROARING_ prefix is for internal macros that a user is unlikely @@ -125,6 +134,12 @@ enum { #ifdef __GLIBC__ #include // this should never be needed but there are some reports that it is needed. #endif +// alignas/alignof are keywords in C++ and in C23+, where is +// deprecated. Only include it for C11..C17. +#if !defined(__cplusplus) && \ + (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 202311L) +#include +#endif #ifdef __cplusplus extern "C" { // portability definitions are in global scope, not a namespace @@ -140,7 +155,14 @@ extern "C" { // portability definitions are in global scope, not a namespace #endif // __restrict__ #endif // CROARING_REGULAR_VISUAL_STUDIO -#if defined(__x86_64__) || defined(_M_X64) +#if defined(__riscv) || defined(_M_RISCV32) || defined(_M_RISCV64) +#define CROARING_IS_RISCV 1 + +#if (defined(__riscv_xlen) && (__riscv_xlen == 64)) || defined(_M_RISCV64) +#define CROARING_IS_RISCV64 1 +#endif + +#elif defined(__x86_64__) || defined(_M_X64) // we have an x64 processor #define CROARING_IS_X64 1 @@ -225,7 +247,7 @@ extern "C" { // portability definitions are in global scope, not a namespace #define CROARING_IS_E2K 1 #endif -#if !CROARING_REGULAR_VISUAL_STUDIO && !defined(CROARING_IS_E2K) +#if !CROARING_REGULAR_VISUAL_STUDIO && !defined(CROARING_IS_E2K) && !__FILC__ /* Non-Microsoft C/C++-compatible compiler, assumes that it supports inline * assembly */ #define CROARING_INLINE_ASM 1 @@ -504,6 +526,7 @@ static inline int roaring_hamming(uint64_t x) { #if defined(__linux__) #define croaring_htobe64(x) bswap_64(x) #elif defined(__FreeBSD__) +#include #define croaring_htobe64(x) bswap64(x) #else #warning "Unknown platform, report as an error" @@ -523,6 +546,55 @@ static inline int roaring_hamming(uint64_t x) { #define croaring_be64toh(x) croaring_htobe64(x) // End of host <-> big endian conversion. +// Host <-> little-endian conversion helpers. +// +// The CRoaring "portable" serialization format (and the regular +// roaring_bitmap_serialize / Roaring64Map::write formats which build on it) +// is defined to be little-endian on the wire. Code that reads or writes +// multi-byte integers to such buffers must convert between host and +// little-endian byte order. On little-endian hosts these are no-ops; on +// big-endian hosts they swap bytes. +// +// The "frozen" format is intentionally non-portable and uses native byte +// order; it must not use these helpers. +#if CROARING_IS_BIG_ENDIAN + +static inline uint16_t croaring_bswap16(uint16_t x) { + return (uint16_t)((x << 8) | (x >> 8)); +} + +static inline uint32_t croaring_bswap32(uint32_t x) { + return ((x & 0x000000FFU) << 24) | ((x & 0x0000FF00U) << 8) | + ((x & 0x00FF0000U) >> 8) | ((x & 0xFF000000U) >> 24); +} + +static inline uint64_t croaring_bswap64(uint64_t x) { + return ((x & 0x00000000000000FFULL) << 56) | + ((x & 0x000000000000FF00ULL) << 40) | + ((x & 0x0000000000FF0000ULL) << 24) | + ((x & 0x00000000FF000000ULL) << 8) | + ((x & 0x000000FF00000000ULL) >> 8) | + ((x & 0x0000FF0000000000ULL) >> 24) | + ((x & 0x00FF000000000000ULL) >> 40) | + ((x & 0xFF00000000000000ULL) >> 56); +} + +#define croaring_htole16(x) croaring_bswap16(x) +#define croaring_htole32(x) croaring_bswap32(x) +#define croaring_htole64(x) croaring_bswap64(x) + +#else // CROARING_IS_BIG_ENDIAN + +#define croaring_htole16(x) (x) +#define croaring_htole32(x) (x) +#define croaring_htole64(x) (x) + +#endif // CROARING_IS_BIG_ENDIAN + +#define croaring_letoh16(x) croaring_htole16(x) +#define croaring_letoh32(x) croaring_htole32(x) +#define croaring_letoh64(x) croaring_htole64(x) + // Defines for the possible CROARING atomic implementations #define CROARING_ATOMIC_IMPL_NONE 1 #define CROARING_ATOMIC_IMPL_CPP 2 @@ -538,13 +610,13 @@ static inline int roaring_hamming(uint64_t x) { #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP #endif //__has_include() #else - // We lack __has_include to check: +// We lack __has_include to check: #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP #endif //__has_include #elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C #elif CROARING_REGULAR_VISUAL_STUDIO - // https://www.technetworkhub.com/c11-atomics-in-visual-studio-2022-version-17/ +// https://www.technetworkhub.com/c11-atomics-in-visual-studio-2022-version-17/ #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C_WINDOWS #endif #endif // !defined(CROARING_ATOMIC_IMPL) @@ -578,6 +650,13 @@ static inline bool croaring_refcount_dec(croaring_refcount_t *val) { static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { return atomic_load_explicit(val, memory_order_relaxed); } + +// Returns true if the caller holds the only reference, and may thus take +// exclusive ownership of the object. The "acquire" operation pairs with the +// "release" operation from dropping the other references. +static inline bool croaring_refcount_is_unique(const croaring_refcount_t *val) { + return atomic_load_explicit(val, memory_order_acquire) == 1; +} #elif CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_CPP #include typedef std::atomic croaring_refcount_t; @@ -598,10 +677,15 @@ static inline bool croaring_refcount_dec(croaring_refcount_t *val) { static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { return val->load(std::memory_order_relaxed); } + +static inline bool croaring_refcount_is_unique(const croaring_refcount_t *val) { + return val->load(std::memory_order_acquire) == 1; +} #elif CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_C_WINDOWS #include #pragma intrinsic(_InterlockedIncrement) #pragma intrinsic(_InterlockedDecrement) +#pragma intrinsic(_InterlockedOr) // _InterlockedIncrement and _InterlockedDecrement take a (signed) long, and // overflow is defined to wrap, so we can pretend it is a uint32_t for our case @@ -623,6 +707,12 @@ static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { // > of the variable updated; all bits are updated in an atomic fashion. return *val; } + +static inline bool croaring_refcount_is_unique(const croaring_refcount_t *val) { + // A plain read would not provide the acquire semantics: _InterlockedOr + // leaves the value alone while acting as a memory barrier. + return _InterlockedOr((croaring_refcount_t *)val, 0) == 1; +} #elif CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_NONE #include typedef uint32_t croaring_refcount_t; @@ -634,12 +724,16 @@ static inline void croaring_refcount_inc(croaring_refcount_t *val) { static inline bool croaring_refcount_dec(croaring_refcount_t *val) { assert(*val > 0); *val -= 1; - return val == 0; + return *val == 0; } static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { return *val; } + +static inline bool croaring_refcount_is_unique(const croaring_refcount_t *val) { + return *val == 1; +} #else #error "Unknown atomic implementation" #endif @@ -678,9 +772,22 @@ static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { #endif /* INCLUDE_PORTABILITY_H_ */ /* end file include/roaring/portability.h */ /* begin file include/roaring/isadetection.h */ +/* + * isadetection.h + * + * This header declares the small interface used to detect instruction-set + * capabilities relevant to CRoaring's optimized kernels on x64 platforms. It + * also defines compile-time feature macros that indicate whether the compiler + * toolchain is capable of building AVX-512 code paths. + * + * The resulting flags are used to decide whether accelerated implementations, + * such as AVX2 or AVX-512 variants, can be selected safely at runtime. + */ #ifndef ROARING_ISADETECTION_H #define ROARING_ISADETECTION_H -#if defined(__x86_64__) || defined(_M_AMD64) // x64 + + +#if CROARING_IS_X64 // x64 #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #ifdef __has_include @@ -718,12 +825,18 @@ int croaring_hardware_support(void); } } // extern "C" { namespace roaring { namespace internal { #endif -#endif // x64 +#endif // CROARING_IS_X64 #endif // ROARING_ISADETECTION_H /* end file include/roaring/isadetection.h */ /* begin file include/roaring/roaring_types.h */ /* - Typedefs used by various components + Shared type definitions used across the CRoaring public API and internal + components. + + This file centralizes common typedefs and small structs that are referenced + by multiple headers, including iterator callback signatures, roaring array + metadata, statistics structures, and compatibility types used to bridge the + C and C++ builds. */ #ifndef ROARING_TYPES_H @@ -762,6 +875,8 @@ struct container_s {}; #define ROARING_FLAG_COW UINT8_C(0x1) #define ROARING_FLAG_FROZEN UINT8_C(0x2) +// 64-bit only: ART node arrays alias a frozen buffer (do not art_free). +#define ROARING_FLAG_FROZEN_ART UINT8_C(0x4) /** * Roaring arrays are array-based key-value pairs having containers as values @@ -875,7 +990,110 @@ typedef struct roaring_container_iterator_s { #endif /* ROARING_TYPES_H */ /* end file include/roaring/roaring_types.h */ +/* begin file include/roaring/utilasm.h */ +/* + * utilasm.h + * + * This file provides optional inline-assembly helpers for low-level bit + * manipulation on supported x86/x64 targets. These macros are used to map a + * few performance-sensitive operations, such as shifting, testing, setting, + * and clearing bits, to specific machine instructions when inline assembly is + * enabled. + * + * The intent is to centralize these architecture-specific primitives behind a + * small interface so the rest of the codebase can use them conditionally while + * keeping the generic implementation paths separate. + */ + +#ifndef INCLUDE_UTILASM_H_ +#define INCLUDE_UTILASM_H_ + + +#ifdef __cplusplus +extern "C" { +namespace roaring { +#endif + +#if defined(CROARING_INLINE_ASM) +#define CROARING_ASMBITMANIPOPTIMIZATION // optimization flag + +#define CROARING_ASM_SHIFT_RIGHT(srcReg, bitsReg, destReg) \ + __asm volatile("shrx %1, %2, %0" \ + : "=r"(destReg) \ + : /* write */ \ + "r"(bitsReg), /* read only */ \ + "r"(srcReg) /* read only */ \ + ) + +#define CROARING_ASM_INPLACESHIFT_RIGHT(srcReg, bitsReg) \ + __asm volatile("shrx %1, %0, %0" \ + : "+r"(srcReg) \ + : /* read/write */ \ + "r"(bitsReg) /* read only */ \ + ) + +#define CROARING_ASM_SHIFT_LEFT(srcReg, bitsReg, destReg) \ + __asm volatile("shlx %1, %2, %0" \ + : "=r"(destReg) \ + : /* write */ \ + "r"(bitsReg), /* read only */ \ + "r"(srcReg) /* read only */ \ + ) +// set bit at position testBit within testByte to 1 and +// copy cmovDst to cmovSrc if that bit was previously clear +#define CROARING_ASM_SET_BIT_INC_WAS_CLEAR(testByte, testBit, count) \ + __asm volatile( \ + "bts %2, %0\n" \ + "sbb $-1, %1\n" \ + : "+r"(testByte), /* read/write */ \ + "+r"(count) \ + : /* read/write */ \ + "r"(testBit) /* read only */ \ + ) + +#define CROARING_ASM_CLEAR_BIT_DEC_WAS_SET(testByte, testBit, count) \ + __asm volatile( \ + "btr %2, %0\n" \ + "sbb $0, %1\n" \ + : "+r"(testByte), /* read/write */ \ + "+r"(count) \ + : /* read/write */ \ + "r"(testBit) /* read only */ \ + ) + +#define CROARING_ASM_BT64(testByte, testBit, count) \ + __asm volatile( \ + "bt %2,%1\n" \ + "sbb %0,%0" /*could use setb */ \ + : "=r"(count) \ + : /* write */ \ + "r"(testByte), /* read only */ \ + "r"(testBit) /* read only */ \ + ) + +#endif + +#ifdef __cplusplus +} +} // extern "C" { namespace roaring { +#endif + +#endif /* INCLUDE_UTILASM_H_ */ +/* end file include/roaring/utilasm.h */ /* begin file include/roaring/bitset/bitset.h */ +/* + * bitset.h + * + * This bitset is a general-purpose dynamic bitmap storing bits in a contiguous + * array of 64-bit words. The array field points to the word buffer, arraysize + * records how many words are currently in use, and capacity records how many + * words are allocated. + * + * Unlike the fixed 16-bit-domain container bitset, this structure can grow to + * cover an arbitrary number of bit positions. It is useful when callers need a + * resizable bitmap with efficient bitwise operations, scans, and shifts over a + * larger or runtime-defined domain. + */ #ifndef CROARING_CBITSET_BITSET_H #define CROARING_CBITSET_BITSET_H @@ -1175,6 +1393,68 @@ inline void bitset_print(const bitset_t *b) { #endif /* end file include/roaring/bitset/bitset.h */ +/* begin file include/roaring/containers/perfparameters.h */ +/* + * perfparameters.h + * + * This header centralizes a small set of performance-tuning constants and + * heuristic defaults used by CRoaring container code. These parameters control + * decisions such as initial container sizing and when lazy or eager operations + * may convert between container representations. + * + * In practice, these values encode trade-offs between memory use, allocation + * overhead, and execution speed for common workloads. + */ +#ifndef PERFPARAMETERS_H_ +#define PERFPARAMETERS_H_ + +#include + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/** +During lazy computations, we can transform array containers into bitset +containers as +long as we can expect them to have ARRAY_LAZY_LOWERBOUND values. +*/ +enum { ARRAY_LAZY_LOWERBOUND = 1024 }; + +/* default initial size of a run container + setting it to zero delays the malloc.*/ +enum { RUN_DEFAULT_INIT_SIZE = 0 }; + +/* default initial size of an array container + setting it to zero delays the malloc */ +enum { ARRAY_DEFAULT_INIT_SIZE = 0 }; + +/* automatic bitset conversion during lazy or */ +#ifndef CROARING_LAZY_OR_BITSET_CONVERSION +#define CROARING_LAZY_OR_BITSET_CONVERSION true +#endif + +/* automatically attempt to convert a bitset to a full run during lazy + * evaluation */ +#ifndef CROARING_LAZY_OR_BITSET_CONVERSION_TO_FULL +#define CROARING_LAZY_OR_BITSET_CONVERSION_TO_FULL true +#endif + +/* automatically attempt to convert a bitset to a full run */ +#ifndef CROARING_OR_BITSET_CONVERSION_TO_FULL +#define CROARING_OR_BITSET_CONVERSION_TO_FULL true +#endif + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif +/* end file include/roaring/containers/perfparameters.h */ /* begin file include/roaring/containers/container_defs.h */ /* * container_defs.h @@ -1276,6 +1556,18 @@ PPDerived movable_CAST_HELPER(Base **ptr_to_ptr) { #endif /* INCLUDE_CONTAINERS_CONTAINER_DEFS_H_ */ /* end file include/roaring/containers/container_defs.h */ /* begin file include/roaring/array_util.h */ +/* + * array_util.h + * + * This header provides low-level utility routines for sorted arrays of + * 16-bit integers, which are used heavily by CRoaring's array-based + * containers and set-operation kernels. It includes search helpers, counting + * helpers, and array intersection/difference primitives. + * + * Some of the routines also have SIMD-accelerated implementations on supported + * platforms, allowing efficient operations on sorted integer arrays that form + * the basis of sparse container processing. + */ #ifndef CROARING_ARRAY_UTIL_H #define CROARING_ARRAY_UTIL_H @@ -1300,29 +1592,95 @@ namespace internal { #endif /* - * Good old binary search. + * Sorted-array search. * Assumes that array is sorted, has logarithmic complexity. * if the result is x, then: * if ( x>0 ) you have array[x] = ikey * if ( x<0 ) then inserting ikey at position -x-1 in array (insuring that * array[-x-1]=ikey) keys the array sorted. + * + * Adapted from array_container_contains: a SIMD-quad block-narrowing + * search at gap=16 (Daniel Lemire, + * https://lemire.me/blog/2026/04/27/you-can-beat-the-binary-search/) + * followed by a scalar in-block scan that recovers the exact insertion + * point required by the binarySearch contract. */ inline int32_t binarySearch(const uint16_t *array, int32_t lenarray, uint16_t ikey) { - int32_t low = 0; - int32_t high = lenarray - 1; - while (low <= high) { - int32_t middleIndex = (low + high) >> 1; - uint16_t middleValue = array[middleIndex]; - if (middleValue < ikey) { - low = middleIndex + 1; - } else if (middleValue > ikey) { - high = middleIndex - 1; - } else { - return middleIndex; + const int32_t gap = 16; + if (lenarray < gap) { + for (int32_t j = 0; j < lenarray; j++) { + if (array[j] >= ikey) { + return (array[j] == ikey) ? j : -(j + 1); + } + } + return -(lenarray + 1); + } + const int32_t num_blocks = lenarray / gap; + int32_t base = 0; + int32_t n = num_blocks; + while (n > 3) { + int32_t quarter = n >> 2; + + int32_t k1 = array[(base + quarter + 1) * gap - 1]; + int32_t k2 = array[(base + 2 * quarter + 1) * gap - 1]; + int32_t k3 = array[(base + 3 * quarter + 1) * gap - 1]; + + int32_t c1 = (k1 < ikey); + int32_t c2 = (k2 < ikey); + int32_t c3 = (k3 < ikey); + + base += (c1 + c2 + c3) * quarter; + n -= 3 * quarter; + } + while (n > 1) { + int32_t half = n >> 1; + base = (array[(base + half + 1) * gap - 1] < ikey) ? base + half : base; + n -= half; + } + int32_t lo = (array[(base + 1) * gap - 1] < ikey) ? base + 1 : base; + + if (lo < num_blocks) { + const int32_t start = lo * gap; +#if defined(CROARING_IS_X64) + // SSE2: subs_epu16 yields zero where lane >= ikey. movemask of an + // epi16 compare gives 2 bits per lane; ctz>>1 = lane index. Scan + // the first 8 lanes first and exit early when they contain the + // answer; otherwise the block-narrowing invariant guarantees the + // second-half mask is non-zero. + __m128i needle = _mm_set1_epi16((short)ikey); + __m128i zero = _mm_setzero_si128(); + __m128i v0 = _mm_loadu_si128((const __m128i *)(array + start)); + __m128i ge0 = _mm_cmpeq_epi16(_mm_subs_epu16(needle, v0), zero); + unsigned m0 = (unsigned)_mm_movemask_epi8(ge0); + if (m0 != 0) { + int32_t j = start + (int32_t)(roaring_trailing_zeroes(m0) >> 1); + return (array[j] == ikey) ? j : -(j + 1); + } + __m128i v1 = _mm_loadu_si128((const __m128i *)(array + start + 8)); + __m128i ge1 = _mm_cmpeq_epi16(_mm_subs_epu16(needle, v1), zero); + unsigned m1 = (unsigned)_mm_movemask_epi8(ge1); + int32_t j = start + 8 + (int32_t)(roaring_trailing_zeroes(m1) >> 1); + return (array[j] == ikey) ? j : -(j + 1); +#else + const int32_t end = start + gap; + for (int32_t j = start; j < end; j++) { + if (array[j] >= ikey) { + return (array[j] == ikey) ? j : -(j + 1); + } } + // Unreachable: the narrowing guarantees the last element of the + // selected block is >= ikey. + return -(end + 1); +#endif } - return -(low + 1); + + for (int32_t j = num_blocks * gap; j < lenarray; j++) { + if (array[j] >= ikey) { + return (array[j] == ikey) ? j : -(j + 1); + } + } + return -(lenarray + 1); } /** @@ -1508,6 +1866,18 @@ size_t union_uint32(const uint32_t *set_1, size_t size_1, const uint32_t *set_2, uint32_t union_vector16(const uint16_t *set_1, uint32_t size_1, const uint16_t *set_2, uint32_t size_2, uint16_t *buffer); + +#if CROARING_COMPILER_SUPPORTS_AVX512 +/** + * AVX-512 union of two sorted uint16 arrays, using a Batcher bitonic merge + * network over 32 lanes. Same contract as union_vector16: `buffer` must have + * room for size_1 + size_2 values. + */ +uint32_t avx512_union_uint16(const uint16_t *set_1, uint32_t size_1, + const uint16_t *set_2, uint32_t size_2, + uint16_t *buffer); +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + /** * A fast SSE-based XOR function. */ @@ -1547,6 +1917,19 @@ bool memequals(const void *s1, const void *s2, size_t n); #endif /* end file include/roaring/array_util.h */ /* begin file include/roaring/bitset_util.h */ +/* + * bitset_util.h + * + * This header collects low-level utility functions for operating on raw + * bitsets represented as arrays of 64-bit words. It includes helpers for + * setting, clearing, flipping, counting, extracting, and combining bit ranges, + * along with architecture-specific SIMD implementations for performance- + * critical routines. + * + * In particular, this file contains accelerated extraction and population- + * count code paths for x64 targets using AVX2 and, when supported by the + * compiler and hardware, AVX-512. + */ #ifndef CROARING_BITSET_UTIL_H #define CROARING_BITSET_UTIL_H @@ -1713,6 +2096,15 @@ size_t bitset_extract_setbits_avx2(const uint64_t *words, size_t length, uint32_t *out, size_t outcapacity, uint32_t base); +/* + * Same as bitset_extract_setbits_avx2, using AVX-512 (VBMI2) decoding. + * + * It writes at most "outcapacity" values and requires no padding beyond them: + * "out" only needs room for the values that are actually produced. + * + * Requires AVX-512 VBMI2 (VPCOMPRESSB): only call it when + * croaring_hardware_support() reports ROARING_SUPPORTS_AVX512. + */ size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, uint32_t *out, size_t outcapacity, uint32_t base); @@ -1748,6 +2140,12 @@ size_t bitset_extract_setbits_sse_uint16(const uint64_t *words, size_t length, uint16_t *out, size_t outcapacity, uint16_t base); +/* + * Same as bitset_extract_setbits_avx512, writing 16-bit values. + * + * Requires AVX-512 VBMI2 (VPCOMPRESSB): only call it when + * croaring_hardware_support() reports ROARING_SUPPORTS_AVX512. + */ size_t bitset_extract_setbits_avx512_uint16(const uint64_t *words, size_t length, uint16_t *out, size_t outcapacity, uint16_t base); @@ -1884,26 +2282,26 @@ inline static uint64_t avx2_harley_seal_popcount256(const __m256i *data, uint64_t i = 0; for (; i < limit; i += 16) { - CSA(&twosA, &ones, ones, _mm256_lddqu_si256(data + i), - _mm256_lddqu_si256(data + i + 1)); - CSA(&twosB, &ones, ones, _mm256_lddqu_si256(data + i + 2), - _mm256_lddqu_si256(data + i + 3)); + CSA(&twosA, &ones, ones, _mm256_loadu_si256(data + i), + _mm256_loadu_si256(data + i + 1)); + CSA(&twosB, &ones, ones, _mm256_loadu_si256(data + i + 2), + _mm256_loadu_si256(data + i + 3)); CSA(&foursA, &twos, twos, twosA, twosB); - CSA(&twosA, &ones, ones, _mm256_lddqu_si256(data + i + 4), - _mm256_lddqu_si256(data + i + 5)); - CSA(&twosB, &ones, ones, _mm256_lddqu_si256(data + i + 6), - _mm256_lddqu_si256(data + i + 7)); + CSA(&twosA, &ones, ones, _mm256_loadu_si256(data + i + 4), + _mm256_loadu_si256(data + i + 5)); + CSA(&twosB, &ones, ones, _mm256_loadu_si256(data + i + 6), + _mm256_loadu_si256(data + i + 7)); CSA(&foursB, &twos, twos, twosA, twosB); CSA(&eightsA, &fours, fours, foursA, foursB); - CSA(&twosA, &ones, ones, _mm256_lddqu_si256(data + i + 8), - _mm256_lddqu_si256(data + i + 9)); - CSA(&twosB, &ones, ones, _mm256_lddqu_si256(data + i + 10), - _mm256_lddqu_si256(data + i + 11)); + CSA(&twosA, &ones, ones, _mm256_loadu_si256(data + i + 8), + _mm256_loadu_si256(data + i + 9)); + CSA(&twosB, &ones, ones, _mm256_loadu_si256(data + i + 10), + _mm256_loadu_si256(data + i + 11)); CSA(&foursA, &twos, twos, twosA, twosB); - CSA(&twosA, &ones, ones, _mm256_lddqu_si256(data + i + 12), - _mm256_lddqu_si256(data + i + 13)); - CSA(&twosB, &ones, ones, _mm256_lddqu_si256(data + i + 14), - _mm256_lddqu_si256(data + i + 15)); + CSA(&twosA, &ones, ones, _mm256_loadu_si256(data + i + 12), + _mm256_loadu_si256(data + i + 13)); + CSA(&twosB, &ones, ones, _mm256_loadu_si256(data + i + 14), + _mm256_loadu_si256(data + i + 15)); CSA(&foursB, &twos, twos, twosA, twosB); CSA(&eightsB, &fours, fours, foursA, foursB); CSA(&sixteens, &eights, eights, eightsA, eightsB); @@ -1921,7 +2319,7 @@ inline static uint64_t avx2_harley_seal_popcount256(const __m256i *data, total = _mm256_add_epi64(total, popcount256(ones)); for (; i < size; i++) total = - _mm256_add_epi64(total, popcount256(_mm256_lddqu_si256(data + i))); + _mm256_add_epi64(total, popcount256(_mm256_loadu_si256(data + i))); return (uint64_t)(_mm256_extract_epi64(total, 0)) + (uint64_t)(_mm256_extract_epi64(total, 1)) + @@ -1944,49 +2342,49 @@ CROARING_UNTARGET_AVX2 const uint64_t limit = size - size % 16; \ uint64_t i = 0; \ for (; i < limit; i += 16) { \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i), \ - _mm256_lddqu_si256(data2 + i)); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 1), \ - _mm256_lddqu_si256(data2 + i + 1)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i), \ + _mm256_loadu_si256(data2 + i)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 1), \ + _mm256_loadu_si256(data2 + i + 1)); \ CSA(&twosA, &ones, ones, A1, A2); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 2), \ - _mm256_lddqu_si256(data2 + i + 2)); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 3), \ - _mm256_lddqu_si256(data2 + i + 3)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 2), \ + _mm256_loadu_si256(data2 + i + 2)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 3), \ + _mm256_loadu_si256(data2 + i + 3)); \ CSA(&twosB, &ones, ones, A1, A2); \ CSA(&foursA, &twos, twos, twosA, twosB); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 4), \ - _mm256_lddqu_si256(data2 + i + 4)); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 5), \ - _mm256_lddqu_si256(data2 + i + 5)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 4), \ + _mm256_loadu_si256(data2 + i + 4)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 5), \ + _mm256_loadu_si256(data2 + i + 5)); \ CSA(&twosA, &ones, ones, A1, A2); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 6), \ - _mm256_lddqu_si256(data2 + i + 6)); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 7), \ - _mm256_lddqu_si256(data2 + i + 7)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 6), \ + _mm256_loadu_si256(data2 + i + 6)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 7), \ + _mm256_loadu_si256(data2 + i + 7)); \ CSA(&twosB, &ones, ones, A1, A2); \ CSA(&foursB, &twos, twos, twosA, twosB); \ CSA(&eightsA, &fours, fours, foursA, foursB); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 8), \ - _mm256_lddqu_si256(data2 + i + 8)); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 9), \ - _mm256_lddqu_si256(data2 + i + 9)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 8), \ + _mm256_loadu_si256(data2 + i + 8)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 9), \ + _mm256_loadu_si256(data2 + i + 9)); \ CSA(&twosA, &ones, ones, A1, A2); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 10), \ - _mm256_lddqu_si256(data2 + i + 10)); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 11), \ - _mm256_lddqu_si256(data2 + i + 11)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 10), \ + _mm256_loadu_si256(data2 + i + 10)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 11), \ + _mm256_loadu_si256(data2 + i + 11)); \ CSA(&twosB, &ones, ones, A1, A2); \ CSA(&foursA, &twos, twos, twosA, twosB); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 12), \ - _mm256_lddqu_si256(data2 + i + 12)); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 13), \ - _mm256_lddqu_si256(data2 + i + 13)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 12), \ + _mm256_loadu_si256(data2 + i + 12)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 13), \ + _mm256_loadu_si256(data2 + i + 13)); \ CSA(&twosA, &ones, ones, A1, A2); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 14), \ - _mm256_lddqu_si256(data2 + i + 14)); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 15), \ - _mm256_lddqu_si256(data2 + i + 15)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 14), \ + _mm256_loadu_si256(data2 + i + 14)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 15), \ + _mm256_loadu_si256(data2 + i + 15)); \ CSA(&twosB, &ones, ones, A1, A2); \ CSA(&foursB, &twos, twos, twosA, twosB); \ CSA(&eightsB, &fours, fours, foursA, foursB); \ @@ -2002,8 +2400,8 @@ CROARING_UNTARGET_AVX2 _mm256_add_epi64(total, _mm256_slli_epi64(popcount256(twos), 1)); \ total = _mm256_add_epi64(total, popcount256(ones)); \ for (; i < size; i++) { \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i), \ - _mm256_lddqu_si256(data2 + i)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i), \ + _mm256_loadu_si256(data2 + i)); \ total = _mm256_add_epi64(total, popcount256(A1)); \ } \ return (uint64_t)(_mm256_extract_epi64(total, 0)) + \ @@ -2025,64 +2423,64 @@ CROARING_UNTARGET_AVX2 const uint64_t limit = size - size % 16; \ uint64_t i = 0; \ for (; i < limit; i += 16) { \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i), \ - _mm256_lddqu_si256(data2 + i)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i), \ + _mm256_loadu_si256(data2 + i)); \ _mm256_storeu_si256(out + i, A1); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 1), \ - _mm256_lddqu_si256(data2 + i + 1)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 1), \ + _mm256_loadu_si256(data2 + i + 1)); \ _mm256_storeu_si256(out + i + 1, A2); \ CSA(&twosA, &ones, ones, A1, A2); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 2), \ - _mm256_lddqu_si256(data2 + i + 2)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 2), \ + _mm256_loadu_si256(data2 + i + 2)); \ _mm256_storeu_si256(out + i + 2, A1); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 3), \ - _mm256_lddqu_si256(data2 + i + 3)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 3), \ + _mm256_loadu_si256(data2 + i + 3)); \ _mm256_storeu_si256(out + i + 3, A2); \ CSA(&twosB, &ones, ones, A1, A2); \ CSA(&foursA, &twos, twos, twosA, twosB); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 4), \ - _mm256_lddqu_si256(data2 + i + 4)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 4), \ + _mm256_loadu_si256(data2 + i + 4)); \ _mm256_storeu_si256(out + i + 4, A1); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 5), \ - _mm256_lddqu_si256(data2 + i + 5)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 5), \ + _mm256_loadu_si256(data2 + i + 5)); \ _mm256_storeu_si256(out + i + 5, A2); \ CSA(&twosA, &ones, ones, A1, A2); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 6), \ - _mm256_lddqu_si256(data2 + i + 6)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 6), \ + _mm256_loadu_si256(data2 + i + 6)); \ _mm256_storeu_si256(out + i + 6, A1); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 7), \ - _mm256_lddqu_si256(data2 + i + 7)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 7), \ + _mm256_loadu_si256(data2 + i + 7)); \ _mm256_storeu_si256(out + i + 7, A2); \ CSA(&twosB, &ones, ones, A1, A2); \ CSA(&foursB, &twos, twos, twosA, twosB); \ CSA(&eightsA, &fours, fours, foursA, foursB); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 8), \ - _mm256_lddqu_si256(data2 + i + 8)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 8), \ + _mm256_loadu_si256(data2 + i + 8)); \ _mm256_storeu_si256(out + i + 8, A1); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 9), \ - _mm256_lddqu_si256(data2 + i + 9)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 9), \ + _mm256_loadu_si256(data2 + i + 9)); \ _mm256_storeu_si256(out + i + 9, A2); \ CSA(&twosA, &ones, ones, A1, A2); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 10), \ - _mm256_lddqu_si256(data2 + i + 10)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 10), \ + _mm256_loadu_si256(data2 + i + 10)); \ _mm256_storeu_si256(out + i + 10, A1); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 11), \ - _mm256_lddqu_si256(data2 + i + 11)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 11), \ + _mm256_loadu_si256(data2 + i + 11)); \ _mm256_storeu_si256(out + i + 11, A2); \ CSA(&twosB, &ones, ones, A1, A2); \ CSA(&foursA, &twos, twos, twosA, twosB); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 12), \ - _mm256_lddqu_si256(data2 + i + 12)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 12), \ + _mm256_loadu_si256(data2 + i + 12)); \ _mm256_storeu_si256(out + i + 12, A1); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 13), \ - _mm256_lddqu_si256(data2 + i + 13)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 13), \ + _mm256_loadu_si256(data2 + i + 13)); \ _mm256_storeu_si256(out + i + 13, A2); \ CSA(&twosA, &ones, ones, A1, A2); \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 14), \ - _mm256_lddqu_si256(data2 + i + 14)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 14), \ + _mm256_loadu_si256(data2 + i + 14)); \ _mm256_storeu_si256(out + i + 14, A1); \ - A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 15), \ - _mm256_lddqu_si256(data2 + i + 15)); \ + A2 = avx_intrinsic(_mm256_loadu_si256(data1 + i + 15), \ + _mm256_loadu_si256(data2 + i + 15)); \ _mm256_storeu_si256(out + i + 15, A2); \ CSA(&twosB, &ones, ones, A1, A2); \ CSA(&foursB, &twos, twos, twosA, twosB); \ @@ -2099,8 +2497,8 @@ CROARING_UNTARGET_AVX2 _mm256_add_epi64(total, _mm256_slli_epi64(popcount256(twos), 1)); \ total = _mm256_add_epi64(total, popcount256(ones)); \ for (; i < size; i++) { \ - A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i), \ - _mm256_lddqu_si256(data2 + i)); \ + A1 = avx_intrinsic(_mm256_loadu_si256(data1 + i), \ + _mm256_loadu_si256(data2 + i)); \ _mm256_storeu_si256(out + i, A1); \ total = _mm256_add_epi64(total, popcount256(A1)); \ } \ @@ -2268,6 +2666,14 @@ CROARING_UNTARGET_AVX512 /* * array.h * + * Array containers store a sparse set of 16-bit integers as a sorted dynamic + * array. The cardinality field tracks how many values are present, capacity + * tracks the allocated length, and array points to the sorted values. + * + * This representation is used for low-cardinality containers because it is + * compact and supports fast iteration and binary-search-based membership + * tests. When the number of stored values grows beyond DEFAULT_MAX_SIZE, + * CRoaring will typically switch to a denser container representation. */ #ifndef INCLUDE_CONTAINERS_ARRAY_H_ @@ -2596,30 +3002,78 @@ static inline bool array_container_remove(array_container_t *arr, /* Check whether x is present. */ inline bool array_container_contains(const array_container_t *arr, uint16_t pos) { - // return binarySearch(arr->array, arr->cardinality, pos) >= 0; - // binary search with fallback to linear search for short ranges - int32_t low = 0; - const uint16_t *carr = (const uint16_t *)arr->array; - int32_t high = arr->cardinality - 1; - // while (high - low >= 0) { - while (high >= low + 16) { - int32_t middleIndex = (low + high) >> 1; - uint16_t middleValue = carr[middleIndex]; - if (middleValue < pos) { - low = middleIndex + 1; - } else if (middleValue > pos) { - high = middleIndex - 1; - } else { - return true; + /** + * SIMD Quad algorithm + * Daniel Lemire, "You can beat the binary search," in Daniel Lemire's blog, + * April 27, 2026, + * https://lemire.me/blog/2026/04/27/you-can-beat-the-binary-search/. + */ + const int32_t gap = 16; + const uint16_t *carr = arr->array; + int32_t cardinality = arr->cardinality; + if (cardinality < gap) { + for (int32_t j = 0; j < cardinality; j++) { + if (carr[j] >= pos) return carr[j] == pos; } + return false; } + int32_t num_blocks = cardinality / gap; + int32_t base = 0; + int32_t n = num_blocks; + while (n > 3) { + int32_t quarter = n >> 2; - for (int i = low; i <= high; i++) { - uint16_t v = carr[i]; - if (v == pos) { - return true; + int32_t k1 = carr[(base + quarter + 1) * gap - 1]; + int32_t k2 = carr[(base + 2 * quarter + 1) * gap - 1]; + int32_t k3 = carr[(base + 3 * quarter + 1) * gap - 1]; + + int32_t c1 = (k1 < pos); + int32_t c2 = (k2 < pos); + int32_t c3 = (k3 < pos); + + base += (c1 + c2 + c3) * quarter; + n -= 3 * quarter; + } + while (n > 1) { + int32_t half = n >> 1; + base = (carr[(base + half + 1) * gap - 1] < pos) ? base + half : base; + n -= half; + } + int32_t lo = (carr[(base + 1) * gap - 1] < pos) ? base + 1 : base; + + if (lo < num_blocks) { + const uint16_t *blk = carr + lo * gap; +#ifdef CROARING_USENEON + uint16x8_t needle = vdupq_n_u16(pos); + uint16x8_t v0 = vld1q_u16(blk); + uint16x8_t v1 = vld1q_u16(blk + 8); + uint16x8_t hit = + vorrq_u16(vceqq_u16(v0, needle), vceqq_u16(v1, needle)); +#ifdef __arm__ + uint16x4_t hit8 = vmax_u16(vget_low_u16(hit), vget_high_u16(hit)); + uint16x4_t hit4 = vpmax_u16(hit8, hit8); + return vget_lane_u32(hit4, 0) != 0; +#else /* __aarch64__ */ + return vmaxvq_u16(hit) != 0; +#endif +#elif defined(CROARING_IS_X64) + __m128i needle = _mm_set1_epi16((short)pos); + __m128i v0 = _mm_loadu_si128((const __m128i *)blk); + __m128i v1 = _mm_loadu_si128((const __m128i *)(blk + 8)); + __m128i hit = _mm_or_si128(_mm_cmpeq_epi16(v0, needle), + _mm_cmpeq_epi16(v1, needle)); + return _mm_movemask_epi8(hit) != 0; +#else + for (int32_t j = 0; j < gap; j++) { + if (blk[j] >= pos) return blk[j] == pos; } - if (v > pos) return false; + return false; +#endif + } + + for (int32_t j = num_blocks * gap; j < cardinality; j++) { + uint16_t v = carr[j]; + if (v >= pos) return (v == pos); } return false; } @@ -2787,6 +3241,15 @@ static inline void array_container_remove_range(array_container_t *array, /* * bitset.h * + * Bitset containers store a set of 16-bit integers as a fixed-size bitmap. + * The words pointer references an array of 64-bit words covering the full + * 16-bit domain, with one bit per possible value. The cardinality field tracks + * the number of set bits; when it is BITSET_UNKNOWN_CARDINALITY, the count must + * be recomputed from the bitmap contents. + * + * This representation is used for denser containers because membership tests, + * set operations, and sequential scans can be implemented efficiently with + * word-level bitwise operations. */ #ifndef INCLUDE_CONTAINERS_BITSET_H_ @@ -2828,6 +3291,10 @@ typedef struct bitset_container_s bitset_container_t; /* Create a new bitset. Return NULL in case of failure. */ bitset_container_t *bitset_container_create(void); +/* Create a bitset without zeroing the words. Caller must overwrite `words` + * before the container is used. Return NULL in case of failure. */ +bitset_container_t *bitset_container_create_uninitialized(void); + /* Free memory. */ void bitset_container_free(bitset_container_t *bitset); @@ -2853,9 +3320,9 @@ static inline void bitset_container_set(bitset_container_t *bitset, uint64_t shift = 6; uint64_t offset; uint64_t p = pos; - ASM_SHIFT_RIGHT(p, shift, offset); + CROARING_ASM_SHIFT_RIGHT(p, shift, offset); uint64_t load = bitset->words[offset]; - ASM_SET_BIT_INC_WAS_CLEAR(load, p, bitset->cardinality); + CROARING_ASM_SET_BIT_INC_WAS_CLEAR(load, p, bitset->cardinality); bitset->words[offset] = load; } @@ -2865,9 +3332,9 @@ static inline void bitset_container_set(bitset_container_t *bitset, uint64_t shift = 6; uint64_t offset; uint64_t p = pos; - ASM_SHIFT_RIGHT(p, shift, offset); + CROARING_ASM_SHIFT_RIGHT(p, shift, offset); uint64_t load = bitset->words[offset]; - ASM_CLEAR_BIT_DEC_WAS_SET(load, p, bitset->cardinality); + CROARING_ASM_CLEAR_BIT_DEC_WAS_SET(load, p, bitset->cardinality); bitset->words[offset] = load; }*/ @@ -2878,11 +3345,11 @@ static inline bool bitset_container_add(bitset_container_t *bitset, uint64_t shift = 6; uint64_t offset; uint64_t p = pos; - ASM_SHIFT_RIGHT(p, shift, offset); + CROARING_ASM_SHIFT_RIGHT(p, shift, offset); uint64_t load = bitset->words[offset]; // could be possibly slightly further optimized const int32_t oldcard = bitset->cardinality; - ASM_SET_BIT_INC_WAS_CLEAR(load, p, bitset->cardinality); + CROARING_ASM_SET_BIT_INC_WAS_CLEAR(load, p, bitset->cardinality); bitset->words[offset] = load; return bitset->cardinality - oldcard; } @@ -2894,11 +3361,11 @@ static inline bool bitset_container_remove(bitset_container_t *bitset, uint64_t shift = 6; uint64_t offset; uint64_t p = pos; - ASM_SHIFT_RIGHT(p, shift, offset); + CROARING_ASM_SHIFT_RIGHT(p, shift, offset); uint64_t load = bitset->words[offset]; // could be possibly slightly further optimized const int32_t oldcard = bitset->cardinality; - ASM_CLEAR_BIT_DEC_WAS_SET(load, p, bitset->cardinality); + CROARING_ASM_CLEAR_BIT_DEC_WAS_SET(load, p, bitset->cardinality); bitset->words[offset] = load; return oldcard - bitset->cardinality; } @@ -2908,7 +3375,7 @@ inline bool bitset_container_get(const bitset_container_t *bitset, uint16_t pos) { uint64_t word = bitset->words[pos >> 6]; const uint64_t p = pos; - ASM_INPLACESHIFT_RIGHT(word, p); + CROARING_ASM_INPLACESHIFT_RIGHT(word, p); return word & 1; } @@ -3298,6 +3765,14 @@ int bitset_container_index_equalorlarger(const bitset_container_t *container, /* * run.h * + * Run containers store a set of 16-bit integers as a sorted array of + * non-overlapping runs. Each run is represented by a starting value and a + * length, encoding one contiguous interval of present integers. + * + * This representation is effective when the data contains long consecutive + * ranges because it compresses many adjacent values into a small number of + * run records while still supporting search and set operations over the + * interval list. */ #ifndef INCLUDE_CONTAINERS_RUN_H_ @@ -4014,6 +4489,15 @@ static inline void run_container_remove_range(run_container_t *run, /* * convert.h * + * This header declares conversion helpers between the different Roaring + * container representations: array, bitset, and run containers. These + * routines are used when an operation produces data better represented in a + * different form, or when the library wants to switch to the most space- + * efficient container type. + * + * In addition to direct conversions, the file also provides helpers that + * choose between candidate result representations based on cardinality and + * storage efficiency. */ #ifndef INCLUDE_CONTAINERS_CONVERT_H_ @@ -4127,6 +4611,12 @@ bool run_container_equals_bitset(const run_container_t* container1, /* * mixed_subset.h * + * This header declares subset-checking routines between different Roaring + * container types. These helpers are used when two containers do not share the + * same representation and a direct type-specific subset predicate is needed. + * + * Each function answers whether all values from one container are contained in + * another, across combinations of array, bitset, and run containers. */ #ifndef CONTAINERS_MIXED_SUBSET_H_ @@ -4180,6 +4670,16 @@ bool bitset_container_is_subset_run(const bitset_container_t* container1, /* begin file include/roaring/containers/mixed_andnot.h */ /* * mixed_andnot.h + * + * This header declares mixed-container difference operations of the form + * `A \ B` (also called andnot) between Roaring container types such as array, + * bitset, and run containers. These helpers are used when the operands have + * different internal representations and the result may need to change + * representation depending on density. + * + * The file includes both allocating and inplace-oriented variants so callers + * can either materialize a fresh result or reuse storage when that is + * efficient and semantically allowed. */ #ifndef INCLUDE_CONTAINERS_MIXED_ANDNOT_H_ #define INCLUDE_CONTAINERS_MIXED_ANDNOT_H_ @@ -4361,6 +4861,15 @@ bool bitset_bitset_container_iandnot(bitset_container_t *src_1, /* * mixed_intersection.h * + * This header declares intersection operations between different Roaring + * container types, such as array, bitset, and run containers. These mixed + * routines are used when the input containers have different internal + * representations and the implementation must choose the appropriate result + * form based on the data. + * + * In addition to materializing intersections, the file also provides helpers + * for intersection cardinality, intersection predicates, and selected inplace + * variants. */ #ifndef INCLUDE_CONTAINERS_MIXED_INTERSECTION_H_ @@ -4461,6 +4970,15 @@ bool bitset_bitset_container_intersection_inplace( /* * mixed_negation.h * + * This header declares negation (complement) operations for Roaring + * containers, both over the full 16-bit container domain and over specified + * subranges. Depending on the input representation and the density of the + * complement, the result may need to switch between array, bitset, and run + * containers. + * + * The file includes both allocating and inplace-oriented variants so callers + * can choose between simple result construction and reuse of an existing + * container when that is practical. */ #ifndef INCLUDE_CONTAINERS_MIXED_NEGATION_H_ @@ -4598,8 +5116,16 @@ int run_container_negation_range_inplace(run_container_t *src, /* end file include/roaring/containers/mixed_negation.h */ /* begin file include/roaring/containers/mixed_union.h */ /* - * mixed_intersection.h + * mixed_union.h + * + * This header declares union operations between different Roaring container + * types, such as array, bitset, and run containers. These mixed-operation + * helpers are used when the two input containers do not share the same + * representation and the result may need to stay in, or be converted to, a + * representation chosen according to the data. * + * The file includes regular, lazy, and inplace variants so callers can select + * between fully maintained results and faster deferred-maintenance paths. */ #ifndef INCLUDE_CONTAINERS_MIXED_UNION_H_ @@ -4716,6 +5242,14 @@ void run_bitset_container_lazy_union(const run_container_t *src_1, /* * mixed_xor.h * + * This header declares XOR operations between different Roaring container + * types, such as array, bitset, and run containers. These "mixed" routines + * handle cases where the two inputs do not share the same representation and + * where the most appropriate output representation may depend on the data. + * + * It includes regular, lazy, and inplace variants so higher-level bitmap code + * can choose between fully normalized results and faster deferred-maintenance + * paths. */ #ifndef INCLUDE_CONTAINERS_MIXED_XOR_H_ @@ -4888,6 +5422,18 @@ int run_run_container_ixor(run_container_t *src_1, const run_container_t *src_2, #endif /* end file include/roaring/containers/mixed_xor.h */ /* begin file include/roaring/containers/containers.h */ +/* + * containers.h + * + * This header is the central internal interface for Roaring container + * operations. It ties together the concrete container types (array, bitset, + * run, and shared containers), their type codes, common helper functions, and + * the mixed-operation headers used to combine different representations. + * + * In practice, it acts as the dispatch layer that lets higher-level bitmap + * code manipulate containers through a uniform interface while still selecting + * type-specific implementations when needed. + */ #ifndef CONTAINERS_CONTAINERS_H #define CONTAINERS_CONTAINERS_H @@ -5432,7 +5978,12 @@ inline bool container_contains( const container_t *c, uint16_t val, uint8_t typecode // !!! should be second argument? ) { - c = container_unwrap_shared(c, &typecode); + if (typecode == SHARED_CONTAINER_TYPE) { + typecode = const_CAST_shared(c)->typecode; + assert(typecode != SHARED_CONTAINER_TYPE); + c = const_CAST_shared(c)->container; + } + switch (typecode) { case BITSET_CONTAINER_TYPE: return bitset_container_get(const_CAST_bitset(c), val); @@ -6094,7 +6645,7 @@ static inline container_t *container_ior(container_t *c1, uint8_t type1, case CONTAINER_PAIR(BITSET, BITSET): bitset_container_or(const_CAST_bitset(c1), const_CAST_bitset(c2), CAST_bitset(c1)); -#ifdef OR_BITSET_CONVERSION_TO_FULL +#if CROARING_OR_BITSET_CONVERSION_TO_FULL if (CAST_bitset(c1)->cardinality == (1 << 16)) { // we convert result = run_container_create_range(0, (1 << 16)); *result_type = RUN_CONTAINER_TYPE; @@ -6201,7 +6752,7 @@ static inline container_t *container_lazy_ior(container_t *c1, uint8_t type1, container_t *result = NULL; switch (PAIR_CONTAINER_TYPES(type1, type2)) { case CONTAINER_PAIR(BITSET, BITSET): -#ifdef LAZY_OR_BITSET_CONVERSION_TO_FULL +#if CROARING_LAZY_OR_BITSET_CONVERSION_TO_FULL // if we have two bitsets, we might as well compute the cardinality bitset_container_or(const_CAST_bitset(c1), const_CAST_bitset(c2), CAST_bitset(c1)); @@ -7312,6 +7863,7 @@ roaring_container_iterator_t container_init_iterator_last(const container_t *c, * Moves the iterator to the next entry. Returns true and sets `value` if a * value is present. */ +CROARING_ALLOW_UNALIGNED inline bool container_iterator_next(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint16_t *value) { @@ -7380,6 +7932,7 @@ inline bool container_iterator_next(const container_t *c, uint8_t typecode, * Moves the iterator to the previous entry. Returns true and sets `value` if a * value is present. */ +CROARING_ALLOW_UNALIGNED inline bool container_iterator_prev(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint16_t *value) { @@ -7471,6 +8024,32 @@ bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, uint32_t count, uint32_t *consumed, uint16_t *value_out); +/** + * Reads up to `count` entries backward from the container, writing them into + * `buf` as `high16 | entry` in descending order. Returns true and sets + * `value_out` if a value is present before the entries read. Sets `consumed` + * to the number of values read. `count` should be greater than zero. + * + * `value_out` must be initialized to the current value yielded by the iterator. + */ +bool container_iterator_read_backward_into_uint32( + const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, + uint32_t high16, uint32_t *buf, uint32_t count, uint32_t *consumed, + uint16_t *value_out); + +/** + * Reads up to `count` entries backward from the container, writing them into + * `buf` as `high48 | entry` in descending order. Returns true and sets + * `value_out` if a value is present before the entries read. Sets `consumed` + * to the number of values read. `count` should be greater than zero. + * + * `value_out` must be initialized to the current value yielded by the iterator. + */ +bool container_iterator_read_backward_into_uint64( + const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, + uint64_t high48, uint64_t *buf, uint32_t count, uint32_t *consumed, + uint16_t *value_out); + /** * Skips the next `skip_count` entries in the container iterator. Returns true * and sets `value_out` if a value is present after skipping. Returns false if @@ -7505,6 +8084,35 @@ bool container_iterator_skip_backward(const container_t *c, uint8_t typecode, uint32_t *consumed_count, uint16_t *value_out); +/** + * Finds the end of the consecutive run starting at the current iterator + * position within a container. Returns the low16 of the last consecutive + * value. If there are more values in the container after the run, + * *has_more is set to true, the iterator is positioned at the next value, + * and *value is updated to that value. Otherwise *has_more is set to false. + * + * *value must be the low 16 bits of the current value at the iterator's + * position on entry. + */ +uint16_t container_iterator_find_run_end(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value, bool *has_more); + +/** + * Finds the start of the consecutive run ending at the current iterator + * position within a container. Returns the low16 of the first consecutive + * value. If there are more values in the container before the run, + * *has_more is set to true, the iterator is positioned at the previous value, + * and *value is updated to that value. Otherwise *has_more is set to false. + * + * *value must be the low 16 bits of the current value at the iterator's + * position on entry. + */ +uint16_t container_iterator_find_run_start(const container_t *c, + uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value, bool *has_more); + #ifdef __cplusplus } } @@ -7514,6 +8122,20 @@ bool container_iterator_skip_backward(const container_t *c, uint8_t typecode, #endif /* end file include/roaring/containers/containers.h */ /* begin file include/roaring/roaring_array.h */ +/* + * roaring_array.h + * + * This file declares the roaring_array helper structure and the operations + * used to manage it. A roaring array is the top-level index used by a 32-bit + * Roaring bitmap: it stores sorted 16-bit high keys alongside the container + * pointers and type codes associated with each key. + * + * In effect, it is the directory that maps each populated 16-bit chunk of the + * 32-bit value space to the container holding that chunk's low 16-bit values. + * The functions in this header handle allocation, lookup, insertion, + * replacement, copying, serialization support, and structural updates on that + * directory. + */ #ifndef INCLUDE_ROARING_ARRAY_H #define INCLUDE_ROARING_ARRAY_H @@ -7818,6 +8440,17 @@ void ra_shift_tail(roaring_array_t *ra, int32_t count, int32_t distance); /* begin file include/roaring/roaring.h */ /* * An implementation of Roaring Bitmaps in C. + * + * This is the main public header for the 32-bit CRoaring API. A Roaring bitmap + * represents a set of unsigned 32-bit integers by partitioning the value space + * into 16-bit chunks and storing each chunk in a container chosen to match the + * local data density. Sparse chunks are typically kept as sorted arrays, + * denser chunks as bitsets, and long consecutive runs as run containers. + * + * This hybrid representation aims to keep bitmaps compact while still + * supporting fast membership tests, iteration, rank/select queries, + * serialization, and set operations such as union, intersection, difference, + * and symmetric difference. */ #ifndef ROARING_H @@ -8436,10 +9069,6 @@ size_t roaring_bitmap_shrink_to_fit(roaring_bitmap_t *r); * * Returns how many bytes written, should be `roaring_bitmap_size_in_bytes(r)`. * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * When serializing data to a file, we recommend that you also use * checksums so that, at deserialization, you can be confident * that you are recovering the correct data. @@ -8452,27 +9081,34 @@ size_t roaring_bitmap_serialize(const roaring_bitmap_t *r, char *buf); * (See `roaring_bitmap_portable_deserialize()` if you want a format that's * compatible with Java and Go implementations). * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * The returned pointer may be NULL in case of errors. */ roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf); /** + * Load a bitmap from a serialized buffer safely (reading up to maxbytes). + * * Use with `roaring_bitmap_serialize()`. * * (See `roaring_bitmap_portable_deserialize_safe()` if you want a format that's * compatible with Java and Go implementations). * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * The difference with `roaring_bitmap_deserialize()` is that this function - * checks that the input buffer is a valid bitmap. If the buffer is too small, - * NULL is returned. + * is guaranteed to not read beyond the provided buffer. If the buffer is too + * small, NULL is returned. + * + * The function itself is safe in the sense that it will not cause buffer + * overflows: it will not read beyond the scope of the provided buffer + * (buf,maxbytes). + * + * However, for correct operations, it is assumed that the bitmap + * read was once serialized from a valid bitmap (i.e., it follows the format + * specification). If you provided an incorrect input (garbage), then the bitmap + * read may not be in a valid state and following operations may not lead to + * sensible results (using it may cause crashes, or it may just give incoherent + * answers). You can call roaring_bitmap_internal_validate to check the validity + * of the bitmap if the source is untrusted. Only after calling + * roaring_bitmap_internal_validate is the bitmap considered safe for use. * * The returned pointer may be NULL in case of errors. */ @@ -8491,15 +9127,20 @@ size_t roaring_bitmap_size_in_bytes(const roaring_bitmap_t *r); * * This function is unsafe in the sense that if there is no valid serialized * bitmap at the pointer, then many bytes could be read, possibly causing a - * buffer overflow. See also roaring_bitmap_portable_deserialize_safe(). + * buffer overflow. In other words, this routine assumes that `buf` points to a + * complete, correctly formatted serialized bitmap and does not take a buffer + * length argument that would let it enforce a read bound. + * + * Use this function only when the input buffer is already trusted, for example + * because it comes from memory that was previously filled by + * `roaring_bitmap_portable_serialize()` and whose size is known by some other + * means. If the source is untrusted, truncated, or otherwise not guaranteed to + * contain a valid serialized bitmap, prefer + * `roaring_bitmap_portable_deserialize_safe()`. * * This is meant to be compatible with the Java and Go versions: * https://github.com/RoaringBitmap/RoaringFormatSpec * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * The returned pointer may be NULL in case of errors. */ roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf); @@ -8533,10 +9174,6 @@ roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf); * corresponds to the serialized bitmap. The CRoaring library does not provide * checksumming. * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * The returned pointer may be NULL in case of errors. */ roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, @@ -8558,9 +9195,11 @@ roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, * This is meant to be compatible with the Java and Go versions: * https://github.com/RoaringBitmap/RoaringFormatSpec * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * Returns NULL on a big-endian system (e.g., a mainframe IBM s390x). The + * portable format is little-endian on every host, and this function uses the + * container payloads where they sit rather than converting them, so there is + * no correct in-place view of them there. Use + * `roaring_bitmap_portable_deserialize_safe()`, which converts as it copies. * * The returned pointer may be NULL in case of errors. */ @@ -8594,10 +9233,6 @@ size_t roaring_bitmap_portable_size_in_bytes(const roaring_bitmap_t *r); * This is meant to be compatible with the Java and Go versions: * https://github.com/RoaringBitmap/RoaringFormatSpec * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * When serializing data to a file, we recommend that you also use * checksums so that, at deserialization, you can be confident * that you are recovering the correct data. @@ -8634,7 +9269,8 @@ size_t roaring_bitmap_frozen_size_in_bytes(const roaring_bitmap_t *r); * * This function is endian-sensitive. If you have a big-endian system (e.g., a * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * compatible with little-endian systems. This is not a bug, it is by design, + *since the format imitates C memory layout * * When serializing data to a file, we recommend that you also use * checksums so that, at deserialization, you can be confident @@ -8655,7 +9291,8 @@ void roaring_bitmap_frozen_serialize(const roaring_bitmap_t *r, char *buf); * * This function is endian-sensitive. If you have a big-endian system (e.g., a * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * compatible with little-endian systems. This is not a bug, it is by design, + *since the format imitates C memory layout of roaring_bitmap_t. */ const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, size_t length); @@ -8676,6 +9313,16 @@ const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, bool roaring_iterate(const roaring_bitmap_t *r, roaring_iterator iterator, void *ptr); +/** + * Like `roaring_iterate`, but the 32-bit values are widened to 64 bits by + * adding `high_bits` (shifted into the upper 32 bits) before being passed to + * the iterator. This is used to build 64-bit iteration on top of 32-bit + * bitmaps. `ptr` (can be NULL) is forwarded as the second argument of each + * call. + * + * Returns true if the iterator returned true throughout (so that all values + * were necessarily visited). + */ bool roaring_iterate64(const roaring_bitmap_t *r, roaring_iterator64 iterator, uint64_t high_bits, void *ptr); @@ -9048,6 +9695,23 @@ CROARING_DEPRECATED static inline uint32_t roaring_read_uint32_iterator( return roaring_uint32_iterator_read(it, buf, count); } +/** + * Reads previous ${count} values from iterator into user-supplied ${buf}. + * Returns the number of read elements. + * This number can be smaller than ${count}, which means that iterator is + * drained. + * + * Values are written in descending order: buf[0] is the highest (current) + * value, buf[ret-1] is the lowest value read. + * + * This function satisfies semantics of reverse iteration and can be used + * together with other iterator functions. + * - first value is copied from ${it}->current_value + * - after function returns, iterator is positioned at the previous element + */ +uint32_t roaring_uint32_iterator_read_backward(roaring_uint32_iterator_t *it, + uint32_t *buf, uint32_t count); + /** * Skip the next ${count} values from iterator. * Returns the number of values actually skipped. @@ -9072,6 +9736,60 @@ uint32_t roaring_uint32_iterator_skip(roaring_uint32_iterator_t *it, uint32_t roaring_uint32_iterator_skip_backward(roaring_uint32_iterator_t *it, uint32_t count); +typedef struct roaring_uint32_range_closed_s { + uint32_t min; + uint32_t max; +} roaring_uint32_range_closed_t; + +/** + * Reads next ${count} ranges from iterator into user-supplied ${buf}. + * A range is defined as a maximal interval of consecutive values. + * For example, the set {1,2,3,5,6} contains two ranges: [1..3] and [5..6]. + * Each range is represented as a struct {min,max}, both endpoints included. + * Consecutive values that span internal container boundaries are merged into + * a single range. + * + * Returns the number of read ranges. + * This number can be smaller than ${count}, which means that the iterator is + * drained. + * + * This function satisfies the semantics of iteration and can be used together + * with other iterator functions. + * - first range will start with ${it}->current_value + * - after the function returns, the iterator is positioned at the next element + * after the end of the last returned range, or ${it}->has_value is false if + * the bitmap is exhausted. + */ +size_t roaring_uint32_iterator_read_ranges(roaring_uint32_iterator_t *it, + roaring_uint32_range_closed_t *buf, + size_t count); + +/** + * Reads previous ${count} ranges from iterator into user-supplied ${buf}. + * A range is defined as a maximal interval of consecutive values. + * For example, the set {1,2,3,5,6} contains two ranges: [1..3] and [5..6]. + * Each range is represented as a struct {min,max}, both endpoints included. + * Consecutive values that span internal container boundaries are merged into + * a single range. + * + * Returns the number of read ranges. + * This number can be smaller than ${count}, which means that the iterator is + * drained. + * + * Ranges are returned in reverse order, e.g. the first range returned is the + * highest range (ending at the current value) + * + * This function satisfies the semantics of reverse iteration and can be used + * together with other iterator functions. + * - first range will end with ${it}->current_value + * - after the function returns, the iterator is positioned at the element + * before the beginning of the last returned range, or ${it}->has_value is + * false if the bitmap is exhausted. + */ +size_t roaring_uint32_iterator_read_prev_ranges( + roaring_uint32_iterator_t *it, roaring_uint32_range_closed_t *buf, + size_t count); + #ifdef __cplusplus } } @@ -9102,6 +9820,18 @@ using namespace ::roaring::api; // in addition to roaring.h. /* end file include/roaring/roaring.h */ /* begin file include/roaring/memory.h */ +/* + * memory.h + * + * This header defines CRoaring's memory-allocation abstraction layer. It + * declares the function pointer types and hook structure used to override the + * library's malloc/realloc/calloc/free and aligned allocation routines, along + * with the wrapper functions used throughout the codebase. + * + * This allows applications to integrate CRoaring with custom allocators, + * memory trackers, arenas, or platform-specific aligned allocation policies + * without changing the rest of the library code. + */ #ifndef INCLUDE_ROARING_MEMORY_H_ #define INCLUDE_ROARING_MEMORY_H_ @@ -9143,6 +9873,15 @@ void roaring_aligned_free(void*); #endif // INCLUDE_ROARING_MEMORY_H_ /* end file include/roaring/memory.h */ /* begin file include/roaring/roaring64.h */ +/* + * roaring64.h + * + * This file declares the 64-bit Roaring bitmap API. A roaring64 bitmap stores + * sets of 64-bit unsigned integers by partitioning the value space by high + * bits and using Roaring containers for the lower bits inside each partition. + * This keeps the structure compact while preserving fast membership tests, + * insertions, iteration, and set operations over large sparse integer sets. + */ #ifndef ROARING64_H #define ROARING64_H @@ -9157,10 +9896,25 @@ namespace roaring { namespace api { #endif +/** An opaque 64-bit Roaring bitmap. Create one with `roaring64_bitmap_create()` + * and release it with `roaring64_bitmap_free()`. */ typedef struct roaring64_bitmap_s roaring64_bitmap_t; +/** Internal leaf type, exposed only for use inside `roaring64_bulk_context_t`. + * Callers should treat it as opaque. */ typedef uint64_t roaring64_leaf_t; +/** An opaque iterator over a 64-bit bitmap. See `roaring64_iterator_create()`. + */ typedef struct roaring64_iterator_s roaring64_iterator_t; +/** The leading members of `roaring64_iterator_t`, so that + * `roaring64_iterator_value()` and `roaring64_iterator_has_value()` can be + * read without a call. The iterator itself stays opaque; do not declare one of + * these, and do not rely on the layout beyond these two members. */ +typedef struct roaring64_iterator_public_s { + uint64_t value; + bool has_value; +} roaring64_iterator_public_t; + /** * A bit of context usable with `roaring64_bitmap_*_bulk()` functions. * @@ -9383,6 +10137,12 @@ bool roaring64_bitmap_contains(const roaring64_bitmap_t *r, uint64_t val); bool roaring64_bitmap_contains_range(const roaring64_bitmap_t *r, uint64_t min, uint64_t max); +/** + * Returns true if all values in the range [min, max] are present. + */ +bool roaring64_bitmap_contains_range_closed(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max); + /** * Check if an item is present using context from a previous insert or search * for faster search. @@ -9715,10 +10475,6 @@ size_t roaring64_bitmap_portable_size_in_bytes(const roaring64_bitmap_t *r); * This is meant to be compatible with other languages: * https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * When serializing data to a file, we recommend that you also use * checksums so that, at deserialization, you can be confident * that you are recovering the correct data. @@ -9763,14 +10519,50 @@ size_t roaring64_bitmap_portable_deserialize_size(const char *buf, * We also recommend that you use checksums to check that serialized data * corresponds to the serialized bitmap. The CRoaring library does not provide * checksumming. - * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. */ roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe(const char *buf, size_t maxbytes); +/** + * Read a bitmap from a portable serialized buffer as a read-only view of the + * container payloads. Headers and the ART index are allocated; bitset/array/run + * payloads alias `buf` and are not copied. + * + * In case of failure, NULL is returned. The function will not read beyond + * `maxbytes`. + * + * The returned bitmap must only be used in a readonly manner. It must be + * freed with `roaring64_bitmap_free()`. The backing buffer must outlive the + * bitmap and must not be freed or modified while it backs it. Calling any + * mutating function on the result is undefined behavior: its container array + * and headers live in a single allocation, so growing it would reallocate an + * interior pointer. + * + * The function itself is safe in the sense that it will not read beyond + * (buf, maxbytes). However, as with + * `roaring64_bitmap_portable_deserialize_safe()`, a bitmap read from garbage + * may not be in a valid state, and subsequent operations on it may not lead + * to sensible results: array containers must be sorted, and run containers + * sorted and non-overlapping, which is guaranteed only when the input came + * from a real serialized bitmap. + * + * If the source is untrusted, you should call + * `roaring64_bitmap_internal_validate` on the result before using it. Only + * after that is the bitmap considered safe for use. We also recommend + * checksumming the serialized data; CRoaring does not provide checksumming. + * + * Returns NULL on a big-endian system (e.g., a mainframe IBM s390x). The + * portable format is little-endian and this function uses the payload bytes + * where they sit, so there is no correct in-place view of them there; use + * `roaring64_bitmap_portable_deserialize_safe()`, which converts as it copies. + * + * Container payloads are used where they sit in the buffer, so they may be + * unaligned. Every access path is either SIMD with unaligned loads or marked + * `CROARING_ALLOW_UNALIGNED`. + */ +roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_frozen( + const char *buf, size_t maxbytes); + /** * Returns the number of bytes required to serialize this bitmap in a "frozen" * format. This is not compatible with any other serialization formats. @@ -9795,7 +10587,8 @@ size_t roaring64_bitmap_frozen_size_in_bytes(const roaring64_bitmap_t *r); * * This function is endian-sensitive. If you have a big-endian system (e.g., a * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * compatible with little-endian systems. This is not a bug, it is by design, + * since the format imitates C memory layout of roaring64_bitmap_t. */ size_t roaring64_bitmap_frozen_serialize(const roaring64_bitmap_t *r, char *buf); @@ -9813,7 +10606,8 @@ size_t roaring64_bitmap_frozen_serialize(const roaring64_bitmap_t *r, * * This function is endian-sensitive. If you have a big-endian system (e.g., a * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * compatible with little-endian systems. This is not a bug, it is by design, + * since the format imitates C memory layout of roaring64_bitmap_t. */ roaring64_bitmap_t *roaring64_bitmap_frozen_view(const char *buf, size_t maxbytes); @@ -9894,14 +10688,22 @@ void roaring64_iterator_free(roaring64_iterator_t *it); /** * Returns true if the iterator currently points to a value. If so, calling * `roaring64_iterator_value()` returns the value. + * + * A pointer to a structure, suitably converted, points to its initial member + * (C17 6.7.2.1p15), and `roaring64_iterator_public_t` is the initial member of + * `roaring64_iterator_t`, so this reads the field directly. */ -bool roaring64_iterator_has_value(const roaring64_iterator_t *it); +inline bool roaring64_iterator_has_value(const roaring64_iterator_t *it) { + return ((const roaring64_iterator_public_t *)it)->has_value; +} /** * Returns the value the iterator currently points to. Should only be called if * `roaring64_iterator_has_value()` returns true. */ -uint64_t roaring64_iterator_value(const roaring64_iterator_t *it); +inline uint64_t roaring64_iterator_value(const roaring64_iterator_t *it) { + return ((const roaring64_iterator_public_t *)it)->value; +} /** * Advance the iterator. If there is a new value, then @@ -9945,6 +10747,75 @@ bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, uint64_t roaring64_iterator_read(roaring64_iterator_t *it, uint64_t *buf, uint64_t count); +/** + * Reads previous ${count} values from iterator into user-supplied ${buf}. + * Returns the number of read elements. + * This number can be smaller than ${count}, which means that iterator is + * drained. + * + * Values are written in descending order: buf[0] is the highest (current) + * value, buf[ret-1] is the lowest value read. + * + * This function satisfies semantics of reverse iteration and can be used + * together with other iterator functions. + * - first value is copied from the current iterator value + * - after function returns, iterator is positioned at the previous element + */ +uint64_t roaring64_iterator_read_backward(roaring64_iterator_t *it, + uint64_t *buf, uint64_t count); + +typedef struct roaring64_range_closed_s { + uint64_t min; + uint64_t max; +} roaring64_range_closed_t; + +/** + * Reads next ${count} ranges from iterator into user-supplied ${buf}. + * A range is defined as a maximal interval of consecutive values. + * For example, the set {1,2,3,5,6} contains two ranges: [1..3] and [5..6]. + * Each range is represented as a struct {min,max}, both endpoints included. + * Consecutive values that span internal container boundaries are merged into + * a single range. + * + * Returns the number of read ranges. + * This number can be smaller than ${count}, which means that the iterator is + * drained. + * + * This function can be used together with other iterator functions. + * - first range will start with the current iterator value + * - after the function returns, the iterator is positioned at the next element + * after the end of the last returned range, or has_value is false if + * the bitmap is exhausted. + */ +size_t roaring64_iterator_read_ranges(roaring64_iterator_t *it, + roaring64_range_closed_t *buf, + size_t count); + +/** + * Reads previous ${count} ranges from iterator into user-supplied ${buf}. + * A range is defined as a maximal interval of consecutive values. + * For example, the set {1,2,3,5,6} contains two ranges: [1..3] and [5..6]. + * Each range is represented as a struct {min,max}, both endpoints included. + * Consecutive values that span internal container boundaries are merged into + * a single range. + * + * Returns the number of read ranges. + * This number can be smaller than ${count}, which means that the iterator is + * drained. + * + * Ranges are returned in reverse order, e.g. the first range returned is the + * highest range (ending at the current value). + * + * This function can be used together with other iterator functions. + * - first range will end with the current iterator value + * - after the function returns, the iterator is positioned at the element + * before the beginning of the last returned range, or has_value is false if + * the bitmap is exhausted. + */ +size_t roaring64_iterator_read_prev_ranges(roaring64_iterator_t *it, + roaring64_range_closed_t *buf, + size_t count); + #ifdef __cplusplus } // extern "C" } // namespace roaring diff --git a/pyroaring/version.pxi b/pyroaring/version.pxi index 6849410..c68196d 100644 --- a/pyroaring/version.pxi +++ b/pyroaring/version.pxi @@ -1 +1 @@ -__version__ = "1.1.0" +__version__ = "1.2.0"