Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion pyiceberg/table/deletion_vector.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,10 @@ def _deserialize_bitmap(payload: bytes) -> list[BitMap]:
def _bitmaps_to_chunked_array(bitmaps: list[BitMap]) -> "pa.ChunkedArray":
import pyarrow as pa

return pa.chunked_array([(key_pos << 32) + pos for pos in bitmap] for key_pos, bitmap in enumerate(bitmaps))
# Use int64 even when the first bitmap is empty or there are no bitmaps.
return pa.chunked_array(
([(key_pos << 32) + pos for pos in bitmap] for key_pos, bitmap in enumerate(bitmaps)), type=pa.int64()
)

def to_vector(self) -> "pa.ChunkedArray":
return self._bitmaps_to_chunked_array(self._bitmaps)
Expand Down
18 changes: 18 additions & 0 deletions tests/table/test_deletion_vector.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from os import path
from pathlib import Path

import pyarrow as pa
import pytest
from pyroaring import BitMap

Expand Down Expand Up @@ -133,6 +134,23 @@ def test_map_empty() -> None:
actual = DeletionVector._deserialize_bitmap(puffin)

assert expected == actual
vector = DeletionVector("data.parquet", actual).to_vector()
assert vector.type == pa.int64()
assert vector.to_pylist() == []


def test_to_vector_with_empty_bitmaps_for_low_keys() -> None:
# Keys without positions have empty bitmaps, such as key 0 when every position is at or above 2**32
for bitmaps, expected in [
([BitMap(), BitMap([5])], [2**32 + 5]),
([BitMap(), BitMap(), BitMap([0, 7])], [2 * 2**32, 2 * 2**32 + 7]),
([BitMap([2]), BitMap(), BitMap([1])], [2, 2 * 2**32 + 1]),
([BitMap()], []),
([BitMap(), BitMap()], []),
]:
vector = DeletionVector("data.parquet", bitmaps).to_vector()
assert vector.type == pa.int64()
assert vector.to_pylist() == expected


def test_map_bitvals() -> None:
Expand Down
Loading