diff --git a/pyiceberg/table/deletion_vector.py b/pyiceberg/table/deletion_vector.py index 9c56af24a2..afe66137c7 100644 --- a/pyiceberg/table/deletion_vector.py +++ b/pyiceberg/table/deletion_vector.py @@ -37,6 +37,7 @@ _DV_BLOB_CRC = struct.Struct(">I") _DV_BLOB_MAGIC_NUMBER = 1681511377 _ROARING_BITMAP_COUNT_SIZE_BYTES = 8 +_ROARING_BITMAP_KEY_SIZE_BYTES = 4 _DV_BLOB_MIN_SIZE_BYTES = _DV_BLOB_LENGTH.size + _DV_BLOB_MAGIC.size + _ROARING_BITMAP_COUNT_SIZE_BYTES + _DV_BLOB_CRC.size @@ -49,30 +50,32 @@ def __init__(self, referenced_data_file: str, bitmaps: list[BitMap]) -> None: self._bitmaps = bitmaps @staticmethod - def _deserialize_bitmap(pl: bytes) -> list[BitMap]: - number_of_bitmaps = int.from_bytes(pl[0:8], byteorder="little") - pl = pl[8:] + def _deserialize_bitmap(payload: bytes) -> list[BitMap]: + payload_view = memoryview(payload) + offset = _ROARING_BITMAP_COUNT_SIZE_BYTES + number_of_bitmaps = int.from_bytes(payload_view[:offset], byteorder="little") bitmaps = [] last_key = -1 + for _ in range(number_of_bitmaps): - key = int.from_bytes(pl[0:4], byteorder="little") + key = int.from_bytes(payload_view[offset : offset + _ROARING_BITMAP_KEY_SIZE_BYTES], byteorder="little") if key < 0: raise ValueError(f"Invalid unsigned key: {key}") if key <= last_key: raise ValueError("Keys must be sorted in ascending order") if key > MAX_JAVA_SIGNED: raise ValueError(f"Key {key} is too large, max {MAX_JAVA_SIGNED} to maintain compatibility with Java impl") - pl = pl[4:] + offset += _ROARING_BITMAP_KEY_SIZE_BYTES while last_key < key - 1: bitmaps.append(EMPTY_BITMAP) last_key += 1 - bm = BitMap().deserialize(pl) - # TODO: Optimize this - pl = pl[len(bm.serialize()) :] - bitmaps.append(bm) + bitmap = BitMap.deserialize(payload_view[offset:]) + # PyRoaring's __sizeof__ returns the portable serialized size. + offset += bitmap.__sizeof__() + bitmaps.append(bitmap) last_key = key diff --git a/pyproject.toml b/pyproject.toml index 8268338fa1..ce1f8cd771 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "fsspec>=2023.1.0", "pyparsing>=3.1.0", "tenacity>=8.2.3", - "pyroaring>=1.0.0", + "pyroaring>=1.1.0", "cachetools>=5.5", "zstandard>=0.13.0" ] diff --git a/tests/table/test_deletion_vector.py b/tests/table/test_deletion_vector.py index 4b38ea8627..ac6ae98256 100644 --- a/tests/table/test_deletion_vector.py +++ b/tests/table/test_deletion_vector.py @@ -164,6 +164,43 @@ def test_map_spread_vals() -> None: assert expected == actual +def test_map_mixed_containers_with_key_gaps() -> None: + array_bitmap = BitMap([1, 3, 5]) + bitset_bitmap = BitMap(range(0, 20_000, 2)) + run_bitmap = BitMap(range(100_000)) + multi_container_bitmap = BitMap([1, 65_537]) + bitmaps_by_key = { + 0: BitMap(), + 2: array_bitmap, + 3: bitset_bitmap, + 5: run_bitmap, + 6: multi_container_bitmap, + } + payload = len(bitmaps_by_key).to_bytes(8, byteorder="little") + b"".join( + key.to_bytes(4, byteorder="little") + bitmap.serialize() for key, bitmap in bitmaps_by_key.items() + ) + + assert DeletionVector._deserialize_bitmap(payload) == [ + BitMap(), + BitMap(), + array_bitmap, + bitset_bitmap, + BitMap(), + run_bitmap, + multi_container_bitmap, + ] + + +def test_map_deserialization_does_not_reserialize(monkeypatch: pytest.MonkeyPatch) -> None: + class NonSerializingBitMap(BitMap): + def serialize(self) -> bytes: + raise AssertionError("Deletion vector deserialization must not serialize bitmaps") + + monkeypatch.setattr("pyiceberg.table.deletion_vector.BitMap", NonSerializingBitMap) + + assert DeletionVector._deserialize_bitmap(_bitmap_payload()) == [BitMap([1, 3, 5])] + + def test_map_high_vals() -> None: puffin = _open_file("64maphighvals.bin") diff --git a/uv.lock b/uv.lock index 7901c5ea12..223eb43745 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -2056,8 +2056,8 @@ name = "httpcore2" version = "2.13.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "h11" }, - { name = "truststore" }, + { name = "h11", marker = "sys_platform != 'emscripten'" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/f3/1db7aa2bc2524062192bb0e0323969492d1883152a232fe36eea65f4e35c/httpcore2-2.13.1.tar.gz", hash = "sha256:e0aa977abe17e69a3b820a24542a6fa88702676d83880b8d194dcd18408e5103", size = 68071, upload-time = "2026-09-23T07:47:22.372Z" } wheels = [ @@ -3877,7 +3877,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -4670,7 +4670,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.0,!=2.4.0,!=2.4.1,!=2.12.0,!=2.12.1,<3.0" }, { name = "pyiceberg-core", marker = "extra == 'pyiceberg-core'", specifier = ">=0.10.1,<0.11.0" }, { name = "pyparsing", specifier = ">=3.1.0" }, - { name = "pyroaring", specifier = ">=1.0.0" }, + { name = "pyroaring", specifier = ">=1.1.0" }, { name = "python-snappy", marker = "extra == 'snappy'", specifier = ">=0.6.0" }, { name = "ray", marker = "extra == 'ray'", specifier = ">=2.10.0" }, { name = "requests", specifier = ">=2.20.0" },