From 2a571d9ce9e09b823f459d622bc98da89d03376d Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Tue, 10 Nov 2026 23:52:07 +0000 Subject: [PATCH 1/4] Fix append to a table partitioned by bucket on a time column The bucket transform returns an int, but the time partition representation only accepted datetime.time, so every append raised AttributeError. Accept the int, as the date, timestamp and uuid handlers already do. Co-authored-by: Isaac --- pyiceberg/partitioning.py | 11 +++++++++-- tests/io/test_pyarrow.py | 23 ++++++++++++++++++++++- tests/test_transforms.py | 11 ++++++++++- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/pyiceberg/partitioning.py b/pyiceberg/partitioning.py index 3d06287b6c..cb648f8798 100644 --- a/pyiceberg/partitioning.py +++ b/pyiceberg/partitioning.py @@ -546,8 +546,15 @@ def _(type: IcebergType, value: int | date | None) -> int | None: @_to_partition_representation.register(TimeType) -def _(type: IcebergType, value: time | None) -> int | None: - return time_to_micros(value) if value is not None else None +def _(type: IcebergType, value: int | time | None) -> int | None: + if value is None: + return None + elif isinstance(value, int): + return value + elif isinstance(value, time): + return time_to_micros(value) + else: + raise ValueError(f"Type not recognized: {value}") @_to_partition_representation.register(UUIDType) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 419d130965..e856d82dc7 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -26,7 +26,7 @@ import zlib from collections.abc import Iterator from dataclasses import replace -from datetime import UTC, date, datetime +from datetime import UTC, date, datetime, time from pathlib import Path from typing import Any, Literal from unittest.mock import MagicMock, patch @@ -3430,6 +3430,27 @@ def test_append_view_types_to_partitioned_table(catalog: InMemoryCatalog) -> Non assert sorted(table.scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == arrow_table.to_pylist() +def test_append_to_table_bucketed_by_time(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=LongType(), required=False), + NestedField(id=2, name="event_time", field_type=TimeType(), required=False), + ) + spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=BucketTransform(4), name="event_time_bucket")) + catalog.create_namespace("default") + table = catalog.create_table("default.test_append_bucketed_by_time", schema=schema, partition_spec=spec) + + arrow_table = pa.table( + { + "id": pa.array([1, 2, 3], type=pa.int64()), + "event_time": pa.array([time(9, 30), time(22, 31, 8), None], type=pa.time64("us")), + } + ) + table.append(arrow_table) + + assert sorted(table.scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == arrow_table.to_pylist() + assert table.scan(row_filter=EqualTo("event_time", "22:31:08")).to_arrow()["id"].to_pylist() == [2] + + def test_inspect_partition_for_nested_field(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="foo", field_type=StringType(), required=True), diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 998d8a224d..a8188525d0 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -17,7 +17,7 @@ # under the License. # pylint: disable=eval-used,protected-access,redefined-outer-name from collections.abc import Callable -from datetime import date, datetime +from datetime import date, datetime, time from decimal import Decimal from typing import Annotated, Any from uuid import UUID @@ -1643,11 +1643,20 @@ def test_to_partition_representation_timestamps(source_type: PrimitiveType, valu assert _to_partition_representation(source_type, value) == expected +def test_to_partition_representation_time() -> None: + assert _to_partition_representation(TimeType(), None) is None + assert _to_partition_representation(TimeType(), time(22, 31, 8)) == 81068000000 + # The bucket transform returns an int, which is already the partition representation + assert _to_partition_representation(TimeType(), 3) == 3 + + def test_to_partition_representation_unrecognized_type_raises() -> None: with pytest.raises(ValueError, match="Type not recognized"): _to_partition_representation(TimestampType(), "not-a-datetime-or-int") with pytest.raises(ValueError, match="Type not recognized"): _to_partition_representation(TimestampNanoType(), "not-a-datetime-or-int") + with pytest.raises(ValueError, match="Type not recognized"): + _to_partition_representation(TimeType(), "not-a-time-or-int") @pytest.mark.parametrize( From 1b46a1547502dbd2782326264354818518743036 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Wed, 11 Nov 2026 17:08:17 +0000 Subject: [PATCH 2/4] Retrigger CI Co-authored-by: Isaac From 8144013bd942fa5d9b49e8bfe2a6a629653abaa6 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Wed, 11 Nov 2026 18:35:04 +0000 Subject: [PATCH 3/4] Retrigger CI Co-authored-by: Isaac From 6969de8459fdd51eabae9381e950ef8cb123e012 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Thu, 12 Nov 2026 07:02:20 +0000 Subject: [PATCH 4/4] Retrigger CI Co-authored-by: Isaac