diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index f62e53a911..1817aee2d4 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -3205,8 +3205,13 @@ def _determine_partitions(spec: PartitionSpec, schema: Schema, arrow_table: pa.T arrow_table = arrow_table.cast(arrow_schema) # Assign unique names to columns where the partition transform has been applied - # to avoid conflicts - partition_fields = [f"_partition_{field.name}" for field in spec.fields] + # to avoid conflicts with the data columns + partition_fields: list[str] = [] + for field in spec.fields: + name = f"_partition_{field.name}" + while name in arrow_table.column_names or name in partition_fields: + name = f"_{name}" + partition_fields.append(name) for partition, name in zip(spec.fields, partition_fields, strict=True): source_field = schema.find_field(partition.source_id) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 419d130965..cf2476002c 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3353,6 +3353,32 @@ def test_determine_partitions_identity_nan() -> None: assert rows_by_partition == {"1.0": [1], "nan": [2, 4], "None": [3]} +def test_determine_partitions_with_column_named_like_partition_helper() -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=LongType(), required=False), + NestedField(id=2, name="_partition_region", field_type=StringType(), required=False), + NestedField(id=3, name="region", field_type=StringType(), required=False), + ) + spec = PartitionSpec(PartitionField(source_id=3, field_id=1000, transform=IdentityTransform(), name="region")) + arrow_table = pa.Table.from_pydict( + {"id": [1, 2, 3], "_partition_region": ["a", "b", "c"], "region": ["us", "eu", "us"]}, + schema=schema.as_arrow(), + ) + + partitions = list(_determine_partitions(spec, schema, arrow_table)) + + rows_by_partition = { + p.partition_key.partition[0]: sorted(p.arrow_table_partition.to_pylist(), key=lambda row: row["id"]) for p in partitions + } + assert rows_by_partition == { + "us": [ + {"id": 1, "_partition_region": "a", "region": "us"}, + {"id": 3, "_partition_region": "c", "region": "us"}, + ], + "eu": [{"id": 2, "_partition_region": "b", "region": "eu"}], + } + + @pytest.mark.parametrize("transform", [IdentityTransform(), BucketTransform(4), TruncateTransform(1)]) def test_determine_partitions_with_view_types(transform: Transform[Any, Any]) -> None: schema = Schema(