diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index 74925c695f..71c0cb3d9b 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -132,12 +132,15 @@ def _cast_if_needed(arr: "ArrayLike") -> "ArrayLike": else: return arr + def _decode_if_dictionary(arr: "ArrayLike") -> "ArrayLike": + return arr.dictionary_decode() if pa.types.is_dictionary(arr.type) else arr + if isinstance(array, pa.Array): - return _cast_if_needed(transform_func(array, *args)) + return _cast_if_needed(transform_func(_decode_if_dictionary(array), *args)) elif isinstance(array, pa.ChunkedArray): result_chunks = [] for arr in array.iterchunks(): - result_chunks.append(_cast_if_needed(transform_func(arr, *args))) + result_chunks.append(_cast_if_needed(transform_func(_decode_if_dictionary(arr), *args))) return pa.chunked_array(result_chunks) else: raise ValueError(f"PyArrow array can only be of type pa.Array or pa.ChunkedArray, but found {type(array)}") diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index 1ed9ea4096..02700f7b21 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -338,6 +338,24 @@ def test_write_pyarrow_schema(catalog: Catalog, test_table_identifier: Identifie table.append(pyarrow_table) +def test_write_dictionary_encoded_column_with_bucket_partition(catalog: Catalog) -> None: + catalog.create_namespace("default") + schema = pa.schema([pa.field("id", pa.int64()), pa.field("s", pa.string())]) + table = catalog.create_table("default.dictionary_bucket", schema=schema) + with table.update_spec() as update: + update.add_field("s", "bucket[8]") + + plain = pa.table({"id": [1, 2], "s": ["a", "b"]}) + encoded = plain.cast(pa.schema([pa.field("id", pa.int64()), pa.field("s", pa.dictionary(pa.int32(), pa.string()))])) + table.append(encoded) + + assert sorted(table.scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == [ + {"id": 1, "s": "a"}, + {"id": 2, "s": "b"}, + ] + assert len(table.inspect.partitions()) == 2 + + @pytest.mark.parametrize( "schema,expected", [ diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 998d8a224d..a8dca09a3a 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -1701,6 +1701,24 @@ def test_bucket_pyarrow_transforms( assert expected == transform.pyarrow_transform(source_type)(input_arr) +def test_pyarrow_transform_decodes_dictionary_arrays() -> None: + transform: Transform[Any, Any] = BucketTransform(num_buckets=8) + plain = pa.array(["a", "b", None]) + encoded = plain.dictionary_encode() + + assert ( + transform.pyarrow_transform(StringType())(encoded).to_pylist() + == transform.pyarrow_transform(StringType())(plain).to_pylist() + ) + + chunked_encoded = pa.chunked_array([encoded, pa.array(["c"]).dictionary_encode()]) + chunked_plain = pa.chunked_array([plain, pa.array(["c"])]) + assert ( + transform.pyarrow_transform(StringType())(chunked_encoded).to_pylist() + == transform.pyarrow_transform(StringType())(chunked_plain).to_pylist() + ) + + def test_bucket_pyarrow_void_transform() -> None: input_arr = pa.chunked_array([pa.array([1, 2], type=pa.int32()), pa.array([3, 4], type=pa.int32())]) output_arr = pa.array([None, None, None, None], type=pa.int32())