Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 32 additions & 1 deletion pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -1848,6 +1848,32 @@ def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[st
return deletes_per_file


def _decode_unrequested_dictionary_columns(batch: pa.RecordBatch, dictionary_columns: set[str]) -> pa.RecordBatch:
"""Decode dictionary-encoded columns that were not requested as dictionary encoded.

Files with dictionary-encoded columns and files with plain columns produce different batch
schemas, which cannot be concatenated. Decoding the columns that were not requested as
dictionaries gives the batches one logical schema, while requested dictionary columns stay as
they are.
"""
to_decode = [field for field in batch.schema if pa.types.is_dictionary(field.type) and field.name not in dictionary_columns]
if not to_decode:
return batch

decode_names = {field.name for field in to_decode}
schema = pa.schema(
[
pa.field(field.name, field.type.value_type, field.nullable, field.metadata) if field.name in decode_names else field
for field in batch.schema
]
)
arrays = [
batch.column(i).dictionary_decode() if field.name in decode_names else batch.column(i)
for i, field in enumerate(batch.schema)
]
return pa.RecordBatch.from_arrays(arrays, schema=schema)


class ArrowScan:
_table_metadata: TableMetadata
_io: FileIO
Expand Down Expand Up @@ -1927,8 +1953,13 @@ def to_table(self, tasks: Iterable[FileScanTask]) -> pa.Table:

# Note: cannot use pa.Table.from_batches(itertools.chain([first_batch], batches)))
# as different batches can use different schema's (due to large_ types)
dictionary_columns = set(self._dictionary_columns)
result = pa.concat_tables(
(pa.Table.from_batches([batch]) for batch in itertools.chain([first_batch], batches)), promote_options="permissive"
(
pa.Table.from_batches([_decode_unrequested_dictionary_columns(batch, dictionary_columns)])
for batch in itertools.chain([first_batch], batches)
),
promote_options="permissive",
)

return result
Expand Down
12 changes: 12 additions & 0 deletions tests/catalog/test_catalog_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,18 @@ def test_write_pyarrow_schema(catalog: Catalog, test_table_identifier: Identifie
table.append(pyarrow_table)


def test_scan_mixed_dictionary_and_plain_string_batches(catalog: Catalog) -> None:
catalog.create_namespace("default")
table = catalog.create_table("default.mixed_dictionary", schema=pa.schema([pa.field("s", pa.string())]))

table.append(pa.table({"s": pa.array(["a"]).dictionary_encode()}))
table.append(pa.table({"s": pa.array(["b"])}))

scanned = table.scan().to_arrow()
assert scanned.schema.field("s").type == pa.string()
assert sorted(scanned["s"].to_pylist()) == ["a", "b"]


@pytest.mark.parametrize(
"schema,expected",
[
Expand Down
Loading