Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -1855,11 +1855,13 @@ def __init__(
limit: int | None = None,
*,
dictionary_columns: tuple[str, ...] = (),
row_filter_schema: Schema | None = None,
) -> None:
self._table_metadata = table_metadata
self._io = io
self._projected_schema = projected_schema
self._bound_row_filter = bind(table_metadata.schema(), row_filter, case_sensitive=case_sensitive)
filter_schema = row_filter_schema if row_filter_schema is not None else table_metadata.schema()
self._bound_row_filter = bind(filter_schema, row_filter, case_sensitive=case_sensitive)
self._case_sensitive = case_sensitive
self._limit = limit
self._downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE)
Expand Down
49 changes: 34 additions & 15 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
from pyiceberg.table.maintenance import MaintenanceTable
from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, TableMetadata
from pyiceberg.table.name_mapping import NameMapping
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType
from pyiceberg.table.snapshots import (
Operation,
Snapshot,
Expand Down Expand Up @@ -943,9 +943,7 @@ def upsert(

# We must use Transaction.table_metadata for the scan. This includes all uncommitted - but relevant - changes.

matched_iceberg_record_batches_scan = DataScan(
table_metadata=self.table_metadata,
io=self._table.io,
matched_iceberg_record_batches_scan = self._scan(
row_filter=matched_predicate,
case_sensitive=case_sensitive,
)
Expand Down Expand Up @@ -2127,6 +2125,9 @@ def __init__(
self.limit = limit
self.encryption_manager = encryption_manager

def _scan_schema(self) -> Schema:
return self.table_metadata.schema()

@abstractmethod
def plan_files(self) -> Iterable[ScanTask]: ...

Expand Down Expand Up @@ -2208,6 +2209,7 @@ class TableScan(BaseScan):
"""A base class for table scans targeting a single snapshot."""

snapshot_id: int | None
_branch_name: str | None
catalog: Catalog | None
table_identifier: Identifier | None
table_config: Properties
Expand All @@ -2226,6 +2228,8 @@ def __init__(
table_identifier: Identifier | None = None,
table_config: Properties = EMPTY_DICT,
encryption_manager: EncryptionManager | None = None,
*,
_branch_name: str | None = None,
):
super().__init__(
table_metadata=table_metadata,
Expand All @@ -2238,6 +2242,7 @@ def __init__(
encryption_manager=encryption_manager,
)
self.snapshot_id = snapshot_id
self._branch_name = _branch_name
self.catalog = catalog
self.table_identifier = table_identifier
self.table_config = table_config
Expand All @@ -2247,9 +2252,9 @@ def snapshot(self) -> Snapshot | None:
return self.table_metadata.snapshot_by_id(self.snapshot_id)
return self.table_metadata.current_snapshot()

def projection(self) -> Schema:
def _scan_schema(self) -> Schema:
current_schema = self.table_metadata.schema()
if self.snapshot_id is not None:
if self.snapshot_id is not None and self._branch_name is None:
snapshot = self.table_metadata.snapshot_by_id(self.snapshot_id)
if snapshot is not None:
if snapshot.schema_id is not None:
Expand All @@ -2261,17 +2266,26 @@ def projection(self) -> Schema:
warnings.warn(f"Metadata does not contain schema with id: {snapshot.schema_id}", stacklevel=2)
else:
raise ValueError(f"Snapshot not found: {self.snapshot_id}")
return current_schema

def projection(self) -> Schema:
current_schema = self._scan_schema()

if "*" in self.selected_fields:
return current_schema

return current_schema.select(*self.selected_fields, case_sensitive=self.case_sensitive)

def use_ref(self: S, name: str) -> S:
"""Select a branch using the table schema or a tag using its snapshot schema."""
if self.snapshot_id is not None:
raise ValueError(f"Cannot override ref, already set snapshot id={self.snapshot_id}")
if snapshot := self.table_metadata.snapshot_by_name(name):
return self.update(snapshot_id=snapshot.snapshot_id)
ref = self.table_metadata.refs[name]
return self.update(
snapshot_id=snapshot.snapshot_id,
_branch_name=name if ref.snapshot_ref_type == SnapshotRefType.BRANCH else None,
)

raise ValueError(f"Cannot scan unknown ref={name}")

Expand Down Expand Up @@ -2426,6 +2440,7 @@ def _to_arrow_via_file_scan_tasks(
scan.case_sensitive,
scan.limit,
dictionary_columns=dictionary_columns,
row_filter_schema=scan._scan_schema(),
).to_table(tasks)


Expand All @@ -2446,6 +2461,7 @@ def _to_arrow_batch_reader_via_file_scan_tasks(
scan.case_sensitive,
scan.limit,
dictionary_columns=dictionary_columns,
row_filter_schema=scan._scan_schema(),
).to_record_batches(tasks)

if dictionary_columns:
Expand Down Expand Up @@ -2475,6 +2491,7 @@ def _manifest_planner(self) -> ManifestGroupPlanner:
row_filter=self.row_filter,
case_sensitive=self.case_sensitive,
options=self.options,
schema=self._scan_schema(),
)

@cached_property
Expand Down Expand Up @@ -2609,6 +2626,7 @@ def count(self) -> int:
projected_schema=self.projection(),
row_filter=self.row_filter,
case_sensitive=self.case_sensitive,
row_filter_schema=self._scan_schema(),
)
tbl = arrow_scan.to_table([task])
res += len(tbl)
Expand Down Expand Up @@ -2816,6 +2834,7 @@ class ManifestGroupPlanner:
row_filter: BooleanExpression
case_sensitive: bool
options: Properties
schema: Schema

def __init__(
self,
Expand All @@ -2824,12 +2843,14 @@ def __init__(
row_filter: str | BooleanExpression = ALWAYS_TRUE,
case_sensitive: bool = True,
options: Properties = EMPTY_DICT,
schema: Schema | None = None,
):
self.table_metadata = table_metadata
self.io = io
self.row_filter = _parse_row_filter(row_filter)
self.case_sensitive = case_sensitive
self.options = options
self.schema = schema if schema is not None else table_metadata.schema()

@cached_property
def partition_filters(self) -> KeyDefaultDict[int, BooleanExpression]:
Expand Down Expand Up @@ -2891,9 +2912,7 @@ def plan_files(
delete_index = DeleteFileIndex()

residual_evaluators: dict[int, ResidualEvaluator] = KeyDefaultDict(self._build_residual_evaluator)
referenced_field_ids = extract_field_ids(
bind(self.table_metadata.schema(), self.row_filter, case_sensitive=self.case_sensitive)
)
referenced_field_ids = extract_field_ids(bind(self.schema, self.row_filter, case_sensitive=self.case_sensitive))
partition_specs = self.table_metadata.specs()
residual_cache_key_positions: dict[int, tuple[int, ...]] = KeyDefaultDict(
lambda spec_id: tuple(
Expand Down Expand Up @@ -2952,16 +2971,16 @@ def residual_for(data_file: DataFile) -> BooleanExpression:
]

def _build_partition_projection(self, spec_id: int) -> BooleanExpression:
project = inclusive_projection(self.table_metadata.schema(), self.table_metadata.specs()[spec_id], self.case_sensitive)
project = inclusive_projection(self.schema, self.table_metadata.specs()[spec_id], self.case_sensitive)
return project(self.row_filter)

def _build_manifest_evaluator(self, spec_id: int) -> Callable[[ManifestFile], bool]:
spec = self.table_metadata.specs()[spec_id]
return manifest_evaluator(spec, self.table_metadata.schema(), self.partition_filters[spec_id], self.case_sensitive)
return manifest_evaluator(spec, self.schema, self.partition_filters[spec_id], self.case_sensitive)

def _build_partition_evaluator(self, spec_id: int) -> Callable[[DataFile], bool]:
spec = self.table_metadata.specs()[spec_id]
partition_type = spec.partition_type(self.table_metadata.schema())
partition_type = spec.partition_type(self.schema)
partition_schema = Schema(*partition_type.fields)
partition_expr = self.partition_filters[spec_id]
evaluator = expression_evaluator(partition_schema, partition_expr, self.case_sensitive)
Expand All @@ -2971,7 +2990,7 @@ def _build_partition_evaluator(self, spec_id: int) -> Callable[[DataFile], bool]
return lambda data_file: evaluator(data_file.partition)

def _build_metrics_evaluator(self) -> Callable[[DataFile], bool]:
schema = self.table_metadata.schema()
schema = self.schema
include_empty_files = strtobool(self.options.get("include_empty_files", "false"))

# Metrics evaluators keep file-specific state local to each call, so one
Expand All @@ -2992,7 +3011,7 @@ def _build_residual_evaluator(self, spec_id: int) -> ResidualEvaluator:
spec=spec,
expr=self.row_filter,
case_sensitive=self.case_sensitive,
schema=self.table_metadata.schema(),
schema=self.schema,
)

@staticmethod
Expand Down
2 changes: 1 addition & 1 deletion pyiceberg/table/inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,7 @@ def partitions(
io=self.tbl.io,
row_filter=row_filter,
case_sensitive=case_sensitive,
snapshot_id=snapshot.snapshot_id,
snapshot_id=snapshot_id,
encryption_manager=self.tbl.encryption_manager,
)

Expand Down
27 changes: 16 additions & 11 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -1440,6 +1440,20 @@ def test_identity_transform_column_projection(tmp_path: str, catalog: InMemoryCa
).to_arrow() == pa.table({"other_field": ["bar"]})
assert table.scan(row_filter="partition_id = 1 AND other_field = 'bar'", selected_fields=("other_field",)).count() == 1

snapshot = table.current_snapshot()
assert snapshot is not None
with table.update_schema() as update:
update.rename_column("partition_id", "renamed_partition_id")

scan = table.scan(
snapshot_id=snapshot.snapshot_id,
row_filter="partition_id = 1 AND other_field = 'bar'",
selected_fields=("other_field",),
)
assert scan.to_arrow() == pa.table({"other_field": ["bar"]})
assert scan.to_arrow_batch_reader().read_all().to_pydict() == {"other_field": ["bar"]}
assert scan.count() == 1


@pytest.mark.parametrize(
"partition_field_type, arrow_partition_type, partition_value, row_filter",
Expand Down Expand Up @@ -5976,10 +5990,9 @@ def test_to_arrow_batch_reader_preserves_dictionary_columns(tmpdir: str) -> None
# RecordBatchReader.cast(target_schema) was called with a plain schema, silently converting
# dictionary arrays back to their value type so to_arrow_batch_reader(dictionary_columns=...)
# .read_all() returned plain strings instead of dictionary-encoded arrays.
from pyiceberg.expressions import AlwaysTrue
from pyiceberg.io.pyarrow import PyArrowFileIO
from pyiceberg.partitioning import PartitionSpec
from pyiceberg.table import FileScanTask, _to_arrow_batch_reader_via_file_scan_tasks
from pyiceberg.table import DataScan, FileScanTask, _to_arrow_batch_reader_via_file_scan_tasks
from pyiceberg.table.metadata import TableMetadataV2

arrow_schema = pa.schema(
Expand Down Expand Up @@ -6007,17 +6020,9 @@ def test_to_arrow_batch_reader_preserves_dictionary_columns(tmpdir: str) -> None
partition_specs=[PartitionSpec()],
)

class _MockScan:
def __init__(self) -> None:
self.table_metadata = table_metadata
self.io = PyArrowFileIO()
self.row_filter = AlwaysTrue()
self.case_sensitive = True
self.limit = None

tasks = [FileScanTask(data_file)]
result = _to_arrow_batch_reader_via_file_scan_tasks(
_MockScan(), # type: ignore[arg-type]
DataScan(table_metadata=table_metadata, io=PyArrowFileIO()),
iceberg_schema,
tasks,
dictionary_columns=("label",),
Expand Down
Loading
Loading