Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions pyiceberg/table/inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType:

partition_record = self.tbl.metadata.specs_struct()
pa_record_struct = schema_to_pyarrow(partition_record)
id_to_name = {field.field_id: field.name for field in partition_record.fields}

entries_schema = pa.schema(
[
Expand Down Expand Up @@ -255,7 +256,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType:

partition = entry.data_file.partition
partition_record_dict = {
field.name: partition[pos]
id_to_name.get(field.field_id, field.name): partition[pos]
for pos, field in enumerate(self.tbl.metadata.specs()[manifest.partition_spec_id].fields)
}

Expand Down Expand Up @@ -386,6 +387,7 @@ def partitions(
spec_ids = {manifest.partition_spec_id for manifest in snapshot.manifests(self.tbl.io, self.tbl.encryption_manager)}
partition_record = self.tbl.metadata.specs_struct(spec_ids=spec_ids)
has_partitions = len(partition_record.fields) > 0
id_to_name = {field.field_id: field.name for field in partition_record.fields}

if has_partitions:
pa_record_struct = schema_to_pyarrow(partition_record)
Expand All @@ -407,13 +409,14 @@ def partitions(
encryption_manager=self.tbl.encryption_manager,
)

partitions_map: dict[tuple[str, Any], Any] = {}
partitions_map: dict[tuple[int, Any], Any] = {}
snapshots_by_id = self._get_snapshots_by_id()

for entry in itertools.chain.from_iterable(scan._plan_manifest_entries()):
partition = entry.data_file.partition
partition_record_dict = {
field.name: partition[pos] for pos, field in enumerate(self.tbl.metadata.specs()[entry.data_file.spec_id].fields)
id_to_name.get(field.field_id, field.name): partition[pos]
for pos, field in enumerate(self.tbl.metadata.specs()[entry.data_file.spec_id].fields)
}
entry_snapshot = snapshots_by_id.get(entry.snapshot_id) if entry.snapshot_id is not None else None
self._update_partitions_map_from_manifest_entry(
Expand All @@ -427,7 +430,7 @@ def partitions(

def _update_partitions_map_from_manifest_entry(
self,
partitions_map: dict[tuple[str, Any], Any],
partitions_map: dict[tuple[int, Any], Any],
file: DataFile,
partition_record_dict: dict[str, Any],
snapshot: Snapshot | None,
Expand All @@ -444,7 +447,7 @@ def _update_partitions_map_from_manifest_entry(
partition_record_dict: Maps partition field names to their values.
snapshot: The owning snapshot, used to track last-updated timestamps.
"""
partition_record_key = _convert_to_hashable_type(partition_record_dict)
partition_record_key = (file.spec_id, _convert_to_hashable_type(partition_record_dict))
if partition_record_key not in partitions_map:
partitions_map[partition_record_key] = {
"partition": partition_record_dict,
Expand Down Expand Up @@ -737,6 +740,7 @@ def _get_files_from_manifest(

files: list[dict[str, Any]] = []
schema = self.tbl.metadata.schema()
id_to_name = {field.field_id: field.name for field in self.tbl.metadata.specs_struct().fields}
io = self.tbl.io

for manifest_entry in manifest_list.fetch_manifest_entry(io):
Expand All @@ -762,7 +766,7 @@ def _get_files_from_manifest(
}
partition = data_file.partition
partition_record_dict = {
field.name: partition[pos]
id_to_name.get(field.field_id, field.name): partition[pos]
for pos, field in enumerate(self.tbl.metadata.specs()[manifest_list.partition_spec_id].fields)
}
files.append(
Expand Down
58 changes: 57 additions & 1 deletion tests/table/test_inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def test_partitions_last_updated_uses_latest_snapshot_regardless_of_order(newest
data_file.spec_id = 0

inspect = InspectTable.__new__(InspectTable)
partitions_map: dict[tuple[str, Any], Any] = {}
partitions_map: dict[tuple[int, Any], Any] = {}
for snapshot in [newer, older] if newest_first else [older, newer]:
inspect._update_partitions_map_from_manifest_entry(partitions_map, data_file, {"part": "a"}, snapshot)

Expand Down Expand Up @@ -138,3 +138,59 @@ def test_inspect_bounds_after_type_promotion(catalog: InMemoryCatalog) -> None:
partition_summaries = tbl.inspect.manifests().to_pydict()["partition_summaries"][0]
assert partition_summaries[0]["lower_bound"] == str(values[0]), name
assert partition_summaries[0]["upper_bound"] == str(values[-1]), name


def test_inspect_partitions_and_files_after_rename_field(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(1, "id", LongType(), required=False),
NestedField(2, "p", StringType(), required=False),
)
spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p"))
tbl = catalog.create_table("default.renamed_partition_col", schema, partition_spec=spec)
tbl.append(pa.table({"id": [1, 2], "p": ["a", "b"]}))

with tbl.update_spec() as spec_update:
spec_update.rename_field("p", "p_renamed")
tbl.append(pa.table({"id": [3], "p": ["c"]}))

partitions = tbl.inspect.partitions().to_pydict()
assert sorted(part["p_renamed"] for part in partitions["partition"]) == ["a", "b", "c"]
assert set(partitions["spec_id"]) == {0, 1}

for files_table in [
tbl.inspect.files(),
tbl.inspect.data_files(),
tbl.inspect.all_files(),
tbl.inspect.all_data_files(),
]:
assert sorted(part["p_renamed"] for part in files_table.to_pydict()["partition"]) == ["a", "b", "c"]

entries = tbl.inspect.entries().to_pydict()
assert sorted(df["partition"]["p_renamed"] for df in entries["data_file"]) == ["a", "b", "c"]


def test_inspect_files_after_v1_partition_field_readd(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(1, "id", LongType(), required=False),
NestedField(2, "p", StringType(), required=False),
)
spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p"))
tbl = catalog.create_table("default.v1_partition_readd", schema, partition_spec=spec, properties={"format-version": "1"})
tbl.append(pa.table({"id": [1, 2], "p": ["a", "b"]}))

with tbl.update_spec() as spec_update:
spec_update.remove_field("p")
with tbl.update_spec() as spec_update:
spec_update.add_identity("p")

# In v1, the voided field is renamed to p_1000 and the new identity field is assigned id 1001 with name p.
# Pre-existing data files from spec 0 belong to field 1000 (p_1000), not the newly added field 1001 (p).
for files_table in [tbl.inspect.files(), tbl.inspect.all_files()]:
partitions = files_table.to_pydict()["partition"]
assert sorted(part["p_1000"] for part in partitions) == ["a", "b"]
assert all(part["p"] is None for part in partitions)

entries = tbl.inspect.entries().to_pydict()
entry_partitions = [df["partition"] for df in entries["data_file"]]
assert sorted(part["p_1000"] for part in entry_partitions) == ["a", "b"]
assert all(part["p"] is None for part in entry_partitions)
Loading