From 2b0925acfb6ff6626ff51cdb53b445b4ca70d3e0 Mon Sep 17 00:00:00 2001 From: Ankit Kumar Date: Thu, 12 Nov 2026 22:48:35 +0530 Subject: [PATCH] fix(inspect): resolve dropped partition source columns in specs_struct When a partition column is removed and subsequently dropped from the table schema, historical partition specs still reference the dropped source_id. specs_struct() previously only looked up field IDs in the current schema, causing inspect metadata tables to fail with ValueError and manifests partition summaries to lose bound values. This updates specs_struct() to fall back to historical schemas when resolving source column types, and updates inspect manifest generation to look up partition types from specs_struct(). --- pyiceberg/table/inspect.py | 14 ++++++-- pyiceberg/table/metadata.py | 16 ++++++--- tests/table/test_inspect.py | 69 ++++++++++++++++++++++++++++++++++++ tests/table/test_metadata.py | 50 ++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 6 deletions(-) diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py index d51079fc9d..2cb942e41c 100644 --- a/pyiceberg/table/inspect.py +++ b/pyiceberg/table/inspect.py @@ -26,7 +26,7 @@ from pyiceberg.manifest import DataFile, DataFileContent, ManifestContent, ManifestFile, PartitionFieldSummary from pyiceberg.partitioning import PartitionSpec from pyiceberg.table.snapshots import Snapshot, ancestors_of -from pyiceberg.types import PrimitiveType +from pyiceberg.types import IcebergType, PrimitiveType from pyiceberg.utils.concurrent import ExecutorFactory from pyiceberg.utils.singleton import _convert_to_hashable_type @@ -553,13 +553,23 @@ def _generate_manifests_table(self, snapshot: Snapshot | None, is_all_manifests_ """ import pyarrow as pa + partition_types_by_spec_id: dict[int, dict[int, IcebergType]] = {} + def _partition_summaries_to_rows( spec: PartitionSpec, partition_summaries: list[PartitionFieldSummary] ) -> list[dict[str, Any]]: + partition_field_types = partition_types_by_spec_id.get(spec.spec_id) + if partition_field_types is None: + partition_field_types = { + struct_field.field_id: struct_field.field_type + for struct_field in self.tbl.metadata.specs_struct([spec.spec_id]).fields + } + partition_types_by_spec_id[spec.spec_id] = partition_field_types + rows = [] for i, field_summary in enumerate(partition_summaries): field = spec.fields[i] - partition_field_type = spec.partition_type(self.tbl.schema()).fields[i].field_type + partition_field_type = partition_field_types[field.field_id] lower_bound = ( ( field.transform.to_human_string( diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index e47faaf672..c0051db754 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -45,7 +45,7 @@ IcebergRootModel, Properties, ) -from pyiceberg.types import NestedField, StructType, transform_dict_value_to_str +from pyiceberg.types import IcebergType, NestedField, StructType, UnknownType, transform_dict_value_to_str from pyiceberg.utils.config import Config from pyiceberg.utils.datetime import datetime_to_millis @@ -298,6 +298,16 @@ def specs(self) -> dict[int, PartitionSpec]: """Return a dict the partition specs this table.""" return {spec.spec_id: spec for spec in self.partition_specs} + def _resolve_field_type(self, field_id: int) -> IcebergType: + """Resolve a field's type from the current schema or historical schemas.""" + if field := self.schema()._lazy_id_to_field.get(field_id): + return field.field_type + # Historical partition specs may reference columns dropped in later schema evolutions. + for schema in reversed(self.schemas): + if field := schema._lazy_id_to_field.get(field_id): + return field.field_type + return UnknownType() + def specs_struct(self, spec_ids: Iterable[int] | None = None) -> StructType: """Produce a struct of the combined PartitionSpecs. @@ -316,13 +326,11 @@ def specs_struct(self, spec_ids: Iterable[int] | None = None) -> StructType: # Collect all the fields struct_fields = {field.field_id: field for spec in selected_specs for field in spec.fields} - schema = self.schema() - nested_fields = [] # Sort them by field_id in order to get a deterministic output for field_id in sorted(struct_fields): field = struct_fields[field_id] - source_type = schema.find_type(field.source_id) + source_type = self._resolve_field_type(field.source_id) result_type = field.transform.result_type(source_type) nested_fields.append(NestedField(field_id=field.field_id, name=field.name, type=result_type, required=False)) diff --git a/tests/table/test_inspect.py b/tests/table/test_inspect.py index 306579754a..68f306ba15 100644 --- a/tests/table/test_inspect.py +++ b/tests/table/test_inspect.py @@ -138,3 +138,72 @@ def test_inspect_bounds_after_type_promotion(catalog: InMemoryCatalog) -> None: partition_summaries = tbl.inspect.manifests().to_pydict()["partition_summaries"][0] assert partition_summaries[0]["lower_bound"] == str(values[0]), name assert partition_summaries[0]["upper_bound"] == str(values[-1]), name + + +def test_inspect_after_dropping_old_partition_column(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(1, "id", LongType(), required=False), + NestedField(2, "p", StringType(), required=False), + NestedField(3, "q", StringType(), required=False), + ) + spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p")) + tbl = catalog.create_table("default.dropped_partition_col", schema, partition_spec=spec) + tbl.append(pa.table({"id": [1, 2], "p": ["a", "b"], "q": ["x", "y"]})) + + with tbl.update_spec() as spec_update: + spec_update.remove_field("p") + with tbl.update_schema() as schema_update: + schema_update.delete_column("p") + + assert tbl.inspect.partitions().to_pydict()["partition"] == [{"p": "a"}, {"p": "b"}] + for table_view in [ + tbl.inspect.entries(), + tbl.inspect.files(), + tbl.inspect.data_files(), + tbl.inspect.all_files(), + tbl.inspect.all_data_files(), + ]: + assert table_view.num_rows == 2 + assert tbl.inspect.delete_files().num_rows == 0 + assert tbl.inspect.all_delete_files().num_rows == 0 + + for manifest_table in [tbl.inspect.manifests(), tbl.inspect.all_manifests()]: + assert manifest_table.to_pydict()["partition_summaries"][0] == [ + {"contains_null": False, "contains_nan": False, "lower_bound": "a", "upper_bound": "b"} + ] + + # Appending under an unpartitioned spec and a newly evolved partition column also stays readable + tbl.append(pa.table({"id": [3], "q": ["z"]})) + with tbl.update_spec() as spec_update: + spec_update.add_identity("q") + tbl.append(pa.table({"id": [4], "q": ["w"]})) + + assert sorted( + [(row["p"], row["q"]) for row in tbl.inspect.partitions().to_pydict()["partition"]], + key=lambda item: (item[0] or "", item[1] or ""), + ) == [(None, None), (None, "w"), ("a", None), ("b", None)] + assert tbl.inspect.entries().num_rows == 4 + assert tbl.inspect.files().num_rows == 4 + assert tbl.inspect.all_files().num_rows == 4 + + +def test_inspect_after_promoting_then_dropping_partition_column(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(1, "id", LongType(), required=False), + NestedField(2, "v", IntegerType(), required=False), + ) + spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "v")) + tbl = catalog.create_table("default.promoted_then_dropped", schema, partition_spec=spec) + tbl.append(pa.table({"id": pa.array([1, 2], pa.int64()), "v": pa.array([10, 20], pa.int32())})) + + with tbl.update_schema() as schema_update: + schema_update.update_column("v", field_type=LongType()) + with tbl.update_spec() as spec_update: + spec_update.remove_field("v") + with tbl.update_schema() as schema_update: + schema_update.delete_column("v") + + assert sorted(row["v"] for row in tbl.inspect.partitions().to_pydict()["partition"]) == [10, 20] + assert tbl.inspect.manifests().to_pydict()["partition_summaries"][0] == [ + {"contains_null": False, "contains_nan": False, "lower_bound": "10", "upper_bound": "20"} + ] diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index fb20f9726a..4a862a8298 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -992,3 +992,53 @@ def test_v3_metadata_with_encryption_keys_serializes_them(example_table_metadata assert table_metadata.model_dump(mode="json")["encryption-keys"] == [ {"key-id": "a", "encrypted-key-metadata": "a2V5", "properties": {}} ] + + +def test_specs_struct_resolves_dropped_partition_source_from_historical_schemas() -> None: + initial_schema = Schema( + NestedField(1, "id", LongType()), + NestedField(2, "p", StringType()), + NestedField(3, "v", IntegerType()), + schema_id=0, + ) + promoted_schema = Schema( + NestedField(1, "id", LongType()), + NestedField(2, "p", StringType()), + NestedField(3, "v", LongType()), + schema_id=1, + ) + dropped_schema = Schema(NestedField(1, "id", LongType()), schema_id=2) + + spec_p = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p"), spec_id=0) + spec_v = PartitionSpec(PartitionField(3, 1001, IdentityTransform(), "v"), spec_id=1) + + table_metadata = TableMetadataV2( + location="s3://bucket/test/location", + schemas=[initial_schema, promoted_schema, dropped_schema], + current_schema_id=2, + partition_specs=[spec_p, spec_v, PartitionSpec(spec_id=2)], + default_spec_id=2, + last_column_id=3, + last_partition_id=1001, + ) + + assert table_metadata.specs_struct() == StructType( + NestedField(1000, "p", StringType(), required=False), + NestedField(1001, "v", LongType(), required=False), + ) + + +def test_specs_struct_falls_back_to_unknown_when_source_missing_from_all_schemas() -> None: + table_metadata = TableMetadataV2( + location="s3://bucket/test/location", + schemas=[Schema(NestedField(1, "id", LongType()), schema_id=0)], + current_schema_id=0, + partition_specs=[PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p"), spec_id=0)], + default_spec_id=0, + last_column_id=2, + last_partition_id=1000, + ) + + assert table_metadata.specs_struct() == StructType( + NestedField(1000, "p", UnknownType(), required=False), + )