From 0193ff4286523a00305faff8b58a1f04c0311948 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <192242476+ShivamSharma43@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:23:28 +0530 Subject: [PATCH] Remove partition statistics of removed snapshots RemoveSnapshotsUpdate dropped the statistics files of removed snapshots but kept their partition statistics files, so expire_snapshots left partition-statistics entries that point at snapshots that no longer exist. Iceberg Java removes both in TableMetadata.Builder. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012XPuJeHi7A5CeHSKHjHmwx --- pyiceberg/table/update/__init__.py | 7 ++++++- tests/table/test_expire_snapshots.py | 29 +++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pyiceberg/table/update/__init__.py b/pyiceberg/table/update/__init__.py index cba3832aae..4cd9feb729 100644 --- a/pyiceberg/table/update/__init__.py +++ b/pyiceberg/table/update/__init__.py @@ -564,7 +564,12 @@ def _(update: RemoveSnapshotsUpdate, base_metadata: TableMetadata, context: _Tab for statistics_file in base_metadata.statistics if statistics_file.snapshot_id in update.snapshot_ids ) - updates = itertools.chain(remove_ref_updates, remove_statistics_updates) + remove_partition_statistics_updates = ( + RemovePartitionStatisticsUpdate(snapshot_id=partition_statistics_file.snapshot_id) + for partition_statistics_file in base_metadata.partition_statistics + if partition_statistics_file.snapshot_id in update.snapshot_ids + ) + updates = itertools.chain(remove_ref_updates, remove_statistics_updates, remove_partition_statistics_updates) new_metadata = base_metadata for upd in updates: new_metadata = _apply_table_update(upd, new_metadata, context) diff --git a/tests/table/test_expire_snapshots.py b/tests/table/test_expire_snapshots.py index 106e5b786c..dd6b6cc2d2 100644 --- a/tests/table/test_expire_snapshots.py +++ b/tests/table/test_expire_snapshots.py @@ -22,7 +22,8 @@ import pytest from pyiceberg.table import CommitTableResponse, Table -from pyiceberg.table.update import RemoveSnapshotsUpdate, update_table_metadata +from pyiceberg.table.statistics import PartitionStatisticsFile +from pyiceberg.table.update import RemoveSnapshotsUpdate, SetPartitionStatisticsUpdate, update_table_metadata from pyiceberg.table.update.snapshot import ExpireSnapshots @@ -316,3 +317,29 @@ def test_update_remove_snapshots_with_statistics(table_v2_with_statistics: Table assert not any(stat.snapshot_id == REMOVE_SNAPSHOT for stat in new_metadata.statistics), ( "Statistics for removed snapshot should be gone" ) + + +def test_update_remove_snapshots_with_partition_statistics(table_v2: Table) -> None: + """Removing a snapshot also removes its partition statistics, as Iceberg Java does.""" + REMOVE_SNAPSHOT = 3051729675574597004 + KEEP_SNAPSHOT = 3055729675574597004 + + metadata = update_table_metadata( + table_v2.metadata, + tuple( + SetPartitionStatisticsUpdate( + partition_statistics=PartitionStatisticsFile( + snapshot_id=snapshot_id, + statistics_path=f"s3://bucket/warehouse/partition-stats-{snapshot_id}.parquet", + file_size_in_bytes=124, + ) + ) + for snapshot_id in (REMOVE_SNAPSHOT, KEEP_SNAPSHOT) + ), + ) + assert len(metadata.partition_statistics) == 2 + + new_metadata = update_table_metadata(metadata, (RemoveSnapshotsUpdate(snapshot_ids=[REMOVE_SNAPSHOT]),)) + + assert [snapshot.snapshot_id for snapshot in new_metadata.snapshots] == [KEEP_SNAPSHOT] + assert [stat.snapshot_id for stat in new_metadata.partition_statistics] == [KEEP_SNAPSHOT]