diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index e7a8c5401e..d497d6fb19 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -675,6 +675,10 @@ def data_file(self) -> DataFile: def data_file(self, value: DataFile) -> None: self._data[4] = value + def is_live(self) -> bool: + """Return whether the entry is live (added or existing) in the snapshot.""" + return self.status in (ManifestEntryStatus.ADDED, ManifestEntryStatus.EXISTING) + PARTITION_FIELD_SUMMARY_TYPE = StructType( NestedField(509, "contains_null", BooleanType(), required=True), @@ -933,7 +937,7 @@ def fetch_manifest_entry( next_row_id = self.first_row_id if self.content == ManifestContent.DATA else None for entry in reader: - if discard_deleted and entry.status == ManifestEntryStatus.DELETED: + if discard_deleted and not entry.is_live(): continue _inherit_from_manifest(entry, self) @@ -941,7 +945,7 @@ def fetch_manifest_entry( data_file = entry.data_file if next_row_id is None: data_file.first_row_id = None - elif entry.status != ManifestEntryStatus.DELETED and data_file.first_row_id is None: + elif entry.is_live() and data_file.first_row_id is None: data_file.first_row_id = next_row_id next_row_id += data_file.record_count @@ -1254,7 +1258,7 @@ def add_entry(self, entry: ManifestEntry) -> ManifestWriter: self._partitions.append(entry.data_file.partition) if ( - (entry.status == ManifestEntryStatus.ADDED or entry.status == ManifestEntryStatus.EXISTING) + entry.is_live() and entry.sequence_number is not None and (self._min_sequence_number is None or entry.sequence_number < self._min_sequence_number) ): diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 6407a09479..0d2f86f907 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -973,7 +973,7 @@ def _create_manifest(self, spec_id: int, manifest_bin: list[ManifestFile]) -> Ma elif entry.status == ManifestEntryStatus.ADDED and entry.snapshot_id == self._snapshot_producer.snapshot_id: # added entries from this snapshot are still added, otherwise they should be existing writer.add(entry) - elif entry.status != ManifestEntryStatus.DELETED: + elif entry.is_live(): # add all non-deleted files from the old manifest as existing files writer.existing(entry) diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index 864b9f25c8..c084e0ccf7 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -236,6 +236,14 @@ def test_fetch_manifest_entry_with_filter(generated_manifest_entry_file: str) -> assert len(no_match) == 0 +@pytest.mark.parametrize( + "status, expected", + [(ManifestEntryStatus.ADDED, True), (ManifestEntryStatus.EXISTING, True), (ManifestEntryStatus.DELETED, False)], +) +def test_manifest_entry_is_live(status: ManifestEntryStatus, expected: bool) -> None: + assert ManifestEntry.from_args(status=status).is_live() is expected + + def test_fetch_manifest_entry_inherits_first_row_id(tmp_path: Path) -> None: """Data files inherit first row IDs from data manifests in file order, and have them cleared otherwise.""" io = PyArrowFileIO()