Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions mkdocs/docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -1634,6 +1634,35 @@ def cleanup_old_snapshots(table_name: str, snapshot_ids: list[int]):
cleanup_old_snapshots("analytics.user_events", [12345, 67890, 11111])
```

### Expiring Branches and Tags

A branch or tag protects the snapshot it points at from expiration, along with that snapshot's
ancestors. A ref left behind by a failed job therefore pins storage indefinitely unless it is
removed.

`remove_expired_refs()` drops refs whose retention period has elapsed, using the ref's own
`max-ref-age-ms` when set and the `history.expire.max-ref-age-ms` table property otherwise.
The `main` branch never expires.

```python
from datetime import datetime, timedelta, timezone

# Give an audit branch a one-day lifetime
table.manage_snapshots().create_branch(
snapshot_id=table.metadata.current_snapshot_id,
branch_name="audit-2024-01-15",
max_ref_age_ms=24 * 60 * 60 * 1000,
).commit()

# Later: drop expired refs, then reclaim the snapshots they were pinning
table.maintenance.expire_snapshots().remove_expired_refs().older_than(
datetime.now(timezone.utc) - timedelta(days=3)
).commit()
```

Call `remove_expired_refs()` and `older_than()` in either order. Snapshots released by the
removed refs are eligible for expiration in the same commit.

## Views

If PyIceberg is unable to automatically determine view support on your REST Catalog, you can manually specify, `"view-endpoints-supported": "true"`:
Expand Down
4 changes: 4 additions & 0 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,10 @@ class TableProperties:
MIN_SNAPSHOTS_TO_KEEP = "history.expire.min-snapshots-to-keep"
MIN_SNAPSHOTS_TO_KEEP_DEFAULT = 1

MAX_REF_AGE_MS = "history.expire.max-ref-age-ms"
# sys.maxsize would be 2**31-1 on a 32-bit build, silently expiring refs after ~25 days.
MAX_REF_AGE_MS_DEFAULT = 2**63 - 1

COMMIT_NUM_RETRIES = "commit.retry.num-retries"
COMMIT_NUM_RETRIES_DEFAULT = 4

Expand Down
71 changes: 63 additions & 8 deletions pyiceberg/table/update/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from collections import defaultdict
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime
from datetime import UTC, datetime
from functools import cached_property
from typing import TYPE_CHECKING, Generic

Expand Down Expand Up @@ -1369,12 +1369,16 @@ class ExpireSnapshots(UpdateTableMetadata["ExpireSnapshots"]):
_updates: tuple[TableUpdate, ...]
_requirements: tuple[TableRequirement, ...]
_snapshot_ids_to_expire: set[int]
_refs_to_remove: dict[str, int]
_expire_older_than_ms: int | None

def __init__(self, transaction: Transaction) -> None:
super().__init__(transaction)
self._updates = ()
self._requirements = ()
self._snapshot_ids_to_expire = set()
self._refs_to_remove = {}
self._expire_older_than_ms = None

def _commit(self) -> UpdatesAndRequirements:
"""
Expand All @@ -1386,9 +1390,21 @@ def _commit(self) -> UpdatesAndRequirements:
Tuple of updates and requirements to be committed,
as required by the calling parent apply functions.
"""
# Remove any protected snapshot IDs from the set to expire, just in case
# Resolved here rather than in older_than() so that snapshots released by
# remove_expired_refs() are expirable no matter which order the two were called in.
protected_ids = self._get_protected_snapshot_ids()
if self._expire_older_than_ms is not None:
for snapshot in self._transaction.table_metadata.snapshots:
if snapshot.timestamp_ms < self._expire_older_than_ms and snapshot.snapshot_id not in protected_ids:
self._snapshot_ids_to_expire.add(snapshot.snapshot_id)

# Remove any protected snapshot IDs from the set to expire, just in case
self._snapshot_ids_to_expire -= protected_ids
# Each removal asserts the snapshot the ref had when it was judged expired, so a ref that
# advanced in the meantime fails the commit instead of being removed.
for ref_name, snapshot_id in self._refs_to_remove.items():
self._updates += (RemoveSnapshotRefUpdate(ref_name=ref_name),)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we need to also add a AssertRefSnapshotId requirement. This helps ensure that if a ref is expired, it's never checked against what's in the catalog at commit time.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in b1bd6ed. Each removed ref now asserts the snapshot it pointed at when it was judged expired, so a branch that advanced meanwhile fails the commit.

self._requirements += (AssertRefSnapshotId(ref=ref_name, snapshot_id=snapshot_id),)
update = RemoveSnapshotsUpdate(snapshot_ids=self._snapshot_ids_to_expire)
self._updates += (update,)
return self._updates, self._requirements
Expand All @@ -1398,16 +1414,57 @@ def _get_protected_snapshot_ids(self) -> set[int]:
Get the IDs of protected snapshots.

These are the HEAD snapshots of all branches and all tagged snapshots. These ids are to be excluded from expiration.
Refs staged for removal by :meth:`remove_expired_refs` do not protect anything.

Returns:
Set of protected snapshot IDs to exclude from expiration.
"""
return {
ref.snapshot_id
for ref in self._transaction.table_metadata.refs.values()
if ref.snapshot_ref_type in [SnapshotRefType.TAG, SnapshotRefType.BRANCH]
for ref_name, ref in self._transaction.table_metadata.refs.items()
if ref.snapshot_ref_type in [SnapshotRefType.TAG, SnapshotRefType.BRANCH] and ref_name not in self._refs_to_remove
}

def remove_expired_refs(self) -> ExpireSnapshots:
"""
Remove branches and tags whose retention period has elapsed.

A ref's age is measured from the timestamp of the snapshot it points at, against its own
``max-ref-age-ms`` when set, otherwise the ``history.expire.max-ref-age-ms`` table property.
The ``main`` branch never expires. A ref pointing at a snapshot that no longer exists is also
removed, since it can no longer be resolved.

This is step 2 of the snapshot retention policy in the Iceberg spec:
https://iceberg.apache.org/spec/#snapshot-retention-policy

Returns:
This for method chaining.
"""
from pyiceberg.table import TableProperties

metadata = self._transaction.table_metadata
default_max_ref_age_ms = property_as_int(
metadata.properties,
TableProperties.MAX_REF_AGE_MS,
TableProperties.MAX_REF_AGE_MS_DEFAULT,
)
now_ms = datetime_to_millis(datetime.now(UTC))

for ref_name, ref in metadata.refs.items():
if ref_name == MAIN_BRANCH:
continue

snapshot = metadata.snapshot_by_id(ref.snapshot_id)
if snapshot is None:
self._refs_to_remove[ref_name] = ref.snapshot_id
continue

max_ref_age_ms = ref.max_ref_age_ms if ref.max_ref_age_ms is not None else default_max_ref_age_ms
if max_ref_age_ms is not None and now_ms - snapshot.timestamp_ms > max_ref_age_ms:
self._refs_to_remove[ref_name] = ref.snapshot_id

return self

def by_id(self, snapshot_id: int) -> ExpireSnapshots:
"""
Expire a snapshot by its ID.
Expand Down Expand Up @@ -1456,9 +1513,7 @@ def older_than(self, dt: datetime) -> ExpireSnapshots:
Returns:
This for method chaining.
"""
protected_ids = self._get_protected_snapshot_ids()
expire_from = datetime_to_millis(dt)
for snapshot in self._transaction.table_metadata.snapshots:
if snapshot.timestamp_ms < expire_from and snapshot.snapshot_id not in protected_ids:
self._snapshot_ids_to_expire.add(snapshot.snapshot_id)
if self._expire_older_than_ms is None or expire_from > self._expire_older_than_ms:
self._expire_older_than_ms = expire_from
return self
65 changes: 65 additions & 0 deletions tests/integration/test_snapshot_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import time
import uuid
from collections.abc import Generator
from datetime import UTC, datetime, timedelta

import pyarrow as pa
import pytest
Expand Down Expand Up @@ -406,3 +408,66 @@ def test_rollback_to_timestamp_chained_with_tag(table_with_snapshots: Table) ->
assert table_with_snapshots.metadata.refs[tag_name] == SnapshotRef(
snapshot_id=current_snapshot.snapshot_id, snapshot_ref_type="tag"
)


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [lf("session_catalog_hive"), lf("session_catalog")])
def test_remove_expired_refs(catalog: Catalog) -> None:
"""An expired branch is removed, and the snapshot it pinned becomes expirable."""
catalog.create_namespace_if_not_exists("default")
identifier = f"default.test_expire_refs_{uuid.uuid4().hex[:8]}"
arrow_schema = pa.schema([pa.field("id", pa.int64(), nullable=False)])
tbl = catalog.create_table(identifier=identifier, schema=arrow_schema)

for i in range(3):
tbl.append(pa.Table.from_pylist([{"id": i}], schema=arrow_schema))
tbl = catalog.load_table(identifier)

pinned_snapshot_id = tbl.metadata.snapshots[0].snapshot_id
tbl.manage_snapshots().create_branch(
snapshot_id=pinned_snapshot_id,
branch_name="audit",
max_ref_age_ms=1,
).commit()

tbl = catalog.load_table(identifier)
assert "audit" in tbl.metadata.refs
time.sleep(0.05)

tbl.maintenance.expire_snapshots().remove_expired_refs().older_than(datetime.now(UTC) + timedelta(days=1)).commit()

tbl = catalog.load_table(identifier)
assert "audit" not in tbl.metadata.refs
assert tbl.snapshot_by_id(pinned_snapshot_id) is None

catalog.drop_table(identifier)


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [lf("session_catalog_hive"), lf("session_catalog")])
def test_remove_expired_refs_keeps_unexpired_branch(catalog: Catalog) -> None:
"""A branch inside its retention window survives and keeps protecting its snapshot."""
catalog.create_namespace_if_not_exists("default")
identifier = f"default.test_expire_refs_keep_{uuid.uuid4().hex[:8]}"
arrow_schema = pa.schema([pa.field("id", pa.int64(), nullable=False)])
tbl = catalog.create_table(identifier=identifier, schema=arrow_schema)

for i in range(3):
tbl.append(pa.Table.from_pylist([{"id": i}], schema=arrow_schema))
tbl = catalog.load_table(identifier)

pinned_snapshot_id = tbl.metadata.snapshots[0].snapshot_id
tbl.manage_snapshots().create_branch(
snapshot_id=pinned_snapshot_id,
branch_name="audit",
max_ref_age_ms=7 * 24 * 60 * 60 * 1000,
).commit()

tbl = catalog.load_table(identifier)
tbl.maintenance.expire_snapshots().remove_expired_refs().older_than(datetime.now(UTC) + timedelta(days=1)).commit()

tbl = catalog.load_table(identifier)
assert "audit" in tbl.metadata.refs
assert tbl.snapshot_by_id(pinned_snapshot_id) is not None

catalog.drop_table(identifier)
Loading
Loading