diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 9ef7519865..1348380b28 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -562,7 +562,7 @@ def append( except ModuleNotFoundError as e: raise ModuleNotFoundError("For writes PyArrow needs to be installed") from e - from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible, _dataframe_to_data_files + from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible if not isinstance(df, (pa.Table, pa.RecordBatchReader)): raise ValueError(f"Expected pa.Table or pa.RecordBatchReader, got: {df}") @@ -576,17 +576,24 @@ def append( ) with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files: - # For pa.Table we can short-circuit empty inputs cheaply. For a - # RecordBatchReader the stream is consumed lazily by - # _dataframe_to_data_files and an empty reader simply yields zero - # data files (the snapshot is still committed for symmetry with the - # pa.Table case where empty inputs also produce a snapshot). - if isinstance(df, pa.RecordBatchReader) or df.shape[0] > 0: - data_files = _dataframe_to_data_files( - table_metadata=self.table_metadata, write_uuid=append_files.commit_uuid, df=df, io=self._table.io - ) - for data_file in data_files: - append_files.append_data_file(data_file) + self._write_append_data(df, append_files) + + def _write_append_data(self, df: pa.Table | pa.RecordBatchReader, append_files: _FastAppendFiles) -> None: + import pyarrow as pa + + from pyiceberg.io.pyarrow import _dataframe_to_data_files + + # For pa.Table we can short-circuit empty inputs cheaply. For a + # RecordBatchReader the stream is consumed lazily by + # _dataframe_to_data_files and an empty reader simply yields zero + # data files (the snapshot is still committed for symmetry with the + # pa.Table case where empty inputs also produce a snapshot). + if isinstance(df, pa.RecordBatchReader) or df.shape[0] > 0: + data_files = _dataframe_to_data_files( + table_metadata=self.table_metadata, write_uuid=append_files.commit_uuid, df=df, io=self._table.io + ) + for data_file in data_files: + append_files.append_data_file(data_file) def dynamic_partition_overwrite( self, df: pa.Table, snapshot_properties: dict[str, str] = EMPTY_DICT, branch: str | None = MAIN_BRANCH @@ -1014,7 +1021,11 @@ def upsert( if when_not_matched_insert_all: insert_row_cnt = len(rows_to_insert) if rows_to_insert: - self.append(rows_to_insert, branch=branch, snapshot_properties=snapshot_properties) + with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files: + append_files.validate_no_conflicting_data( + upsert_util.create_match_filter(rows_to_insert, join_cols), case_sensitive + ) + self._write_append_data(rows_to_insert, append_files) return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt) diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 0c9831450e..fbd4592401 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -690,8 +690,41 @@ def _refresh_for_retry(self) -> None: class _FastAppendFiles(_SnapshotProducer["_FastAppendFiles"]): + _conflict_detection_filter: BooleanExpression | None = None + _conflict_detection_case_sensitive: bool = True + + def validate_no_conflicting_data(self, data_filter: BooleanExpression, case_sensitive: bool = True) -> None: + """Fail a retry when a concurrent commit added data matching the filter.""" + self._conflict_detection_filter = data_filter + self._conflict_detection_case_sensitive = case_sensitive + def _validate_concurrency(self) -> None: - """Skip validation; appends do not conflict with other operations.""" + """Skip validation for plain appends. + + An append that declares a conflict detection filter fails when a concurrent commit added + data matching it, instead of replaying the append on retry. + """ + if self._conflict_detection_filter is None or self._commit_window is None or self._commit_window.is_empty(): + return + + from pyiceberg.table.snapshots import IsolationLevel + from pyiceberg.table.update.validate import _validate_added_data_files + + table = self._transaction._table + if table.metadata.isolation_level(Operation.OVERWRITE) != IsolationLevel.SERIALIZABLE: + return + + catalog_head = self._commit_window.head + if catalog_head is None: + return + + _validate_added_data_files( + table, + catalog_head, + self._conflict_detection_filter, + self._commit_window.base, + self._conflict_detection_case_sensitive, + ) def _existing_manifests(self) -> list[ManifestFile]: """To determine if there are any existing manifest files. diff --git a/tests/table/test_upsert.py b/tests/table/test_upsert.py index 4291c18a55..3a666d5351 100644 --- a/tests/table/test_upsert.py +++ b/tests/table/test_upsert.py @@ -23,17 +23,17 @@ from pyarrow import Table as pa_table from pyiceberg.catalog import Catalog -from pyiceberg.exceptions import NoSuchTableError +from pyiceberg.exceptions import NoSuchTableError, ValidationException from pyiceberg.expressions import AlwaysTrue, And, EqualTo, Reference from pyiceberg.expressions.literals import LongLiteral from pyiceberg.io.pyarrow import schema_to_pyarrow from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table import Table, UpsertResult +from pyiceberg.table import Table, TableProperties, UpsertResult from pyiceberg.table.snapshots import Operation from pyiceberg.table.upsert_util import create_match_filter, get_rows_to_update from pyiceberg.transforms import DayTransform -from pyiceberg.types import IntegerType, NestedField, StringType, StructType, TimestampType +from pyiceberg.types import IntegerType, LongType, NestedField, StringType, StructType, TimestampType from tests.catalog.test_base import InMemoryCatalog @@ -427,6 +427,59 @@ def test_upsert_into_empty_table(catalog: Catalog) -> None: assert upd.rows_inserted == 4 +def test_upsert_retry_with_a_concurrently_inserted_key(catalog: Catalog) -> None: + identifier = "default.test_upsert_retry_with_a_concurrently_inserted_key" + _drop_table(catalog, identifier) + + schema = Schema( + NestedField(1, "k", LongType(), required=True), + NestedField(2, "v", StringType(), required=False), + identifier_field_ids=[1], + ) + arrow_schema = schema_to_pyarrow(schema) + catalog.create_table(identifier, schema=schema).append(pa.Table.from_pylist([{"k": 1, "v": "a"}], schema=arrow_schema)) + + writer_a = catalog.load_table(identifier) + writer_b = catalog.load_table(identifier) + writer_b.upsert(pa.Table.from_pylist([{"k": 2, "v": "b"}], schema=arrow_schema)) + + with pytest.raises(ValidationException): + writer_a.upsert(pa.Table.from_pylist([{"k": 2, "v": "a"}], schema=arrow_schema)) + + assert sorted(catalog.load_table(identifier).scan().to_arrow().to_pylist(), key=str) == [ + {"k": 1, "v": "a"}, + {"k": 2, "v": "b"}, + ] + + +def test_upsert_retry_under_snapshot_isolation_does_not_validate_inserted_keys(catalog: Catalog) -> None: + """The insert half follows the update isolation level, which tolerates concurrently added rows.""" + identifier = "default.test_upsert_retry_snapshot_isolation" + _drop_table(catalog, identifier) + + schema = Schema( + NestedField(1, "k", LongType(), required=True), + NestedField(2, "v", StringType(), required=False), + identifier_field_ids=[1], + ) + arrow_schema = schema_to_pyarrow(schema) + catalog.create_table(identifier, schema=schema, properties={TableProperties.WRITE_UPDATE_ISOLATION_LEVEL: "snapshot"}).append( + pa.Table.from_pylist([{"k": 1, "v": "a"}], schema=arrow_schema) + ) + + writer_a = catalog.load_table(identifier) + writer_b = catalog.load_table(identifier) + writer_b.upsert(pa.Table.from_pylist([{"k": 2, "v": "b"}], schema=arrow_schema)) + + writer_a.upsert(pa.Table.from_pylist([{"k": 2, "v": "a"}], schema=arrow_schema)) + + assert sorted(catalog.load_table(identifier).scan().to_arrow().to_pylist(), key=str) == [ + {"k": 1, "v": "a"}, + {"k": 2, "v": "a"}, + {"k": 2, "v": "b"}, + ] + + def test_create_match_filter_single_condition() -> None: """ Test create_match_filter with a composite key where the source yields exactly one unique key.