From 2c267a8504d736561cb44987996931a49e58a30d Mon Sep 17 00:00:00 2001 From: Brad Heller Date: Thu, 16 Apr 2026 13:05:31 +0200 Subject: [PATCH 1/2] Add commit retries to insert and delete table writes Parallel writes to the same Iceberg table can fail with CommitFailedException when the branch changes during active writes. This adds the same retry-with-refresh logic that upsert already has to both insert() and delete(), resolving concurrent write conflicts automatically. --- src/tower/_tables.py | 81 ++++++++++++++++++++----- tests/tower/test_tables.py | 120 +++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 14 deletions(-) diff --git a/src/tower/_tables.py b/src/tower/_tables.py index 902476ba..18b82312 100644 --- a/src/tower/_tables.py +++ b/src/tower/_tables.py @@ -153,9 +153,14 @@ def rows_affected(self) -> RowsAffectedInformation: """ return self._stats - def insert(self, data: pa.Table) -> TTable: + def insert( + self, + data: pa.Table, + max_retries: int = 5, + retry_delay_seconds: float = 0.5, + ) -> TTable: """ - Inserts new rows into the Iceberg table. + Inserts new rows into the Iceberg table. In case of commit conflicts, reloads the metadata and retries. This method appends the provided data to the table. The data must be provided as a PyArrow table with a schema that matches the table's schema. The operation is @@ -164,10 +169,17 @@ def insert(self, data: pa.Table) -> TTable: Args: data (pa.Table): The data to insert into the table. The schema of this table must match the schema of the target table. + max_retries (int): Maximum number of retry attempts on commit conflicts. + Defaults to 5. + retry_delay_seconds (float): Wait time in seconds between retries. + Defaults to 0.5 seconds. Returns: TTable: The table instance with the newly inserted rows, allowing for method chaining. + Raises: + CommitFailedException: If all retry attempts are exhausted. + Example: >>> table = tables("my_table").load() >>> # Create a PyArrow table with new data @@ -182,9 +194,23 @@ def insert(self, data: pa.Table) -> TTable: >>> stats = table.rows_affected() >>> print(f"Inserted {stats.inserts} rows") """ - self._table.append(data) - self._stats.inserts += data.num_rows - return self + last_exception = None + + for attempt in range(max_retries + 1): + try: + if attempt > 0: + self._table.refresh() + + self._table.append(data) + self._stats.inserts += data.num_rows + return self + + except CommitFailedException as e: + last_exception = e + if attempt < max_retries: + time.sleep(retry_delay_seconds) + + raise last_exception def upsert( self, @@ -268,9 +294,15 @@ def upsert( raise last_exception - def delete(self, filters: Union[str, List[pc.Expression]]) -> TTable: + def delete( + self, + filters: Union[str, List[pc.Expression]], + max_retries: int = 5, + retry_delay_seconds: float = 0.5, + ) -> TTable: """ Deletes rows from the Iceberg table that match the specified filter conditions. + In case of commit conflicts, reloads the metadata and retries. This method removes rows from the table based on the provided filter expressions. The operation is always case-sensitive. Note that the number of deleted rows @@ -282,10 +314,17 @@ def delete(self, filters: Union[str, List[pc.Expression]]) -> TTable: - A single PyArrow compute expression - A list of PyArrow compute expressions (combined with AND) - A string expression + max_retries (int): Maximum number of retry attempts on commit conflicts. + Defaults to 5. + retry_delay_seconds (float): Wait time in seconds between retries. + Defaults to 0.5 seconds. Returns: TTable: The table instance with the deleted rows, allowing for method chaining. + Raises: + CommitFailedException: If all retry attempts are exhausted. + Note: - The operation is always case-sensitive - The number of deleted rows cannot be tracked in the table statistics @@ -309,16 +348,30 @@ def delete(self, filters: Union[str, List[pc.Expression]]) -> TTable: next_filters = convert_pyarrow_expressions(filters) filters = next_filters - self._table.delete( - delete_filter=filters, - # We want this to always be the case. Not sure why you wouldn't? - case_sensitive=True, - ) + last_exception = None + + for attempt in range(max_retries + 1): + try: + if attempt > 0: + self._table.refresh() + + self._table.delete( + delete_filter=filters, + # We want this to always be the case. Not sure why you wouldn't? + case_sensitive=True, + ) + + # NOTE: There is, unfortunately, no way to get the number of rows + # deleted besides comparing the two snapshots that were created. - # NOTE: There is, unfortunately, no way to get the number of rows - # deleted besides comparing the two snapshots that were created. + return self - return self + except CommitFailedException as e: + last_exception = e + if attempt < max_retries: + time.sleep(retry_delay_seconds) + + raise last_exception def schema(self) -> pa.Schema: """ diff --git a/tests/tower/test_tables.py b/tests/tower/test_tables.py index f39d5b49..7207c4db 100644 --- a/tests/tower/test_tables.py +++ b/tests/tower/test_tables.py @@ -321,6 +321,126 @@ def tracked_refresh(): assert final_counter in [1, 2, 3, 4, 5] +def test_insert_concurrent_writes_with_retry(sql_catalog): + """Test that concurrent inserts succeed with retry logic handling conflicts.""" + schema = pa.schema( + [ + pa.field("ticker", pa.string()), + pa.field("date", pa.string()), + pa.field("price", pa.float64()), + ] + ) + + ref = tower.tables("concurrent_insert_test", catalog=sql_catalog) + table = ref.create_if_not_exists(schema) + + retry_count = {"value": 0} + retry_lock = threading.Lock() + + def insert_ticker(ticker: str, price: float): + t = tower.tables("concurrent_insert_test", catalog=sql_catalog).load() + + original_refresh = t._table.refresh + + def tracked_refresh(): + with retry_lock: + retry_count["value"] += 1 + return original_refresh() + + t._table.refresh = tracked_refresh + + data = pa.Table.from_pylist( + [{"ticker": ticker, "date": "2024-01-01", "price": price}], + schema=schema, + ) + t.insert(data) + return ticker + + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: + futures = [ + executor.submit(insert_ticker, "AAPL", 150.0), + executor.submit(insert_ticker, "GOOGL", 250.0), + executor.submit(insert_ticker, "MSFT", 350.0), + ] + results = [f.result() for f in concurrent.futures.as_completed(futures)] + + assert len(results) == 3 + assert ( + retry_count["value"] > 0 + ), "Expected at least one retry due to concurrent conflicts" + + final_table = tower.tables("concurrent_insert_test", catalog=sql_catalog).load() + df = final_table.read() + + assert len(df) == 3 + + ticker_prices = {row["ticker"]: row["price"] for row in df.iter_rows(named=True)} + + assert ticker_prices["AAPL"] == 150.0 + assert ticker_prices["GOOGL"] == 250.0 + assert ticker_prices["MSFT"] == 350.0 + + +def test_delete_concurrent_writes_with_retry(sql_catalog): + """Test that concurrent deletes succeed with retry logic handling conflicts.""" + schema = pa.schema( + [ + pa.field("ticker", pa.string()), + pa.field("date", pa.string()), + pa.field("price", pa.float64()), + ] + ) + + ref = tower.tables("concurrent_delete_test", catalog=sql_catalog) + table = ref.create_if_not_exists(schema) + + initial_data = pa.Table.from_pylist( + [ + {"ticker": "AAPL", "date": "2024-01-01", "price": 100.0}, + {"ticker": "GOOGL", "date": "2024-01-01", "price": 200.0}, + {"ticker": "MSFT", "date": "2024-01-01", "price": 300.0}, + ], + schema=schema, + ) + table.insert(initial_data) + + retry_count = {"value": 0} + retry_lock = threading.Lock() + + def delete_ticker(ticker: str): + t = tower.tables("concurrent_delete_test", catalog=sql_catalog).load() + + original_refresh = t._table.refresh + + def tracked_refresh(): + with retry_lock: + retry_count["value"] += 1 + return original_refresh() + + t._table.refresh = tracked_refresh + + t.delete(filters=f"ticker = '{ticker}'") + return ticker + + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: + futures = [ + executor.submit(delete_ticker, "AAPL"), + executor.submit(delete_ticker, "GOOGL"), + executor.submit(delete_ticker, "MSFT"), + ] + results = [f.result() for f in concurrent.futures.as_completed(futures)] + + assert len(results) == 3 + assert ( + retry_count["value"] > 0 + ), "Expected at least one retry due to concurrent conflicts" + + final_table = tower.tables("concurrent_delete_test", catalog=sql_catalog).load() + df = final_table.read() + + assert len(df) == 0 + + def test_delete_from_tables(in_memory_catalog): schema = pa.schema( [ From 50263b1ca820712311465a66c698bed32fe6d7d6 Mon Sep 17 00:00:00 2001 From: Brad Heller Date: Thu, 16 Apr 2026 16:00:51 +0200 Subject: [PATCH 2/2] Address PR #250 feedback: validate retry args and make tests deterministic Add input validation for max_retries and retry_delay_seconds across insert(), upsert(), and delete() to prevent confusing errors from negative values. Make concurrent insert/delete tests deterministic by forcing CommitFailedException on the first attempt rather than relying on real concurrency conflicts. --- src/tower/_tables.py | 12 ++++++++++++ tests/tower/test_tables.py | 33 +++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/tower/_tables.py b/src/tower/_tables.py index 18b82312..de5bca46 100644 --- a/src/tower/_tables.py +++ b/src/tower/_tables.py @@ -153,6 +153,13 @@ def rows_affected(self) -> RowsAffectedInformation: """ return self._stats + @staticmethod + def _validate_retry_args(max_retries: int, retry_delay_seconds: float) -> None: + if max_retries < 0: + raise ValueError("max_retries must be >= 0") + if retry_delay_seconds < 0: + raise ValueError("retry_delay_seconds must be >= 0") + def insert( self, data: pa.Table, @@ -194,6 +201,8 @@ def insert( >>> stats = table.rows_affected() >>> print(f"Inserted {stats.inserts} rows") """ + self._validate_retry_args(max_retries, retry_delay_seconds) + last_exception = None for attempt in range(max_retries + 1): @@ -265,6 +274,8 @@ def upsert( >>> print(f"Updated {stats.updates} rows") >>> print(f"Inserted {stats.inserts} rows") """ + self._validate_retry_args(max_retries, retry_delay_seconds) + last_exception = None for attempt in range(max_retries + 1): @@ -342,6 +353,7 @@ def delete( >>> # Delete rows using a string expression >>> table.delete("age > 30 AND department = 'IT'") """ + self._validate_retry_args(max_retries, retry_delay_seconds) if isinstance(filters, list): # We need to convert the pc.Expression into PyIceberg diff --git a/tests/tower/test_tables.py b/tests/tower/test_tables.py index 7207c4db..931d6afe 100644 --- a/tests/tower/test_tables.py +++ b/tests/tower/test_tables.py @@ -12,6 +12,7 @@ import pyarrow as pa from pyiceberg.catalog.memory import InMemoryCatalog from pyiceberg.catalog.sql import SqlCatalog +from pyiceberg.exceptions import CommitFailedException import concurrent.futures @@ -340,13 +341,24 @@ def test_insert_concurrent_writes_with_retry(sql_catalog): def insert_ticker(ticker: str, price: float): t = tower.tables("concurrent_insert_test", catalog=sql_catalog).load() + original_append = t._table.append original_refresh = t._table.refresh + first_attempt = {"done": False} + refresh_called = {"value": False} + + def failing_then_succeeding_append(data): + if not first_attempt["done"]: + first_attempt["done"] = True + raise CommitFailedException("Simulated concurrent write conflict") + return original_append(data) def tracked_refresh(): + refresh_called["value"] = True with retry_lock: retry_count["value"] += 1 return original_refresh() + t._table.append = failing_then_succeeding_append t._table.refresh = tracked_refresh data = pa.Table.from_pylist( @@ -354,6 +366,7 @@ def tracked_refresh(): schema=schema, ) t.insert(data) + assert refresh_called["value"], "Expected refresh() to be called before retry" return ticker with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: @@ -366,8 +379,8 @@ def tracked_refresh(): assert len(results) == 3 assert ( - retry_count["value"] > 0 - ), "Expected at least one retry due to concurrent conflicts" + retry_count["value"] >= 3 + ), "Expected at least one refresh per thread (one simulated failure each)" final_table = tower.tables("concurrent_insert_test", catalog=sql_catalog).load() df = final_table.read() @@ -410,16 +423,28 @@ def test_delete_concurrent_writes_with_retry(sql_catalog): def delete_ticker(ticker: str): t = tower.tables("concurrent_delete_test", catalog=sql_catalog).load() + original_delete = t._table.delete original_refresh = t._table.refresh + first_attempt = {"done": False} + refresh_called = {"value": False} + + def failing_then_succeeding_delete(**kwargs): + if not first_attempt["done"]: + first_attempt["done"] = True + raise CommitFailedException("Simulated concurrent write conflict") + return original_delete(**kwargs) def tracked_refresh(): + refresh_called["value"] = True with retry_lock: retry_count["value"] += 1 return original_refresh() + t._table.delete = failing_then_succeeding_delete t._table.refresh = tracked_refresh t.delete(filters=f"ticker = '{ticker}'") + assert refresh_called["value"], "Expected refresh() to be called before retry" return ticker with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: @@ -432,8 +457,8 @@ def tracked_refresh(): assert len(results) == 3 assert ( - retry_count["value"] > 0 - ), "Expected at least one retry due to concurrent conflicts" + retry_count["value"] >= 3 + ), "Expected at least one refresh per thread (one simulated failure each)" final_table = tower.tables("concurrent_delete_test", catalog=sql_catalog).load() df = final_table.read()