diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 1dd816d5d3..fd0ddb722f 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -69,7 +69,7 @@ from pyiceberg.conversions import to_bytes from pyiceberg.exceptions import ResolveError -from pyiceberg.expressions import AlwaysTrue, BooleanExpression, BoundIsNaN, BoundIsNull, BoundTerm, Not, Or +from pyiceberg.expressions import AlwaysFalse, AlwaysTrue, BooleanExpression, BoundIsNaN, BoundTerm, Or from pyiceberg.expressions.literals import Literal from pyiceberg.expressions.visitors import ( BoundBooleanExpressionVisitor, @@ -986,11 +986,7 @@ def visit_or(self, left_result: pc.Expression, right_result: pc.Expression) -> p return left_result | right_result -class _NullNaNUnmentionedTermsCollector(BoundBooleanExpressionVisitor[None]): - # BoundTerms which have either is_null or is_not_null appearing at least once in the boolean expr. - is_null_or_not_bound_terms: set[BoundTerm] - # The remaining BoundTerms appearing in the boolean expr. - null_unmentioned_bound_terms: set[BoundTerm] +class _NaNUnmentionedTermsCollector(BoundBooleanExpressionVisitor[None]): # BoundTerms which have either is_nan or is_not_nan appearing at least once in the boolean expr. is_nan_or_not_bound_terms: set[BoundTerm] # The remaining BoundTerms appearing in the boolean expr. @@ -998,22 +994,9 @@ class _NullNaNUnmentionedTermsCollector(BoundBooleanExpressionVisitor[None]): def __init__(self) -> None: super().__init__() - self.is_null_or_not_bound_terms = set() - self.null_unmentioned_bound_terms = set() self.is_nan_or_not_bound_terms = set() self.nan_unmentioned_bound_terms = set() - def _handle_explicit_is_null_or_not(self, term: BoundTerm) -> None: - """Handle the predicate case where either is_null or is_not_null is included.""" - if term in self.null_unmentioned_bound_terms: - self.null_unmentioned_bound_terms.remove(term) - self.is_null_or_not_bound_terms.add(term) - - def _handle_null_unmentioned(self, term: BoundTerm) -> None: - """Handle the predicate case where neither is_null or is_not_null is included.""" - if term not in self.is_null_or_not_bound_terms: - self.null_unmentioned_bound_terms.add(term) - def _handle_explicit_is_nan_or_not(self, term: BoundTerm) -> None: """Handle the predicate case where either is_nan or is_not_nan is included.""" if term in self.nan_unmentioned_bound_terms: @@ -1026,59 +1009,45 @@ def _handle_nan_unmentioned(self, term: BoundTerm) -> None: self.nan_unmentioned_bound_terms.add(term) def visit_in(self, term: BoundTerm, literals: set[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_not_in(self, term: BoundTerm, literals: set[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_is_nan(self, term: BoundTerm) -> None: - self._handle_null_unmentioned(term) self._handle_explicit_is_nan_or_not(term) def visit_not_nan(self, term: BoundTerm) -> None: - self._handle_null_unmentioned(term) self._handle_explicit_is_nan_or_not(term) def visit_is_null(self, term: BoundTerm) -> None: - self._handle_explicit_is_null_or_not(term) self._handle_nan_unmentioned(term) def visit_not_null(self, term: BoundTerm) -> None: - self._handle_explicit_is_null_or_not(term) self._handle_nan_unmentioned(term) def visit_equal(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_not_equal(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_greater_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_greater_than(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_less_than(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_less_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_starts_with(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_not_starts_with(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_true(self) -> None: @@ -1100,9 +1069,9 @@ def collect( self, expr: BooleanExpression, ) -> None: - """Collect bound references categorized by null predicates. + """Collect bound references categorized by NaN predicates. - Categorizes by having at least one is_null or is_not_null in the expr and the remaining. + Categorizes by having at least one is_nan or is_not_nan in the expr and the remaining. """ boolean_expression_visit(expr, self) @@ -1125,25 +1094,24 @@ def _expression_to_complementary_pyarrow(expr: BooleanExpression, schema: Schema """Complementary filter conversion function of expression_to_pyarrow. Could not use expression_to_pyarrow(Not(expr)) to achieve this complementary effect because - ~ in pyarrow.compute.Expression does not handle null. + ~ in pyarrow.compute.Expression does not handle null: where expr evaluates to null, Not(expr) is null too + and the row is dropped. Only the rows where expr is true are deleted, so null is read as false. """ - collector = _NullNaNUnmentionedTermsCollector() + collector = _NaNUnmentionedTermsCollector() collector.collect(expr) # Convert the set of terms to a sorted list so that layout of the expression to build is deterministic. - null_unmentioned_bound_terms: list[BoundTerm] = sorted( - collector.null_unmentioned_bound_terms, key=lambda term: term.ref().field.name - ) nan_unmentioned_bound_terms: list[BoundTerm] = sorted( collector.nan_unmentioned_bound_terms, key=lambda term: term.ref().field.name ) - preserve_expr: BooleanExpression = Not(expr) - for term in null_unmentioned_bound_terms: - preserve_expr = Or(preserve_expr, BoundIsNull(term=term)) + preserve_expr = ~pc.coalesce(expression_to_pyarrow(expr, schema), pc.scalar(False)) + nan_expr: BooleanExpression = AlwaysFalse() for term in nan_unmentioned_bound_terms: - preserve_expr = Or(preserve_expr, BoundIsNaN(term=term)) - return expression_to_pyarrow(preserve_expr, schema) + nan_expr = Or(nan_expr, BoundIsNaN(term=term)) + if nan_expr == AlwaysFalse(): + return preserve_expr + return preserve_expr | expression_to_pyarrow(nan_expr, schema) @lru_cache diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index 1148ca8527..394c9efc15 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -1295,6 +1295,31 @@ def test_overwrite_record_batch_reader(catalog: Catalog) -> None: assert len(tbl.scan().to_arrow()) == total_rows +def test_delete_and_overwrite_filter_with_null(catalog: Catalog) -> None: + catalog.create_namespace("default") + pa_table = pa.table({"x": [1, 2, 3], "p": [1, 2, 2], "v": [None, None, "c"]}) + filters = [ + # A null v makes the second branch null for x=2, which matches neither branch + "(p == 1 AND v IS NULL) OR (p == 2 AND v == 'c')", + # A null v leaves the filter true for x=1 + "p == 1 OR v == 'c'", + "v != 'c'", + "NOT (v == 'c')", + ] + for i, row_filter in enumerate(filters): + tbl = catalog.create_table(identifier=f"default.delete_filter_with_null_{i}_{catalog.name}", schema=pa_table.schema) + tbl.append(pa_table) + matched = set(tbl.scan(row_filter=row_filter).to_arrow()["x"].to_pylist()) + expected = sorted({1, 2, 3} - matched) + + tbl.delete(row_filter) + assert sorted(tbl.scan().to_arrow()["x"].to_pylist()) == expected, row_filter + + tbl.overwrite(pa_table, overwrite_filter="x > 0") + tbl.overwrite(pa_table.slice(0, 0), overwrite_filter=row_filter) + assert sorted(tbl.scan().to_arrow()["x"].to_pylist()) == expected, row_filter + + def test_append_record_batch_reader_to_partitioned_table_raises(catalog: Catalog) -> None: catalog.create_namespace("default") identifier = f"default.append_record_batch_reader_partitioned_{catalog.name}" diff --git a/tests/integration/test_deletes.py b/tests/integration/test_deletes.py index 20205c59fb..64b14c031a 100644 --- a/tests/integration/test_deletes.py +++ b/tests/integration/test_deletes.py @@ -677,7 +677,7 @@ def test_delete_overwrite_table_with_nan(session_catalog: RestCatalog) -> None: This means if we set the test case as floats == 2.0 (equal predicate as in test_delete_overwrite_table_with_null), test will pass even without the logic under test - in _NullNaNUnmentionedTermsCollector (a helper of _expression_to_complementary_pyarrow + in _NaNUnmentionedTermsCollector (a helper of _expression_to_complementary_pyarrow to handle revert of iceberg expression of is_null/not_null/is_nan/not_nan). Instead, we test the filter of !=, so that the revert is == which exposes the issue. """ diff --git a/tests/io/test_pyarrow_visitor.py b/tests/io/test_pyarrow_visitor.py index e98d76e262..00eb1f1caf 100644 --- a/tests/io/test_pyarrow_visitor.py +++ b/tests/io/test_pyarrow_visitor.py @@ -38,7 +38,7 @@ _ConvertToIcebergWithoutIDs, _expression_to_complementary_pyarrow, _HasIds, - _NullNaNUnmentionedTermsCollector, + _NaNUnmentionedTermsCollector, _pyarrow_schema_ensure_large_types, _pyarrow_schema_ensure_small_types, pyarrow_to_schema, @@ -789,19 +789,12 @@ def test_collect_null_nan_unmentioned_terms( bound_expr = And( Or(And(bound_eq_str_field, bound_is_nan_float_field), bound_is_null_double_field), Not(bound_is_nan_float_field) ) - collector = _NullNaNUnmentionedTermsCollector() + collector = _NaNUnmentionedTermsCollector() collector.collect(bound_expr) - assert {t.ref().field.name for t in collector.null_unmentioned_bound_terms} == { - "float_field", - "string_field", - } assert {t.ref().field.name for t in collector.nan_unmentioned_bound_terms} == { "string_field", "double_field", } - assert {t.ref().field.name for t in collector.is_null_or_not_bound_terms} == { - "double_field", - } assert {t.ref().field.name for t in collector.is_nan_or_not_bound_terms} == {"float_field"} @@ -821,19 +814,12 @@ def test_collect_null_nan_unmentioned_terms_with_multiple_predicates_on_the_same ), Not(bound_is_null_double_field), ) - collector = _NullNaNUnmentionedTermsCollector() + collector = _NaNUnmentionedTermsCollector() collector.collect(bound_expr) - assert {t.ref().field.name for t in collector.null_unmentioned_bound_terms} == { - "float_field", - "string_field", - } assert {t.ref().field.name for t in collector.nan_unmentioned_bound_terms} == { "string_field", "double_field", } - assert {t.ref().field.name for t in collector.is_null_or_not_bound_terms} == { - "double_field", - } assert {t.ref().field.name for t in collector.is_nan_or_not_bound_terms} == {"float_field"} @@ -853,13 +839,16 @@ def test_expression_to_complementary_pyarrow( Not(bound_is_null_double_field), ) result = _expression_to_complementary_pyarrow(bound_expr) - # Notice an isNan predicate on a str column is automatically converted to always false and removed from Or - # and thus will not appear in the pc.expr. - expected_repr = ( - ' 100)) ' - "or ((is_nan(float_field) and (double_field == 0)) or (float_field > 100))) " - "and invert(is_null(double_field, {nan_is_null=false})))) " - "or is_null(float_field, {nan_is_null=false})) " - "or is_null(string_field, {nan_is_null=false})) or is_nan(double_field))>" + table = pa.table( + { + "id": pa.array([1, 2, 3, 4, 5], type=pa.int32()), + "string_field": pa.array(["hello", "hello", "x", "x", None], type=pa.string()), + "float_field": pa.array([200.0, 50.0, float("nan"), 200.0, None], type=pa.float32()), + "double_field": pa.array([1.0, 1.0, 0.0, None, 1.0], type=pa.float64()), + } ) - assert repr(result) == expected_repr + # The complement keeps exactly the rows where bound_expr is not true: + # 1 matches (float_field > 100), 3 matches (is_nan(float_field) and double_field == 0), + # 2 does not match, 4 does not match because double_field is null, + # and 5 evaluates to null because string_field and float_field are null. + assert table.filter(result)["id"].to_pylist() == [2, 4, 5]