Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 13 additions & 45 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@

from pyiceberg.conversions import to_bytes
from pyiceberg.exceptions import ResolveError
from pyiceberg.expressions import AlwaysTrue, BooleanExpression, BoundIsNaN, BoundIsNull, BoundTerm, Not, Or
from pyiceberg.expressions import AlwaysFalse, AlwaysTrue, BooleanExpression, BoundIsNaN, BoundTerm, Or
from pyiceberg.expressions.literals import Literal
from pyiceberg.expressions.visitors import (
BoundBooleanExpressionVisitor,
Expand Down Expand Up @@ -986,34 +986,17 @@ def visit_or(self, left_result: pc.Expression, right_result: pc.Expression) -> p
return left_result | right_result


class _NullNaNUnmentionedTermsCollector(BoundBooleanExpressionVisitor[None]):
# BoundTerms which have either is_null or is_not_null appearing at least once in the boolean expr.
is_null_or_not_bound_terms: set[BoundTerm]
# The remaining BoundTerms appearing in the boolean expr.
null_unmentioned_bound_terms: set[BoundTerm]
class _NaNUnmentionedTermsCollector(BoundBooleanExpressionVisitor[None]):
# BoundTerms which have either is_nan or is_not_nan appearing at least once in the boolean expr.
is_nan_or_not_bound_terms: set[BoundTerm]
# The remaining BoundTerms appearing in the boolean expr.
nan_unmentioned_bound_terms: set[BoundTerm]

def __init__(self) -> None:
super().__init__()
self.is_null_or_not_bound_terms = set()
self.null_unmentioned_bound_terms = set()
self.is_nan_or_not_bound_terms = set()
self.nan_unmentioned_bound_terms = set()

def _handle_explicit_is_null_or_not(self, term: BoundTerm) -> None:
"""Handle the predicate case where either is_null or is_not_null is included."""
if term in self.null_unmentioned_bound_terms:
self.null_unmentioned_bound_terms.remove(term)
self.is_null_or_not_bound_terms.add(term)

def _handle_null_unmentioned(self, term: BoundTerm) -> None:
"""Handle the predicate case where neither is_null or is_not_null is included."""
if term not in self.is_null_or_not_bound_terms:
self.null_unmentioned_bound_terms.add(term)

def _handle_explicit_is_nan_or_not(self, term: BoundTerm) -> None:
"""Handle the predicate case where either is_nan or is_not_nan is included."""
if term in self.nan_unmentioned_bound_terms:
Expand All @@ -1026,59 +1009,45 @@ def _handle_nan_unmentioned(self, term: BoundTerm) -> None:
self.nan_unmentioned_bound_terms.add(term)

def visit_in(self, term: BoundTerm, literals: set[Any]) -> None:
self._handle_null_unmentioned(term)
self._handle_nan_unmentioned(term)

def visit_not_in(self, term: BoundTerm, literals: set[Any]) -> None:
self._handle_null_unmentioned(term)
self._handle_nan_unmentioned(term)

def visit_is_nan(self, term: BoundTerm) -> None:
self._handle_null_unmentioned(term)
self._handle_explicit_is_nan_or_not(term)

def visit_not_nan(self, term: BoundTerm) -> None:
self._handle_null_unmentioned(term)
self._handle_explicit_is_nan_or_not(term)

def visit_is_null(self, term: BoundTerm) -> None:
self._handle_explicit_is_null_or_not(term)
self._handle_nan_unmentioned(term)

def visit_not_null(self, term: BoundTerm) -> None:
self._handle_explicit_is_null_or_not(term)
self._handle_nan_unmentioned(term)

def visit_equal(self, term: BoundTerm, literal: Literal[Any]) -> None:
self._handle_null_unmentioned(term)
self._handle_nan_unmentioned(term)

def visit_not_equal(self, term: BoundTerm, literal: Literal[Any]) -> None:
self._handle_null_unmentioned(term)
self._handle_nan_unmentioned(term)

def visit_greater_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> None:
self._handle_null_unmentioned(term)
self._handle_nan_unmentioned(term)

def visit_greater_than(self, term: BoundTerm, literal: Literal[Any]) -> None:
self._handle_null_unmentioned(term)
self._handle_nan_unmentioned(term)

def visit_less_than(self, term: BoundTerm, literal: Literal[Any]) -> None:
self._handle_null_unmentioned(term)
self._handle_nan_unmentioned(term)

def visit_less_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> None:
self._handle_null_unmentioned(term)
self._handle_nan_unmentioned(term)

def visit_starts_with(self, term: BoundTerm, literal: Literal[Any]) -> None:
self._handle_null_unmentioned(term)
self._handle_nan_unmentioned(term)

def visit_not_starts_with(self, term: BoundTerm, literal: Literal[Any]) -> None:
self._handle_null_unmentioned(term)
self._handle_nan_unmentioned(term)

def visit_true(self) -> None:
Expand All @@ -1100,9 +1069,9 @@ def collect(
self,
expr: BooleanExpression,
) -> None:
"""Collect bound references categorized by null predicates.
"""Collect bound references categorized by NaN predicates.

Categorizes by having at least one is_null or is_not_null in the expr and the remaining.
Categorizes by having at least one is_nan or is_not_nan in the expr and the remaining.
"""
boolean_expression_visit(expr, self)

Expand All @@ -1125,25 +1094,24 @@ def _expression_to_complementary_pyarrow(expr: BooleanExpression, schema: Schema
"""Complementary filter conversion function of expression_to_pyarrow.

Could not use expression_to_pyarrow(Not(expr)) to achieve this complementary effect because
~ in pyarrow.compute.Expression does not handle null.
~ in pyarrow.compute.Expression does not handle null: where expr evaluates to null, Not(expr) is null too
and the row is dropped. Only the rows where expr is true are deleted, so null is read as false.
"""
collector = _NullNaNUnmentionedTermsCollector()
collector = _NaNUnmentionedTermsCollector()
collector.collect(expr)

# Convert the set of terms to a sorted list so that layout of the expression to build is deterministic.
null_unmentioned_bound_terms: list[BoundTerm] = sorted(
collector.null_unmentioned_bound_terms, key=lambda term: term.ref().field.name
)
nan_unmentioned_bound_terms: list[BoundTerm] = sorted(
collector.nan_unmentioned_bound_terms, key=lambda term: term.ref().field.name
)

preserve_expr: BooleanExpression = Not(expr)
for term in null_unmentioned_bound_terms:
preserve_expr = Or(preserve_expr, BoundIsNull(term=term))
preserve_expr = ~pc.coalesce(expression_to_pyarrow(expr, schema), pc.scalar(False))
nan_expr: BooleanExpression = AlwaysFalse()
for term in nan_unmentioned_bound_terms:
preserve_expr = Or(preserve_expr, BoundIsNaN(term=term))
return expression_to_pyarrow(preserve_expr, schema)
nan_expr = Or(nan_expr, BoundIsNaN(term=term))
if nan_expr == AlwaysFalse():
return preserve_expr
return preserve_expr | expression_to_pyarrow(nan_expr, schema)


@lru_cache
Expand Down
25 changes: 25 additions & 0 deletions tests/catalog/test_catalog_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -1295,6 +1295,31 @@ def test_overwrite_record_batch_reader(catalog: Catalog) -> None:
assert len(tbl.scan().to_arrow()) == total_rows


def test_delete_and_overwrite_filter_with_null(catalog: Catalog) -> None:
catalog.create_namespace("default")
pa_table = pa.table({"x": [1, 2, 3], "p": [1, 2, 2], "v": [None, None, "c"]})
filters = [
# A null v makes the second branch null for x=2, which matches neither branch
"(p == 1 AND v IS NULL) OR (p == 2 AND v == 'c')",
# A null v leaves the filter true for x=1
"p == 1 OR v == 'c'",
"v != 'c'",
"NOT (v == 'c')",
]
for i, row_filter in enumerate(filters):
tbl = catalog.create_table(identifier=f"default.delete_filter_with_null_{i}_{catalog.name}", schema=pa_table.schema)
tbl.append(pa_table)
matched = set(tbl.scan(row_filter=row_filter).to_arrow()["x"].to_pylist())
expected = sorted({1, 2, 3} - matched)

tbl.delete(row_filter)
assert sorted(tbl.scan().to_arrow()["x"].to_pylist()) == expected, row_filter

tbl.overwrite(pa_table, overwrite_filter="x > 0")
tbl.overwrite(pa_table.slice(0, 0), overwrite_filter=row_filter)
assert sorted(tbl.scan().to_arrow()["x"].to_pylist()) == expected, row_filter


def test_append_record_batch_reader_to_partitioned_table_raises(catalog: Catalog) -> None:
catalog.create_namespace("default")
identifier = f"default.append_record_batch_reader_partitioned_{catalog.name}"
Expand Down
2 changes: 1 addition & 1 deletion tests/integration/test_deletes.py
Original file line number Diff line number Diff line change
Expand Up @@ -677,7 +677,7 @@ def test_delete_overwrite_table_with_nan(session_catalog: RestCatalog) -> None:

This means if we set the test case as floats == 2.0 (equal predicate as in test_delete_overwrite_table_with_null),
test will pass even without the logic under test
in _NullNaNUnmentionedTermsCollector (a helper of _expression_to_complementary_pyarrow
in _NaNUnmentionedTermsCollector (a helper of _expression_to_complementary_pyarrow
to handle revert of iceberg expression of is_null/not_null/is_nan/not_nan).
Instead, we test the filter of !=, so that the revert is == which exposes the issue.
"""
Expand Down
41 changes: 15 additions & 26 deletions tests/io/test_pyarrow_visitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
_ConvertToIcebergWithoutIDs,
_expression_to_complementary_pyarrow,
_HasIds,
_NullNaNUnmentionedTermsCollector,
_NaNUnmentionedTermsCollector,
_pyarrow_schema_ensure_large_types,
_pyarrow_schema_ensure_small_types,
pyarrow_to_schema,
Expand Down Expand Up @@ -789,19 +789,12 @@ def test_collect_null_nan_unmentioned_terms(
bound_expr = And(
Or(And(bound_eq_str_field, bound_is_nan_float_field), bound_is_null_double_field), Not(bound_is_nan_float_field)
)
collector = _NullNaNUnmentionedTermsCollector()
collector = _NaNUnmentionedTermsCollector()
collector.collect(bound_expr)
assert {t.ref().field.name for t in collector.null_unmentioned_bound_terms} == {
"float_field",
"string_field",
}
assert {t.ref().field.name for t in collector.nan_unmentioned_bound_terms} == {
"string_field",
"double_field",
}
assert {t.ref().field.name for t in collector.is_null_or_not_bound_terms} == {
"double_field",
}
assert {t.ref().field.name for t in collector.is_nan_or_not_bound_terms} == {"float_field"}


Expand All @@ -821,19 +814,12 @@ def test_collect_null_nan_unmentioned_terms_with_multiple_predicates_on_the_same
),
Not(bound_is_null_double_field),
)
collector = _NullNaNUnmentionedTermsCollector()
collector = _NaNUnmentionedTermsCollector()
collector.collect(bound_expr)
assert {t.ref().field.name for t in collector.null_unmentioned_bound_terms} == {
"float_field",
"string_field",
}
assert {t.ref().field.name for t in collector.nan_unmentioned_bound_terms} == {
"string_field",
"double_field",
}
assert {t.ref().field.name for t in collector.is_null_or_not_bound_terms} == {
"double_field",
}
assert {t.ref().field.name for t in collector.is_nan_or_not_bound_terms} == {"float_field"}


Expand All @@ -853,13 +839,16 @@ def test_expression_to_complementary_pyarrow(
Not(bound_is_null_double_field),
)
result = _expression_to_complementary_pyarrow(bound_expr)
# Notice an isNan predicate on a str column is automatically converted to always false and removed from Or
# and thus will not appear in the pc.expr.
expected_repr = (
'<pyarrow.compute.Expression (((invert(((((string_field == "hello") and (float_field > 100)) '
"or ((is_nan(float_field) and (double_field == 0)) or (float_field > 100))) "
"and invert(is_null(double_field, {nan_is_null=false})))) "
"or is_null(float_field, {nan_is_null=false})) "
"or is_null(string_field, {nan_is_null=false})) or is_nan(double_field))>"
table = pa.table(
{
"id": pa.array([1, 2, 3, 4, 5], type=pa.int32()),
"string_field": pa.array(["hello", "hello", "x", "x", None], type=pa.string()),
"float_field": pa.array([200.0, 50.0, float("nan"), 200.0, None], type=pa.float32()),
"double_field": pa.array([1.0, 1.0, 0.0, None, 1.0], type=pa.float64()),
}
)
assert repr(result) == expected_repr
# The complement keeps exactly the rows where bound_expr is not true:
# 1 matches (float_field > 100), 3 matches (is_nan(float_field) and double_field == 0),
# 2 does not match, 4 does not match because double_field is null,
# and 5 evaluates to null because string_field and float_field are null.
assert table.filter(result)["id"].to_pylist() == [2, 4, 5]
Loading