From 066c42541b412d1de6be79ed062879f98aa09182 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Wed, 11 Nov 2026 16:48:15 +0000 Subject: [PATCH] Widen decimal literal precision in scan filters A decimal filter literal with more digits than the column, such as amt < 1000.00 on decimal(5, 2), failed with ArrowInvalid because the Arrow literal took the column's precision. Widen the literal's precision to fit its digits and keep the column's scale, so the comparison is by value. Co-authored-by: Isaac --- pyiceberg/io/pyarrow.py | 19 ++++++++++++++--- tests/io/test_pyarrow.py | 44 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index f62e53a911..1e90fd0de3 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -41,6 +41,7 @@ from collections.abc import Callable, Iterable, Iterator from copy import copy from dataclasses import dataclass +from decimal import Decimal from enum import Enum from functools import lru_cache, singledispatch from typing import ( @@ -886,10 +887,22 @@ def visit_geography(self, geography_type: GeographyType) -> pa.DataType: return pa.large_binary() +def _literal_arrow_type(iceberg_type: IcebergType, values: Iterable[Any]) -> pa.DataType: + """Return the Arrow type for literals compared with a column of the given type. + + A decimal literal can have more digits than the column, e.g. 1000.00 for decimal(5, 2), + so the precision is widened to hold every literal. The scale always stays the column's. + """ + if isinstance(iceberg_type, DecimalType): + digits = [len(value.as_tuple().digits) for value in values if isinstance(value, Decimal)] + return pa.decimal128(max([iceberg_type.precision, *digits]), iceberg_type.scale) + return visit(iceberg_type, _ConvertToArrowSchema()) + + def _convert_scalar(value: Any, iceberg_type: IcebergType) -> pa.scalar: if not isinstance(iceberg_type, PrimitiveType): raise ValueError(f"Expected primitive type, got: {iceberg_type}") - return pa.scalar(value=value, type=schema_to_pyarrow(iceberg_type)) + return pa.scalar(value=value, type=_literal_arrow_type(iceberg_type, [value])) class _ConvertToArrowExpression(BoundBooleanExpressionVisitor[pc.Expression]): @@ -926,11 +939,11 @@ def _get_field_name(self, term: BoundTerm) -> str | tuple[str, ...]: return term.ref().field.name def visit_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) + pyarrow_literals = pa.array(literals, type=_literal_arrow_type(term.ref().field.field_type, literals)) return pc.field(self._get_field_name(term)).isin(pyarrow_literals) def visit_not_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression: - pyarrow_literals = pa.array(literals, type=schema_to_pyarrow(term.ref().field.field_type)) + pyarrow_literals = pa.array(literals, type=_literal_arrow_type(term.ref().field.field_type, literals)) return ~pc.field(self._get_field_name(term)).isin(pyarrow_literals) def visit_is_nan(self, term: BoundTerm) -> pc.Expression: diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 419d130965..122e2f8cff 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -27,6 +27,7 @@ from collections.abc import Iterator from dataclasses import replace from datetime import UTC, date, datetime +from decimal import Decimal from pathlib import Path from typing import Any, Literal from unittest.mock import MagicMock, patch @@ -940,6 +941,49 @@ def test_always_false_to_pyarrow(bound_reference: BoundReference) -> None: assert repr(expression_to_pyarrow(AlwaysFalse())) == "" +def test_decimal_literal_wider_than_column_to_pyarrow() -> None: + schema = Schema(NestedField(1, "amt", DecimalType(5, 2), required=False)) + ref = BoundReference(schema.find_field(1), schema.accessor_for_field(1)) + wide = literal(Decimal("1000.00")).to(DecimalType(5, 2)) + narrow = literal(Decimal("1.25")).to(DecimalType(5, 2)) + arrow_table = pa.table({"amt": pa.array([Decimal("1.25"), Decimal("99.99")], pa.decimal128(5, 2))}) + for bound_expr, expected in [ + (BoundLessThan(term=ref, literal=wide), [Decimal("1.25"), Decimal("99.99")]), + (BoundNotEqualTo(term=ref, literal=wide), [Decimal("1.25"), Decimal("99.99")]), + (BoundIn(term=ref, literals={wide, narrow}), [Decimal("1.25")]), + (BoundNotIn(term=ref, literals={wide, narrow}), [Decimal("99.99")]), + ]: + assert arrow_table.filter(expression_to_pyarrow(bound_expr))["amt"].to_pylist() == expected, bound_expr + + +def test_row_filter_decimal_literal_wider_than_column(catalog: InMemoryCatalog) -> None: + catalog.create_namespace("default") + table = catalog.create_table( + "default.test_decimal_literal_wider_than_column", + schema=Schema( + NestedField(1, "id", IntegerType(), required=False), + NestedField(2, "amt", DecimalType(5, 2), required=False), + ), + ) + table.append( + pa.table( + { + "id": pa.array([1, 2, 3], pa.int32()), + "amt": pa.array([Decimal("1.25"), Decimal("99.99"), None], pa.decimal128(5, 2)), + } + ) + ) + + # 1000.00 needs one more digit than decimal(5, 2) holds, but every value is below it + for row_filter, expected in [ + ("amt < 1000.00", [1, 2]), + ("amt > -1000.00", [1, 2]), + ("amt < 50.00", [1]), + ("amt IN (1.25, 1000.00)", [1]), + ]: + assert sorted(table.scan(row_filter=row_filter).to_arrow()["id"].to_pylist()) == expected, row_filter + + @pytest.fixture def schema_int() -> Schema: return Schema(NestedField(1, "id", IntegerType(), required=False))