diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index 9952fd82d4..377f6c2b4a 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -26,7 +26,7 @@ from datetime import date, datetime, time from decimal import ROUND_HALF_UP, Decimal from functools import singledispatchmethod -from math import isfinite, isnan +from math import isfinite, isinf, isnan from typing import Any, Generic from uuid import UUID @@ -433,6 +433,8 @@ def _(self, _: DoubleType) -> Literal[float]: @to.register(FloatType) def _(self, _: FloatType) -> Literal[float]: + if isinf(self.value): + return FloatLiteral(self.value) if FloatType.max < self.value: return FloatAboveMax() elif FloatType.min > self.value: diff --git a/pyiceberg/utils/datetime.py b/pyiceberg/utils/datetime.py index 70ffb1f303..b187d8cac5 100644 --- a/pyiceberg/utils/datetime.py +++ b/pyiceberg/utils/datetime.py @@ -229,22 +229,39 @@ def micros_to_hours(micros: int) -> int: return micros // 3_600_000_000 +def _civil_from_days(days: int) -> tuple[int, int]: + """Return the year and month of the number of days from 1970-01-01. + + The conversion uses integer arithmetic, so values outside the range of ``datetime.date``, such + as the day after 9999-12-31, are converted as well. Based on + https://howardhinnant.github.io/date_algorithms.html#civil_from_days. + """ + z = days + 719468 + era = z // 146097 + doe = z - era * 146097 + yoe = (doe - doe // 1460 + doe // 36524 - doe // 146096) // 365 + doy = doe - (365 * yoe + yoe // 4 - yoe // 100) + mp = (5 * doy + 2) // 153 + month = mp + 3 if mp < 10 else mp - 9 + year = yoe + era * 400 + (1 if month <= 2 else 0) + return year, month + + def days_to_months(days: int) -> int: - d = days_to_date(days) - return (d.year - EPOCH_DATE.year) * 12 + (d.month - EPOCH_DATE.month) + year, month = _civil_from_days(days) + return (year - EPOCH_DATE.year) * 12 + (month - EPOCH_DATE.month) def micros_to_months(micros: int) -> int: - dt = micros_to_timestamp(micros) - return (dt.year - EPOCH_TIMESTAMP.year) * 12 + (dt.month - EPOCH_TIMESTAMP.month) + return days_to_months(micros // 86_400_000_000) def days_to_years(days: int) -> int: - return days_to_date(days).year - EPOCH_DATE.year + return _civil_from_days(days)[0] - EPOCH_DATE.year def micros_to_years(micros: int) -> int: - return micros_to_timestamp(micros).year - EPOCH_TIMESTAMP.year + return days_to_years(micros // 86_400_000_000) def nanos_to_timestamp(nanos: int) -> datetime: @@ -254,12 +271,11 @@ def nanos_to_timestamp(nanos: int) -> datetime: def nanos_to_years(nanos: int) -> int: - return nanos_to_timestamp(nanos).year - EPOCH_TIMESTAMP.year + return days_to_years(nanos // 86_400_000_000_000) def nanos_to_months(nanos: int) -> int: - dt = nanos_to_timestamp(nanos) - return (dt.year - EPOCH_TIMESTAMP.year) * 12 + (dt.month - EPOCH_TIMESTAMP.month) + return days_to_months(nanos // 86_400_000_000_000) def nanos_to_days(nanos: int) -> int: diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 7e29f8d25d..4b18daa868 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -17,6 +17,7 @@ # pylint:disable=eval-used import datetime +import math import uuid from decimal import Decimal from typing import ( @@ -204,6 +205,11 @@ def test_long_to_float_outside_bound() -> None: assert below_min_lit == FloatBelowMin() +def test_double_to_float_infinity() -> None: + assert literal(math.inf).to(FloatType()) == FloatLiteral(math.inf) + assert literal(-math.inf).to(FloatType()) == FloatLiteral(-math.inf) + + def test_long_to_double_conversion() -> None: lit = literal(34).to(LongType()) dbl_lit = lit.to(DoubleType()) diff --git a/tests/expressions/test_projection.py b/tests/expressions/test_projection.py index 4d0c2c1346..5d787fe5c6 100644 --- a/tests/expressions/test_projection.py +++ b/tests/expressions/test_projection.py @@ -16,6 +16,8 @@ # under the License. # pylint:disable=redefined-outer-name +from datetime import date + import pytest from pyiceberg.expressions import ( @@ -43,6 +45,7 @@ HourTransform, IdentityTransform, TruncateTransform, + YearTransform, ) from pyiceberg.types import ( DateType, @@ -237,6 +240,18 @@ def test_day_projection(schema: Schema, day_spec: PartitionSpec) -> None: assert expected[index] == expr, predicate +def test_year_projection_at_date_limits(schema: Schema) -> None: + year_spec = PartitionSpec(PartitionField(3, 1000, YearTransform(), "event_year")) + max_date = date(9999, 12, 31) + min_date = date(1, 1, 1) + + project = inclusive_projection(schema, year_spec) + assert project(LessThanOrEqual("event_date", max_date)) == LessThanOrEqual("event_year", 8029) + assert project(GreaterThan("event_date", max_date)) == GreaterThanOrEqual("event_year", 8030) + assert project(LessThan("event_date", min_date)) == LessThanOrEqual("event_year", -1970) + assert project(GreaterThanOrEqual("event_date", min_date)) == GreaterThanOrEqual("event_year", -1969) + + def test_date_day_projection(schema: Schema, day_spec: PartitionSpec) -> None: predicates = [ NotNull("event_date"), diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index ca21fd05f0..f0d62447d3 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -17,6 +17,7 @@ # pylint: disable=protected-access,unused-argument,redefined-outer-name import json import logging +import math import os import struct import sys @@ -70,8 +71,13 @@ BoundStartsWith, EqualTo, GreaterThan, + GreaterThanOrEqual, + In, IsNull, + LessThan, + LessThanOrEqual, Not, + NotEqualTo, Or, ) from pyiceberg.expressions.literals import literal @@ -115,7 +121,7 @@ from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID from pyiceberg.table.name_mapping import create_mapping_from_schema from pyiceberg.table.puffin import MAGIC_BYTES, PuffinFile -from pyiceberg.transforms import BucketTransform, HourTransform, IdentityTransform, Transform, TruncateTransform +from pyiceberg.transforms import BucketTransform, HourTransform, IdentityTransform, Transform, TruncateTransform, YearTransform from pyiceberg.typedef import UTF8, Properties, Record, TableVersion from pyiceberg.types import ( BinaryType, @@ -3713,6 +3719,55 @@ def test_append_to_table_bucketed_by_time(catalog: InMemoryCatalog) -> None: assert table.scan(row_filter=EqualTo("event_time", "22:31:08")).to_arrow()["id"].to_pylist() == [2] +def test_scan_filters_on_infinity(catalog: InMemoryCatalog) -> None: + schema = Schema(NestedField(id=1, name="x", field_type=FloatType(), required=False)) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_infinity", schema=schema) + table.append(pa.table({"x": pa.array([1.5, math.inf, -math.inf], type=pa.float32())})) + + assert table.scan(row_filter=EqualTo("x", math.inf)).to_arrow()["x"].to_pylist() == [math.inf] + assert table.scan(row_filter=NotEqualTo("x", math.inf)).to_arrow()["x"].to_pylist() == [1.5, -math.inf] + assert sorted(table.scan(row_filter=In("x", {math.inf, 1.5})).to_arrow()["x"].to_pylist()) == [1.5, math.inf] + + +def test_scan_and_delete_filters_on_infinity(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=IntegerType(), required=False), + NestedField(id=2, name="x", field_type=FloatType(), required=False), + ) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_and_delete_infinity", schema=schema) + table.append( + pa.table( + { + "id": pa.array([1, 2, 3, 4, 5], type=pa.int32()), + "x": pa.array([1.5, math.inf, -math.inf, math.nan, None], type=pa.float32()), + } + ) + ) + + assert table.scan(row_filter=LessThan("x", math.inf)).to_arrow()["id"].to_pylist() == [1, 3] + assert table.scan(row_filter=LessThanOrEqual("x", math.inf)).to_arrow()["id"].to_pylist() == [1, 2, 3] + assert table.scan(row_filter=GreaterThan("x", -math.inf)).to_arrow()["id"].to_pylist() == [1, 2] + assert table.scan(row_filter=GreaterThanOrEqual("x", math.inf)).to_arrow()["id"].to_pylist() == [2] + + table.delete(LessThan("x", math.inf)) + + assert table.scan().to_arrow()["id"].to_pylist() == [2, 4, 5] + + +def test_scan_filters_at_the_end_of_the_date_range(catalog: InMemoryCatalog) -> None: + schema = Schema(NestedField(id=1, name="x", field_type=DateType(), required=False)) + spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=YearTransform(), name="x_year")) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_date_range", schema=schema, partition_spec=spec) + last_day = date(9999, 12, 31) + table.append(pa.table({"x": pa.array([date(2024, 5, 1), last_day], type=pa.date32())})) + + assert table.scan(row_filter=LessThanOrEqual("x", last_day)).count() == 2 + assert table.scan(row_filter=GreaterThan("x", last_day)).count() == 0 + + def test_append_counts_nan_values(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="f", field_type=FloatType(), required=False), diff --git a/tests/utils/test_datetime.py b/tests/utils/test_datetime.py index 6cbbf9c1c4..01b6a330bd 100644 --- a/tests/utils/test_datetime.py +++ b/tests/utils/test_datetime.py @@ -24,7 +24,11 @@ datetime_to_millis, datetime_to_nanos, days_to_date, + days_to_months, + days_to_years, micros_to_days, + micros_to_months, + micros_to_years, millis_to_datetime, nanos_to_hours, nanos_to_micros, @@ -290,3 +294,27 @@ def test_to_human_timestamptz() -> None: (1_740_342_104_375_612, "2025-02-23T20:21:44.375612+00:00"), ]: assert to_human_timestamptz(micros) == expected + + +@pytest.mark.parametrize( + "days, years, months", + [ + (0, 0, 0), + (-1, -1, -1), + (2932896, 8029, 96359), + (2932897, 8030, 96360), + (-719162, -1969, -23628), + (-719163, -1970, -23629), + (-719529, -1971, -23641), + ], +) +def test_days_to_years_and_months(days: int, years: int, months: int) -> None: + assert days_to_years(days) == years + assert days_to_months(days) == months + + +def test_years_and_months_beyond_the_date_range() -> None: + day_micros = 86_400_000_000 + + assert micros_to_years(2932897 * day_micros) == 8030 + assert micros_to_months(2932897 * day_micros) == 96360