diff --git a/pyathena/converter.py b/pyathena/converter.py index dfca02c1b..671f29866 100644 --- a/pyathena/converter.py +++ b/pyathena/converter.py @@ -15,6 +15,7 @@ from typing import Any, ClassVar from zoneinfo import ZoneInfo, ZoneInfoNotFoundError +from pyathena.error import DataError from pyathena.parser import ( TypedValueConverter, TypeNode, @@ -94,44 +95,70 @@ def _parse_utc_offset(value: str) -> timezone | None: @functools.cache -def _parse_time_zone(value: str) -> tzinfo | None: +def _parse_time_zone(value: str) -> tzinfo: """Parse the time zone of an Athena TIMESTAMP WITH TIME ZONE value. - Results are cached per text, so an unknown zone name logs a warning only on its - first lookup. - Args: value: A ``+HH:MM`` or ``-HH:MM`` UTC offset or an IANA time zone name. Returns: - The fixed-offset time zone or ``ZoneInfo``, or None if the IANA time zone - database available to Python cannot provide a zone with the name. + The fixed-offset time zone or ``ZoneInfo``. + + Raises: + DataError: If the IANA time zone database available to Python cannot + provide a zone with the name. """ offset = _parse_utc_offset(value) if offset is not None: return offset try: return ZoneInfo(value) - except (ZoneInfoNotFoundError, ValueError, OSError): - _logger.warning(f"Unknown time zone {value!r}; returning naive datetimes for it.") - return None + except (ZoneInfoNotFoundError, ValueError, OSError) as e: + raise DataError( + f"Unknown time zone {value!r}. " + "Update the tzdata package or the system time zone database." + ) from e def _to_datetime_with_tz(varchar_value: str | None) -> datetime | None: """Convert an Athena TIMESTAMP WITH TIME ZONE value to an aware datetime. + Athena writes a value with a zone name as the local time in that zone. A local + time that a change of the zone's UTC offset, such as DST, repeats or skips does + not identify one instant, so it is not converted. + Args: varchar_value: The value as text with a trailing zone name or ``+HH:MM`` / ``-HH:MM`` UTC offset, or None. An empty string, which pandas and the Arrow CSV reader return for NULL, is None. Returns: - The aware datetime, a naive datetime if the zone name is unknown, or None. + The aware datetime, or None. + + Raises: + DataError: If the zone name is unknown, or the local time is ambiguous or + does not exist in the zone. """ if not varchar_value: return None datetime_, _, tz = varchar_value.rpartition(" ") - return _parse_datetime(datetime_).replace(tzinfo=_parse_time_zone(tz)) + zone = _parse_time_zone(tz) + value = _parse_datetime(datetime_).replace(tzinfo=zone) + if isinstance(zone, timezone): + return value + # PEP 495: fold=0 and fold=1 give different offsets only at a transition. At a + # repeated local time, fold=0 is the earlier instant; at a skipped one, the later. + folded = value.replace(fold=1) + if value.utcoffset() != folded.utcoffset(): + if value.timestamp() < folded.timestamp(): + reason = "is ambiguous: the local time occurs twice" + else: + reason = "does not exist: the local time is skipped" + raise DataError( + f"{varchar_value!r} {reason} in {tz}. Convert the value to UTC in SQL, " + "for example with AT TIME ZONE 'UTC'." + ) + return value def _parse_time(value: str) -> time: diff --git a/pyproject.toml b/pyproject.toml index 7997e2755..5ebc49443 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "botocore>=1.43.31", "tenacity>=8.0.1", "fsspec>=2026.9.0", - "tzdata", + "tzdata>=2024.2", ] requires-python = ">=3.11" readme = "README.md" diff --git a/tests/pyathena/arrow/test_cursor.py b/tests/pyathena/arrow/test_cursor.py index 162a146a3..180a1c281 100644 --- a/tests/pyathena/arrow/test_cursor.py +++ b/tests/pyathena/arrow/test_cursor.py @@ -22,12 +22,13 @@ from pyathena.arrow.converter import DefaultArrowTypeConverter from pyathena.arrow.cursor import ArrowCursor from pyathena.arrow.result_set import AthenaArrowResultSet -from pyathena.error import DatabaseError, ProgrammingError +from pyathena.error import DatabaseError, DataError, ProgrammingError from pyathena.model import AthenaQueryExecution from pyathena.util import RetryConfig from tests import ENV from tests.pyathena.conftest import connect from tests.pyathena.util import ( + AMBIGUOUS_TIME_ZONE_QUERY, CONVERTED_VALUES_QUERY, CONVERTED_VALUES_ROW, RESULT_FILE_VALUES_QUERY, @@ -1066,6 +1067,15 @@ def test_fetch_all_rows(self, arrow_cursor): "2024-02-29 23:59:58.123 +05:30" ] + def test_ambiguous_time_zone_value(self, arrow_cursor): + """Fetch methods raise DataError for a local time that its zone repeats. + + The table keeps the text, as for every TIMESTAMP WITH TIME ZONE value. + """ + arrow_cursor.execute(AMBIGUOUS_TIME_ZONE_QUERY) + with pytest.raises(DataError, match="is ambiguous"): + arrow_cursor.fetchall() + @pytest.mark.skipif(not ENV.managed_work_group, reason="AWS_ATHENA_MANAGED_WORKGROUP not set") def test_managed_results_match_result_file(self): """Managed results have the types and values of the CSV result file. diff --git a/tests/pyathena/pandas/test_cursor.py b/tests/pyathena/pandas/test_cursor.py index a61d3cf18..0dbd79e72 100644 --- a/tests/pyathena/pandas/test_cursor.py +++ b/tests/pyathena/pandas/test_cursor.py @@ -14,7 +14,7 @@ import pytest from pandas.io.parsers import TextFileReader -from pyathena.error import DatabaseError, OperationalError, ProgrammingError +from pyathena.error import DatabaseError, DataError, OperationalError, ProgrammingError from pyathena.filesystem.s3 import S3FileSystem from pyathena.model import AthenaQueryExecution from pyathena.pandas.converter import DefaultPandasTypeConverter @@ -28,6 +28,7 @@ from tests import ENV from tests.pyathena.conftest import connect from tests.pyathena.util import ( + AMBIGUOUS_TIME_ZONE_QUERY, CONVERTED_VALUES_QUERY, CONVERTED_VALUES_ROW, RESULT_FILE_VALUES_QUERY, @@ -1903,6 +1904,24 @@ def test_fetch_all_rows(self, pandas_cursor): pandas_cursor.execute(CONVERTED_VALUES_QUERY) assert pandas_cursor.fetchall() == [CONVERTED_VALUES_ROW] + def test_ambiguous_time_zone_value(self, pandas_cursor): + """A local time that its zone repeats raises instead of choosing one. + + A whole read converts the values in execute() and wraps the converter's + DataError in OperationalError. + """ + with pytest.raises(DatabaseError, match="is ambiguous"): + pandas_cursor.execute(AMBIGUOUS_TIME_ZONE_QUERY) + + @pytest.mark.parametrize( + "pandas_cursor", [{"cursor_kwargs": {"chunksize": 1}}], indirect=["pandas_cursor"] + ) + def test_ambiguous_time_zone_value_chunked(self, pandas_cursor): + """A chunked read raises DataError when it reads the chunk with the value.""" + pandas_cursor.execute(AMBIGUOUS_TIME_ZONE_QUERY) + with pytest.raises(DataError, match="is ambiguous"): + pandas_cursor.fetchall() + @pytest.mark.skipif(not ENV.managed_work_group, reason="AWS_ATHENA_MANAGED_WORKGROUP not set") def test_managed_results_match_result_file(self): """Managed results have the dtypes and values of the CSV result file. diff --git a/tests/pyathena/polars/test_cursor.py b/tests/pyathena/polars/test_cursor.py index c8183f07b..e84be4990 100644 --- a/tests/pyathena/polars/test_cursor.py +++ b/tests/pyathena/polars/test_cursor.py @@ -17,7 +17,7 @@ import polars as pl import pytest -from pyathena.error import DatabaseError, ProgrammingError +from pyathena.error import DatabaseError, DataError, ProgrammingError from pyathena.model import AthenaQueryExecution from pyathena.polars.converter import DefaultPolarsTypeConverter from pyathena.polars.cursor import PolarsCursor @@ -26,6 +26,7 @@ from tests import ENV from tests.pyathena.conftest import connect from tests.pyathena.util import ( + AMBIGUOUS_TIME_ZONE_QUERY, CONVERTED_VALUES_QUERY, CONVERTED_VALUES_ROW, RESULT_FILE_VALUES_QUERY, @@ -902,6 +903,15 @@ def test_fetch_all_rows(self, polars_cursor): "2024-02-29 23:59:58.123 +05:30" ] + def test_ambiguous_time_zone_value(self, polars_cursor): + """Fetch methods raise DataError for a local time that its zone repeats. + + The table keeps the text, as for every TIMESTAMP WITH TIME ZONE value. + """ + polars_cursor.execute(AMBIGUOUS_TIME_ZONE_QUERY) + with pytest.raises(DataError, match="is ambiguous"): + polars_cursor.fetchall() + @pytest.mark.skipif(not ENV.managed_work_group, reason="AWS_ATHENA_MANAGED_WORKGROUP not set") def test_managed_results_match_result_file(self): """Managed results have the types and values of the CSV result file. diff --git a/tests/pyathena/s3fs/test_cursor.py b/tests/pyathena/s3fs/test_cursor.py index d5f56e677..fbc335864 100644 --- a/tests/pyathena/s3fs/test_cursor.py +++ b/tests/pyathena/s3fs/test_cursor.py @@ -10,7 +10,7 @@ import pytest from pyathena.converter import _to_default -from pyathena.error import DatabaseError, ProgrammingError +from pyathena.error import DatabaseError, DataError, ProgrammingError from pyathena.model import AthenaQueryExecution from pyathena.s3fs.converter import DefaultS3FSTypeConverter from pyathena.s3fs.cursor import S3FSCursor @@ -19,7 +19,12 @@ from pyathena.util import RetryConfig from tests import ENV from tests.pyathena.conftest import connect -from tests.pyathena.util import CONVERTED_VALUES_QUERY, CONVERTED_VALUES_ROW, cached_file_systems +from tests.pyathena.util import ( + AMBIGUOUS_TIME_ZONE_QUERY, + CONVERTED_VALUES_QUERY, + CONVERTED_VALUES_ROW, + cached_file_systems, +) def _s3fs_converter_with_json_text(): @@ -541,6 +546,12 @@ def test_fetch_all_rows(self, s3fs_cursor): s3fs_cursor.execute(CONVERTED_VALUES_QUERY) assert s3fs_cursor.fetchall() == [CONVERTED_VALUES_ROW] + def test_ambiguous_time_zone_value(self, s3fs_cursor): + """A local time that its zone repeats raises DataError instead of choosing one.""" + s3fs_cursor.execute(AMBIGUOUS_TIME_ZONE_QUERY) + with pytest.raises(DataError, match="is ambiguous"): + s3fs_cursor.fetchall() + @pytest.mark.parametrize( "s3fs_cursor", [ diff --git a/tests/pyathena/test_converter.py b/tests/pyathena/test_converter.py index 2edfada83..bac5a8e0c 100644 --- a/tests/pyathena/test_converter.py +++ b/tests/pyathena/test_converter.py @@ -1,5 +1,5 @@ -import logging -from datetime import date, datetime, time, timedelta, timezone +import re +from datetime import UTC, date, datetime, time, timedelta, timezone from decimal import Decimal from unittest.mock import patch from zoneinfo import ZoneInfo @@ -8,7 +8,6 @@ from pyathena.converter import ( DefaultTypeConverter, - _parse_time_zone, _to_array, _to_datetime, _to_datetime_with_tz, @@ -18,6 +17,7 @@ _to_time, _to_time_with_tz, ) +from pyathena.error import DataError @pytest.mark.parametrize( @@ -724,34 +724,74 @@ def test_to_datetime_with_tz_offsets_and_zone_names(input_value, expected): @pytest.mark.parametrize("zone_name", ["Foo/Bar", "../etc"]) -def test_to_datetime_with_tz_unknown_zone_name(caplog, zone_name): - """An unknown zone name gives a naive datetime and one warning per name.""" - _parse_time_zone.cache_clear() - with caplog.at_level(logging.WARNING, logger="pyathena.converter"): - results = [ - _to_datetime_with_tz(f"2024-02-29 23:59:58.123 {zone_name}"), - _to_datetime_with_tz(f"2024-02-29 23:59:59 {zone_name}"), - ] - assert results == [ - datetime(2024, 2, 29, 23, 59, 58, 123000), - datetime(2024, 2, 29, 23, 59, 59), - ] - assert [record.getMessage() for record in caplog.records] == [ - f"Unknown time zone {zone_name!r}; returning naive datetimes for it." - ] +def test_to_datetime_with_tz_unknown_zone_name(zone_name): + """An unknown zone name raises DataError instead of giving a naive datetime.""" + with pytest.raises(DataError, match=f"Unknown time zone '{re.escape(zone_name)}'"): + _to_datetime_with_tz(f"2024-02-29 23:59:58.123 {zone_name}") def test_to_datetime_with_tz_unreadable_zone_name(): - """A zone name the database cannot load gives a naive datetime. + """A zone name the database cannot load raises DataError. With only the tzdata package, ``ZoneInfo("America")`` opens a directory and raises ``IsADirectoryError``. """ - _parse_time_zone.cache_clear() - with patch("pyathena.converter.ZoneInfo", side_effect=IsADirectoryError("America")): - result = _to_datetime_with_tz("2024-02-29 23:59:58 America") - _parse_time_zone.cache_clear() - assert result == datetime(2024, 2, 29, 23, 59, 58) + with ( + patch("pyathena.converter.ZoneInfo", side_effect=IsADirectoryError("America")), + pytest.raises(DataError, match="Unknown time zone 'America'"), + ): + _to_datetime_with_tz("2024-02-29 23:59:58 America") + + +@pytest.mark.parametrize( + ("input_value", "match"), + [ + # Athena writes 05:30 and 06:30 UTC both as this text. + ("2020-11-01 01:30:00.000 America/New_York", "is ambiguous"), + ("2020-11-01 01:00:00.000 America/New_York", "is ambiguous"), + ("2020-11-01 01:59:59.999999 America/New_York", "is ambiguous"), + ("2020-10-25 01:30:00.000 Europe/London", "is ambiguous"), + # A 30-minute DST. + ("2020-04-05 01:45:00.000 Australia/Lord_Howe", "is ambiguous"), + ("2020-03-08 02:30:00.000 America/New_York", "does not exist"), + ("2020-03-08 02:00:00.000 America/New_York", "does not exist"), + ("2020-03-29 01:30:00.000 Europe/London", "does not exist"), + ], +) +def test_to_datetime_with_tz_repeated_or_skipped_local_time(input_value, match): + """A local time that a DST transition repeats or skips raises DataError.""" + with pytest.raises(DataError, match=match): + _to_datetime_with_tz(input_value) + + +def test_typed_datetime_with_tz_elements_ambiguous_local_time(): + """An ambiguous element of a type-hinted array raises DataError.""" + with pytest.raises(DataError, match="is ambiguous"): + DefaultTypeConverter().convert( + "array", + "[2020-11-01 01:30:00.000 America/New_York, null]", + type_hint="array(timestamp(3) with time zone)", + ) + + +@pytest.mark.parametrize( + ("input_value", "expected_utc"), + [ + ("2020-11-01 00:59:59.999999 America/New_York", datetime(2020, 11, 1, 4, 59, 59, 999999)), + ("2020-11-01 02:00:00.000 America/New_York", datetime(2020, 11, 1, 7)), + ("2020-03-08 01:59:59.999999 America/New_York", datetime(2020, 3, 8, 6, 59, 59, 999999)), + ("2020-03-08 03:00:00.000 America/New_York", datetime(2020, 3, 8, 7)), + ("2020-11-01 01:30:00.000 UTC", datetime(2020, 11, 1, 1, 30)), + ("2020-11-01 01:30:00.000 -04:00", datetime(2020, 11, 1, 5, 30)), + ], +) +def test_to_datetime_with_tz_around_dst_transitions(input_value, expected_utc): + """Local times next to a DST transition convert to one instant. + + So does the repeated local time with a zone or offset that has no transition. + """ + result = _to_datetime_with_tz(input_value) + assert result.astimezone(UTC) == expected_utc.replace(tzinfo=UTC) @pytest.mark.parametrize( diff --git a/tests/pyathena/test_cursor.py b/tests/pyathena/test_cursor.py index c484786d4..f1578fd1f 100644 --- a/tests/pyathena/test_cursor.py +++ b/tests/pyathena/test_cursor.py @@ -33,7 +33,13 @@ from pyathena.async_cursor import AsyncCursor from pyathena.converter import _to_array, _to_map, _to_struct from pyathena.cursor import Cursor, DictCursor -from pyathena.error import DatabaseError, NotSupportedError, OperationalError, ProgrammingError +from pyathena.error import ( + DatabaseError, + DataError, + NotSupportedError, + OperationalError, + ProgrammingError, +) from pyathena.model import AthenaQueryExecution from pyathena.result_set import AthenaDictResultSet from pyathena.util import RetryConfig @@ -41,6 +47,7 @@ from tests.pyathena.conftest import connect from tests.pyathena.tables import TABLES, VIEWS from tests.pyathena.util import ( + AMBIGUOUS_TIME_ZONE_QUERY, CONVERTED_VALUES_QUERY, CONVERTED_VALUES_ROW, EVENT_TIMEOUT, @@ -1687,6 +1694,29 @@ def test_fetch_all_rows(self, cursor): cursor.execute(CONVERTED_VALUES_QUERY) assert cursor.fetchall() == [CONVERTED_VALUES_ROW] + @pytest.mark.parametrize( + "cursor", + [ + pytest.param({}, id="default"), + pytest.param( + {"work_group": ENV.managed_work_group, "s3_staging_dir": ""}, + id="managed", + marks=pytest.mark.skipif( + not ENV.managed_work_group, + reason="AWS_ATHENA_MANAGED_WORKGROUP not set", + ), + ), + ], + indirect=["cursor"], + ) + def test_ambiguous_time_zone_value(self, cursor): + """A local time that its zone repeats raises DataError instead of choosing one. + + The first page of rows is converted by execute(). + """ + with pytest.raises(DataError, match="is ambiguous"): + cursor.execute(AMBIGUOUS_TIME_ZONE_QUERY) + @pytest.mark.parametrize( "cursor", [ diff --git a/tests/pyathena/util.py b/tests/pyathena/util.py index cf227edb4..49edb95a0 100644 --- a/tests/pyathena/util.py +++ b/tests/pyathena/util.py @@ -222,6 +222,12 @@ def unreachable_glue(connection): None, ) +# A TIMESTAMP WITH TIME ZONE value in the hour that a DST transition repeats. Athena +# writes 05:30 and 06:30 UTC both as 2020-11-01 01:30:00.000 America/New_York. +AMBIGUOUS_TIME_ZONE_QUERY = """ +SELECT at_timezone(TIMESTAMP '2020-11-01 06:30:00 UTC', 'America/New_York') AS col_timestamp_tz +""" + # Values that the cursors reading CSV result files must read the same way from the # GetQueryResults rows of managed query result storage. RESULT_FILE_VALUES_QUERY = """ diff --git a/uv.lock b/uv.lock index b9d779bcc..8dc272a66 100644 --- a/uv.lock +++ b/uv.lock @@ -1307,7 +1307,7 @@ requires-dist = [ { name = "sqlalchemy", marker = "extra == 'sqlalchemy'", specifier = ">=2.0.0" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'aiosqlalchemy'", specifier = ">=2.0.0" }, { name = "tenacity", specifier = ">=8.0.1" }, - { name = "tzdata" }, + { name = "tzdata", specifier = ">=2024.2" }, ] provides-extras = ["aiosqlalchemy", "arrow", "pandas", "polars", "sqlalchemy"]