From 6b072428e15b7413b849d13fdfcc4582945a9cc4 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Sat, 17 Oct 2026 08:17:09 +0900 Subject: [PATCH 1/3] Raise DataError for time zone values without one instant Athena writes a TIMESTAMP WITH TIME ZONE value with a zone name as the local time in that zone. The converter returned a naive datetime for a zone name unknown to Python's time zone database and silently chose fold=0 for a local time that a DST transition repeats, although Athena writes two instants as the same text there. The converter now raises DataError for an unknown zone name, an ambiguous local time, and a local time that a DST transition skips, which Athena never writes and which therefore means that Python's rules differ from Athena's. The message suggests converting the value to UTC in SQL. Require tzdata>=2024.2 (IANA 2024b), which knows every zone name that Athena accepts. Closes #1220 Co-Authored-By: Claude Opus 5.5 --- pyathena/converter.py | 45 ++++++++++---- pyproject.toml | 2 +- tests/pyathena/arrow/test_cursor.py | 12 +++- tests/pyathena/pandas/test_cursor.py | 21 ++++++- tests/pyathena/polars/test_cursor.py | 12 +++- tests/pyathena/s3fs/test_cursor.py | 15 ++++- tests/pyathena/test_converter.py | 92 ++++++++++++++++++++++------ tests/pyathena/test_cursor.py | 32 +++++++++- tests/pyathena/util.py | 6 ++ uv.lock | 2 +- 10 files changed, 200 insertions(+), 39 deletions(-) diff --git a/pyathena/converter.py b/pyathena/converter.py index dfca02c1b..f890ab6a3 100644 --- a/pyathena/converter.py +++ b/pyathena/converter.py @@ -15,6 +15,7 @@ from typing import Any, ClassVar from zoneinfo import ZoneInfo, ZoneInfoNotFoundError +from pyathena.error import DataError from pyathena.parser import ( TypedValueConverter, TypeNode, @@ -94,44 +95,66 @@ def _parse_utc_offset(value: str) -> timezone | None: @functools.cache -def _parse_time_zone(value: str) -> tzinfo | None: +def _parse_time_zone(value: str) -> tzinfo: """Parse the time zone of an Athena TIMESTAMP WITH TIME ZONE value. - Results are cached per text, so an unknown zone name logs a warning only on its - first lookup. - Args: value: A ``+HH:MM`` or ``-HH:MM`` UTC offset or an IANA time zone name. Returns: - The fixed-offset time zone or ``ZoneInfo``, or None if the IANA time zone - database available to Python cannot provide a zone with the name. + The fixed-offset time zone or ``ZoneInfo``. + + Raises: + DataError: If the IANA time zone database available to Python cannot + provide a zone with the name. """ offset = _parse_utc_offset(value) if offset is not None: return offset try: return ZoneInfo(value) - except (ZoneInfoNotFoundError, ValueError, OSError): - _logger.warning(f"Unknown time zone {value!r}; returning naive datetimes for it.") - return None + except (ZoneInfoNotFoundError, ValueError, OSError) as e: + raise DataError( + f"Unknown time zone {value!r}. " + "Update the tzdata package or the system time zone database." + ) from e def _to_datetime_with_tz(varchar_value: str | None) -> datetime | None: """Convert an Athena TIMESTAMP WITH TIME ZONE value to an aware datetime. + Athena writes a value with a zone name as the local time in that zone. The + local time of a DST transition does not identify one instant, so it is not + converted. + Args: varchar_value: The value as text with a trailing zone name or ``+HH:MM`` / ``-HH:MM`` UTC offset, or None. An empty string, which pandas and the Arrow CSV reader return for NULL, is None. Returns: - The aware datetime, a naive datetime if the zone name is unknown, or None. + The aware datetime, or None. + + Raises: + DataError: If the zone name is unknown, or the local time is ambiguous or + does not exist in the zone. """ if not varchar_value: return None datetime_, _, tz = varchar_value.rpartition(" ") - return _parse_datetime(datetime_).replace(tzinfo=_parse_time_zone(tz)) + value = _parse_datetime(datetime_).replace(tzinfo=_parse_time_zone(tz)) + # PEP 495: fold=0 and fold=1 give different offsets only at a transition. At a + # repeated local time, fold=0 is the earlier instant; at a skipped one, the later. + if value.utcoffset() != value.replace(fold=1).utcoffset(): + if value.timestamp() < value.replace(fold=1).timestamp(): + reason = "is ambiguous: the local time occurs twice" + else: + reason = "does not exist: the local time is skipped" + raise DataError( + f"{varchar_value!r} {reason} in {tz}. Convert the value to UTC in SQL, " + "for example with AT TIME ZONE 'UTC'." + ) + return value def _parse_time(value: str) -> time: diff --git a/pyproject.toml b/pyproject.toml index 7997e2755..5ebc49443 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "botocore>=1.43.31", "tenacity>=8.0.1", "fsspec>=2026.9.0", - "tzdata", + "tzdata>=2024.2", ] requires-python = ">=3.11" readme = "README.md" diff --git a/tests/pyathena/arrow/test_cursor.py b/tests/pyathena/arrow/test_cursor.py index 162a146a3..180a1c281 100644 --- a/tests/pyathena/arrow/test_cursor.py +++ b/tests/pyathena/arrow/test_cursor.py @@ -22,12 +22,13 @@ from pyathena.arrow.converter import DefaultArrowTypeConverter from pyathena.arrow.cursor import ArrowCursor from pyathena.arrow.result_set import AthenaArrowResultSet -from pyathena.error import DatabaseError, ProgrammingError +from pyathena.error import DatabaseError, DataError, ProgrammingError from pyathena.model import AthenaQueryExecution from pyathena.util import RetryConfig from tests import ENV from tests.pyathena.conftest import connect from tests.pyathena.util import ( + AMBIGUOUS_TIME_ZONE_QUERY, CONVERTED_VALUES_QUERY, CONVERTED_VALUES_ROW, RESULT_FILE_VALUES_QUERY, @@ -1066,6 +1067,15 @@ def test_fetch_all_rows(self, arrow_cursor): "2024-02-29 23:59:58.123 +05:30" ] + def test_ambiguous_time_zone_value(self, arrow_cursor): + """Fetch methods raise DataError for a local time that its zone repeats. + + The table keeps the text, as for every TIMESTAMP WITH TIME ZONE value. + """ + arrow_cursor.execute(AMBIGUOUS_TIME_ZONE_QUERY) + with pytest.raises(DataError, match="is ambiguous"): + arrow_cursor.fetchall() + @pytest.mark.skipif(not ENV.managed_work_group, reason="AWS_ATHENA_MANAGED_WORKGROUP not set") def test_managed_results_match_result_file(self): """Managed results have the types and values of the CSV result file. diff --git a/tests/pyathena/pandas/test_cursor.py b/tests/pyathena/pandas/test_cursor.py index a61d3cf18..0dbd79e72 100644 --- a/tests/pyathena/pandas/test_cursor.py +++ b/tests/pyathena/pandas/test_cursor.py @@ -14,7 +14,7 @@ import pytest from pandas.io.parsers import TextFileReader -from pyathena.error import DatabaseError, OperationalError, ProgrammingError +from pyathena.error import DatabaseError, DataError, OperationalError, ProgrammingError from pyathena.filesystem.s3 import S3FileSystem from pyathena.model import AthenaQueryExecution from pyathena.pandas.converter import DefaultPandasTypeConverter @@ -28,6 +28,7 @@ from tests import ENV from tests.pyathena.conftest import connect from tests.pyathena.util import ( + AMBIGUOUS_TIME_ZONE_QUERY, CONVERTED_VALUES_QUERY, CONVERTED_VALUES_ROW, RESULT_FILE_VALUES_QUERY, @@ -1903,6 +1904,24 @@ def test_fetch_all_rows(self, pandas_cursor): pandas_cursor.execute(CONVERTED_VALUES_QUERY) assert pandas_cursor.fetchall() == [CONVERTED_VALUES_ROW] + def test_ambiguous_time_zone_value(self, pandas_cursor): + """A local time that its zone repeats raises instead of choosing one. + + A whole read converts the values in execute() and wraps the converter's + DataError in OperationalError. + """ + with pytest.raises(DatabaseError, match="is ambiguous"): + pandas_cursor.execute(AMBIGUOUS_TIME_ZONE_QUERY) + + @pytest.mark.parametrize( + "pandas_cursor", [{"cursor_kwargs": {"chunksize": 1}}], indirect=["pandas_cursor"] + ) + def test_ambiguous_time_zone_value_chunked(self, pandas_cursor): + """A chunked read raises DataError when it reads the chunk with the value.""" + pandas_cursor.execute(AMBIGUOUS_TIME_ZONE_QUERY) + with pytest.raises(DataError, match="is ambiguous"): + pandas_cursor.fetchall() + @pytest.mark.skipif(not ENV.managed_work_group, reason="AWS_ATHENA_MANAGED_WORKGROUP not set") def test_managed_results_match_result_file(self): """Managed results have the dtypes and values of the CSV result file. diff --git a/tests/pyathena/polars/test_cursor.py b/tests/pyathena/polars/test_cursor.py index c8183f07b..e84be4990 100644 --- a/tests/pyathena/polars/test_cursor.py +++ b/tests/pyathena/polars/test_cursor.py @@ -17,7 +17,7 @@ import polars as pl import pytest -from pyathena.error import DatabaseError, ProgrammingError +from pyathena.error import DatabaseError, DataError, ProgrammingError from pyathena.model import AthenaQueryExecution from pyathena.polars.converter import DefaultPolarsTypeConverter from pyathena.polars.cursor import PolarsCursor @@ -26,6 +26,7 @@ from tests import ENV from tests.pyathena.conftest import connect from tests.pyathena.util import ( + AMBIGUOUS_TIME_ZONE_QUERY, CONVERTED_VALUES_QUERY, CONVERTED_VALUES_ROW, RESULT_FILE_VALUES_QUERY, @@ -902,6 +903,15 @@ def test_fetch_all_rows(self, polars_cursor): "2024-02-29 23:59:58.123 +05:30" ] + def test_ambiguous_time_zone_value(self, polars_cursor): + """Fetch methods raise DataError for a local time that its zone repeats. + + The table keeps the text, as for every TIMESTAMP WITH TIME ZONE value. + """ + polars_cursor.execute(AMBIGUOUS_TIME_ZONE_QUERY) + with pytest.raises(DataError, match="is ambiguous"): + polars_cursor.fetchall() + @pytest.mark.skipif(not ENV.managed_work_group, reason="AWS_ATHENA_MANAGED_WORKGROUP not set") def test_managed_results_match_result_file(self): """Managed results have the types and values of the CSV result file. diff --git a/tests/pyathena/s3fs/test_cursor.py b/tests/pyathena/s3fs/test_cursor.py index d5f56e677..fbc335864 100644 --- a/tests/pyathena/s3fs/test_cursor.py +++ b/tests/pyathena/s3fs/test_cursor.py @@ -10,7 +10,7 @@ import pytest from pyathena.converter import _to_default -from pyathena.error import DatabaseError, ProgrammingError +from pyathena.error import DatabaseError, DataError, ProgrammingError from pyathena.model import AthenaQueryExecution from pyathena.s3fs.converter import DefaultS3FSTypeConverter from pyathena.s3fs.cursor import S3FSCursor @@ -19,7 +19,12 @@ from pyathena.util import RetryConfig from tests import ENV from tests.pyathena.conftest import connect -from tests.pyathena.util import CONVERTED_VALUES_QUERY, CONVERTED_VALUES_ROW, cached_file_systems +from tests.pyathena.util import ( + AMBIGUOUS_TIME_ZONE_QUERY, + CONVERTED_VALUES_QUERY, + CONVERTED_VALUES_ROW, + cached_file_systems, +) def _s3fs_converter_with_json_text(): @@ -541,6 +546,12 @@ def test_fetch_all_rows(self, s3fs_cursor): s3fs_cursor.execute(CONVERTED_VALUES_QUERY) assert s3fs_cursor.fetchall() == [CONVERTED_VALUES_ROW] + def test_ambiguous_time_zone_value(self, s3fs_cursor): + """A local time that its zone repeats raises DataError instead of choosing one.""" + s3fs_cursor.execute(AMBIGUOUS_TIME_ZONE_QUERY) + with pytest.raises(DataError, match="is ambiguous"): + s3fs_cursor.fetchall() + @pytest.mark.parametrize( "s3fs_cursor", [ diff --git a/tests/pyathena/test_converter.py b/tests/pyathena/test_converter.py index 2edfada83..96a397d9a 100644 --- a/tests/pyathena/test_converter.py +++ b/tests/pyathena/test_converter.py @@ -1,11 +1,12 @@ -import logging -from datetime import date, datetime, time, timedelta, timezone +import re +from datetime import UTC, date, datetime, time, timedelta, timezone from decimal import Decimal from unittest.mock import patch from zoneinfo import ZoneInfo import pytest +from pyathena import DataError from pyathena.converter import ( DefaultTypeConverter, _parse_time_zone, @@ -724,34 +725,85 @@ def test_to_datetime_with_tz_offsets_and_zone_names(input_value, expected): @pytest.mark.parametrize("zone_name", ["Foo/Bar", "../etc"]) -def test_to_datetime_with_tz_unknown_zone_name(caplog, zone_name): - """An unknown zone name gives a naive datetime and one warning per name.""" +def test_to_datetime_with_tz_unknown_zone_name(zone_name): + """An unknown zone name raises DataError instead of giving a naive datetime.""" _parse_time_zone.cache_clear() - with caplog.at_level(logging.WARNING, logger="pyathena.converter"): - results = [ - _to_datetime_with_tz(f"2024-02-29 23:59:58.123 {zone_name}"), - _to_datetime_with_tz(f"2024-02-29 23:59:59 {zone_name}"), - ] - assert results == [ - datetime(2024, 2, 29, 23, 59, 58, 123000), - datetime(2024, 2, 29, 23, 59, 59), - ] - assert [record.getMessage() for record in caplog.records] == [ - f"Unknown time zone {zone_name!r}; returning naive datetimes for it." - ] + with pytest.raises(DataError, match=f"Unknown time zone '{re.escape(zone_name)}'"): + _to_datetime_with_tz(f"2024-02-29 23:59:58.123 {zone_name}") def test_to_datetime_with_tz_unreadable_zone_name(): - """A zone name the database cannot load gives a naive datetime. + """A zone name the database cannot load raises DataError. With only the tzdata package, ``ZoneInfo("America")`` opens a directory and raises ``IsADirectoryError``. """ _parse_time_zone.cache_clear() - with patch("pyathena.converter.ZoneInfo", side_effect=IsADirectoryError("America")): - result = _to_datetime_with_tz("2024-02-29 23:59:58 America") + with ( + patch("pyathena.converter.ZoneInfo", side_effect=IsADirectoryError("America")), + pytest.raises(DataError, match="Unknown time zone 'America'"), + ): + _to_datetime_with_tz("2024-02-29 23:59:58 America") _parse_time_zone.cache_clear() - assert result == datetime(2024, 2, 29, 23, 59, 58) + + +@pytest.mark.parametrize( + "input_value", + [ + # Athena writes 05:30 and 06:30 UTC both as this text. + "2020-11-01 01:30:00.000 America/New_York", + "2020-11-01 01:00:00.000 America/New_York", + "2020-11-01 01:59:59.999999 America/New_York", + "2020-10-25 01:30:00.000 Europe/London", + # A 30-minute DST. + "2020-04-05 01:45:00.000 Australia/Lord_Howe", + ], +) +def test_to_datetime_with_tz_ambiguous_local_time(input_value): + """A local time that occurs twice in its zone raises DataError.""" + with pytest.raises(DataError, match="is ambiguous"): + _to_datetime_with_tz(input_value) + + +def test_typed_datetime_with_tz_elements_ambiguous_local_time(): + """An ambiguous element of a type-hinted array raises DataError.""" + with pytest.raises(DataError, match="is ambiguous"): + DefaultTypeConverter().convert( + "array", + "[2020-11-01 01:30:00.000 America/New_York, null]", + type_hint="array(timestamp(3) with time zone)", + ) + + +@pytest.mark.parametrize( + "input_value", + [ + "2020-03-08 02:30:00.000 America/New_York", + "2020-03-08 02:00:00.000 America/New_York", + "2020-03-29 01:30:00.000 Europe/London", + ], +) +def test_to_datetime_with_tz_nonexistent_local_time(input_value): + """A local time that a DST transition skips raises DataError.""" + with pytest.raises(DataError, match="does not exist"): + _to_datetime_with_tz(input_value) + + +@pytest.mark.parametrize( + ("input_value", "expected_utc"), + [ + ("2020-11-01 00:59:59.999999 America/New_York", datetime(2020, 11, 1, 4, 59, 59, 999999)), + ("2020-11-01 02:00:00.000 America/New_York", datetime(2020, 11, 1, 7)), + ("2020-03-08 01:59:59.999999 America/New_York", datetime(2020, 3, 8, 6, 59, 59, 999999)), + ("2020-03-08 03:00:00.000 America/New_York", datetime(2020, 3, 8, 7)), + ("2020-11-01 01:30:00.000 UTC", datetime(2020, 11, 1, 1, 30)), + ("2020-11-01 01:30:00.000 -04:00", datetime(2020, 11, 1, 5, 30)), + ], +) +def test_to_datetime_with_tz_around_dst_transitions(input_value, expected_utc): + """Local times next to a DST transition, and offsets, convert to one instant.""" + result = _to_datetime_with_tz(input_value) + assert result.astimezone(UTC) == expected_utc.replace(tzinfo=UTC) @pytest.mark.parametrize( diff --git a/tests/pyathena/test_cursor.py b/tests/pyathena/test_cursor.py index c484786d4..f1578fd1f 100644 --- a/tests/pyathena/test_cursor.py +++ b/tests/pyathena/test_cursor.py @@ -33,7 +33,13 @@ from pyathena.async_cursor import AsyncCursor from pyathena.converter import _to_array, _to_map, _to_struct from pyathena.cursor import Cursor, DictCursor -from pyathena.error import DatabaseError, NotSupportedError, OperationalError, ProgrammingError +from pyathena.error import ( + DatabaseError, + DataError, + NotSupportedError, + OperationalError, + ProgrammingError, +) from pyathena.model import AthenaQueryExecution from pyathena.result_set import AthenaDictResultSet from pyathena.util import RetryConfig @@ -41,6 +47,7 @@ from tests.pyathena.conftest import connect from tests.pyathena.tables import TABLES, VIEWS from tests.pyathena.util import ( + AMBIGUOUS_TIME_ZONE_QUERY, CONVERTED_VALUES_QUERY, CONVERTED_VALUES_ROW, EVENT_TIMEOUT, @@ -1687,6 +1694,29 @@ def test_fetch_all_rows(self, cursor): cursor.execute(CONVERTED_VALUES_QUERY) assert cursor.fetchall() == [CONVERTED_VALUES_ROW] + @pytest.mark.parametrize( + "cursor", + [ + pytest.param({}, id="default"), + pytest.param( + {"work_group": ENV.managed_work_group, "s3_staging_dir": ""}, + id="managed", + marks=pytest.mark.skipif( + not ENV.managed_work_group, + reason="AWS_ATHENA_MANAGED_WORKGROUP not set", + ), + ), + ], + indirect=["cursor"], + ) + def test_ambiguous_time_zone_value(self, cursor): + """A local time that its zone repeats raises DataError instead of choosing one. + + The first page of rows is converted by execute(). + """ + with pytest.raises(DataError, match="is ambiguous"): + cursor.execute(AMBIGUOUS_TIME_ZONE_QUERY) + @pytest.mark.parametrize( "cursor", [ diff --git a/tests/pyathena/util.py b/tests/pyathena/util.py index cf227edb4..49edb95a0 100644 --- a/tests/pyathena/util.py +++ b/tests/pyathena/util.py @@ -222,6 +222,12 @@ def unreachable_glue(connection): None, ) +# A TIMESTAMP WITH TIME ZONE value in the hour that a DST transition repeats. Athena +# writes 05:30 and 06:30 UTC both as 2020-11-01 01:30:00.000 America/New_York. +AMBIGUOUS_TIME_ZONE_QUERY = """ +SELECT at_timezone(TIMESTAMP '2020-11-01 06:30:00 UTC', 'America/New_York') AS col_timestamp_tz +""" + # Values that the cursors reading CSV result files must read the same way from the # GetQueryResults rows of managed query result storage. RESULT_FILE_VALUES_QUERY = """ diff --git a/uv.lock b/uv.lock index b9d779bcc..8dc272a66 100644 --- a/uv.lock +++ b/uv.lock @@ -1307,7 +1307,7 @@ requires-dist = [ { name = "sqlalchemy", marker = "extra == 'sqlalchemy'", specifier = ">=2.0.0" }, { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'aiosqlalchemy'", specifier = ">=2.0.0" }, { name = "tenacity", specifier = ">=8.0.1" }, - { name = "tzdata" }, + { name = "tzdata", specifier = ">=2024.2" }, ] provides-extras = ["aiosqlalchemy", "arrow", "pandas", "polars", "sqlalchemy"] From 0ebd096be610f8346dd4f7d5bef0bd5ace2b6d91 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Sat, 17 Oct 2026 08:19:22 +0900 Subject: [PATCH 2/3] Describe offset changes other than DST in the converter docstring Co-Authored-By: Claude Opus 5.5 --- pyathena/converter.py | 6 +++--- tests/pyathena/test_converter.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pyathena/converter.py b/pyathena/converter.py index f890ab6a3..142c860bd 100644 --- a/pyathena/converter.py +++ b/pyathena/converter.py @@ -123,9 +123,9 @@ def _parse_time_zone(value: str) -> tzinfo: def _to_datetime_with_tz(varchar_value: str | None) -> datetime | None: """Convert an Athena TIMESTAMP WITH TIME ZONE value to an aware datetime. - Athena writes a value with a zone name as the local time in that zone. The - local time of a DST transition does not identify one instant, so it is not - converted. + Athena writes a value with a zone name as the local time in that zone. A local + time that a change of the zone's UTC offset, such as DST, repeats or skips does + not identify one instant, so it is not converted. Args: varchar_value: The value as text with a trailing zone name or ``+HH:MM`` / diff --git a/tests/pyathena/test_converter.py b/tests/pyathena/test_converter.py index 96a397d9a..65add9cb9 100644 --- a/tests/pyathena/test_converter.py +++ b/tests/pyathena/test_converter.py @@ -727,7 +727,6 @@ def test_to_datetime_with_tz_offsets_and_zone_names(input_value, expected): @pytest.mark.parametrize("zone_name", ["Foo/Bar", "../etc"]) def test_to_datetime_with_tz_unknown_zone_name(zone_name): """An unknown zone name raises DataError instead of giving a naive datetime.""" - _parse_time_zone.cache_clear() with pytest.raises(DataError, match=f"Unknown time zone '{re.escape(zone_name)}'"): _to_datetime_with_tz(f"2024-02-29 23:59:58.123 {zone_name}") From cabcb4e309df03d71cc766d7363c37610cfcb321 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Sat, 17 Oct 2026 08:30:06 +0900 Subject: [PATCH 3/3] Skip the transition check for fixed UTC offsets A fixed offset has no transitions, so values with one return before the fold comparison and keep master's per-value cost. Merge the repeated and skipped local time tests into one parametrized test, and drop cache resets that no longer guard anything because failed lookups are not cached. Co-Authored-By: Claude Opus 5.5 --- pyathena/converter.py | 10 ++++--- tests/pyathena/test_converter.py | 45 ++++++++++++-------------------- 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/pyathena/converter.py b/pyathena/converter.py index 142c860bd..671f29866 100644 --- a/pyathena/converter.py +++ b/pyathena/converter.py @@ -142,11 +142,15 @@ def _to_datetime_with_tz(varchar_value: str | None) -> datetime | None: if not varchar_value: return None datetime_, _, tz = varchar_value.rpartition(" ") - value = _parse_datetime(datetime_).replace(tzinfo=_parse_time_zone(tz)) + zone = _parse_time_zone(tz) + value = _parse_datetime(datetime_).replace(tzinfo=zone) + if isinstance(zone, timezone): + return value # PEP 495: fold=0 and fold=1 give different offsets only at a transition. At a # repeated local time, fold=0 is the earlier instant; at a skipped one, the later. - if value.utcoffset() != value.replace(fold=1).utcoffset(): - if value.timestamp() < value.replace(fold=1).timestamp(): + folded = value.replace(fold=1) + if value.utcoffset() != folded.utcoffset(): + if value.timestamp() < folded.timestamp(): reason = "is ambiguous: the local time occurs twice" else: reason = "does not exist: the local time is skipped" diff --git a/tests/pyathena/test_converter.py b/tests/pyathena/test_converter.py index 65add9cb9..bac5a8e0c 100644 --- a/tests/pyathena/test_converter.py +++ b/tests/pyathena/test_converter.py @@ -6,10 +6,8 @@ import pytest -from pyathena import DataError from pyathena.converter import ( DefaultTypeConverter, - _parse_time_zone, _to_array, _to_datetime, _to_datetime_with_tz, @@ -19,6 +17,7 @@ _to_time, _to_time_with_tz, ) +from pyathena.error import DataError @pytest.mark.parametrize( @@ -737,30 +736,31 @@ def test_to_datetime_with_tz_unreadable_zone_name(): With only the tzdata package, ``ZoneInfo("America")`` opens a directory and raises ``IsADirectoryError``. """ - _parse_time_zone.cache_clear() with ( patch("pyathena.converter.ZoneInfo", side_effect=IsADirectoryError("America")), pytest.raises(DataError, match="Unknown time zone 'America'"), ): _to_datetime_with_tz("2024-02-29 23:59:58 America") - _parse_time_zone.cache_clear() @pytest.mark.parametrize( - "input_value", + ("input_value", "match"), [ # Athena writes 05:30 and 06:30 UTC both as this text. - "2020-11-01 01:30:00.000 America/New_York", - "2020-11-01 01:00:00.000 America/New_York", - "2020-11-01 01:59:59.999999 America/New_York", - "2020-10-25 01:30:00.000 Europe/London", + ("2020-11-01 01:30:00.000 America/New_York", "is ambiguous"), + ("2020-11-01 01:00:00.000 America/New_York", "is ambiguous"), + ("2020-11-01 01:59:59.999999 America/New_York", "is ambiguous"), + ("2020-10-25 01:30:00.000 Europe/London", "is ambiguous"), # A 30-minute DST. - "2020-04-05 01:45:00.000 Australia/Lord_Howe", + ("2020-04-05 01:45:00.000 Australia/Lord_Howe", "is ambiguous"), + ("2020-03-08 02:30:00.000 America/New_York", "does not exist"), + ("2020-03-08 02:00:00.000 America/New_York", "does not exist"), + ("2020-03-29 01:30:00.000 Europe/London", "does not exist"), ], ) -def test_to_datetime_with_tz_ambiguous_local_time(input_value): - """A local time that occurs twice in its zone raises DataError.""" - with pytest.raises(DataError, match="is ambiguous"): +def test_to_datetime_with_tz_repeated_or_skipped_local_time(input_value, match): + """A local time that a DST transition repeats or skips raises DataError.""" + with pytest.raises(DataError, match=match): _to_datetime_with_tz(input_value) @@ -774,20 +774,6 @@ def test_typed_datetime_with_tz_elements_ambiguous_local_time(): ) -@pytest.mark.parametrize( - "input_value", - [ - "2020-03-08 02:30:00.000 America/New_York", - "2020-03-08 02:00:00.000 America/New_York", - "2020-03-29 01:30:00.000 Europe/London", - ], -) -def test_to_datetime_with_tz_nonexistent_local_time(input_value): - """A local time that a DST transition skips raises DataError.""" - with pytest.raises(DataError, match="does not exist"): - _to_datetime_with_tz(input_value) - - @pytest.mark.parametrize( ("input_value", "expected_utc"), [ @@ -800,7 +786,10 @@ def test_to_datetime_with_tz_nonexistent_local_time(input_value): ], ) def test_to_datetime_with_tz_around_dst_transitions(input_value, expected_utc): - """Local times next to a DST transition, and offsets, convert to one instant.""" + """Local times next to a DST transition convert to one instant. + + So does the repeated local time with a zone or offset that has no transition. + """ result = _to_datetime_with_tz(input_value) assert result.astimezone(UTC) == expected_utc.replace(tzinfo=UTC)