Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 35 additions & 8 deletions python/pyspark/sql/_pandas_input.py
Original file line number Diff line number Diff line change
Expand Up @@ -286,10 +286,11 @@ def timestamp_columns(frame, schema):
buffers separately so NaT's sentinel never participates in alignment tests.
"""
import sys
from types import GetSetDescriptorType, WrapperDescriptorType
from types import FunctionType, GetSetDescriptorType, WrapperDescriptorType

import numpy as np
import pandas as pd
from pandas.core import frame as frame_module, generic as generic_module
try:
from pandas.core.arrays.datetimes import DatetimeArray
from pandas.core.dtypes.dtypes import DatetimeTZDtype
Expand All @@ -305,7 +306,23 @@ def timestamp_columns(frame, schema):
from pyspark.sql import types as sql_types

if (pd.DataFrame is not DataFrame or type(frame) is not DataFrame
or type(schema) is not NativeStructType or not len(frame)):
or type(schema) is not NativeStructType or not len(frame)
or any(name in vars(frame) for name in ("astype", "notna", "where"))):
return None
# The row path calls these normalization methods before reading cells.
# Preserve customized transformations and errors before any Arrow Config RPC.
for name, module, qualname in (
("astype", generic_module, "NDFrame.astype"),
("notna", frame_module, "DataFrame.notna"),
("where", generic_module, "NDFrame.where"),
):
method = getattr(DataFrame, name)
if (type(method) is not FunctionType or method.__name__ != name
or method.__qualname__ != qualname or method.__module__ != module.__name__
or method.__globals__ is not vars(module)):
return None
if (DataFrame.astype is not generic_module.NDFrame.astype
or DataFrame.where is not generic_module.NDFrame.where):
return None
if not _has_canonical_columns(frame.columns):
return None
Expand Down Expand Up @@ -390,6 +407,10 @@ def timestamp_columns(frame, schema):
if unit == "ns" and np.any(column.array.asi8[column.notna()] % 1000):
return None
columns.append(column)
# Reject unsupported host zones before Rust queries the session time zone.
# Their row fallback builds a local relation without any configuration RPC.
if not timestamp_columns_are_aware(columns) and not _timestamp_host_is_utc():
return None
return columns


Expand All @@ -400,20 +421,26 @@ def timestamp_columns_are_aware(columns):
return all(type(column.dtype) is DatetimeTZDtype for column in columns)


def _timestamp_host_is_utc():
"""Require an explicit UTC host setting and an initialized UTC runtime zone."""
import os
import time

return os.environ.get("TZ") == "UTC" and time.tzname == ("UTC", "UTC")


def prepare_timestamp_table(columns, schema, timezone):
"""Preserve instants without row materialization or a host-zone conversion.

Aware columns carry their own timezone. Naive columns are deliberately
restricted to UTC host/session pairs: the existing row path uses the host
zone, unlike Python Connect's session-zone conversion (and DST fold policy).
restricted to literal ``TZ=UTC`` and session ``UTC``, with a UTC runtime
zone: the existing row path uses the host zone, unlike Python Connect's
session-zone conversion (and DST fold policy).
"""
import os
import time
import pyarrow as pa

if not timestamp_columns_are_aware(columns):
if (timezone != "UTC" or os.environ.get("TZ") != "UTC"
or time.tzname != ("UTC", "UTC")):
if timezone != "UTC" or not _timestamp_host_is_utc():
return None
return pa.Table.from_arrays(
[pa.array(column, type=pa.timestamp("us", tz="UTC"), from_pandas=True)
Expand Down
148 changes: 135 additions & 13 deletions python/tests/test_arrow_handoff.py
Original file line number Diff line number Diff line change
Expand Up @@ -2087,10 +2087,6 @@ def test_pandas_timestamp_ipc_preserves_every_microsecond(protocol, monkeypatch,
schema = T.StructType([T.StructField("renamed_a", T.TimestampType(), True, {"source": "test"}),
T.StructField("b", T.TimestampType())])

def no_rows(*args, **kwargs):
raise AssertionError("supported timestamps must not materialize Python rows")

monkeypatch.setattr(pd.DataFrame, "astype", no_rows)
with _arrow_input_service(protocol, capture_tables=True, **{"spark.sql.session.timeZone": zone}) as (spark, state):
result = spark.createDataFrame(frame, schema)
# Aware columns carry their zone: only the importer's settings are read.
Expand All @@ -2117,7 +2113,6 @@ def test_pandas_naive_timestamp_utc_ipc(protocol, monkeypatch):
frame = pd.DataFrame({"a": pd.to_datetime(["1969-12-31 23:59:59.999999", None,
"2020-01-01 00:00:00.123456"])})
schema = T.StructType([T.StructField("a", T.TimestampType())])
monkeypatch.setattr(pd.DataFrame, "astype", lambda *a, **k: pytest.fail("row conversion"))
with _arrow_input_service(protocol, capture_tables=True) as (spark, state):
result = spark.createDataFrame(frame, schema)
# A naive column reads the session time zone, then the importer its settings.
Expand All @@ -2132,9 +2127,13 @@ def test_pandas_naive_timestamp_utc_ipc(protocol, monkeypatch):

@pytest.mark.parametrize("dtype", ["datetime64[ns]", "datetime64[us]", "datetime64[ms]",
"datetime64[s]", "object"])
def test_pandas_timestamp_eligibility(dtype):
def test_pandas_timestamp_eligibility(monkeypatch, dtype):
import time
from pyspark.sql import types as T
from pyspark.sql._pandas_input import timestamp_columns

monkeypatch.setenv("TZ", "UTC")
monkeypatch.setattr(time, "tzname", ("UTC", "UTC"))
frame = pd.DataFrame({"a": pd.Series([datetime(2020, 1, 1), None], dtype=dtype)})
schema = T.StructType([T.StructField("a", T.TimestampType())])
assert (timestamp_columns(frame, schema) is not None) == (dtype in (
Expand All @@ -2146,22 +2145,74 @@ def test_pandas_timestamp_eligibility(dtype):


def test_pandas_timestamp_precision_and_timezone_fallback(monkeypatch):
import time
from pyspark.sql import types as T
from pyspark.sql._pandas_input import timestamp_columns, prepare_timestamp_table

monkeypatch.setenv("TZ", "UTC")
monkeypatch.setattr(time, "tzname", ("UTC", "UTC"))
schema = T.StructType([T.StructField("a", T.TimestampType())])
for nanos in (-1, 1, 1001):
frame = pd.DataFrame({"a": pd.to_datetime([nanos, None], unit="ns")})
assert timestamp_columns(frame, schema) is None
frame = pd.DataFrame({"a": pd.to_datetime([0, None], unit="ns")})
columns = timestamp_columns(frame, schema)
monkeypatch.setenv("TZ", "UTC")
assert columns is not None
assert prepare_timestamp_table(columns, schema, "America/Los_Angeles") is None
monkeypatch.setenv("TZ", "America/Los_Angeles")
assert prepare_timestamp_table(columns, schema, "UTC") is None
monkeypatch.delenv("TZ")
assert prepare_timestamp_table(columns, schema, "UTC") is None


@pytest.mark.parametrize("unit,mixed", [("ns", False), ("us", True)])
@pytest.mark.parametrize("host_tz,host_names", [
(None, ("UTC", "UTC")),
("America/Los_Angeles", ("PST", "PDT")),
("UTC", ("PST", "PDT")),
], ids=["unset-tz", "non-utc-tz", "mismatched-host-zone"])
def test_timestamp_non_utc_host_declines_and_converts_as_pyspark(
protocol, monkeypatch, unit, mixed, host_tz, host_names
):
# A host zone other than a literal UTC declines the timestamp fast path before its
# session time zone RPC; the frame then converts as PySpark's Arrow conversion does,
# which reads the configs first and takes naive values in the session time zone (UTC).
import json
import time

if host_tz is None:
monkeypatch.delenv("TZ", raising=False)
else:
monkeypatch.setenv("TZ", host_tz)
monkeypatch.setattr(time, "tzname", host_names)
micros = [-2000000, 0, 123456, None]
columns = {}
if mixed:
columns["aware"] = pd.to_datetime(micros, unit="us", utc=True).as_unit(unit)
columns["naive"] = pd.to_datetime(micros, unit="us").as_unit(unit)
frame = pd.DataFrame(columns)
schema = T.StructType([
T.StructField(f"renamed_{name}", T.TimestampType(), True, {"source": name})
for name in columns
])
from pyspark.sql._pandas_input import timestamp_columns

assert timestamp_columns(frame, schema) is None
expected = [micros for _ in columns]
with _arrow_input_service(protocol) as (spark, state):
spark.createDataFrame(frame, schema).toArrow()
assert state["config_requests"]
assert state["uploads"] == 0
assert state["column_names"] == schema.names
assert len(state["input_tables"]) == 1
table = state["input_tables"][0]
assert table.num_rows == len(micros)
for column, values in zip(table.columns, expected):
assert column.type == pa.timestamp("us", tz="UTC")
assert column.cast(pa.int64()).to_pylist() == values
assert column.is_null().to_pylist() == [value is None for value in values]


@pytest.mark.parametrize("has_null", [False, True])
def test_pandas_timestamp_non_nullable_schema(protocol, has_null):
from pyspark.sql import types as T
Expand Down Expand Up @@ -2218,6 +2269,81 @@ def test_timestamp_nonnullable_nat_reads_config_before_cast_error(protocol, conf
assert state["uploads"] == 0


@pytest.mark.parametrize("unit,aware", [("ns", True), ("us", False)])
@pytest.mark.parametrize("location,method", [
("instance", "notna"), ("instance", "astype"), ("instance", "where"),
("class", "notna"), ("class", "astype"), ("class", "where"),
])
@pytest.mark.parametrize("behavior", ["mutate", "raise"])
def test_timestamp_custom_normalization_hooks_keep_native_row_path(
protocol, monkeypatch, unit, aware, location, method, behavior
):
import json
import time
from types import MethodType
import pyspark.sql._pandas_input as helper

monkeypatch.setenv("TZ", "UTC")
monkeypatch.setattr(time, "tzname", ("UTC", "UTC"))
values = pd.to_datetime([-2000000, -1000000, 0, 1000000, None], unit="us", utc=aware)
frame = pd.DataFrame({"original": values.as_unit(unit)})
schema = T.StructType([
T.StructField("renamed", T.TimestampType(), True, {"source": "normalization"})
])
original = getattr(pd.DataFrame, method)
calls, tables = [], []
# astype returns a new frame, so the row path never calls an instance where.
invokes_hook = (location, method) != ("instance", "where")
expected = [None if pd.isna(value) else round(value.to_pydatetime().timestamp() * 1000000)
for value in frame["original"]]
if invokes_hook:
expected[0] = None if method == "notna" else 7000000
message = f"custom timestamp {method} precedes configuration"

def customized(self, *args, **kwargs):
calls.append(method)
if behavior == "raise":
raise RuntimeError(message)
result = original(self, *args, **kwargs)
result.iat[0, 0] = (False if method == "notna"
else datetime(1970, 1, 1, 0, 0, 7, tzinfo=timezone.utc))
return result

if location == "class":
monkeypatch.setattr(pd.DataFrame, method, customized)
else:
monkeypatch.setitem(vars(frame), method, MethodType(customized, frame))

# Compare with native row conversion, then restore ordinary dispatch. A failing
# configuration service makes any premature Arrow configuration read observable.
for bypass in (True, False):
calls.clear()
with monkeypatch.context() as dispatch:
if bypass:
dispatch.setattr(helper, "timestamp_columns", lambda *_: None)
with _arrow_input_service(protocol, config_error=True) as (spark, state):
if behavior == "raise" and invokes_hook:
with pytest.raises(RuntimeError, match=message):
spark.createDataFrame(frame, schema)
assert state["relations"] == []
assert state["input_tables"] == []
else:
spark.createDataFrame(frame, schema).toArrow()
assert len(state["input_tables"]) == 1
table = state["input_tables"][0]
assert state["column_names"] == schema.names
assert table.column(0).cast(pa.int64()).to_pylist() == expected
assert table.column(0).is_null().to_pylist() == [
value is None for value in expected]
assert json.loads(state["relations"][0].local_relation.schema) == json.loads(schema.json())
tables.append(table)
assert calls == ([method] if invokes_hook else [])
assert state["config_requests"] == []
assert state["uploads"] == 0
if behavior == "mutate" or not invokes_hook:
assert tables[0].equals(tables[1], check_metadata=True)


@pytest.mark.parametrize("custom", ["dtype", "array"])
def test_timestamp_custom_datetime_array_keeps_row_values(protocol, custom):
from pandas.core.arrays.datetimes import DatetimeArray
Expand Down Expand Up @@ -2427,8 +2553,8 @@ def prepare_table(table, schema, *configs):
spark.createDataFrame(pa.table({"warm": [1]})).toArrow()
monkeypatch.setitem(sys.modules, importer.__name__, substitute)
assert timestamp_columns(frame, schema) is not None
monkeypatch.setattr(pd.DataFrame, "astype", lambda *a, **k: pytest.fail("row conversion"))
spark.createDataFrame(frame, schema).toArrow()
assert len(state["config_requests"]) == 2
table = state["tables"][-1]
assert table.column_names == ["renamed"]
assert table.column(0).cast(pa.int64()).to_pylist() == [0, 1000, None]
Expand Down Expand Up @@ -2669,7 +2795,7 @@ def set_data(index, value):

@pytest.mark.parametrize("unit", ["ns", "us"])
@pytest.mark.parametrize("storage", ["object", "default", "python", "pyarrow", "python_nan", "pyarrow_nan"])
def test_timestamp_canonical_columns_storage_stays_columnar(protocol, monkeypatch, unit, storage):
def test_timestamp_canonical_columns_storage_stays_columnar(protocol, unit, storage):
from pyspark.sql.pandas.types import to_arrow_schema
import json

Expand All @@ -2682,10 +2808,6 @@ def test_timestamp_canonical_columns_storage_stays_columnar(protocol, monkeypatc
if storage.endswith("_nan") else pd.StringDtype(storage=kind))
frame.columns = pd.Index(pd.array(["original"], dtype=dtype))

def no_rows(*args, **kwargs):
raise AssertionError("canonical timestamp labels must retain columnar construction")

monkeypatch.setattr(pd.DataFrame, "astype", no_rows)
with _arrow_input_service(protocol) as (spark, state):
spark.createDataFrame(frame, schema).toArrow()
assert state["config_requests"]
Expand Down
Loading