Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
dc29a19
[SPARK-60676][CONNECT] Use Arrow columns for signed int64 pandas inputs
dbtsai Oct 29, 2026
3e79f91
[SPARK-60676][CONNECT] Validate integer upload scales against mock RPC
dbtsai Oct 29, 2026
8c705c0
[SPARK-60676][CONNECT] Cover leading null inference and non-nullable …
dbtsai Oct 29, 2026
fb996d9
[SPARK-60676][CONNECT] Exercise named schemas through mock rename rel…
dbtsai Oct 29, 2026
79d3faf
[SPARK-60676][CONNECT] Avoid Arrow configuration overhead for small N…
dbtsai Oct 29, 2026
e266e8e
[SPARK-60676][CONNECT] Retain IPC row fallback for NUL field names
dbtsai Oct 29, 2026
9942f21
[SPARK-60676][CONNECT] Include pandas regressions in the existing Arr…
dbtsai Oct 29, 2026
5edd700
[SPARK-60676][CONNECT] Preserve integer pandas schema and fallback se…
dbtsai Oct 29, 2026
080aa16
[SPARK-60676][TESTS] Preserve subclass column labels on pandas 3
dbtsai Oct 30, 2026
abbc23a
[SPARK-60676][CONNECT] Preserve customized pandas columns hooks
dbtsai Oct 30, 2026
7961bde
[SPARK-60676][CONNECT] Preserve rebound pandas index storage ordering
dbtsai Oct 30, 2026
3e3a8f5
Preserve integer pandas fallback with missing Arrow aliases
dbtsai Oct 31, 2026
f058ac9
[SPARK-60676][TESTS] Retain current master coverage after rebase
dbtsai Nov 5, 2026
121a141
Preserve current pandas label iteration and fallback order after rebase
dbtsai Nov 5, 2026
aca1d02
Keep canonical string storage guard module identity
dbtsai Nov 5, 2026
ecdd67a
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
e7ae71a
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
40e5266
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
9b99bec
[SPARK-60686][CONNECT] Use Arrow columns for supported pandas timestamps
dbtsai Oct 29, 2026
4a8d032
[SPARK-60686][CONNECT] Preserve timestamp row fallback for customized…
dbtsai Oct 29, 2026
70aefe3
[SPARK-60686][CONNECT][PYTHON] Preserve timestamp columns hook order
dbtsai Oct 30, 2026
fc04109
[SPARK-60686][CONNECT] Preserve current Arrow alias fallback semantics
dbtsai Nov 5, 2026
ed0dec8
Align timestamp label regression tests with current native iteration
dbtsai Nov 5, 2026
9491c38
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
a34d64e
Merge remote-tracking branch 'apache/master' into upd-130
HyukjinKwon Nov 6, 2026
19a8d99
[SPARK-60686][CONNECT] Read the session time zone only for naive pand…
HyukjinKwon Nov 6, 2026
5687f90
[SPARK-60686][CONNECT] Do not pin the reads of master's pandas row pa…
HyukjinKwon Nov 6, 2026
9888234
[SPARK-60686][CONNECT] Test the configuration reads of aware and naiv…
HyukjinKwon Nov 6, 2026
91807b3
Merge remote-tracking branch 'apache/master' into upd-130
HyukjinKwon Nov 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions crates/pyspark-rs/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -835,6 +835,42 @@ impl PySparkSession {
_ => frame,
});
}
let helper = py.import("pyspark.sql._pandas_input")?;
let columns = helper.getattr("timestamp_columns")?.call1((data, schema))?;
if !columns.is_none() {
// Only a naive column needs the session time zone; aware columns carry theirs.
let aware = helper
.getattr("timestamp_columns_are_aware")?
.call1((&columns,))?
.is_truthy()?;
let timezone = if aware {
None
} else {
py.detach(|| {
self.session
.conf()
.get("spark.sql.session.timeZone")
.to_pyerr()
})?
};
let table = helper
.getattr("prepare_timestamp_table")?
.call1((columns, schema, timezone))?;
if !table.is_none() {
let frame =
crate::pyarrow_import::create_dataframe(py, &self.session, &table, schema)?;
return Ok(match &spec {
Spec::Struct(fields) if !fields.is_empty() && table.len()? > 0 => {
PyDataFrame::new(
frame.dataframe.to_df(
fields.iter().map(|field| field.name.as_str()).collect(),
),
)
}
_ => frame,
});
}
}
return self.create_from_pandas(py, data, spec);
}
// Preserve schema parsing, ignored-argument warnings, and pandas/Arrow routing.
Expand Down
144 changes: 144 additions & 0 deletions python/pyspark/sql/_pandas_input.py
Original file line number Diff line number Diff line change
Expand Up @@ -276,3 +276,147 @@ def prepare_int64_table(frame, schema):
[pa.array(column, type=pa.int64(), from_pandas=True) for column in columns],
names=names,
)


def timestamp_columns(frame, schema):
"""Select only explicit, microsecond-aligned datetime64 timestamp columns.

Inference (including TimestampNTZ preferences), object/mixed columns, other
units and submicrosecond truncation retain the row path. Check the integer
buffers separately so NaT's sentinel never participates in alignment tests.
"""
import sys
from types import GetSetDescriptorType, WrapperDescriptorType

import numpy as np
import pandas as pd
try:
from pandas.core.arrays.datetimes import DatetimeArray
from pandas.core.dtypes.dtypes import DatetimeTZDtype
from pandas.core.frame import DataFrame
from pandas.core.series import Series
except ImportError:
return None
from pyspark._pyspark import (
AtomicType, BooleanType, ByteType, ShortType, IntegerType, LongType, FloatType,
DoubleType, DecimalType, StringType, BinaryType, DateType, TimestampType,
StructField, StructType as NativeStructType,
)
from pyspark.sql import types as sql_types

if (pd.DataFrame is not DataFrame or type(frame) is not DataFrame
or type(schema) is not NativeStructType or not len(frame)):
return None
if not _has_canonical_columns(frame.columns):
return None
# Native row conversion reads the native schema directly. The Arrow importer
# also consults Python descriptors and public or captured converter aliases.
if sys.modules.get("pyspark.sql.types") is not sql_types:
return None
importer = sys.modules.get("pyspark.sql._arrow_input")
if vars(sql_types).get("AtomicType") is not AtomicType:
return None
if importer is not None:
if getattr(importer, "StructType", None) is not NativeStructType:
return None
if vars(importer).get("AtomicType") is not AtomicType:
return None
converter = sys.modules.get("pyspark.sql.pandas.types")
namespaces = [vars(sql_types)]
if converter is not None:
namespaces.append(vars(converter))
if importer is not None:
for name in ("from_arrow_schema", "to_arrow_schema",
"_check_arrow_table_timestamps_localize"):
# Newer importers resolve converters locally within prepare_table.
if name not in vars(importer):
continue
namespace = getattr(getattr(importer, name, None), "__globals__", None)
if type(namespace) is not dict:
return None
namespaces.append(namespace)
for name, cls in (
("BooleanType", BooleanType), ("ByteType", ByteType), ("ShortType", ShortType),
("IntegerType", IntegerType), ("LongType", LongType), ("FloatType", FloatType),
("DoubleType", DoubleType), ("DecimalType", DecimalType), ("StringType", StringType),
("BinaryType", BinaryType), ("DateType", DateType), ("TimestampType", TimestampType),
("StructField", StructField), ("StructType", NativeStructType),
):
if any(namespace.get(name) is not cls for namespace in namespaces):
return None
for cls in (TimestampType, StructField, NativeStructType):
if (cls.__getattribute__ is not object.__getattribute__
or any("__class__" in vars(base) for base in cls.__mro__ if base is not object)):
return None
for cls, names, descriptor_type in (
(NativeStructType, ("fields", "names"), GetSetDescriptorType),
(NativeStructType, ("__iter__",), WrapperDescriptorType),
(StructField, ("name", "dataType", "nullable", "metadata"), GetSetDescriptorType),
):
for name in names:
descriptor = vars(cls).get(name)
if (type(descriptor) is not descriptor_type
or descriptor.__objclass__ is not cls or descriptor.__name__ != name):
return None

width = len(frame.columns)
fields = schema.fields
if not width or len(fields) != width or not frame.columns.is_unique:
return None
if not all(type(name) is str and "\x00" not in name for name in frame.columns):
return None
if not all(type(field) is StructField and type(field.dataType) is TimestampType
and type(field.name) is str and "\x00" not in field.name for field in fields):
return None
if len(set(schema.names)) != width:
return None
columns = []
datetime_dtype = type(np.dtype("datetime64[ns]"))
for i, dtype in enumerate(frame.dtypes):
if type(dtype) is DatetimeTZDtype:
unit = dtype.unit
elif type(dtype) is datetime_dtype and dtype.kind == "M":
unit = np.datetime_data(dtype)[0]
else:
return None
if unit not in ("ns", "us"):
return None
column = frame.iloc[:, i]
if type(column) is not Series or type(column.array) is not DatetimeArray:
return None
# Preserve the row path's local null error before any timezone Config RPC.
if not fields[i].nullable and column.hasnans:
return None
if unit == "ns" and np.any(column.array.asi8[column.notna()] % 1000):
return None
columns.append(column)
return columns


def timestamp_columns_are_aware(columns):
"""Whether every column carries its time zone, so the session's is not needed."""
from pandas.core.dtypes.dtypes import DatetimeTZDtype

return all(type(column.dtype) is DatetimeTZDtype for column in columns)


def prepare_timestamp_table(columns, schema, timezone):
"""Preserve instants without row materialization or a host-zone conversion.

Aware columns carry their own timezone. Naive columns are deliberately
restricted to UTC host/session pairs: the existing row path uses the host
zone, unlike Python Connect's session-zone conversion (and DST fold policy).
"""
import os
import time
import pyarrow as pa

if not timestamp_columns_are_aware(columns):
if (timezone != "UTC" or os.environ.get("TZ") != "UTC"
or time.tzname != ("UTC", "UTC")):
return None
return pa.Table.from_arrays(
[pa.array(column, type=pa.timestamp("us", tz="UTC"), from_pandas=True)
for column in columns],
names=schema.names,
)
Loading
Loading