Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,9 @@ Pandas frames of at least 100,000 rows whose columns are all NumPy `float64`,
with an explicit `StructType` of `DoubleType` fields, are encoded to Arrow IPC
from their columns, with pandas' missing values as nulls, and sent inline as the
row path sends them, without the importer's configuration RPC.
Pandas object columns containing exact `datetime.date` values or `None` use the
Arrow importer at 10,000 rows or more with an explicit `StructType` of `DateType`
fields. Unsupported values and customized inputs retain the row conversion path.
Pandas frames of at least 10,000 rows whose columns are all kept in pandas' Arrow
string storage (`string[pyarrow]`, and the `str` dtype of pandas 3), with an
explicit `StructType` of `StringType` fields, pass that storage to this importer
Expand Down
18 changes: 15 additions & 3 deletions python/pyspark/sql/_pandas_input.py
Original file line number Diff line number Diff line change
Expand Up @@ -796,19 +796,31 @@ def prepare_date_table(frame, schema):
except ImportError:
return None
columns = [frame.iloc[:, i] for i in range(width)]
arrays = []
for column, field in zip(columns, schema.fields):
if type(column) is not Series or type(column.array) is not NumpyExtensionArray:
if type(column) is not Series:
return None
backing = column.array
if type(backing) is not NumpyExtensionArray:
return None
array = backing._ndarray
if (type(array) is not np.ndarray or array.ndim != 1
or type(array.dtype) is not type(np.dtype("object"))
or array.dtype != np.dtype("object") or len(array) != len(frame)):
return None
# Arrow rejects nulls in non-nullable fields; let the row path retain
# its existing behavior for that input instead.
if not all(type(value) is date or (value is None and field.nullable)
for value in column.to_numpy(copy=False)):
for value in array):
return None
arrays.append(array)

import pyarrow as pa

return pa.Table.from_arrays(
[pa.array(column, type=pa.date32(), from_pandas=False) for column in columns],
# Series accessors can be customized independently of the DataFrame row
# path. Emit the same concrete storage whose cells were checked above.
[pa.array(array, type=pa.date32(), from_pandas=False) for array in arrays],
names=schema.names,
)

Expand Down
32 changes: 32 additions & 0 deletions python/tests/test_arrow_handoff.py
Original file line number Diff line number Diff line change
Expand Up @@ -4763,6 +4763,38 @@ def test_pandas_inferred_int_beyond_int64_raises_as_pyspark(protocol, schema):

# Exact built-in date columns reuse Arrow without materializing pandas rows.

@pytest.mark.parametrize("accessor", ["values", "to_numpy"])
def test_date_series_accessors_do_not_replace_validated_cells(monkeypatch, accessor):
from pyspark.sql._pandas_input import prepare_date_table

dates = [date.min, date.max, date(1969, 12, 31), date(2000, 2, 29)] * 2500
nullable = [None if i % 7 == 0 else value for i, value in enumerate(dates)]
expected = {"renamed": dates, "nullable": nullable}
frame = pd.DataFrame({"original": dates, "nullable": nullable}, dtype=object)
schema = T.StructType([
T.StructField("renamed", T.DateType(), False),
T.StructField("nullable", T.DateType()),
])
row_values = frame.astype(object).where(frame.notna(), None).values.tolist()
calls = []

def replacement(series, *args, **kwargs):
calls.append(accessor)
return np.full(len(series), date(2021, 2, 3), dtype=object)

monkeypatch.setattr(pd.Series, accessor,
property(replacement) if accessor == "values" else replacement)
table = prepare_date_table(frame, schema)
assert table is not None
sink = pa.BufferOutputStream()
with pa.ipc.new_stream(sink, table.schema) as writer:
writer.write_table(table)
actual = pa.ipc.open_stream(sink.getvalue()).read_all()
assert actual.to_pydict() == expected
assert frame.astype(object).where(frame.notna(), None).values.tolist() == row_values
assert calls == []


@pytest.mark.parametrize("cached", [False, True])
@pytest.mark.parametrize("verify", [False, True])
def test_date_columns_preserve_every_ipc_cell(protocol, monkeypatch, cached, verify):
Expand Down
Loading