From 34f6116409756ce6aebe72104c8c9bcff5a3d42d Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 6 Nov 2026 17:18:47 -0800 Subject: [PATCH] [SPARK-60716][CONNECT][PYTHON] Use validated backing arrays for pandas dates Validate and serialize the same concrete object arrays so customized Series accessors cannot replace dates between the eligibility scan and Arrow output. Add IPC regression coverage and document the existing date input path. Co-authored-by: Isaac --- docs/architecture.md | 3 +++ python/pyspark/sql/_pandas_input.py | 18 +++++++++++++--- python/tests/test_arrow_handoff.py | 32 +++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index ee734f0d..7c2234d4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -159,6 +159,9 @@ Pandas frames of at least 100,000 rows whose columns are all NumPy `float64`, with an explicit `StructType` of `DoubleType` fields, are encoded to Arrow IPC from their columns, with pandas' missing values as nulls, and sent inline as the row path sends them, without the importer's configuration RPC. +Pandas object columns containing exact `datetime.date` values or `None` use the +Arrow importer at 10,000 rows or more with an explicit `StructType` of `DateType` +fields. Unsupported values and customized inputs retain the row conversion path. Inputs at or above `spark.sql.session.localRelationCacheThreshold` are serialized into independent IPC streams and uploaded as session-scoped cache artifacts. The plan carries `ChunkedCachedLocalRelation` hashes instead of embedding the diff --git a/python/pyspark/sql/_pandas_input.py b/python/pyspark/sql/_pandas_input.py index 8cf6d98a..dff3e611 100644 --- a/python/pyspark/sql/_pandas_input.py +++ b/python/pyspark/sql/_pandas_input.py @@ -754,18 +754,30 @@ def prepare_date_table(frame, schema): except ImportError: return None columns = [frame.iloc[:, i] for i in range(width)] + arrays = [] for column, field in zip(columns, schema.fields): - if type(column) is not Series or type(column.array) is not NumpyExtensionArray: + if type(column) is not Series: + return None + backing = column.array + if type(backing) is not NumpyExtensionArray: + return None + array = backing._ndarray + if (type(array) is not np.ndarray or array.ndim != 1 + or type(array.dtype) is not type(np.dtype("object")) + or array.dtype != np.dtype("object") or len(array) != len(frame)): return None # Arrow rejects nulls in non-nullable fields; let the row path retain # its existing behavior for that input instead. if not all(type(value) is date or (value is None and field.nullable) - for value in column.to_numpy(copy=False)): + for value in array): return None + arrays.append(array) import pyarrow as pa return pa.Table.from_arrays( - [pa.array(column, type=pa.date32(), from_pandas=False) for column in columns], + # Series accessors can be customized independently of the DataFrame row + # path. Emit the same concrete storage whose cells were checked above. + [pa.array(array, type=pa.date32(), from_pandas=False) for array in arrays], names=schema.names, ) diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 9424ac5e..5342350a 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -4383,6 +4383,38 @@ def test_string_large_inline_input_keeps_row_transport_and_schema(protocol): # Exact built-in date columns reuse Arrow without materializing pandas rows. +@pytest.mark.parametrize("accessor", ["values", "to_numpy"]) +def test_date_series_accessors_do_not_replace_validated_cells(monkeypatch, accessor): + from pyspark.sql._pandas_input import prepare_date_table + + dates = [date.min, date.max, date(1969, 12, 31), date(2000, 2, 29)] * 2500 + nullable = [None if i % 7 == 0 else value for i, value in enumerate(dates)] + expected = {"renamed": dates, "nullable": nullable} + frame = pd.DataFrame({"original": dates, "nullable": nullable}, dtype=object) + schema = T.StructType([ + T.StructField("renamed", T.DateType(), False), + T.StructField("nullable", T.DateType()), + ]) + row_values = frame.astype(object).where(frame.notna(), None).values.tolist() + calls = [] + + def replacement(series, *args, **kwargs): + calls.append(accessor) + return np.full(len(series), date(2021, 2, 3), dtype=object) + + monkeypatch.setattr(pd.Series, accessor, + property(replacement) if accessor == "values" else replacement) + table = prepare_date_table(frame, schema) + assert table is not None + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, table.schema) as writer: + writer.write_table(table) + actual = pa.ipc.open_stream(sink.getvalue()).read_all() + assert actual.to_pydict() == expected + assert frame.astype(object).where(frame.notna(), None).values.tolist() == row_values + assert calls == [] + + @pytest.mark.parametrize("cached", [False, True]) @pytest.mark.parametrize("verify", [False, True]) def test_date_columns_preserve_every_ipc_cell(protocol, monkeypatch, cached, verify):