diff --git a/python/pyspark/sql/_pandas_input.py b/python/pyspark/sql/_pandas_input.py index f4891c23..f517a7dc 100644 --- a/python/pyspark/sql/_pandas_input.py +++ b/python/pyspark/sql/_pandas_input.py @@ -474,10 +474,15 @@ def prepare_float64_table(frame, schema): if any(type(column) is not Series or type(column.array) is not NumpyExtensionArray for column in columns): return None + # NumPy subclasses can customize the row path's astype conversion even + # inside pandas' concrete NumPy wrapper. Keep their hooks, accepting the + # row path's cost for subclasses such as np.memmap. + if any(type(column.array._ndarray) is not np.ndarray for column in columns): + return None # Native row conversion reads Rust fields, bypassing mutable Python accessors. for name in ("DoubleType", "StructField", "StructType"): cls = getattr(native, name) - if (getattr(sql_types, name, None) is not cls + if (vars(sql_types).get(name) is not cls or cls.__getattribute__ is not object.__getattribute__ or any("__class__" in vars(base) for base in cls.__mro__ if base is not object)): return None diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index af993c41..370a691c 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -3321,6 +3321,57 @@ def __arrow_array__(self, type=None): +@pytest.mark.parametrize("raises", [False, True]) +def test_float64_ndarray_subclass_converts_as_pyspark(protocol, raises): + # The fast path declines a NumPy subclass behind pandas' concrete wrapper; the frame then + # converts as PySpark's Arrow conversion does, which reads the raw storage and does not + # call the subclass's astype, whatever it would do. + from pyspark.sql._pandas_input import prepare_float64_table + + calls = [] + + class Values(np.ndarray): + def astype(self, *args, **kwargs): + calls.append("astype") + if raises: + raise ValueError("custom ndarray astype") + return np.ndarray.astype(self, *args, **kwargs) + 7.0 + + values = np.arange(100000, dtype=np.float64) + # Avoid pandas consolidating the subclass storage into a plain ndarray. + frame = pd.DataFrame({"x": values.view(Values)}, copy=False) + schema = T.StructType([T.StructField("x", T.DoubleType())]) + assert type(frame["x"].array._ndarray) is Values + assert prepare_float64_table(frame, schema) is None + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + _assert_float64_ipc(state["input_tables"][0], [values], schema) + assert calls == [] + assert state["uploads"] == 0 + + +def test_float64_missing_schema_alias_keeps_native_row_path(protocol, monkeypatch): + values = np.arange(100000, dtype=np.float64) + frame = pd.DataFrame({"x": values}) + schema = T.StructType([T.StructField("x", T.DoubleType())]) + calls = [] + + def lazy_attribute(name): + if name == "DoubleType": + calls.append(name) + raise RuntimeError("schema alias hook must not run before row conversion") + raise AttributeError(name) + + monkeypatch.delitem(vars(T), "DoubleType") + monkeypatch.setattr(T, "__getattr__", lazy_attribute, raising=False) + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + _assert_float64_ipc(state["input_tables"][0], [values], schema) + assert calls == [] + assert state["uploads"] == 0 + + + def test_float64_missing_concrete_numpy_backing_class_keeps_fallback(monkeypatch): import pandas.core.arrays.numpy_ as numpy_arrays from pyspark.sql._pandas_input import prepare_float64_table