diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs index 7f632c30..549565de 100644 --- a/crates/pyspark-rs/src/lib.rs +++ b/crates/pyspark-rs/src/lib.rs @@ -36,6 +36,7 @@ mod group; mod infer; mod ml; mod observation; +mod pandas_input; mod pipelines; mod profiler; mod pyarrow_export; @@ -130,6 +131,8 @@ fn _pyspark(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(types::parse_datatype_json, m)?)?; m.add_function(wrap_pyfunction!(column::column_reference_name, m)?)?; + m.add_function(wrap_pyfunction!(pandas_input::_pandas_string_ipc, m)?)?; + // Spark Declarative Pipelines (SDP) command execution m.add_class::()?; m.add_function(wrap_pyfunction!( diff --git a/crates/pyspark-rs/src/pandas_input.rs b/crates/pyspark-rs/src/pandas_input.rs new file mode 100644 index 00000000..4d02adba --- /dev/null +++ b/crates/pyspark-rs/src/pandas_input.rs @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Fused validation and inline encoding of guarded pandas string columns. + +use std::sync::Arc; + +use arrow::array::{ArrayRef, StringBuilder}; +use arrow::datatypes::{DataType as ArrowType, Field, Schema}; +use arrow::ipc::writer::StreamWriter; +use arrow::record_batch::RecordBatch; +use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyString}; + +use crate::types::PyStructType; + +/// The Python eligibility helper authenticates pandas storage and native schema. +/// Validate each object before accepting it: extraction must never call a custom +/// numeric/string conversion hook or turn a pandas missing sentinel into null. +#[pyfunction] +pub(crate) fn _pandas_string_ipc<'py>( + py: Python<'py>, + columns: Vec>, + schema: PyRef<'py, PyStructType>, +) -> PyResult>> { + if columns.len() != schema.fields.len() { + return Ok(None); + } + // Reject unsupported objects before copying any string data. Otherwise a + // late subclass would pay for most of an Arrow build before the row path. + // Keep this pass to exact type/null checks; UTF-8 extraction stays below. + for (column, field) in columns.iter().zip(&schema.fields) { + if !matches!(&field.data_type, spark_connect::types::DataType::String { collation } + if collation == "UTF8_BINARY") + { + return Ok(None); + } + for value in column.try_iter()? { + let value = value?; + if !(value.is_none() && field.nullable || value.is_exact_instance_of::()) { + return Ok(None); + } + } + } + // Encode each string once, while building. A lone surrogate cannot be encoded: it + // falls back to the row path, which drops the partial Arrow buffers. The type pass + // above keeps a late subclass from paying for the build. + let mut arrays: Vec = Vec::with_capacity(columns.len()); + let mut fields = Vec::with_capacity(columns.len()); + for (column, field) in columns.iter().zip(&schema.fields) { + let mut builder = StringBuilder::with_capacity(column.len()?, column.len()? * 8); + for value in column.try_iter()? { + let value = value?; + if value.is_none() && field.nullable { + builder.append_null(); + } else if let Ok(value) = value.cast_exact::() { + let Ok(bytes) = value.encode_utf8() else { + return Ok(None); + }; + let Ok(text) = std::str::from_utf8(bytes.as_bytes()) else { + return Ok(None); + }; + // append_value copies while the bytes object and GIL are live. + // This avoids to_cow's owned String on the abi3-py39 build. + builder.append_value(text); + } else { + return Ok(None); + } + } + arrays.push(Arc::new(builder.finish())); + // Like the row encoder, Spark metadata lives in the logical schema; + // Arrow fields carry the name, standard string type and nullability. + fields.push(Field::new(&field.name, ArrowType::Utf8, field.nullable)); + } + let bytes = py + .detach(move || -> Result, arrow::error::ArrowError> { + let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays)?; + let mut writer = StreamWriter::try_new(Vec::new(), &batch.schema())?; + writer.write(&batch)?; + writer.into_inner() + }) + .map_err(|error| pyo3::exceptions::PyRuntimeError::new_err(error.to_string()))?; + Ok(Some(PyBytes::new(py, &bytes))) +} diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index 25363993..68a55148 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -896,6 +896,31 @@ impl PySparkSession { frame.to_df(fields.iter().map(|field| field.name.as_str()).collect()), )); } + let ipc = py + .import("pyspark.sql._pandas_input")? + .getattr("prepare_string_input")? + .call1((data, schema))?; + if !ipc.is_none() { + let fields = match &spec { + Spec::Struct(fields) => fields, + _ => { + return Err(pyo3::exceptions::PyValueError::new_err( + "string input requires a struct schema", + )); + } + }; + let schema = DataType::Struct { + fields: fields.clone(), + }; + let frame = self + .session + .create_dataframe_from_inline_arrow_ipc(ipc.extract()?, schema); + // Match pandas row input: explicit nonempty field names travel + // through toDF as well as the local relation's logical schema. + return Ok(PyDataFrame::new( + frame.to_df(fields.iter().map(|field| field.name.as_str()).collect()), + )); + } return self.create_from_pandas(py, data, spec); } // Preserve schema parsing, ignored-argument warnings, and pandas/Arrow routing. diff --git a/python/pyspark/sql/_pandas_input.py b/python/pyspark/sql/_pandas_input.py index 2c97e5fa..df89d961 100644 --- a/python/pyspark/sql/_pandas_input.py +++ b/python/pyspark/sql/_pandas_input.py @@ -530,3 +530,102 @@ def prepare_float64_input(frame, schema): with pa.ipc.new_stream(sink, table.schema) as writer: writer.write_table(table) return sink.getvalue().to_pybytes() + + +def prepare_string_input(frame, schema): + """Prepare inline IPC for explicit columns of exact strings or None. + + Inspect every object before Arrow conversion: str subclasses can override + conversion hooks, and pandas treats several other objects as missing values. + Those cases must keep the existing row conversion and its error ordering. + """ + import numpy as np + import pandas as pd + from pandas.core import frame as frame_module, generic as generic_module + from pandas.core.frame import DataFrame + from pandas.core.series import Series + from types import FunctionType, GetSetDescriptorType, MethodDescriptorType, WrapperDescriptorType + + from pyspark._pyspark import StringType, StructField, StructType as NativeStructType + from pyspark.sql import types as sql_types + + # Keep small frames on the established row path. + if (pd.DataFrame is not DataFrame or type(frame) is not DataFrame + or type(schema) is not NativeStructType + or len(frame) < 10000 + or any(name in vars(frame) for name in ("astype", "notna", "where"))): + return None + # Preserve the direct normalization hooks used by the existing row path. + for name, module in (("astype", generic_module), ("notna", frame_module), + ("where", generic_module)): + method = getattr(DataFrame, name) + if (type(method) is not FunctionType or method.__name__ != name + or method.__module__ != module.__name__ or method.__globals__ is not vars(module)): + return None + if (DataFrame.astype is not generic_module.NDFrame.astype + or DataFrame.where is not generic_module.NDFrame.where): + return None + # The row path reads native fields directly. Class attributes are mutable, + # so customized Python schema access must not select a different Arrow path. + for name, cls in (("StringType", StringType), ("StructField", StructField), + ("StructType", NativeStructType)): + if (getattr(sql_types, name, None) is not cls + or cls.__getattribute__ is not object.__getattribute__ + or any("__class__" in vars(base) for base in cls.__mro__ if base is not object)): + return None + for cls, names, descriptor_type in ( + (NativeStructType, ("fields", "names"), GetSetDescriptorType), + (NativeStructType, ("__iter__",), WrapperDescriptorType), + (StructField, ("name", "dataType", "nullable", "metadata"), GetSetDescriptorType), + (StringType, ("isUTF8BinaryCollation",), MethodDescriptorType), + ): + for name in names: + descriptor = vars(cls).get(name) + if (type(descriptor) is not descriptor_type + or descriptor.__objclass__ is not cls or descriptor.__name__ != name): + return None + if not _has_canonical_columns(frame.columns): + return None + width = len(frame.columns) + if not width or len(schema.fields) != width or not frame.columns.is_unique: + return None + if not all(type(name) is str and "\x00" not in name for name in frame.columns): + return None + if not all( + type(field) is StructField and type(field.dataType) is StringType + and field.dataType.isUTF8BinaryCollation() + and type(field.name) is str and "\x00" not in field.name + for field in schema.fields + ) or len(set(schema.names)) != width: + return None + if not all(type(dtype) is type(np.dtype("object")) and dtype == np.dtype("object") + for dtype in frame.dtypes): + return None + # A custom ExtensionArray may report a genuine NumPy dtype while its + # Arrow hook emits different values from to_numpy(). Trust only pandas' + # concrete NumPy backing array, including its name in older pandas releases. + try: + from pandas.core.arrays.numpy_ import NumpyExtensionArray + except ImportError: + try: + from pandas.core.arrays.numpy_ import PandasArray as NumpyExtensionArray + except ImportError: + return None + columns = [frame.iloc[:, i] for i in range(width)] + if any(type(column) is not Series or type(column.array) is not NumpyExtensionArray + for column in columns): + return None + + from pyspark._pyspark import _pandas_string_ipc + + # Preflight exact object types before the native Arrow buffer traversal. + # Unsupported values, non-nullable None, and surrogates return to the row + # path before serialization. Keep its inline transport and configuration + # behavior rather than selecting the public Arrow import/upload policy. + arrays = [column.array._ndarray for column in columns] + if any(type(array) is not np.ndarray or array.ndim != 1 + or type(array.dtype) is not type(np.dtype("object")) + or array.dtype != np.dtype("object") or len(array) != len(frame) + for array in arrays): + return None + return _pandas_string_ipc(arrays, schema) diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index eebb17c8..ca77f9c7 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -3494,3 +3494,842 @@ def no_rows(*args, **kwargs): with _arrow_input_service(protocol) as (spark, state): spark.createDataFrame(frame, schema).toArrow() _assert_float64_columns_guard_payload(state, schema, values) + + +@pytest.mark.parametrize("location", [ + "subclass", "class", "owner", "backing_subclass_item", "backing_class_iter", + "backing_class_getitem", "backing_instance_box", "arrow_class_iter", +]) +@pytest.mark.parametrize("behavior", ["raise", "mutate"]) +def test_string_columns_iterator_hooks_keep_native_order( + protocol, monkeypatch, location, behavior +): + from pandas.core.base import IndexOpsMixin + + values = ["plain"] * 10000 + frame = pd.DataFrame({"original": values}, dtype=object) + schema = T.StructType([T.StructField("renamed", T.StringType())]) + original_iter = pd.Index.__iter__ + original_notna, original_astype = frame.notna, frame.astype + calls = [] + invoked = False + + class ColumnsError(RuntimeError): + pass + + def notna(*args, **kwargs): + calls.append("notna") + return original_notna(*args, **kwargs) + + def astype(*args, **kwargs): + calls.append("astype") + return original_astype(*args, **kwargs) + + def before_hook(): + nonlocal invoked + if invoked: + return + invoked = True + calls.append("columns iterator") + if behavior == "raise": + raise ColumnsError("columns hook precedes pandas normalization") + # Install spies only when native iteration reaches the columns hook. + # Before eligibility, normalization methods are the canonical methods. + monkeypatch.setitem(vars(frame), "notna", notna) + monkeypatch.setitem(vars(frame), "astype", astype) + frame.iat[0, 0], frame.iat[-1, 0] = "changed", None + + def iterate(index): + if index is columns: + before_hook() + return original_iter(index) + + if location == "subclass": + class Columns(pd.Index): + pass + monkeypatch.setattr(Columns, "__iter__", iterate) + labels = pd.Index(["original"], dtype=object) + frame.columns = Columns._simple_new(labels._values, name=None) + assert type(frame.columns) is Columns + elif location == "backing_subclass_item": + # IndexOpsMixin iteration uses ndarray.item, not ndarray.__iter__. + class Names(np.ndarray): + def item(self, *args): + before_hook() + return super().item(*args) + backing = np.asarray(["original"], dtype=object).view(Names) + frame.columns = pd.Index._simple_new(backing, name=None) + assert type(frame.columns._values) is Names + elif location.startswith(("backing_", "arrow_")): + storage = "pyarrow" if location.startswith("arrow_") else "python" + frame.columns = pd.Index(pd.array(["original"], dtype=pd.StringDtype(storage=storage))) + backing = frame.columns._values + if location.endswith("class_iter"): + original = type(backing).__iter__ + + def backing_iter(array): + if array is backing: + before_hook() + return original(array) + + monkeypatch.setattr(type(backing), "__iter__", backing_iter) + elif location.endswith("class_getitem"): + original = type(backing).__getitem__ + + def backing_getitem(array, key): + if array is backing: + before_hook() + return original(array, key) + + monkeypatch.setattr(type(backing), "__getitem__", backing_getitem) + else: + original = backing._box_func + + def box(value): + before_hook() + return original(value) + + monkeypatch.setitem(vars(backing), "_box_func", box) + else: + monkeypatch.setattr(pd.Index if location == "class" else IndexOpsMixin, + "__iter__", iterate) + columns = frame.columns + + with _arrow_input_service(protocol) as (spark, state): + if behavior == "raise": + with pytest.raises(ColumnsError, + match="columns hook precedes pandas normalization"): + spark.createDataFrame(frame, schema) + assert calls == ["columns iterator"] + assert state["config_requests"] == [] + assert state["input_tables"] == [] + assert state["uploads"] == 0 + else: + spark.createDataFrame(frame, schema).toArrow() + expected = values.copy() + expected[0], expected[-1] = "changed", None + import json + + table = state["input_tables"][0] + assert table.schema.names == ["renamed"] + assert table.column(0).type == pa.string() + assert table.column(0).to_pylist() == expected + assert table.column(0).is_null().to_pylist() == [False] * 9999 + [True] + assert table.schema.equals(pa.schema([pa.field("renamed", pa.string())]), + check_metadata=True) + assert json.loads(state["relations"][0].local_relation.schema) == json.loads(schema.json()) + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert calls == ["columns iterator", "notna", "astype"] + + + +def test_string_rebound_index_values_keeps_native_error(protocol, monkeypatch): + + frame = pd.DataFrame({"original": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("renamed", T.StringType())]) + columns = frame.columns + original = pd.Index._values.fget + calls = [] + + def values(index): + if index is columns: + calls.append("_values") + raise RuntimeError("columns backing hook precedes normalization") + return original(index) + + monkeypatch.setattr(pd.Index, "_values", property(values)) + assert calls == [] + with _arrow_input_service(protocol) as (spark, state): + with pytest.raises(RuntimeError, + match="columns backing hook precedes normalization"): + spark.createDataFrame(frame, schema) + assert calls == ["_values"] + assert state["config_requests"] == [] + assert state["input_tables"] == [] + + + +@pytest.mark.parametrize("nullable", [False, True]) +def test_string_rebound_index_data_keeps_native_read_order(protocol, monkeypatch, nullable): + import json + + values = ["plain", "é", "漢字", ""] * 2500 + if nullable: + values[1] = None + frame = pd.DataFrame({"original": values}, dtype=object) + schema = T.StructType([ + T.StructField("renamed", T.StringType(), nullable, {"source": "native"}) + ]) + columns, backing = frame.columns, vars(frame.columns)["_data"] + original_notna, original_astype = frame.notna, frame.astype + calls = [] + normalizing = False + allowed_reads = None + + class ColumnsDataError(RuntimeError): + pass + + def notna(*args, **kwargs): + nonlocal normalizing + normalizing = True + calls.append("notna") + return original_notna(*args, **kwargs) + + def astype(*args, **kwargs): + calls.append("astype") + return original_astype(*args, **kwargs) + + def data(index): + if index is columns: + if not normalizing: + calls.append("_data") + if allowed_reads is not None and calls.count("_data") > allowed_reads: + raise ColumnsDataError("eligibility added a columns storage read") + # Canonical normalization remains intact before eligibility. + monkeypatch.setitem(vars(frame), "notna", notna) + monkeypatch.setitem(vars(frame), "astype", astype) + return backing + return vars(index)["_data"] + + def set_data(index, value): + # Keep ordinary storage initialization for indexes created by pandas. + vars(index)["_data"] = value + + monkeypatch.setattr(pd.Index, "_data", property(data, set_data), raising=False) + import pyspark.sql._pandas_input as helper + + with _arrow_input_service(protocol) as (spark, state): + # Measure the current native fallback's storage reads, rather than the + # former tolist path's single-read contract. + with monkeypatch.context() as baseline: + baseline.setattr(helper, "prepare_string_input", lambda *_: None) + spark.createDataFrame(frame, schema).toArrow() + baseline_calls = calls.copy() + allowed_reads = baseline_calls.count("_data") + assert allowed_reads > 0 and "notna" in baseline_calls + calls.clear() + normalizing = False + # Restore canonical methods before the candidate eligibility check; + # native label iteration will install the spies through the storage hook. + monkeypatch.delitem(vars(frame), "notna") + monkeypatch.delitem(vars(frame), "astype") + spark.createDataFrame(frame, schema).toArrow() + # The rebound storage declines the string path without reading the columns + # more than the row path does. pandas caches some of these reads, so a later + # row conversion of the same frame can read them fewer times than the first. + assert "notna" in calls + assert calls.count("_data") <= allowed_reads + assert not state["config_requests"] + assert state["uploads"] == 0 + assert len(state["input_tables"]) == 2 + assert len(state["relations"]) == 2 + table = state["input_tables"][1] + assert table.equals(state["input_tables"][0], check_metadata=True) + assert table.column(0).to_pylist() == values + assert table.column(0).is_null().to_pylist() == [value is None for value in values] + assert table.schema.equals( + pa.schema([pa.field("renamed", pa.string(), nullable=nullable)]), + check_metadata=True, + ) + assert json.loads(state["relations"][1].local_relation.schema) == { + "type": "struct", "fields": [ + {"name": "renamed", "type": "string", "nullable": nullable, + "metadata": {"source": "native"}} + ], + } + + + +@pytest.mark.parametrize("kind", [ + "object", "default", "python", "pyarrow", "python_numpy", "pyarrow_numpy", +]) +def test_string_canonical_column_storage_stays_eligible(kind): + from pyspark.sql._pandas_input import _has_canonical_columns, prepare_string_input + + frame = pd.DataFrame({"original": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("renamed", T.StringType())]) + if kind == "object": + frame.columns = pd.Index(np.asarray(["original"], dtype=object), dtype=object) + elif kind == "default": + frame.columns = pd.Index(["original"]) + elif kind.endswith("_numpy"): + from pandas.core.arrays import string_ as strings, string_arrow as arrow_strings + + module = strings if kind.startswith("python") else arrow_strings + name = ("StringArrayNumpySemantics" if kind.startswith("python") + else "ArrowStringArrayNumpySemantics") + cls = getattr(module, name, None) + if cls is None: + pytest.skip("older infer-string storage class is absent") + frame.columns = pd.Index._simple_new(cls._from_sequence(["original"]), name=None) + else: + frame.columns = pd.Index(pd.array(["original"], dtype=pd.StringDtype(storage=kind))) + assert _has_canonical_columns(frame.columns) + assert type(prepare_string_input(frame, schema)) is bytes + + + +@pytest.mark.parametrize("missing", ["module", "owner", "base"]) +def test_string_absent_private_columns_storage_keeps_row_fallback(monkeypatch, missing): + import builtins + from pandas.core.arrays import numpy_ as numpy_module, string_ as string_module + from pyspark.sql._pandas_input import _has_canonical_columns + + columns = pd.Index(pd.array(["original"], dtype=pd.StringDtype(storage="python"))) + if missing == "module": + original = builtins.__import__ + + def without_arrow(name, *args, **kwargs): + if name == "pandas.core.arrays.arrow": + raise ImportError("private storage module is absent") + return original(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", without_arrow) + elif missing == "owner": + monkeypatch.delattr(numpy_module, "NumpyExtensionArray") + else: + monkeypatch.delattr(string_module, "BaseStringArray") + assert not _has_canonical_columns(columns) + + + +@pytest.mark.parametrize("cached", [False, True]) +@pytest.mark.parametrize("verify", [False, True]) +@pytest.mark.parametrize("large", ["false", "true"]) +def test_string_columns_preserve_every_ipc_cell(protocol, monkeypatch, cached, verify, large): + import json + + from pyspark.sql.pandas.types import from_arrow_metadata + + strings = ["", "plain", "é", "e\u0301", "\x00embedded\x00", "漢字", "🙂", + "\U0010ffff", "\uffff", "\u202eright-to-left"] * 1000 + nullable = [None if i % 7 == 0 else value for i, value in enumerate(reversed(strings))] + expected = [strings, nullable, [None] * len(strings)] + frame = pd.DataFrame(dict(zip(["original", "b", "c"], expected)), dtype=object) + schema = T.StructType([ + T.StructField("renamed.`text", T.StringType(), False, {"source": "string", "id": 3}), + T.StructField("nullable", T.StringType(), True, {"tag": [1, 2]}), + T.StructField("all_null", T.StringType()), + ]) + + def no_rows(*args, **kwargs): + raise AssertionError("string fast path must not materialize Python rows") + + for method in ("to_numpy", "iterrows", "itertuples"): + monkeypatch.setattr(pd.DataFrame, method, no_rows) + monkeypatch.setattr(pd.DataFrame, "values", property(no_rows)) + monkeypatch.setattr(pd.Series, "tolist", no_rows) + overrides = { + "spark.sql.session.localRelationCacheThreshold": "1" if cached else "999999999", + "spark.sql.execution.arrow.useLargeVarTypes": large, + "spark.sql.session.localRelationChunkSizeRows": "997", + } + with _arrow_input_service(protocol, **overrides) as (spark, state): + result = spark.createDataFrame(frame, schema, verifySchema=verify) + assert not result.isLocal() + result.toArrow() + assert state["column_names"] == schema.names + table = state["input_tables"][0] + assert table.num_rows == len(strings) + assert table.schema.names == schema.names + for column, values, field in zip(table.columns, expected, schema.fields): + assert column.type == pa.string() + assert column.to_pylist() == values + actual_field = table.schema.field(field.name) + assert actual_field.nullable == field.nullable + assert from_arrow_metadata(actual_field.metadata) is None + spark_field = json.loads(state["relations"][0].local_relation.schema)["fields"][ + schema.names.index(field.name)] + assert spark_field["metadata"] == field.metadata + assert state["config_requests"] == [] + assert state["uploads"] == 0 + + + +@pytest.mark.parametrize("reason", [ + "small", "empty", "inferred", "names", "ddl", "cast", "atomic", "mixed_schema", + "width", "duplicate_input", "duplicate_schema", "input_nul", "schema_nul", + "input_number", "datetime", "date", "nan", "pd_na", "nat", "mixed", "bytes", + "nonnullable_null", "frame_subclass", "string_subclass", "string_dtype", + "arrow_string_dtype", "late_unsupported", "surrogate", "late_surrogate", "collation", +]) +def test_string_column_guard_preserves_row_fallback(reason): + from pyspark.sql._pandas_input import prepare_string_input + + class Frame(pd.DataFrame): + pass + + class Text(str): + def __str__(self): + raise AssertionError("guard must not call conversion hooks") + + class Unsupported: + def __str__(self): + raise AssertionError("guard must not stringify unsupported objects") + + values = ["plain"] * 10000 + frame = pd.DataFrame({"text": values}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + if reason == "small": + frame = frame.iloc[:9999] + elif reason == "empty": + frame = frame.iloc[:0] + elif reason in ("inferred", "names", "ddl", "cast", "atomic"): + schema = {"inferred": None, "names": ["text"], "ddl": "text STRING", + "cast": T.StructType([T.StructField("text", T.LongType())]), + "atomic": T.StringType()}[reason] + elif reason == "mixed_schema": + frame["long"] = np.arange(10000) + schema.add("long", T.LongType()) + elif reason == "width": + schema.add("extra", T.StringType()) + elif reason in ("duplicate_input", "duplicate_schema"): + frame["other"] = values + schema.add("other", T.StringType()) + if reason == "duplicate_input": + frame.columns = ["text", "text"] + else: + schema = T.StructType([T.StructField("text", T.StringType())] * 2) + elif reason in ("input_nul", "input_number"): + frame.columns = ["text\x00" if reason == "input_nul" else 1] + elif reason == "schema_nul": + schema = T.StructType([T.StructField("text\x00", T.StringType())]) + elif reason == "frame_subclass": + frame = Frame(frame) + elif reason in ("string_dtype", "arrow_string_dtype"): + frame = pd.DataFrame({"text": pd.Series( + values, dtype=pd.StringDtype(storage="pyarrow" if reason == "arrow_string_dtype" + else "python"))}) + elif reason == "collation": + schema = T.StructType([T.StructField("text", T.StringType("UTF8_LCASE"))]) + else: + value = { + "datetime": datetime(2020, 1, 1), "date": date(2020, 1, 1), + "nan": float("nan"), "pd_na": pd.NA, "nat": pd.NaT, "mixed": 42, + "bytes": b"plain", "nonnullable_null": None, "string_subclass": Text("plain"), + "late_unsupported": Unsupported(), "surrogate": "\ud800", "late_surrogate": "\udfff", + }[reason] + if reason == "nonnullable_null": + schema = T.StructType([T.StructField("text", T.StringType(), False)]) + if reason in ("late_unsupported", "late_surrogate"): + # pandas 3 infers StringDtype for a newly assigned list column. + # Keep these unsupported-object fixtures on concrete object storage. + frame["other"] = pd.Series(values, dtype=object) + schema.add("other", T.StringType()) + backing = frame["other"].array._ndarray + assert type(backing) is np.ndarray and backing.dtype == np.dtype("object") + frame.iat[-1, -1] = value + if reason in ("late_unsupported", "late_surrogate"): + assert frame.iat[-1, -1] is value + assert prepare_string_input(frame, schema) is None + + + +@pytest.mark.parametrize("reason", [ + "string_subclass", "frame_subclass", "nan", "pd_na", "nat", "string_dtype", + "nonnullable_null", "inferred", "names", "ddl", "small", "collation", "cast", "mixed", +]) +def test_string_public_fallback_keeps_row_semantics(protocol, monkeypatch, reason): + calls = [] + + class Text(str): + def __str__(self): + calls.append("str") + return "changed" + + class Frame(pd.DataFrame): + pass + + values = ["plain"] * (9999 if reason == "small" else 10000) + if reason in ("nan", "pd_na", "nat", "nonnullable_null", "mixed"): + values[-1] = {"nan": float("nan"), "pd_na": pd.NA, "nat": pd.NaT, + "nonnullable_null": None, "mixed": 42}[reason] + elif reason == "string_subclass": + values[-1] = Text("plain") + frame_type = Frame if reason == "frame_subclass" else pd.DataFrame + frame = frame_type({"text": values}, dtype="string" if reason == "string_dtype" else object) + schema = T.StructType([T.StructField("text", T.StringType(), reason != "nonnullable_null")]) + if reason == "inferred": + schema = None + elif reason == "names": + schema = ["renamed"] + elif reason == "ddl": + schema = "text STRING" + elif reason == "collation": + schema = T.StructType([T.StructField("text", T.StringType("UTF8_LCASE"))]) + elif reason == "cast": + schema = T.StructType([T.StructField("text", T.LongType())]) + astype = pd.DataFrame.astype + + def record_rows(self, *args, **kwargs): + calls.append("astype") + return astype(self, *args, **kwargs) + + monkeypatch.setattr(pd.DataFrame, "astype", record_rows) + with _arrow_input_service(protocol) as (spark, state): + if reason in ("nonnullable_null", "cast", "mixed"): + from pyspark.errors import PySparkRuntimeError + + message = ("Column 'text' is declared as non-nullable but contains null values" + if reason == "nonnullable_null" else "Type mismatch in row data") + with pytest.raises(PySparkRuntimeError, match=message): + spark.createDataFrame(frame, schema) + assert calls == ["astype"] + assert state["config_requests"] == [] + assert state["relations"] == [] + return + spark.createDataFrame(frame, schema).toArrow() + assert calls == ["astype"] # Native extraction keeps str.__str__ overrides uncalled. + table = state["input_tables"][0] + assert table.schema.names == ["renamed" if reason == "names" else "text"] + expected = ["plain"] * len(values) + if reason in ("nan", "pd_na", "nat"): + expected[-1] = None + assert table.column(0).to_pylist() == expected + assert state["config_requests"] == [] + + + +@pytest.mark.parametrize("surrogate", ["\ud800", "\udfff"]) +def test_string_surrogate_fallback_precedes_importer_configuration(protocol, monkeypatch, surrogate): + values = ["plain"] * 10000 + values[-1] = surrogate + frame = pd.DataFrame({"text": values}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + astype = pd.DataFrame.astype + calls = [] + + def record_rows(self, *args, **kwargs): + calls.append("astype") + return astype(self, *args, **kwargs) + + monkeypatch.setattr(pd.DataFrame, "astype", record_rows) + with _arrow_input_service(protocol) as (spark, state): + state["config_error"] = grpc.StatusCode.INVALID_ARGUMENT + with pytest.raises(TypeError, match="unsupported value type for createDataFrame: str"): + spark.createDataFrame(frame, schema) + assert calls == ["astype"] + assert state["config_requests"] == [] + assert state["relations"] == [] + assert state["uploads"] == 0 + + + +@pytest.mark.parametrize("class_name,attribute", [ + ("StructType", "names"), ("StructType", "fields"), ("StructType", "__iter__"), + ("StructField", "name"), ("StructField", "dataType"), ("StructField", "nullable"), + ("StructField", "metadata"), ("StringType", "isUTF8BinaryCollation"), + ("StructType", "__getattribute__"), ("StructField", "__getattribute__"), + ("StringType", "__getattribute__"), ("StructType", "__class__"), + ("StructField", "__class__"), ("StringType", "__class__"), + ("StructType", "alias"), ("StructField", "alias"), ("StringType", "alias"), +]) +@pytest.mark.parametrize("before_import", [False, True]) +def test_string_custom_schema_descriptors_keep_native_row_path( + protocol, class_name, attribute, before_import +): + import sys + + script = r''' +import sys +import tempfile +from pathlib import Path +import pandas as pd +from pyspark.sql import types as T + +import importlib.util +spec = importlib.util.spec_from_file_location("string_schema_capture", sys.argv[4]) +capture = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = capture +spec.loader.exec_module(capture) +_arrow_input_service, protocol = capture._arrow_input_service, capture.protocol +frame = pd.DataFrame({"text": ["plain"] * 10000}, dtype=object) +schema = T.StructType([T.StructField("text", T.StringType(), True, {"source": "native"})]) +class_name, attribute, before_import = sys.argv[1:4] +if before_import == "True": + # The capture module imports prepare_int64_table; force the public call to + # import a new helper after the class or public alias has been customized. + sys.modules.pop("pyspark.sql._pandas_input", None) +assert ("pyspark.sql._pandas_input" in sys.modules) == (before_import == "False") +cls = getattr(T, class_name) +if attribute == "alias": + setattr(T, class_name, object) +else: + if attribute == "__getattribute__": + replacement = lambda self, name: object.__getattribute__(self, name) + elif attribute == "__iter__": + replacement = lambda self: iter([T.StructField("changed", T.StringType())]) + elif attribute == "__class__": + attribute = "__getattribute__" + replacement = lambda self, name: object if name == "__class__" else ( + object.__getattribute__(self, name)) + elif attribute == "isUTF8BinaryCollation": + replacement = lambda self: False + else: + value = {"names": ["changed"], "fields": [], "name": "changed", + "dataType": T.StringType(), "nullable": False, "metadata": {"changed": True}}[attribute] + replacement = property(lambda self: value) + setattr(cls, attribute, replacement) +class Factory: + def mktemp(self, name): + return Path(tempfile.mkdtemp(prefix=name)) +with _arrow_input_service(protocol.__wrapped__(Factory())) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["config_requests"] == [], state["config_requests"] + table = state["input_tables"][0] + assert table.schema.names == ["text"], table.schema + assert table.column(0).to_pylist() == ["plain"] * 10000 +''' + result = subprocess.run([sys.executable, "-c", script, class_name, attribute, + str(before_import), __file__], + capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stdout + result.stderr + + + +def test_string_rebound_pandas_alias_keeps_subclass_fallback(monkeypatch): + from pyspark.sql._pandas_input import prepare_string_input + + class Frame(pd.DataFrame): + pass + + frame = Frame({"text": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + monkeypatch.setattr(pd, "DataFrame", Frame) + assert prepare_string_input(frame, schema) is None + + + +@pytest.mark.parametrize("dtype_mode", ["equal", "class_spoof", "exact_numpy"]) +def test_string_custom_extension_dtype_cannot_impersonate_numpy_object(protocol, dtype_mode): + from pandas.api.extensions import ExtensionArray, ExtensionDtype, take + from pyspark.sql._pandas_input import prepare_string_input + + arrow_calls = [] + + class ObjectEqualDtype(ExtensionDtype): + name = "object_equal_string" + type = str + kind = "O" + na_value = None + + @classmethod + def construct_array_type(cls): + return DifferentArrowArray + + def __eq__(self, other): + return (isinstance(other, ObjectEqualDtype) + or isinstance(other, np.dtype) and other.kind == "O") + + def __hash__(self): + return hash(self.name) + + class DifferentArrowArray(ExtensionArray): + def __init__(self, values): + self._values = np.asarray(values, dtype=object) + + @property + def dtype(self): + return ObjectEqualDtype() + + @property + def nbytes(self): + return self._values.nbytes + + def __len__(self): + return len(self._values) + + def __getitem__(self, key): + result = self._values[key] + return type(self)(result) if isinstance(result, np.ndarray) else result + + def isna(self): + return pd.isna(self._values) + + def take(self, indices, allow_fill=False, fill_value=None): + return type(self)(take(self._values, indices, allow_fill=allow_fill, + fill_value=fill_value)) + + def copy(self): + return type(self)(self._values.copy()) + + @classmethod + def _from_sequence(cls, scalars, *, dtype=None, copy=False): + values = np.asarray(scalars, dtype=object) + return cls(values.copy() if copy else values) + + def to_numpy(self, dtype=None, copy=False, na_value=None): + if dtype is not None: + return self._values.astype(dtype, copy=copy) + return self._values.copy() if copy else self._values + + def __arrow_array__(self, type=None): + arrow_calls.append("arrow") + # A scan of to_numpy() cannot establish the values this hook emits. + return pa.array(["different"] * len(self), type=type) + + array = DifferentArrowArray(["plain"] * 10000) + frame = pd.DataFrame({"text": pd.Series(array, copy=False)}) + schema = T.StructType([T.StructField("text", T.StringType())]) + if dtype_mode == "exact_numpy": + # An ExtensionArray may change its advertised dtype after construction. + # The dtype and NumPy scan both pass while its Arrow hook changes cells. + DifferentArrowArray.dtype = property(lambda self: np.dtype("object")) + DifferentArrowArray.__array__ = lambda self, dtype=None, copy=None: self._values + elif dtype_mode == "class_spoof": + def spoof_class(self, name): + if name == "__class__": + return type(np.dtype("object")) + return object.__getattribute__(self, name) + + ObjectEqualDtype.__getattribute__ = spoof_class + dtype = frame.dtypes.iloc[0] + assert type(frame) is pd.DataFrame + assert type(frame.iloc[:, 0].array) is DifferentArrowArray + assert type(dtype) is (type(np.dtype("object")) if dtype_mode == "exact_numpy" + else ObjectEqualDtype) + assert isinstance(dtype, np.dtype) is (dtype_mode != "equal") + assert dtype == np.dtype("object") + assert prepare_string_input(frame, schema) is None + with _arrow_input_service(protocol) as (spark, state): + if dtype_mode == "exact_numpy": + # This deliberately inconsistent EA already fails pandas' row-path + # astype(object). Preserve that error before importer configuration. + with pytest.raises(TypeError, match="cannot set astype"): + spark.createDataFrame(frame, schema) + assert state["input_tables"] == [] + else: + spark.createDataFrame(frame, schema).toArrow() + assert state["input_tables"][0].column(0).to_pylist() == ["plain"] * 10000 + assert state["config_requests"] == [] + assert arrow_calls == [] + + + +@pytest.mark.parametrize("method", ["astype", "notna"]) +def test_string_instance_conversion_hooks_keep_row_fallback(protocol, method): + frame = pd.DataFrame({"text": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + original = getattr(frame, method) + calls = [] + expected = ["plain"] * 10000 + if method == "astype": + expected[0] = "changed" + + def customized(*args, **kwargs): + calls.append(method) + result = original(*args, **kwargs) + result.iat[0, 0] = "changed" + return result + else: + expected[0] = None + + def customized(*args, **kwargs): + calls.append(method) + result = original(*args, **kwargs) + result.iat[0, 0] = False + return result + setattr(frame, method, customized) + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert calls == [method] + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["input_tables"][0].column(0).to_pylist() == expected + + + +@pytest.mark.parametrize("method", ["astype", "notna", "where"]) +@pytest.mark.parametrize("before_import", [False, True]) +def test_string_class_conversion_hooks_keep_row_fallback( + protocol, monkeypatch, method, before_import +): + import importlib + import sys + + name = "pyspark.sql._pandas_input" + if before_import: + monkeypatch.delitem(sys.modules, name, raising=False) + else: + importlib.import_module(name) + frame = pd.DataFrame({"text": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + original = getattr(pd.DataFrame, method) + calls = [] + expected = ["plain"] * 10000 + expected[0] = None if method == "notna" else "changed" + + def customized(self, *args, **kwargs): + calls.append(method) + result = original(self, *args, **kwargs) + result.iat[0, 0] = False if method == "notna" else "changed" + return result + + monkeypatch.setattr(pd.DataFrame, method, customized) + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert calls == [method] + assert state["config_requests"] == [] + assert state["uploads"] == 0 + assert state["input_tables"][0].column(0).to_pylist() == expected + + + +def test_string_instance_where_override_keeps_existing_row_behavior(protocol): + from pyspark.sql._pandas_input import prepare_string_input + + frame = pd.DataFrame({"text": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + + def customized(*args, **kwargs): + raise AssertionError("row normalization calls where on the astype result") + + frame.where = customized + assert prepare_string_input(frame, schema) is None + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["config_requests"] == [] + assert state["input_tables"][0].column(0).to_pylist() == ["plain"] * 10000 + + + +def test_string_native_backing_does_not_add_series_conversion_hooks(protocol, monkeypatch): + frame = pd.DataFrame({"text": ["plain"] * 10000}, dtype=object) + schema = T.StructType([T.StructField("text", T.StringType())]) + + def customized(*args, **kwargs): + raise AssertionError("native string input must not call Series.to_numpy") + + monkeypatch.setattr(pd.Series, "to_numpy", customized) + with _arrow_input_service(protocol) as (spark, state): + spark.createDataFrame(frame, schema).toArrow() + assert state["config_requests"] == [] + assert state["input_tables"][0].column(0).to_pylist() == ["plain"] * 10000 + + + +def test_string_large_inline_input_keeps_row_transport_and_schema(protocol): + values = ["漢字-🙂-" + str(i) for i in range(100000)] + frame = pd.DataFrame({"text": values}, dtype=object) + schema = T.StructType([T.StructField("renamed", T.StringType(), False, + {"source": "inline-string", "size": len(values)})]) + with _arrow_input_service(protocol, **{ + "spark.sql.session.localRelationCacheThreshold": "1", + "spark.sql.session.localRelationSizeLimit": "1", + "spark.sql.execution.arrow.useLargeVarTypes": "true", + }) as (spark, state): + result = spark.createDataFrame(frame, schema) + assert not result.isLocal() + result.toArrow() + assert state["column_names"] == schema.names + assert state["config_requests"] == [] + assert state["uploads"] == 0 + table = state["input_tables"][0] + assert table.schema.names == ["renamed"] + assert table.column(0).type == pa.string() + assert table.schema.field(0).nullable is False + assert table.schema.field(0).metadata is None + assert table.column(0).to_pylist() == values + assert __import__("json").loads(state["relations"][0].local_relation.schema) == ( + __import__("json").loads(schema.json()))