From 24d430235477810c6e6ed79306ed564119e1961f Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Wed, 18 Nov 2026 14:12:27 +0000 Subject: [PATCH] [SPARK-62004][CONNECT][PYTHON] Pickle DataFrames of failing plans and table arguments, keep the schema that a copy requests, rebuild same-session deep copies, raise frame-bound errors at the build, and report Column._expr to Python's GC, as PySpark - A DataFrame whose plan fails with an AttributeError or another PySpark error class pickles, and its unpickle raises the error (NOT_ITERABLE of its schema request), as PySpark's. - A copy, deep copy or unpickle keeps the schema that it requests, which schema, printSchema, the set operations and numeric aggregates read, as PySpark's _cached_schema_serialized. - A TableArg pickles of its relation, levels, partitioning, ordering and single partition, and loads through _of_relation, or as PySpark Connect's TableArg on a worker. - A deep copy into the same session rebuilds its relation at each build, so its warnings and origins are PySpark's. - Window frame bounds that the core holds raise VALUE_NOT_BETWEEN when the plan is built. - Column._expr and TableArg._subquery_expr report their pickler objects to Python's GC, and copy as PySpark's. - The docs describe these and what remains. Co-authored-by: Isaac --- crates/pyspark-rs/src/column.rs | 116 ++++++++++-- crates/pyspark-rs/src/dataframe.rs | 146 ++++++++++++--- crates/pyspark-rs/src/group.rs | 21 ++- crates/pyspark-rs/src/tablearg.rs | 138 ++++++++++++-- crates/pyspark-rs/src/window.rs | 104 ++++++----- crates/spark-connect/src/dataframe.rs | 37 ++-- crates/spark-connect/src/table_arg.rs | 60 +++++++ dev/parity/known-differences.md | 19 +- docs/compatibility.md | 91 ++++++---- python/pyspark/sql/connect/_column_reduce.py | 96 ++++++++-- python/pyspark/sql/connect/dataframe.py | 15 +- python/tests/test_dropin_offline.py | 168 +++++++++++++++--- .../test_dropin_types_columns_offline.py | 14 +- .../test_dropin_window_tablearg_offline.py | 64 +++++++ 14 files changed, 892 insertions(+), 197 deletions(-) diff --git a/crates/pyspark-rs/src/column.rs b/crates/pyspark-rs/src/column.rs index 322a1573..dfeacd6d 100644 --- a/crates/pyspark-rs/src/column.rs +++ b/crates/pyspark-rs/src/column.rs @@ -129,17 +129,68 @@ fn is_literal_operand(value: &Bound<'_, PyAny>, with_timedelta: bool) -> PyResul #[pyclass(name = "_ColumnExpression", module = "pyspark.sql.connect.expressions")] pub struct PyColumnExpression { column: Column, + /// Set once the value is written, as `__traverse__` reads `column` (see + /// [`crate::gc::GcReady`]). + gc_ready: crate::gc::GcReady, } impl PyColumnExpression { /// The expression of `column`. pub(crate) fn new(column: Column) -> Self { - PyColumnExpression { column } + PyColumnExpression { + column, + gc_ready: crate::gc::GcReady::new(), + } } } #[pymethods] impl PyColumnExpression { + /// The shallow copy of the expression, as PySpark's `copy.copy` of an expression, a Python + /// object, which shares its fields: of the same expression, as the copy of a Column shares it. + fn __copy__(&self) -> PyColumnExpression { + PyColumnExpression::new(self.column.clone()) + } + + /// The deep copy of the expression, as PySpark's `copy.deepcopy` of an expression, which + /// copies its fields with `memo`, of the function of each UDF of it and of the plan of a + /// subquery, as the deep copy of a Column copies them (`deep_copied_picklers`): a function is + /// the same object, and a callable object, a bound method or a partial is a copy. + fn __deepcopy__( + &self, + py: Python<'_>, + memo: &Bound<'_, PyAny>, + ) -> PyResult { + let expression = self.column.expression(); + let copies = crate::udf_command::deep_copied_picklers(py, expression.picklers(), memo)?; + Ok(PyColumnExpression::new(if copies.is_empty() { + self.column.clone() + } else { + Column::new(expression.with_pickler_copies(&copies)) + })) + } + + /// As PySpark's expression, a Python object whose fields Python's garbage collector sees: the + /// functions of its UDFs, also of the plan of a subquery, that only this expression keeps + /// alive are reported, as those of a Column, so that a reference cycle through a function and + /// the expression is collected, as PySpark collects it. + fn __traverse__(&self, visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { + if !self.gc_ready.get() { + return Ok(()); + } + crate::udf_command::visit_pickler_objects( + self.column.expression().exclusive_pickler_objects(), + &visit, + ) + } + + /// Break a reference cycle that the garbage collector found unreachable: the expression is + /// replaced by one that holds nothing, which drops the objects that `__traverse__` reported. + fn __clear__(&mut self) { + if self.gc_ready.get() { + self.column = Column::new(spark_connect::expression::Expression::column("")); + } + } /// As PySpark's `__repr__` of the expression, which raises its error of an expression that it /// does not render (a `ColumnAlias` of names that are not all `str`). fn __repr__(&self) -> PyResult { @@ -801,9 +852,7 @@ impl PyColumn { /// its metrics. #[getter] fn _expr(&self) -> PyColumnExpression { - PyColumnExpression { - column: self.column.clone(), - } + PyColumnExpression::new(self.column.clone()) } /// The `spark.connect.Expression` of this column, as PySpark Connect's `Column.to_plan`: that @@ -1335,9 +1384,31 @@ fn error_description<'py>( }) } +/// The kinds of the errors of PySpark's error classes that a plan or an expression carries, as +/// [`crate::errors::pyspark_class_error`] gives them, by the names of their classes in +/// `pyspark.errors`, which a pickle of the error keeps. +const PYSPARK_ERROR_CLASSES: [(spark_connect_core::error::SparkErrorKind, &str); 10] = { + use spark_connect_core::error::SparkErrorKind as K; + [ + (K::ValueError, "PySparkValueError"), + (K::TypeError, "PySparkTypeError"), + (K::IndexError, "PySparkIndexError"), + (K::AttributeError, "PySparkAttributeError"), + (K::KeyError, "PySparkKeyError"), + (K::RuntimeError, "PySparkRuntimeError"), + (K::NotImplementedError, "PySparkNotImplementedError"), + (K::AssertionError, "PySparkAssertionError"), + (K::PicklingError, "PySparkPicklingError"), + (K::ImportError, "PySparkImportError"), + ] +}; + /// The description of `error`, of a plan or an expression that PySpark's client fails to build, /// as [`error_description`] gives it, of which `pyspark.sql.connect._column_reduce.raise_error` -/// raises the error again: None for the error of another kind, which no pickle keeps. +/// raises the error again: a builtin `TypeError`, `ValueError`, `OverflowError` or +/// `AttributeError` of its message, the error of UTF-8 of its value, or an error of one of +/// PySpark's error classes, of its error class and parameters. None for the error of another +/// kind, which no pickle keeps. pub(crate) fn described_error<'py>( py: Python<'py>, error: &spark_connect_core::error::SparkError, @@ -1350,6 +1421,7 @@ pub(crate) fn described_error<'py>( K::BuiltinTypeError => ("TypeError", error.message.clone(), none()), K::BuiltinValueError => ("ValueError", error.message.clone(), none()), K::BuiltinOverflowError => ("OverflowError", error.message.clone(), none()), + K::BuiltinAttributeError => ("AttributeError", error.message.clone(), none()), // The str that UTF-8 does not encode, and the bytes that it does not decode. K::UnicodeEncode => { let utf32le = error.unencodable_value().unwrap_or_default(); @@ -1365,12 +1437,14 @@ pub(crate) fn described_error<'py>( PyBytes::new(py, &bytes).into_any(), ) } - K::ValueError => ( - "PySparkValueError", - error.error_class.clone(), - error.params.clone().into_bound_py_any(py)?, - ), - _ => return Ok(None), + kind => match PYSPARK_ERROR_CLASSES.iter().find(|(of, _)| *of == kind) { + Some((_, class)) if !error.error_class.is_empty() => ( + *class, + error.error_class.clone(), + error.params.clone().into_bound_py_any(py)?, + ), + _ => return Ok(None), + }, }; PyTuple::new( py, @@ -1384,7 +1458,7 @@ pub(crate) fn described_error<'py>( } /// The error of `description`, of [`error_description`]. -fn error_of_description( +pub(crate) fn error_of_description( description: &Bound<'_, PyAny>, ) -> PyResult { use spark_connect_core::error::{SparkError, SparkErrorKind as K}; @@ -1397,23 +1471,27 @@ fn error_of_description( "TypeError" => builtin(K::BuiltinTypeError), "ValueError" => builtin(K::BuiltinValueError), "OverflowError" => builtin(K::BuiltinOverflowError), + "AttributeError" => builtin(K::BuiltinAttributeError), "UnicodeEncodeError" | "UnicodeDecodeError" => { crate::dataframe::unbuildable_str_error(&payload)?.ok_or_else(|| { pyo3::exceptions::PyValueError::new_err(format!("no {kind} of the value")) })? } - "PySparkValueError" => { + class => { + let Some((of_class, _)) = PYSPARK_ERROR_CLASSES + .iter() + .find(|(_, name)| *name == class) + else { + return Err(pyo3::exceptions::PyValueError::new_err(format!( + "no error of the kind {class}" + ))); + }; let params: std::collections::BTreeMap = payload.extract()?; let params: Vec<(&str, &str)> = params .iter() .map(|(key, value)| (key.as_str(), value.as_str())) .collect(); - SparkError::classed(K::ValueError, &text, ¶ms) - } - _ => { - return Err(pyo3::exceptions::PyValueError::new_err(format!( - "no error of the kind {kind}" - ))) + SparkError::classed(*of_class, &text, ¶ms) } }) } diff --git a/crates/pyspark-rs/src/dataframe.rs b/crates/pyspark-rs/src/dataframe.rs index b8ac567b..e3e4fc0b 100644 --- a/crates/pyspark-rs/src/dataframe.rs +++ b/crates/pyspark-rs/src/dataframe.rs @@ -57,6 +57,16 @@ pub struct PyDataFrame { /// (`_execution_info`), which each access returns until another execution replaces it. execution_info_object: std::sync::Mutex)>>, + /// The schema that the copy, the deep copy or the unpickle of which this DataFrame is made + /// requested, as PySpark's `_cached_schema_serialized`: its reconstruction reads the columns + /// of the new DataFrame, which requests the schema and keeps it, before the state sets the + /// `_cached_schema` of the DataFrame that it is of. `schema` gives it, and so do + /// `printSchema`, `toArrow`, the schema that a set operation keeps and the numeric + /// aggregations of grouped data, which read `schema`; `columns`, `dtypes`, `__repr__`, the + /// column lookups and the DataFrames of methods that copy the cached schema read the cached + /// one, as PySpark's `_schema`. It is kept with a cached schema only: without one, PySpark's + /// next read of `_schema` requests the schema and keeps it as both. + copy_schema: Option, } /// The answer kept in `cell`, else that of `ask`, which is kept unless it fails, as @@ -266,6 +276,24 @@ impl PyDataFrame { input_files: std::sync::OnceLock::new(), schema_object: None, execution_info_object: std::sync::Mutex::new(None), + copy_schema: None, + } + } + + /// The schema that the copy or the unpickle of which this DataFrame is made requested, which + /// PySpark's `schema` gives (see `copy_schema`): the one place that reads it, for + /// `schema`, `printSchema`, the merge of a set operation and the numeric aggregations of + /// grouped data. + pub(crate) fn copy_schema(&self) -> Option { + self.copy_schema.clone() + } + + /// Keep `requested`, the schema that the copy or the unpickle of which this DataFrame is made + /// requested, if this DataFrame has a cached schema, as PySpark's `_cached_schema_serialized` + /// is read only then (see `copy_schema`). + fn keep_copy_schema(&mut self, requested: spark_connect::types::DataType) { + if self.schema_object.is_some() || self.dataframe.known_schema().is_some() { + self.copy_schema = Some(requested); } } @@ -320,14 +348,19 @@ impl PyDataFrame { /// The schema request that PySpark's `copy` of a DataFrame makes for the new DataFrame of its /// reconstruction, `hasattr(copy, "__setstate__")`, which reads the columns of a DataFrame that - /// has no schema yet, even when this one has one. The request is that of a DataFrame of the - /// same relation, which has no schema cached, so that it fails as PySpark's does. - fn fresh_schema_request(&self, py: Python<'_>) -> PyResult<()> { + /// has no schema yet, even when this one has one: the schema that it gives, which the copy + /// keeps (see `copy_schema`). The request is that of a DataFrame of the same relation, + /// which has no schema cached, so that it fails as PySpark's does, of the error of its plan + /// as the read of the columns raises it ([`columns_read_error`]). + fn fresh_schema_request(&self, py: Python<'_>) -> PyResult { let session = self.dataframe.spark_session(); - let relation = self.dataframe.to_relation().to_pyerr()?; + let relation = self + .dataframe + .to_relation() + .to_pyerr() + .map_err(|err| columns_read_error(py, err))?; let fresh = DataFrame::from_sent_relation(session.clone(), relation); - interruptible(py, &session, || fresh.schema())?; - Ok(()) + interruptible(py, &session, || fresh.schema()) } /// The number of rows and the truncation of the eager evaluation, as PySpark's `int` of @@ -1044,15 +1077,17 @@ impl PyDataFrame { /// The shallow copy of a DataFrame, as PySpark's `copy.copy` of its `__reduce__`: the same /// plan, and so the same Observations, as PySpark's copy shares its plan, with the /// `_support_repr_html` and the schema that its state carries, the one that it keeps as an - /// object, of `to` or of a map function, as the same object, and an `executionInfo` of its - /// own, None until it executes, as that of PySpark's `DataFrame(plan, session)`. + /// object, of `to` or of a map function, as the same object, the schema that its + /// reconstruction requests (see `copy_schema`), and an `executionInfo` of its own, None + /// until it executes, as that of PySpark's `DataFrame(plan, session)`. fn __copy__(&self, py: Python<'_>) -> PyResult { - self.fresh_schema_request(py)?; + let requested = self.fresh_schema_request(py)?; let mut copy = self.copy_of(self.dataframe.shallow_copy()); copy.schema_object = self .schema_object .as_ref() .map(|schema| schema.clone_ref(py)); + copy.keep_copy_schema(requested); Ok(copy) } @@ -1067,16 +1102,19 @@ impl PyDataFrame { /// a deep copy raises `NO_ACTIVE_SESSION` where no session is active, as on another thread, /// and `PySparkAssertionError` where another one is, while `copy.copy` keeps the session. /// PySpark's copy of the plan deep-copies the function of each UDF of it, with `memo`, before - /// the session: so do these (`deep_copied_picklers`), of which the plan of a copy in the - /// session of this DataFrame is made, which pickles them at each request, as PySpark's copy - /// does; a copy in another session is of the relation (`deep_copy_with_pickler_copies`). + /// the session: so do these (`deep_copied_picklers`), of which, and of the copies of the + /// Observations, the plan of a copy in the session of this DataFrame is made, which its + /// requests build, as PySpark's copy does: they pickle the copied functions, warn of a window + /// of no partitions, and give the `and` of the conditions of a join the origin of its call + /// site then. A copy in another session is of the relation (`deep_copy_with_pickler_copies`). fn __deepcopy__(&self, py: Python<'_>, memo: &Bound<'_, PyAny>) -> PyResult { // As PySpark's: `copy.deepcopy` looks up `__deepcopy__` through `__getattr__`, whose check // reads the columns, so the schema is requested unless it is cached. On a stopped session, // that request raises the SparkConnectException of its closed client, before the session // is copied, which raises a PySparkRuntimeError. let session = self.dataframe.spark_session(); - interruptible(py, &session, || self.dataframe.schema())?; + interruptible(py, &session, || self.dataframe.schema()) + .map_err(|err| columns_read_error(py, err))?; // As PySpark's copy of the plan, which copies the objects of `pyspark.ml` that it keeps // (see `__reduce_ex__`): the TypeError of the lock of a model, after a copy of the // `_summary_dataset` of the training summary of a DataFrame of a method of the summary, of @@ -1117,10 +1155,10 @@ impl PyDataFrame { }) .to_pyerr()?; // As PySpark's reconstruction of the copy, which requests its schema, as the copy has none, - // before it sets the deep copy of the state, of the schema that it keeps as an object too, - // which the deep copy of the plan shares in `memo`. + // and keeps it, before it sets the deep copy of the state, of the schema that it keeps as + // an object too, which the deep copy of the plan shares in `memo`. let mut copy = self.copy_of(copy); - copy.fresh_schema_request(py)?; + let requested = copy.fresh_schema_request(py)?; if let Some(schema) = &self.schema_object { copy.schema_object = Some( py.import("copy")? @@ -1128,16 +1166,18 @@ impl PyDataFrame { .unbind(), ); } + copy.keep_copy_schema(requested); Ok(copy) } /// The DataFrame of `relation`, the bytes of a relation of `levels` levels of `session`, as a /// pickled DataFrame unpickles, with the `_support_repr_html` and the `_cached_schema` of its /// state. As PySpark's unpickle, whose lookup of the `__setstate__` of the new DataFrame reads - /// its columns, it requests the schema of the relation, whose error it raises, and then the - /// DataFrame has the cached schema of the pickled one, none when it had none, so that its - /// `schema` requests it again then. Without `request_schema`, of an object that PySpark - /// pickles of the plan of the DataFrame, as a DataStreamWriter, the schema is not requested. + /// its columns, it requests the schema of the relation, whose error it raises, and keeps it + /// (see `copy_schema`), and then the DataFrame has the cached schema of the pickled one, + /// none when it had none, so that its `schema` requests it again then. Without + /// `request_schema`, of an object that PySpark pickles of the plan of the DataFrame, as a + /// DataStreamWriter, the schema is not requested. #[staticmethod] #[pyo3(signature = ( session, relation, levels=1, support_repr_html=false, cached_schema=None, request_schema=true @@ -1166,11 +1206,17 @@ impl PyDataFrame { }) .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))? .to_pyerr()?; - if request_schema { + let requested = if request_schema { let schema = interruptible(py, &dataframe.spark_session(), || dataframe.schema())?; check_schema_conversion(py, Some(&schema))?; + Some(schema) + } else { + None + }; + let mut dataframe = PyDataFrame::new(dataframe.with_known_schema(known_schema)); + if let Some(requested) = requested { + dataframe.keep_copy_schema(requested); } - let dataframe = PyDataFrame::new(dataframe.with_known_schema(known_schema)); dataframe .support_repr_html .store(support_repr_html, std::sync::atomic::Ordering::SeqCst); @@ -2927,9 +2973,10 @@ impl PyDataFrame { /// argument. Mirrors `DataFrame.asTable()`. #[pyo3(name = "asTable")] fn as_table(&self) -> crate::tablearg::PyTableArg { - crate::tablearg::PyTableArg::new(spark_connect::table_arg::TableArg::new( - self.dataframe.clone(), - )) + crate::tablearg::PyTableArg::new( + spark_connect::table_arg::TableArg::new(self.dataframe.clone()), + self.dataframe.spark_session().session_id().to_string(), + ) } /// Deprecated alias for dropDuplicates. @@ -3432,6 +3479,12 @@ impl PyDataFrame { /// that is an object, a deep copy of it, as PySpark's (see `property_schema`). #[getter] fn schema(slf: &Bound<'_, Self>) -> PyResult> { + // As PySpark's loads of `_cached_schema_serialized`, of the schema that the copy or the + // unpickle of which this DataFrame is made requested. + if let Some(schema) = slf.borrow().copy_schema() { + check_schema_conversion(slf.py(), Some(&schema))?; + return crate::types::data_type_to_py(slf.py(), &schema); + } Self::property_schema(slf)?.schema(slf.py()) } @@ -3594,7 +3647,16 @@ impl PyDataFrame { Some(level) if level.is_truthy()? => Some(level), _ => None, }; - let tree = match Self::property_schema(slf)? { + // `self.schema` is the schema that a copy or an unpickle requested, if any (see + // `copy_schema`), else that of `property_schema`. + let property = match slf.borrow().copy_schema() { + Some(schema) => { + check_schema_conversion(py, Some(&schema))?; + PropertySchema::Core(schema) + } + None => Self::property_schema(slf)?, + }; + let tree = match property { // A schema that is an object (see `property_schema`) is printed as PySpark's // `print(self.schema.treeString(level))`, of a copy of it, whose `treeString` raises // its own AttributeError, and that of a Column is a Column, whose call raises. @@ -7084,7 +7146,8 @@ pub(crate) fn check_schema_conversion( /// equal, else none. The core keeps that of two cached structs; of a schema that a DataFrame /// keeps as a Python object, the schema of `to` that is not a DataType or the `StructType` of a /// map function, the objects are compared as they are now, as PySpark compares them with `==`, -/// and the result keeps a deep copy of that of `this`, as `this.schema` gives. +/// and the result keeps a deep copy of that of `this`, as `this.schema` gives. Of a copy or an +/// unpickle, `this.schema` is the schema that it requested (see `copy_schema`). fn set_operation_result( py: Python<'_>, this: &PyDataFrame, @@ -7092,7 +7155,12 @@ fn set_operation_result( dataframe: DataFrame, ) -> PyResult { if this.schema_object.is_none() && other.schema_object.is_none() { - return Ok(PyDataFrame::new(dataframe)); + return Ok(PyDataFrame::new( + match (this.copy_schema(), dataframe.known_schema()) { + (Some(requested), Some(_)) => dataframe.with_known_schema(Some(requested)), + _ => dataframe, + }, + )); } let cached = |df: &PyDataFrame| -> PyResult>> { match &df.schema_object { @@ -7107,6 +7175,11 @@ fn set_operation_result( }; if let (Some(left), Some(right)) = (cached(this)?, cached(other)?) { if left.bind(py).eq(right.bind(py))? { + if let Some(requested) = this.copy_schema() { + return Ok(PyDataFrame::new( + dataframe.with_known_schema(Some(requested)), + )); + } return Ok(match &this.schema_object { Some(schema) => { let mut result = PyDataFrame::new(dataframe.with_known_schema(None)); @@ -7239,6 +7312,23 @@ fn object_dtypes<'py>(py: Python<'py>, schema: &Bound<'py, PyAny>) -> PyResult, err: PyErr) -> PyErr { + if !err.is_instance_of::(py) { + return err; + } + crate::column::raise_pyspark( + py, + "PySparkTypeError", + "NOT_ITERABLE", + &[("objectName", "Column")], + ) +} + /// The error of a value that protobuf refuses in a field of a plan, which PySpark's plan raises as /// a request sends it: a builtin `TypeError`, `OverflowError` or `ValueError`, as [`proto_int`] /// and [`proto_bool`] raise them. Another error is raised at the call. diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs index b5fdb5b9..55c80762 100644 --- a/crates/pyspark-rs/src/group.rs +++ b/crates/pyspark-rs/src/group.rs @@ -211,7 +211,8 @@ impl PyGroupedData { /// `sum`, `avg`, `mean`, `min` and `max`, as PySpark's `_numeric_agg`: of column names, /// which it asserts, every numeric column without any, and `NOT_NUMERIC_COLUMNS` for a - /// column that is not numeric. + /// column that is not numeric, of the columns of `self._df.schema`, which of a copy or an + /// unpickle is the schema that it requested (see `PyDataFrame::copy_schema`). fn numeric_agg( &self, py: Python<'_>, @@ -225,7 +226,23 @@ impl PyGroupedData { }; names.push(name.to_cow().ok().map(|name| name.into_owned())); } - let grouped = &self.grouped_data; + let requested = self.frame.bind(py).borrow().copy_schema(); + let of_requested; + let grouped = match requested { + Some(schema) => { + let grouped = &self.grouped_data; + of_requested = RustGroupedData::from_parts( + grouped.dataframe().with_known_schema(Some(schema)), + grouped.grouping_columns().to_vec(), + grouped.group_type(), + grouped.pivot_column(), + grouped.pivot_value_columns(), + grouped.grouping_set_columns().to_vec(), + ); + &of_requested + } + None => &self.grouped_data, + }; if names.iter().any(Option::is_none) { // A name that UTF-8 cannot encode is no column of the schema, which the server // sends: PySpark reads the schema, and raises `NOT_NUMERIC_COLUMNS` of the `str` of diff --git a/crates/pyspark-rs/src/tablearg.rs b/crates/pyspark-rs/src/tablearg.rs index b9ef7148..1c568169 100644 --- a/crates/pyspark-rs/src/tablearg.rs +++ b/crates/pyspark-rs/src/tablearg.rs @@ -2,6 +2,7 @@ use pyo3::prelude::*; use pyo3::types::PyDict; +use spark_connect::dataframe::DataFrame; use spark_connect::table_arg::TableArg; use crate::errors::ResultExt; @@ -19,18 +20,29 @@ use crate::window::window_columns; )] pub struct PyTableArg { pub(crate) inner: TableArg, + /// The ID of the session of the DataFrame of the table argument, which its pickle keeps: it + /// unpickles of the active session of that ID (see `__reduce__`). + session_id: String, /// Set once the value is written, as `__traverse__` reads `inner`, which is not valid while /// the object is created (see [`crate::gc::GcReady`]). gc_ready: crate::gc::GcReady, } impl PyTableArg { - pub fn new(inner: TableArg) -> Self { + /// The table argument `inner` of a DataFrame of the session of `session_id`. + pub fn new(inner: TableArg, session_id: String) -> Self { PyTableArg { inner, + session_id, gc_ready: crate::gc::GcReady::new(), } } + + /// The table argument `inner`, of the session of this one, as the methods of PySpark's table + /// argument give a new one of the same plan. + fn of_same_session(&self, inner: TableArg) -> Self { + PyTableArg::new(inner, self.session_id.clone()) + } } #[pymethods] @@ -42,9 +54,7 @@ impl PyTableArg { // PySpark checks the partitioning before it converts the columns. self.inner.check_partition_by().to_pyerr()?; let columns = window_columns(cols)?; - Ok(PyTableArg::new( - self.inner.clone().partition_by(columns).to_pyerr()?, - )) + Ok(self.of_same_session(self.inner.clone().partition_by(columns).to_pyerr()?)) } /// Order the (partitioned) table argument by the given columns. @@ -54,9 +64,7 @@ impl PyTableArg { // PySpark checks the partitioning before it converts the columns. self.inner.check_order_by().to_pyerr()?; let columns = window_columns(cols)?; - Ok(PyTableArg::new( - self.inner.clone().order_by(columns).to_pyerr()?, - )) + Ok(self.of_same_session(self.inner.clone().order_by(columns).to_pyerr()?)) } /// The `spark.connect.Expression` of the table argument, as PySpark Connect's @@ -85,15 +93,17 @@ impl PyTableArg { /// Mark the table argument as requiring a single partition. #[allow(non_snake_case)] fn withSinglePartition(&self) -> PyResult { - Ok(PyTableArg::new( - self.inner.clone().with_single_partition().to_pyerr()?, - )) + Ok(self.of_same_session(self.inner.clone().with_single_partition().to_pyerr()?)) } /// The shallow copy of the table argument, as PySpark's `copy.copy`: of the same subquery, /// and of the same user-set attributes. fn __copy__<'py>(slf: &Bound<'py, Self>) -> PyResult> { - let copy = Bound::new(slf.py(), PyTableArg::new(slf.borrow().inner.clone()))?; + let copy = { + let this = slf.borrow(); + this.of_same_session(this.inner.clone()) + }; + let copy = Bound::new(slf.py(), copy)?; copy_instance_state(copy.as_any(), slf.as_any())?; Ok(copy) } @@ -107,8 +117,11 @@ impl PyTableArg { memo: &Bound<'py, PyAny>, ) -> PyResult> { let py = slf.py(); - let inner = slf.borrow().inner.clone(); - let copy = Bound::new(py, PyTableArg::new(inner.clone()))?; + let (inner, copy) = { + let this = slf.borrow(); + (this.inner.clone(), this.of_same_session(this.inner.clone())) + }; + let copy = Bound::new(py, copy)?; memo.set_item(slf.as_ptr() as usize, ©)?; let copies = crate::udf_command::deep_copied_picklers(py, inner.picklers(), memo)?; if !copies.is_empty() { @@ -123,6 +136,105 @@ impl PyTableArg { Ok(copy) } + /// A table argument pickles, as PySpark's, which pickles the objects of its subquery + /// expression, its plan, its partitioning, its ordering and its single partition: of the ID of + /// its session and the relation of its plan, as a DataFrame pickles them, of the Columns of + /// its partitioning and its ordering and of its single partition, of which + /// `pyspark.sql.connect._column_reduce.table_arg_of` gives back the table argument, as + /// `_of_relation` and the calls that made it, with no request, as PySpark's unpickle sends + /// none. A plan that does not build pickles of its error, which the requests of the + /// unpickled table argument raise ([`crate::column::described_error`]), as PySpark's raise it + /// when they build the plan. It keeps the ID of its session rather than the session, which + /// pickles of a local function, as PySpark's table argument holds no session: so a plain + /// pickle of it succeeds. A user-set attribute is the state, as of a plain object. + fn __reduce__<'py>(slf: &Bound<'py, Self>) -> PyResult> { + use pyo3::types::PyBytes; + use spark_connect::expression::ExpressionNode; + let py = slf.py(); + let this = slf.borrow(); + let (relation, levels, build_error) = + match crate::origin::pickling(|| this.inner.relation_with_levels()) { + Ok((relation, levels)) => ( + PyBytes::new(py, &crate::encode_on_grown_stack(relation)), + levels, + None, + ), + Err(error) => match crate::column::described_error(py, &error)? { + Some(description) => (PyBytes::new(py, b""), 1, Some(description)), + None => return Err(crate::errors::spark_error_to_py_exception(error)), + }, + }; + let subquery = this.inner.to_column(); + let ExpressionNode::Subquery(subquery) = subquery.expression().node() else { + unreachable!("a table argument is a subquery expression"); + }; + let columns = |expressions: &[spark_connect::expression::Expression]| { + expressions + .iter() + .map(|expression| { + Py::new( + py, + crate::column::PyColumn::new(spark_connect::column::Column::new( + expression.clone(), + )), + ) + }) + .collect::>>() + }; + let args = ( + this.session_id.clone(), + relation, + levels, + columns(&subquery.partition_spec)?, + columns(&subquery.order_spec)?, + subquery.with_single_partition == Some(true), + build_error, + ) + .into_pyobject(py)?; + let table_arg_of = py + .import("pyspark.sql.connect.dataframe")? + .getattr("_column_reduce")? + .getattr("table_arg_of")?; + let reduce = pyo3::types::PyTuple::new(py, [table_arg_of, args.into_any()])?.into_any(); + crate::group::reduce_with_state(&reduce, slf.as_any()) + } + + /// The table argument of `relation`, the bytes of the relation of `levels` levels of the plan + /// of a DataFrame of `session`, as a pickled table argument unpickles + /// (`_column_reduce.table_arg_of`): the `asTable()` of the DataFrame of the relation, of its + /// plan ID, as `DataFrame._of_relation` gives it, with no request. Of `build_error`, the + /// description of the error of a plan that did not build, each request of a plan of it + /// raises the error. + #[staticmethod] + #[pyo3(signature = (session, relation, levels=1, build_error=None))] + fn _of_relation( + session: &Bound<'_, crate::session::PySparkSession>, + relation: &[u8], + levels: usize, + build_error: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let session = session.borrow().session.clone(); + let session_id = session.session_id().to_string(); + let dataframe = match build_error { + Some(description) => { + let error = crate::column::error_of_description(description)?; + DataFrame::from_sent_relation(session, Default::default()).unbuildable(error) + } + None => { + use prost::Message; + // Decoded, and then built or dropped, on a stack that holds its levels. + crate::on_grown_stack(|| { + spark_connect_proto::Relation::decode(relation).map(|relation| { + DataFrame::from_sent_relation_levels(session, relation, levels) + }) + }) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))? + .to_pyerr()? + } + }; + Ok(PyTableArg::new(TableArg::new(dataframe), session_id)) + } + /// As PySpark's table argument, a Python object that holds the plan of its DataFrame and its /// partitioning and ordering columns, which Python's garbage collector sees: the functions of /// the UDFs that the table argument alone keeps alive are reported, those of the plan once its diff --git a/crates/pyspark-rs/src/window.rs b/crates/pyspark-rs/src/window.rs index 209a6ec6..0e21361f 100644 --- a/crates/pyspark-rs/src/window.rs +++ b/crates/pyspark-rs/src/window.rs @@ -169,56 +169,75 @@ fn boundary_not_between(py: Python<'_>, name: &str) -> PyErr { ) } -/// The bound `name` of a frame argument, as the plan of PySpark's `WindowExpression` converts -/// it, in a row frame if `rows`, else in a range frame: an int of 0 (False too) is the current -/// row; a row frame sets any other int of an int32 in the int32 literal, which protobuf's default -/// (upb) backend refuses of a bool, and raises `VALUE_NOT_BETWEEN` of an int beyond an int32; -/// and a range frame sets it in the long literal, which refuses a bool and an int beyond a long, -/// as [`crate::stat_arguments::proto_int`] does. A `FrameBound` of a row frame is checked as -/// such an int. +/// A frame bound as the window of the core holds it, or, of a bound that it does not hold, the +/// error that the plan of PySpark's `WindowExpression` raises of it. +enum ConvertedBound { + Held(FrameBound), + Unheld(PyErr), +} + +/// The bound `name` of a frame argument, in a row frame if `rows`, else in a range frame, as the +/// window of the core holds it: 0 (False too) is the current row, and another int of a long the +/// offset of its sign, which the core converts when a request builds the plan, as the plan of +/// PySpark's `WindowExpression` converts it then (an int beyond an int32 of a row frame fails +/// with `VALUE_NOT_BETWEEN`). The core does not hold True, which PySpark sets in the int field of +/// the literal, which protobuf's default (upb) backend refuses of a bool, nor an int beyond a +/// long, of `VALUE_NOT_BETWEEN` in a row frame and of the `ValueError` of upb's long field in a +/// range frame: these give the error. fn converted_frame_bound( py: Python<'_>, argument: FrameArgument<'_, '_>, name: &str, rows: bool, -) -> PyResult { +) -> PyResult { let value = match argument { - FrameArgument::Bound(bound) => { - let offset = match &bound { - FrameBound::Preceding(n) => Some(n.saturating_neg()), - FrameBound::Following(n) => Some(*n), - _ => None, - }; - if rows && offset.is_some_and(|n| i32::try_from(n).is_err()) { - return Err(boundary_not_between(py, name)); - } - return Ok(bound); - } + FrameArgument::Bound(bound) => return Ok(ConvertedBound::Held(bound)), FrameArgument::Value(value) => value, }; - let long = value.extract::().ok(); - if long == Some(0) { - return Ok(FrameBound::CurrentRow); - } - if !rows { - return crate::stat_arguments::proto_int::(value, RANGE_BOUNDARY_FIELD) - .map(frame_bound_of); - } - match long.and_then(|n| i32::try_from(n).ok()) { - Some(_) if value.is_instance_of::() => Err( - crate::stat_arguments::bool_not_proto_int(ROW_BOUNDARY_FIELD), - ), - Some(n) => Ok(frame_bound_of(n.into())), - None => Err(boundary_not_between(py, name)), - } + if value.is_instance_of::() { + let field = if rows { + ROW_BOUNDARY_FIELD + } else { + RANGE_BOUNDARY_FIELD + }; + return Ok(if value.is_truthy()? { + ConvertedBound::Unheld(crate::stat_arguments::bool_not_proto_int(field)) + } else { + ConvertedBound::Held(FrameBound::CurrentRow) + }); + } + Ok(match value.extract::() { + Ok(n) => ConvertedBound::Held(frame_bound_of(n)), + Err(_) if rows => ConvertedBound::Unheld(boundary_not_between(py, name)), + Err(_) => ConvertedBound::Unheld(crate::session::value_out_of_range(value)), + }) +} + +/// The error that a request raises of the held bound `bound` of a row frame, its start if +/// `lower`, else its end, as the core converts it when it builds the plan +/// (`to_proto_frame_boundary`): an offset beyond an int32 that is neither the current row nor +/// the unbounded bound of its side is `VALUE_NOT_BETWEEN`. A range frame holds no failing bound. +fn held_bound_error(py: Python<'_>, bound: &FrameBound, rows: bool, lower: bool) -> Option { + let offset = match bound { + FrameBound::UnboundedPreceding => i128::from(i64::MIN), + FrameBound::Preceding(n) => -i128::from(*n), + FrameBound::CurrentRow => 0, + FrameBound::Following(n) => i128::from(*n), + FrameBound::UnboundedFollowing => i128::from(i64::MAX), + }; + let unbounded = i128::from(if lower { i64::MIN } else { i64::MAX }); + let fails = rows && offset != 0 && offset != unbounded && i32::try_from(offset).is_err(); + fails.then(|| boundary_not_between(py, if lower { "start" } else { "end" })) } /// The bounds of `rowsBetween(start, end)` if `rows`, else of `rangeBetween(start, end)`, a /// plain int (negative is preceding, positive following) or a `FrameBound` each, as PySpark's: /// it compares the start and then the end with the thresholds of their sides, and WindowFrame /// then asserts that the start and then the end that is not past its threshold is an int (a -/// bool included), a bare `AssertionError` of a NumPy int, a float or a Decimal. The errors of -/// the plan, which converts the start and then the end, are raised here, after those. +/// bool included), a bare `AssertionError` of a NumPy int, a float or a Decimal. The plan converts +/// the start and then the end when a request builds it, as the window of the core does of the +/// bounds that it holds (see [`converted_frame_bound`]). Of a bound that it does not hold, the +/// error that PySpark's build raises first, of the start and then of the end, is raised here. fn frame_bounds( py: Python<'_>, start: &Bound<'_, PyAny>, @@ -234,10 +253,15 @@ fn frame_bounds( } } } - Ok(( - converted_frame_bound(py, start, "start", rows)?, - converted_frame_bound(py, end, "end", rows)?, - )) + let start = converted_frame_bound(py, start, "start", rows)?; + let end = converted_frame_bound(py, end, "end", rows)?; + match (start, end) { + (ConvertedBound::Held(start), ConvertedBound::Held(end)) => Ok((start, end)), + (ConvertedBound::Unheld(error), _) => Err(error), + (ConvertedBound::Held(start), ConvertedBound::Unheld(error)) => { + Err(held_bound_error(py, &start, rows, true).unwrap_or(error)) + } + } } #[pymethods] diff --git a/crates/spark-connect/src/dataframe.rs b/crates/spark-connect/src/dataframe.rs index 9193da1f..680b6afe 100644 --- a/crates/spark-connect/src/dataframe.rs +++ b/crates/spark-connect/src/dataframe.rs @@ -1727,9 +1727,12 @@ impl DataFrame { /// As [`DataFrame::deep_copy_with`], of `copies`, the copies of the picklers of the UDFs of the /// plan, as PySpark's deep copy copies the functions of the UDFs of its plan: the copy is of /// this plan with the copies of its picklers and of its Observations, of the same plan IDs, - /// which pickles the copied functions at each request, as PySpark's copy does. A copy in - /// another session, of plan IDs of its own, is that of [`DataFrame::deep_copy_with`], whose - /// relation carries the plan IDs. Not a stable API. + /// which pickles the copied functions at each request, as PySpark's copy does. A copy of no + /// such copies shares the plan, of which its requests build the relation, as those of + /// PySpark's copy build its plan objects: each warns of a window of no partitions and gives + /// the `and` of the conditions of a join the origin of its call site. A copy in another + /// session, of plan IDs of its own, is that of [`DataFrame::deep_copy_with`], whose relation + /// carries the plan IDs. Not a stable API. #[doc(hidden)] pub fn deep_copy_with_pickler_copies( &self, @@ -1737,9 +1740,7 @@ impl DataFrame { observations: Vec<(String, crate::observation::Observation)>, copies: &crate::udf::PicklerCopies, ) -> Result { - if copies.is_empty() - || !Arc::ptr_eq(self.session.plan_ids_shared(), session.plan_ids_shared()) - { + if !Arc::ptr_eq(self.session.plan_ids_shared(), session.plan_ids_shared()) { return self.deep_copy_with(session, observations); } let plan = crate::udf::PicklerRebuild::new(copies, &observations) @@ -6981,8 +6982,9 @@ mod plan_construction_tests { /// As PySpark's `copy.deepcopy` of a DataFrame of UDFs, which copies the functions of the UDFs /// of its plan with the plan: the plan of the copy holds the copies of the picklers and of the /// Observations that it is given, of the plan IDs of the DataFrame, so its requests pickle - /// the copies and its actions set the copied Observations. A copy in another session is of - /// the relation, as a copy without UDFs. + /// the copies and its actions set the copied Observations. So is a copy of no copies of + /// picklers, of the plan, which its requests build; a copy in another session is of the + /// relation. #[test] fn a_deep_copy_of_copies_of_picklers_holds_them_of_the_plan_ids_as_pyspark_deep_copies() { use crate::observation::Observation; @@ -7026,16 +7028,31 @@ mod plan_construction_tests { report_count(©, 3); assert_eq!(counts(&given), [3]); assert!(counts(&observation).is_empty()); - // Of no copies, or in another session: the copy of the relation. + // In another session: the copy of the relation. let other = spark.new_session(); let moved = df .deep_copy_with_pickler_copies(other, Vec::new(), &copies) .unwrap(); assert!(matches!(moved.plan.as_ref(), LogicalPlan::Raw { .. })); + // Of no copies of picklers: the plan, rebuilt above the copy of the Observation, which + // sends the relation of the DataFrame, and the plan itself of no Observation. + let again = observation.detached(); let plain = df + .deep_copy_with_pickler_copies( + spark.clone(), + vec![("c".into(), again.clone())], + &PicklerCopies::new(), + ) + .unwrap(); + assert!(!matches!(plain.plan.as_ref(), LogicalPlan::Raw { .. })); + assert!(!Arc::ptr_eq(&plain.plan, &df.plan)); + assert_eq!(sent(&plain), sent(&df)); + assert!(plain.observations()[0].1.same(&again)); + let unobserved = spark.range(3).unwrap(); + let shared = unobserved .deep_copy_with_pickler_copies(spark.clone(), Vec::new(), &PicklerCopies::new()) .unwrap(); - assert!(matches!(plain.plan.as_ref(), LogicalPlan::Raw { .. })); + assert!(Arc::ptr_eq(&shared.plan, &unobserved.plan)); } /// As PySpark's unpickled plan, which has copies of the Observations of the pickled one: a diff --git a/crates/spark-connect/src/table_arg.rs b/crates/spark-connect/src/table_arg.rs index 350e1aff..d97b8231 100644 --- a/crates/spark-connect/src/table_arg.rs +++ b/crates/spark-connect/src/table_arg.rs @@ -48,6 +48,22 @@ impl TableArg { } } + /// The relation of the plan of this table argument, of the plan IDs of the session of its + /// DataFrame, as that DataFrame sends it, and the levels of the plan, which a pickle of the + /// table argument keeps: [`DataFrame::from_sent_relation_levels`] gives back a DataFrame of + /// the plan ID of the plan, whose table argument refers to the plan as this one does. A plan + /// that a request fails to build gives the error of its build. Not a stable API. + #[doc(hidden)] + pub fn relation_with_levels(&self) -> Result<(spark_connect_proto::Relation, usize)> { + let ids = self + .subquery + .plan_ids + .clone() + .unwrap_or_else(|| std::sync::Arc::new(crate::plan::PlanIds::new())); + let relation = crate::plan::relation_with_ids(&self.subquery.plan, &ids)?; + Ok((relation, self.subquery.plan.depth())) + } + /// The picklers of the UDFs of the plan and of the expressions of this table argument, each /// once, which a deep copy of it copies. Not a stable API. #[doc(hidden)] @@ -442,6 +458,50 @@ mod tests { )); } + /// As PySpark's pickle of a table argument, which keeps the objects of its plan, of its plan + /// ID: the relation that a pickle keeps is the one that its DataFrame sends, of its levels, + /// and the table argument of the DataFrame of that relation refers to it by the same plan + /// ID, of which a UDTF of it sends the same relation. + #[test] + fn the_relation_of_a_table_argument_gives_back_its_plan_of_its_plan_id() { + let spark = crate::session::SparkSession::builder() + .remote("sc://localhost:15002") + .get_or_create() + .expect("failed to build session"); + let df = spark.range(3).unwrap().select([col("id")]); + let table = TableArg::new(df.clone()) + .partition_by(vec![col("id")]) + .unwrap(); + let (relation, levels) = table.relation_with_levels().unwrap(); + assert_eq!(relation, df.to_relation().unwrap()); + assert_eq!(levels, df.plan_levels()); + let sent = TableArg::new( + DataFrame::from_sent_relation_levels(spark.clone(), relation, levels).unwrap(), + ) + .partition_by(vec![col("id")]) + .unwrap(); + assert_eq!(sent.subquery.plan_id, table.subquery.plan_id); + let reference = |table: &TableArg| { + let udtf = spark.tvf().udtf( + "t", + [table.to_column()], + None, + 300, + vec![], + "3.11".into(), + false, + ); + let Some(proto::relation::RelType::WithRelations(with)) = + udtf.relation().unwrap().rel_type + else { + panic!("a WithRelations"); + }; + with.references[0].clone() + }; + assert_eq!(reference(&sent), reference(&table)); + assert_eq!(subquery_proto(&sent), subquery_proto(&table)); + } + #[test] fn to_proto_single_partition_sets_flag_and_preserves_desc_order() { let t = arg() diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index a4c454d5..3dfd1e97 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -157,14 +157,14 @@ difference is intentional or tracked for a fix. differs. - **What a plan build does for a pickle or a deep copy** (intentional; DESIGN). A pickled - DataFrame or Column, and a deep copy of a DataFrame whose plan holds no function of a UDF or - that is copied into another session, carry the relation that the drop-in builds for them, - where PySpark's keep their plan objects and build them at each request. (A deep copy of a - DataFrame of such functions in its session is a copy of its plan, of copies of them, which - its requests build, as PySpark's; see "A deep copy of a DataFrame into another session".) - So the requests of an unpickled DataFrame or Column, or of such a deep copy, do not warn of a - window of no partitions (`WARN WindowExpression: No Partition Defined ...`), which - PySpark's do at each build, and the `and` of the conditions of a join of a list of + DataFrame, Column or table argument, and a deep copy of a DataFrame that is copied into another + session, carry the relation that the drop-in builds for them, where PySpark's keep their plan + objects and build them at each request. (A deep copy of a DataFrame in its session is a copy of + its plan, of copies of the functions of its UDFs and of its Observations, which its requests + build, as PySpark's; see "A deep copy of a DataFrame into another session".) + So the requests of an unpickled DataFrame, Column or table argument, or of such a deep copy, do + not warn of a window of no partitions (`WARN WindowExpression: No Partition Defined ...`), + which PySpark's do at each build, and the `and` of the conditions of a join of a list of Columns keeps the origin of its call site at the pickle or the copy, where PySpark's has the call site of each request. The pickle itself does not warn, as PySpark's does not, but PySpark's unpickle of a DataFrame warns once, as it builds the plan to read the columns of the @@ -288,7 +288,8 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan - **Garbage collection** (tracked). A DataFrame or a Column reports the functions of its UDFs that it alone holds to Python's garbage collector, and so do grouped data and their cogroups, a table argument, a window specification, the writers, `na`, `stat`, a merge builder and its clauses, - and the rows of `toLocalIterator`, of the Columns and the plans that they hold. Those that hold + the rows of `toLocalIterator`, and the expression of a Column and of a table argument (`_expr`, + `_subquery_expr`) and its copies, of the Columns and the plans that they hold. Those that hold their DataFrame report it, which reports the functions of the plan that they share (`crate::gc::Holders`). A function that a Column and a DataFrame of it share, or two DataFrames, or a DataFrame and a table argument of it, which holds the plan and not the DataFrame, is diff --git a/docs/compatibility.md b/docs/compatibility.md index e9197adf..579f7d10 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -1872,9 +1872,13 @@ Column share its expression, of which a deep copy makes again only the levels ab functions it copies, so they copy a Column of any depth, as PySpark's `copy.copy` does; PySpark's `copy.deepcopy` raises a `RecursionError` from about 165 levels. -The frame bounds of `Window.rowsBetween` and `rangeBetween`, which PySpark's client converts when -it builds the plan, the drop-in converts when it builds the window: their errors are PySpark's, -raised at the call rather than when the plan is built. The type of `cast` and `try_cast` is converted when the plan is +The frame bounds of `Window.rowsBetween` and `rangeBetween` are converted when the plan is built, +as in PySpark: a bound of a row frame beyond an int, such as a start of `-2**40` or of +`Window.unboundedFollowing`, makes a window that shows it as it is, of which each request of a plan +raises `VALUE_NOT_BETWEEN`, of the start and then of the end. A bound of `True`, which protobuf's +int field refuses, or of an int beyond a long, which the window of the drop-in does not hold, raises +at the call the error that PySpark raises when it builds the plan, the first of the start and of +the end. The type of `cast` and `try_cast` is converted when the plan is built, as in PySpark: a `StructField`, the base `DataType()`, an abstract type such as `AtomicType()` and a Python subclass of them, also one nested in the type, make a Column that renders as the `simpleString` of the type, and whose plans raise PySpark's `UNSUPPORTED_OPERATION`. @@ -2094,12 +2098,23 @@ which JSON has no value of, raises the `ValueError` of serde_json when the field (`expected value at line 1 column 7`), where PySpark's makes the field: the server refuses such metadata in every request, so only the local uses of the field differ. -The pickle of a `TableArg`, of `DataFrame.asTable()`, raises a `TypeError`, where PySpark's -pickles; `copy.copy` and `copy.deepcopy` copy it as PySpark's. Its `_subquery_expr` is an -expression of `pyspark.sql.connect.expressions`, as a Column's `_expr` is, of PySpark's `to_plan` -and `repr`, which is made at each access as a property of the native class: `vars()` of the table -argument does not show it and it cannot be set, where PySpark's is an instance attribute, a -`SubqueryExpression` with the fields of the subquery. `_is_partitioned()` is PySpark's. +A `TableArg`, of `DataFrame.asTable()`, pickles at every protocol, plainly and by cloudpickle, as +PySpark's, of its user-set attributes, and `copy.copy` and `copy.deepcopy` copy it as PySpark's. Its +pickle keeps the relation of the plan of its DataFrame, as a pickled DataFrame does (see "What a +plan build does for a pickle or a deep copy" in `dev/parity/known-differences.md`), and its +partitioning, ordering and single partition; the unpickle sends no request, as PySpark's, and the +table argument that it gives sends the same plan and subquery, of the same plan ID. A plan that does +not build pickles too, and the requests of the unpickled table argument raise its error, as +PySpark's. The pickle names the session of its DataFrame by its ID, so it unpickles only where the +session of that ID is active, as a pickled DataFrame does: it raises `NO_ACTIVE_SESSION` where none +is, such as on another thread, and `PySparkAssertionError` where another one is, where PySpark's, +whose plan holds no session, unpickles anywhere. On a +Python worker of the server it unpickles as PySpark Connect's table argument of the relation, which +needs no session. Its `_subquery_expr` is an expression of `pyspark.sql.connect.expressions`, as a +Column's `_expr` is, of PySpark's `to_plan` and `repr`, which is made at each access as a property +of the native class: `vars()` of the table argument does not show it and it cannot be set, where +PySpark's is an instance attribute, a `SubqueryExpression` with the fields of the subquery. +`_is_partitioned()` is PySpark's. The `__repr__` of a Column of a subquery names its plan by its plan ID, `SubqueryExpression(plan=, type=scalar)`, where PySpark's shows the repr of its plan @@ -2286,10 +2301,13 @@ PySpark's is the schema. A DataFrame whose plan fails to build pickles, also by cloudpickle, and its unpickle raises the error, as PySpark's: its pickle keeps the objects of the plan, and its unpickle builds the plan to -read the columns of the new DataFrame. `alias`, a hint, `dropDuplicates`, `observe`, `filter`, -`sort`, `sortWithinPartitions` and the repartitions of such a value keep the cached schema of the -DataFrame that they are of, as PySpark copies it, so their `schema` is known without a request, -and their other requests raise the error. +read the columns of the new DataFrame. Of an `AttributeError`, such as of a join on a tuple or on a +list of conditions whose `&` fails, PySpark's read of the columns falls back from its `_schema` to +the Column of `__getattr__`, whose `fields` is a Column, which is not iterable: so the unpickle, a +`copy.copy` and a `copy.deepcopy`, which read the columns too, raise `NOT_ITERABLE`, in both. +`alias`, a hint, `dropDuplicates`, `observe`, `filter`, `sort`, `sortWithinPartitions` and the +repartitions of such a value keep the cached schema of the DataFrame that they are of, as PySpark +copies it, so their `schema` is known without a request, and their other requests raise the error. The `barrier` of `mapInPandas` and `mapInArrow` is a protobuf bool, which their plan sets once it pickled the function, when it is built: an int is taken, as `barrier=1`, and another value, such @@ -2384,18 +2402,20 @@ DataFrame or the Column that holds it, such as through a bound method of an obje DataFrame, is collected. So do the objects that hold such Columns, or the plan of a DataFrame, as PySpark's do: grouped data and their cogroups, a table argument, a window specification, the writers of `write`, `writeTo` and `writeStream`, `na` and `stat`, a merge builder and its clauses, -and the generator of `toLocalIterator`, which holds its DataFrame until it ends, as PySpark's -generator holds `self`. Those that hold their DataFrame report it, and the DataFrame reports the -functions of the plan that they share, so a cycle through the DataFrame alone, such as -`df.back = df.groupBy("id")`, is collected too. A function that a Column and a DataFrame of it -share, or two DataFrames, or a DataFrame and a table argument of it, which holds its plan and not -the DataFrame, as PySpark's does, is reported by neither, so that no reference is counted twice: a -cycle through it is freed only when it is broken, where PySpark collects it. A stream writer holds -its DataFrame, where PySpark's holds the plan, so a DataFrame that keeps its stream writer is freed -by the collector, where PySpark frees it with its last reference. The functions of `foreachBatch` -and `foreach` are kept as their pickles, as PySpark's stream writer keeps them, so a function that -keeps its writer is freed with its last reference, and a copy or a pickle of the writer carries the -pickle of the call. The registration of `dataSource` reports its session, and the builder of +the generator of `toLocalIterator`, which holds its DataFrame until it ends, as PySpark's +generator holds `self`, and the expression of a Column (`Column._expr`) and of a table argument +(`TableArg._subquery_expr`), which `copy.copy` and `copy.deepcopy` copy as they copy a Column. +Those that hold their DataFrame report it, and the DataFrame reports the functions of the plan +that they share, so a cycle through the DataFrame alone, such as `df.back = df.groupBy("id")`, is +collected too. A function that a Column and a DataFrame of it share, or two DataFrames, or a +DataFrame and a table argument of it, which holds its plan and not the DataFrame, as PySpark's +does, is reported by neither, so that no reference is counted twice: a cycle through it is freed +only when it is broken, where PySpark collects it. A stream writer holds its DataFrame, where +PySpark's holds the plan, so a DataFrame that keeps its stream writer is freed by the collector, +where PySpark frees it with its last reference. The functions of `foreachBatch` and `foreach` are +kept as their pickles, as PySpark's stream writer keeps them, so a function that keeps its writer +is freed with its last reference, and a copy or a pickle of the writer carries the pickle of the +call. The registration of `dataSource` reports its session, and the builder of `pyspark.sql.connect.session.SparkSession` its channel builder and its hook factories, as PySpark's plain objects hold them, so a session that keeps its `dataSource`, or a channel builder or a hook factory that keeps its builder, is collected. One that a copy of the builder shares is reported by @@ -2510,16 +2530,17 @@ again. So does the unpickle of an object that PySpark pickles with its DataFrame callback, grouped data, `na` and `stat`; a stream writer, which PySpark's pickles of the plan of the DataFrame, requests nothing. -The drop-in does not keep the schema that a copy, a deep copy or an unpickle of a DataFrame -requests, where PySpark's keeps it apart from the cached schema of the state, as the -`_cached_schema_serialized` that its `schema` returns. So the two differ when the schema on the -server changed since the DataFrame cached it, such as that of a view that was replaced: PySpark's -copy gives the schema that it requested for `schema`, `printSchema`, `toArrow` and the set -operations of equal cached schemas (`c.union(c).schema`), and the cached schema for `columns`, -`dtypes`, `repr`, the column lookups and the methods that copy the cached schema, such as -`filter`; the drop-in's copy gives the cached schema for all, so its `toArrow` raises the -`ValueError` of the cast of the table to it. Of a DataFrame that had no cached schema, both give -the schema of the server. +A copy, a deep copy or an unpickle of a DataFrame keeps the schema that it requests apart from the +cached schema of the state, as PySpark's keeps it as the `_cached_schema_serialized` that its +`schema` returns. So the two differ when the schema on the server changed since the DataFrame +cached it, such as that of a view that was replaced, and the copy gives, as PySpark's, the schema +that it requested for `schema`, `printSchema`, `toArrow`, the set operations of equal cached +schemas (`c.union(c).schema`) and the numeric aggregations of its grouped data (`sum()` of no +columns), and the cached schema for `columns`, `dtypes`, `repr`, the column lookups and the +methods that copy the cached schema, such as `filter`. A copy of such a copy keeps the schema that +its own reconstruction requests, and the cached schema, which a pickle of it carries too. Of a +DataFrame that had no cached schema, both give the schema of the server, which the next read of the +copy requests again. `to` caches its schema whatever it is, as the object that it is, as PySpark's, so of a DataType that is not a struct, such as `LongType()`, `schema` gives a copy of it without a request, while its plan diff --git a/python/pyspark/sql/connect/_column_reduce.py b/python/pyspark/sql/connect/_column_reduce.py index 94e1afcd..8430e6ad 100644 --- a/python/pyspark/sql/connect/_column_reduce.py +++ b/python/pyspark/sql/connect/_column_reduce.py @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""The function that a pickled Column of the client unpickles with, and the parser of the deep -messages that a pickled Column or DataFrame carries. +"""The functions that a pickled Column and table argument of the client unpickle with, and the +parser of the deep messages that a pickled Column, DataFrame or table argument carries. ``pyspark.sql.connect.column`` registers this module with cloudpickle to pickle by value, so a Column that a UDF or a ``foreachBatch`` function captures unpickles on a Python worker of the @@ -207,20 +207,92 @@ def __repr__(self): # type: ignore[no-untyped-def] ) +def table_arg_of( + session_id: str, + relation: bytes, + levels: int, + partition: list[Any], + order: list[Any], + single_partition: bool, + build_error: Optional[tuple[str, str, Any]] = None, +) -> Any: + """The table argument of ``relation``, the bytes of the relation of ``levels`` levels of the + plan of a DataFrame of the session of ``session_id``, partitioned by the Columns of + ``partition``, or of a single partition, and ordered by the Columns of ``order``, as a + pickled table argument unpickles: with no request, as PySpark's unpickle of the objects of + its plan sends none. Of ``build_error``, the description of the error of a plan that did not + build, the requests of a plan of it raise that error, as PySpark's raise it when they build + the plan. + + On the client, it is the ``asTable()`` of the DataFrame of the relation, of its plan ID, of + the active session of that ID (``SparkSession._getActiveSessionIfMatches``), and of the calls + that made it: ``partitionBy`` or ``withSinglePartition``, then ``orderBy``. On a Python worker + of the server, whose PySpark has no Rust client and no session, it is PySpark Connect's table + argument of the relation, of the same calls, as PySpark's table argument, which holds no + session, unpickles there. A function of the module, so that a plain pickle of a table argument + pickles it by its name, as a plain pickle of PySpark's table argument succeeds.""" + try: + from pyspark import _pyspark + except ImportError: + from pyspark.sql.connect import proto + from pyspark.sql.connect.expressions import SubqueryExpression + from pyspark.sql.connect.plan import LogicalPlan + from pyspark.sql.connect.table_arg import TableArg + + # Without annotations: the class body looks its names up by name, which cloudpickle + # does not pickle with the function. + class Relation(LogicalPlan): + """The plan of a relation as it was sent, of its plan ID, which the subquery of the + table argument refers to.""" + + def __init__(self, relation): # type: ignore[no-untyped-def] + super().__init__(None) + self._relation = relation + if relation.common.HasField("plan_id"): + self._root_plan_id = relation.common.plan_id + + def plan(self, session): # type: ignore[no-untyped-def] + if build_error is not None: + raise_error(build_error) + sent = proto.Relation() + sent.CopyFrom(self._relation) + return sent + + plan = Relation(parse_deep(proto.Relation, relation)) + table = TableArg(SubqueryExpression(plan, subquery_type="table_arg")) + else: + from pyspark.sql.connect.session import SparkSession + + session = SparkSession._getActiveSessionIfMatches(session_id) + table = _pyspark.TableArg._of_relation(session, relation, levels, build_error) + if partition: + table = table.partitionBy(*partition) + if single_partition: + table = table.withSinglePartition() + if order: + table = table.orderBy(*order) + return table + + def raise_error(error: tuple[str, str, Any]) -> None: - """Raise the error of ``error``, the ``(kind, text, payload)`` of a pickled Column of an - expression that a request did not build, or whose ``__repr__`` failed, as PySpark raised it: - a builtin error of its message, the error of UTF-8 of the str or bytes of the payload, or - ``PySparkValueError`` of its error class and parameters.""" + """Raise the error of ``error``, the ``(kind, text, payload)`` of a pickled Column or + DataFrame of an expression or a plan that a request did not build, or of an expression whose + ``__repr__`` failed, as PySpark raised it: a builtin error of its message, the error of UTF-8 + of the str or bytes of the payload, or the error of one of PySpark's error classes, such as + ``PySparkAttributeError``, of its error class and parameters.""" kind, text, payload = error if kind == "UnicodeEncodeError": payload.encode("utf-8") if kind == "UnicodeDecodeError": payload.decode("utf-8") - if kind == "PySparkValueError": - from pyspark.errors import PySparkValueError + if kind.startswith("PySpark"): + import pyspark.errors - raise PySparkValueError(errorClass=text, messageParameters=payload) - raise {"TypeError": TypeError, "ValueError": ValueError, "OverflowError": OverflowError}[kind]( - text - ) + raise getattr(pyspark.errors, kind)(errorClass=text, messageParameters=payload) + builtins = { + "TypeError": TypeError, + "ValueError": ValueError, + "OverflowError": OverflowError, + "AttributeError": AttributeError, + } + raise builtins[kind](text) diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py index 1af8bde8..bce0b4d8 100644 --- a/python/pyspark/sql/connect/dataframe.py +++ b/python/pyspark/sql/connect/dataframe.py @@ -72,9 +72,12 @@ def _reduce_dataframe( pickle looks its ``__setstate__`` up through its ``__getattr__``, which reads its columns, the function requests the schema of the relation, whose error it raises, and then sets the ``_cached_schema`` of the state. A plan that did not build is pickled of ``build_error``, the - description of its error, which it raises instead (``_column_reduce.raise_error``). Without - ``request_schema``, of an object that PySpark pickles of the plan of the DataFrame and not of - the DataFrame, such as a DataStreamWriter, the schema is not requested. + description of its error, which it raises instead (``_column_reduce.raise_error``): but for + an ``AttributeError``, of which PySpark's ``_schema`` falls back to the Column of the + ``__getattr__`` of the DataFrame, whose ``fields``, a Column too, its columns iterate, which + raises ``NOT_ITERABLE``. Without ``request_schema``, of an object that PySpark pickles of the + plan of the DataFrame and not of the DataFrame, such as a DataStreamWriter, the schema is not + requested. On a Python worker of the server, whose PySpark has no Rust client, such as the worker of a ``foreachBatch`` function, which has the active session of the query, the function gives @@ -91,6 +94,12 @@ def creator( request_schema: bool = True, ) -> Any: if build_error is not None: + if request_schema and build_error[0] in ("AttributeError", "PySparkAttributeError"): + from pyspark.errors import PySparkTypeError + + raise PySparkTypeError( + errorClass="NOT_ITERABLE", messageParameters={"objectName": "Column"} + ) _raise_error(build_error) try: from pyspark import _pyspark diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index caac760e..aa8a1143 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -10570,6 +10570,63 @@ def schema_requests(call): assert schema_requests(lambda: copy.deepcopy(cached)) == 1 +def test_copies_keep_the_schema_that_they_request_apart_from_the_cached_one_as_pyspark( + mock_connect): + # As PySpark's reconstruction of a copy, a deep copy or an unpickle of a DataFrame, which reads + # the columns of the new DataFrame, so that it requests its schema and keeps it as its + # `_cached_schema_serialized`, before the state sets the `_cached_schema` of the DataFrame: + # `schema`, `printSchema`, the set operations of equal cached schemas and the numeric + # aggregations of grouped data give the schema that it requested, and `columns`, `dtypes`, + # `repr` and the DataFrames of the methods that copy the cached schema give the cached one, + # with no more request. They differ when the schema on the server changed since it was + # cached. So do the copies of a copy, of the schema that they request. A copy of a DataFrame of + # no cached schema requests it again, as PySpark's `_schema` does, which keeps it as both. + import contextlib + import copy + import io + + import cloudpickle + from pyspark.sql.connect.proto import Relation + + spark, service = mock_connect + service.confs = {"spark.sql.repl.eagerEval.enabled": "false", + "spark.sql.repl.eagerEval.maxNumRows": "20", + "spark.sql.repl.eagerEval.truncate": "20"} + df = spark.range(1) + df.schema + service.data_type = ddl_data_type("a int, b string") + requested = "struct" + copies = [copy.copy, copy.deepcopy, lambda frame: cloudpickle.loads(cloudpickle.dumps(frame))] + for make in copies: + before = len(service.analyze_requests) + c = make(df) + assert len(service.analyze_requests) == before + 1 + assert c.schema.simpleString() == requested + printed = io.StringIO() + with contextlib.redirect_stdout(printed): + c.printSchema() + assert printed.getvalue() == c.schema.treeString() + "\n" + assert c.columns == ["id"] and c.dtypes == [("id", "bigint")] + assert repr(c) == "DataFrame[id: bigint]" + for result in (c.union(c), c.intersect(c), c.subtract(c), c.unionByName(c), c.union(df)): + assert result.schema.simpleString() == requested + for result in (df.union(c), c.filter("true"), c.alias("t"), c.limit(1)): + assert result.schema.simpleString() == "struct" + aggregate = Relation.FromString(c.groupBy().sum()._relation_bytes()).aggregate + assert [expression.unresolved_function.arguments[0].unresolved_attribute.unparsed_identifier + for expression in aggregate.aggregate_expressions] == ["a"] + assert len(service.analyze_requests) == before + 1 + service.data_type = ddl_data_type("z string") + again = copy.copy(c) + assert again.schema.simpleString() == "struct" and again.columns == ["id"] + service.data_type = ddl_data_type("a int, b string") + fresh = spark.range(1) + before = len(service.analyze_requests) + c = copy.copy(fresh) + assert c.schema.simpleString() == requested and c.columns == ["a", "b"] + assert len(service.analyze_requests) == before + 2 + + def test_reader_strings_that_utf8_cannot_encode_fail_the_plan_as_pyspark(mock_connect): # As PySpark's DataSource, which builds its plan at a request: its format is set before its # paths, and protobuf's string fields raise the UnicodeEncodeError of a str that UTF-8 cannot @@ -12700,8 +12757,9 @@ def test_cycles_through_the_functions_of_udfs_are_collected_as_pyspark(mock_conn # of a UDF, a map function and a UDTF. So it is through the objects that hold such a Column or # the plan of such a DataFrame, as PySpark's do: grouped data and their cogroups, a table # argument, a window specification, the writers, the null and statistics functions, a merge - # builder and its clauses, and the generator of toLocalIterator. A function that live Columns - # and DataFrames share is kept, and pickles at their requests. + # builder and its clauses, the generator of toLocalIterator, and the expression of a Column + # and of a table argument, and their copies. A function that live Columns and DataFrames share + # is kept, and pickles at their requests. import copy import gc import weakref @@ -12759,6 +12817,11 @@ def udf_column(job, returns="long"): lambda job: df.mergeInto("t", F.lit(True)).whenMatched(udf_column(job, "boolean")), lambda job: df.mergeInto("t", F.lit(True)).whenNotMatched().insert( {"id": udf_column(job)}), + lambda job: udf_column(job)._expr, + lambda job: copy.copy(udf_column(job)._expr), + lambda job: copy.deepcopy(udf_column(job)._expr), + lambda job: udf_frame(job).asTable()._subquery_expr, + lambda job: df.asTable().partitionBy(udf_column(job))._subquery_expr, ] for holder in holders: job = Job() @@ -12809,14 +12872,18 @@ def udf_column(job, returns="long"): state = [""] shared = F.udf(lambda x: state[0], "long")("id") - kept = [df.select(shared), copy.copy(shared), copy.deepcopy(shared)] + kept = [df.select(shared), copy.copy(shared), copy.deepcopy(shared), shared._expr, + copy.copy(shared._expr), copy.deepcopy(shared._expr)] for _ in range(3): gc.collect() - for column in [shared] + kept[1:]: + for column in [shared] + kept[1:3]: df.select(column).explain() root = base_pb2.AnalyzePlanRequest.FromString(service.analyze_requests[-1]).explain.plan.root command = root.project.expressions[0].common_inline_user_defined_function.python_udf.command assert b"" in command + for expression in kept[3:]: + udf = expression.to_plan(None).common_inline_user_defined_function + assert b"" in udf.python_udf.command def test_cycles_through_the_dataframes_of_their_objects_are_collected_as_pyspark(mock_connect): @@ -13942,12 +14009,17 @@ def test_unpickled_dataframes_request_their_schema_and_keep_the_cached_one_as_py # columns, and then sets the `_cached_schema` of the state of the pickle: one request at each # unpickle, whose error the unpickle raises, and then the schema of a DataFrame that had one # is known, while one that had none is requested again. A plan that does not build pickles, - # and its unpickle raises the error of its build, with no request. + # and its unpickle raises the error of its build, with no request: of an AttributeError, such + # as of a join on a tuple or on a Row, PySpark's `_schema` falls back to the Column of + # `__getattr__`, whose `fields` the columns iterate, so the unpickle, a copy and a deep copy, + # which read the columns too, raise NOT_ITERABLE. + import copy import pickle - from pyspark.errors import PySparkValueError + from pyspark.errors import PySparkTypeError, PySparkValueError from pyspark.errors.exceptions.connect import SparkConnectGrpcException from pyspark.serializers import CloudPickleSerializer + from pyspark.sql import Row spark, service = mock_connect serializer = CloudPickleSerializer() @@ -13975,6 +14047,17 @@ def test_unpickled_dataframes_request_their_schema_and_keep_the_cached_one_as_py assert type(error.value) is error_type and str(error.value) == message with pytest.raises(AttributeError, match="Can't pickle local object"): pickle.dumps(df) + joins = [cached.join(cached, ("id",)), cached.join(cached, [cached.id == cached.id, Row(a=1)])] + for df in joins: + data = serializer.dumps(df) + for call in (lambda: serializer.loads(data), lambda: copy.copy(df), + lambda: copy.deepcopy(df)): + with pytest.raises(PySparkTypeError) as error: + call() + assert error.value.getCondition() == "NOT_ITERABLE" + assert error.value.getMessageParameters() == {"objectName": "Column"} + with pytest.raises(AttributeError, match="Can't pickle local object"): + pickle.dumps(df) assert (len(service.analyze_requests), len(service.execute_requests)) == requests data = serializer.dumps(spark.range(5)) service.failure = "rpc" @@ -17509,8 +17592,9 @@ def test_aggregates_range_partitions_and_join_conditions_carry_pysparks_origins( # `F.col`, and their functions have no origin; repartitionByRange sorts a column by its # `asc`, whose sort order is of `asc_nulls_first`; and the plans of join and _joinAsOf of a # list of Columns reduce them with `&` each time they are built, so their `and`s are of - # `__and__` at the call site of the request, not of the join. In a new interpreter, whose - # first column of a session reads spark.python.sql.dataFrameDebugging.enabled. + # `__and__` at the call site of the request, not of the join, also of a deep copy, as + # PySpark's copy of the plan objects builds them at each request too. In a new interpreter, + # whose first column of a session reads spark.python.sql.dataFrameDebugging.enabled. import subprocess import sys from pathlib import Path @@ -17558,6 +17642,8 @@ def root(dataframe): condition = root(df._joinAsOf(df, "id", "id", on=[df.id == df.id, df.id == df.id])) condition = condition.as_of_join.join_expr found.append((origin(condition), arguments(condition))) + condition = root(__import__("copy").deepcopy(joined)).join.join_condition + found.append((origin(condition), arguments(condition))) print(found) """ tests = str(Path(__file__).resolve().parent) @@ -17570,6 +17656,7 @@ def root(dataframe): [("asc_nulls_first", 35), ("desc_nulls_last", 35)], (("__and__", 22), [("__and__", 22), ("__eq__", 37)]), (("__and__", 22), [("__eq__", 40), ("__eq__", 40)]), + (("__and__", 22), [("__and__", 22), ("__eq__", 37)]), ]) @@ -21973,7 +22060,8 @@ def test_objects_traversed_while_they_are_created_do_not_crash(mock_connect): # the allocations before each object shifted by one, meets each kind of object there: also the # grouped data, the cogroups, the table arguments, the window specifications, the null and # statistics functions, the merge clauses, the stream writers, the data source registrations - # and the session builders, and their copies, and the rows of toLocalIterator. + # and the session builders, and their copies, the rows of toLocalIterator, and the expressions + # of Columns and table arguments, and their copies. import copy import gc @@ -21993,6 +22081,7 @@ def test_objects_traversed_while_they_are_created_do_not_crash(mock_connect): data_source = spark.dataSource builder = ConnectSession.builder._registerHook(lambda session: None).channelBuilder( DefaultChannelBuilder("sc://localhost")) + column = F.col("id") + 1 makers = [ lambda: df.write, lambda: copy.copy(writer), lambda: df.writeTo("t"), lambda: copy.copy(writer_v2), lambda: df.mergeInto("t", F.lit(True)), @@ -22010,6 +22099,8 @@ def test_objects_traversed_while_they_are_created_do_not_crash(mock_connect): lambda: copy.copy(data_source), lambda: type(data_source)(spark), lambda: ConnectSession.builder, lambda: ConnectSession.Builder(), lambda: copy.copy(builder), lambda: df.toLocalIterator(), + lambda: F.col("id")._expr, lambda: copy.copy(column._expr), + lambda: copy.deepcopy(column._expr), lambda: table._subquery_expr, ] thresholds = gc.get_threshold() gc.collect() @@ -26041,6 +26132,7 @@ def test_windows_of_no_partitions_warn_at_each_plan_build_as_pyspark(mock_connec # pickle, which keeps the plan objects without building them. A warning of one place, which # the default filter shows once, and which the error filter raises from the request, before # it is sent. + import copy import warnings import cloudpickle @@ -26072,6 +26164,15 @@ def test_windows_of_no_partitions_warn_at_each_plan_build_as_pyspark(mock_connec df.collect() df.collect() assert len(record) == 1 + # A deep copy is of the plan, which its requests build, so each of them warns, as PySpark's + # copy of the plan objects does: the request of the schema of its reconstruction, and those + # after it. + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + copied = copy.deepcopy(df) + copied.collect() + copied.collect() + assert [str(warning.message) for warning in record] == [message] * 6 requests = len(service.execute_requests) with warnings.catch_warnings(): warnings.simplefilter("error") @@ -26126,12 +26227,14 @@ def test_chained_partition_by_and_order_by_replace_the_spec_as_pyspark(): def test_window_frames_and_columns_are_checked_as_pyspark(): # As PySpark's: a start of -sys.maxsize or less is unbounded preceding and an end of # sys.maxsize or more unbounded following; a row frame raises VALUE_NOT_BETWEEN for a - # boundary out of the range of an int, as a start of Window.unboundedFollowing; and the + # boundary out of the range of an int, as a start of Window.unboundedFollowing, at each build + # of a plan of it, of the window as it was given, which a later frame replaces; and the # columns are Columns or column names, else NOT_EXPECTED_TYPE of `col`. import sys from pyspark.errors import PySparkTypeError, PySparkValueError from pyspark.sql import Window + from pyspark.sql import functions as F o = Window.orderBy("id") assert repr(o.rowsBetween(-sys.maxsize, sys.maxsize)) == ( @@ -26143,12 +26246,21 @@ def test_window_frames_and_columns_are_checked_as_pyspark(): "WindowSpec(OrderBy(id ASC NULLS FIRST), WindowFrame(RANGE_FRAME, -1099511627776, 0))") for start, end, name in ((-(1 << 31) - 1, 0, "start"), (0, 1 << 31, "end"), (Window.unboundedFollowing, 0, "start"), - (0, Window.unboundedPreceding, "end")): - with pytest.raises(PySparkValueError) as error: - o.rowsBetween(start, end) - assert error.value.getCondition() == "VALUE_NOT_BETWEEN" - assert error.value.getMessageParameters() == { - "arg_name": name, "min": "-2147483648", "max": "2147483647"} + (0, Window.unboundedPreceding, "end"), + (-(1 << 40), 1 << 40, "start")): + spec = o.rowsBetween(start, end) + assert repr(spec) == ( + f"WindowSpec(OrderBy(id ASC NULLS FIRST), WindowFrame(ROW_FRAME, {start}, {end}))") + assert repr(spec._frame) == f"WindowFrame(ROW_FRAME, {start}, {end})" + for column in (F.sum("id").over(spec), F.sum("id").over(spec.partitionBy("id"))): + for _ in range(2): + with pytest.raises(PySparkValueError) as error: + column._expr.to_plan(None) + assert error.value.getCondition() == "VALUE_NOT_BETWEEN" + assert error.value.getMessageParameters() == { + "arg_name": name, "min": "-2147483648", "max": "2147483647"} + replaced = F.sum("id").over(spec.rowsBetween(-1, 1))._expr.to_plan(None) + assert replaced.window.frame_spec.lower.value.literal.integer == -1 with pytest.raises(AssertionError): o.rowsBetween(1.5, 2) # A float bound past the threshold of its side is unbounded, as PySpark compares it with @@ -26177,10 +26289,13 @@ def test_window_frame_bounds_are_compared_asserted_and_converted_as_pyspark(): # thresholds of their sides, with Python's <= and >=, whose TypeError a type that does not # compare with an int raises; WindowFrame then asserts that the start and then the end that # is not past its threshold is an int, a bare AssertionError of a NumPy int or a Decimal; - # and the plan converts the start and then the end, whose errors are raised at the call: 0, - # False too, is the current row, a row frame sets an int of an int32 in the integer literal, - # and a range frame any other int in the long literal, both of which protobuf's default - # (upb) backend refuses of a bool, and the long of an int beyond a long. + # and the plan converts the start and then the end when it is built, whose errors its + # requests raise: 0, False too, is the current row, a row frame sets an int of an int32 in + # the integer literal, and raises VALUE_NOT_BETWEEN of an int beyond, and a range frame any + # other int in the long literal, both of which protobuf's default (upb) backend refuses of a + # bool, and the long of an int beyond a long. The window of the drop-in holds no bound of True + # nor of an int beyond a long: of such a frame, the call raises the error that PySpark's build + # raises first, of the start and then of the end. from pyspark.errors import PySparkValueError from pyspark.sql import Window from pyspark.sql import functions as F @@ -26227,9 +26342,16 @@ def bound(boundary): with pytest.raises(TypeError) as error: between(start, end) assert str(error.value) == integer - with pytest.raises(PySparkValueError) as error: - between(2**40, True) - assert error.value.getMessageParameters()["arg_name"] == "start" + for start, end, name in [(2**40, True, "start"), (2**64, 0, "start"), + (0, -(2**64), "end"), (2**40, -(2**64), "start")]: + with pytest.raises(PySparkValueError) as error: + between(start, end) + assert error.value.getMessageParameters()["arg_name"] == name + for start, end, name in [(2**40, 0, "start"), (0, -(2**40), "end"), + (Window.unboundedFollowing, Window.unboundedPreceding, "start")]: + with pytest.raises(PySparkValueError) as error: + bounds(between(start, end)) + assert error.value.getMessageParameters()["arg_name"] == name for between in ranges: assert bounds(between(-(2**31) - 1, 2**40)) == (("long", -(2**31) - 1), ("long", 2**40)) assert bounds(between(Window.unboundedFollowing, Window.unboundedPreceding)) == ( diff --git a/python/tests/test_dropin_types_columns_offline.py b/python/tests/test_dropin_types_columns_offline.py index 6b090322..cd11afc9 100644 --- a/python/tests/test_dropin_types_columns_offline.py +++ b/python/tests/test_dropin_types_columns_offline.py @@ -19,7 +19,7 @@ import pytest from _mock_connect_server import mock_connect_session -from pyspark.errors import PySparkValueError +from pyspark.errors import PySparkAttributeError, PySparkTypeError, PySparkValueError from pyspark.sql import functions as F from pyspark.sql import types as T from pyspark.sql.connect.proto import base_pb2 @@ -2019,17 +2019,25 @@ def test_a_column_of_an_alias_that_fails_copies_and_pickles_as_pysparks(): (("TypeError", "no", None), TypeError, "no"), (("ValueError", "no", None), ValueError, "no"), (("OverflowError", "no", None), OverflowError, "no"), + (("AttributeError", "no", None), AttributeError, "no"), (("UnicodeEncodeError", "", "\ud800"), UnicodeEncodeError, "surrogates not allowed"), (("UnicodeDecodeError", "", b"\xff"), UnicodeDecodeError, "invalid start byte"), (("PySparkValueError", "CANNOT_PROVIDE_METADATA", {}), PySparkValueError, "single column"), + (("PySparkAttributeError", "ATTRIBUTE_NOT_SUPPORTED", {"attr_name": "_expr"}), + PySparkAttributeError, "Attribute `_expr` is not supported"), + (("PySparkTypeError", "NOT_ITERABLE", {"objectName": "Column"}), PySparkTypeError, + "Column is not iterable"), ], ) def test_a_pickled_columns_errors_raise_on_a_worker_as_pysparks(error, raised, message): - # The errors that a Python worker of the server raises of a pickled Column, of PySpark's. + # The errors that a Python worker of the server raises of a pickled Column or DataFrame, of + # PySpark's: a builtin error, the error of UTF-8 of a value, or the error of one of PySpark's + # error classes, of its error class and parameters, as the plan raised it. from pyspark.sql.connect._column_reduce import raise_error - with pytest.raises(raised, match=message): + with pytest.raises(raised, match=message) as caught: raise_error(error) + assert type(caught.value) is raised def test_alias_takes_bytes_as_pyspark(): diff --git a/python/tests/test_dropin_window_tablearg_offline.py b/python/tests/test_dropin_window_tablearg_offline.py index 89ad1d77..20e11b5c 100644 --- a/python/tests/test_dropin_window_tablearg_offline.py +++ b/python/tests/test_dropin_window_tablearg_offline.py @@ -158,6 +158,70 @@ def test_table_arg_deepcopy_keeps_user_attributes(spark): assert deep.foo is not table.foo +def test_table_arg_pickles_as_pyspark(spark, _shared_mock_connect): + # As PySpark's table argument, a plain object of the objects of its plan, its partitioning, + # its ordering and its single partition, which pickles at every protocol, plainly and by + # cloudpickle, of no session: the unpickled table argument is of the relation of the plan, of + # its plan ID, as the calls that made it, and of its user-set attributes, with no request, as + # PySpark's unpickle sends none, and a UDTF of it sends what one of the table argument sends. + # Of the active session of the ID of the session of the pickled one, it raises + # NO_ACTIVE_SESSION where none is, as on another thread. Of a plan that does not build, it + # unpickles, and its requests raise the error of the plan, as PySpark's do. + import threading + + import cloudpickle + from pyspark.errors import PySparkRuntimeError + from pyspark.sql import functions as F + from pyspark.sql.connect.proto import Relation + + _, service = _shared_mock_connect + + @F.udtf(returnType="id: bigint") + class Echo: + def eval(self, row): + yield (row["id"],) + + def references(table): + return list(Relation.FromString(Echo(table)._relation_bytes()).with_relations.references) + + df = spark.range(3).withColumn("k", F.col("id") % 2) + tables = [ + df.asTable(), + df.asTable().partitionBy("k"), + df.asTable().partitionBy(F.col("k"), "id").orderBy(F.col("id").desc(), "k"), + df.asTable().withSinglePartition().orderBy(F.col("id").asc_nulls_last()), + ] + tables[0].note = [1, 2] + requests = (len(service.analyze_requests), len(service.execute_requests)) + for table in tables: + pickles = [pickle.dumps(table, protocol) for protocol in range(pickle.HIGHEST_PROTOCOL + 1)] + for loaded in [pickle.loads(data) for data in pickles] + [ + cloudpickle.loads(cloudpickle.dumps(table))]: + assert type(loaded) is type(table) + assert loaded.to_plan(spark.client) == table.to_plan(spark.client) + assert loaded._is_partitioned() == table._is_partitioned() + assert vars(loaded) == vars(table) + assert references(loaded) == references(table) + assert (len(service.analyze_requests), len(service.execute_requests)) == requests + bad = spark.range(1).alias(5).asTable() + loaded = cloudpickle.loads(cloudpickle.dumps(bad)) + for table in (bad, loaded): + with pytest.raises(TypeError, match="^bad argument type for built-in operation$"): + Echo(table)._relation_bytes() + errors = [] + + def unpickle(data): + try: + cloudpickle.loads(data) + except PySparkRuntimeError as error: + errors.append(error.getCondition()) + + thread = threading.Thread(target=unpickle, args=(cloudpickle.dumps(tables[1]),)) + thread.start() + thread.join() + assert errors == ["NO_ACTIVE_SESSION"] + + def test_window_objects_and_members_are_those_of_pyspark(): # As PySpark's Window, a plain class of static methods and of the thresholds of the frames, # and PySpark Connect's subclass of `_spec`: its objects, also of a subclass, have no state,