From 9aae07ec60a0d40b5f762e3f72f14bacab0b37c1 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Wed, 18 Nov 2026 20:12:08 +0000 Subject: [PATCH] [SPARK-62037][CONNECT][PYTHON] Share the options and hooks of a builder's shallow copies, hold PySpark's session and builder locks, let SparkSession, Catalog and DataSourceRegistration be subclassed, and load PySpark's pickles of readers - The options of the Connect builder are one shared state of its shallow copies, and its hook factories one shared list, as PySpark's __dict__; the channel builder is each builder's own; __traverse__ reports the list and the channel builder. - SparkSession._lock is held for getOrCreate, around the default session of create, and for stop, and Builder._lock for config, remote, appName, channelBuilder and _registerHook, as PySpark holds them, through their __enter__/__exit__; getOrCreate of a new session applies its options again, as PySpark's. - SparkSession, Catalog and DataSourceRegistration are subclassable, with PySpark's __init__ and pickle formats. - DataFrameReader and DataStreamReader load PySpark's pickles of them (argument-less __new__ and a __setstate__ of PySpark's state). - LogicalPlan.to_proto, as PySpark's. - The docs: the builder copies, the locks, the subclasses, a channel builder without a host, the rendering of deep plans, PySpark's hang at exit of a session that is not stopped, the port of the channel builder and the extra names of pyspark.sql.connect.client. Co-authored-by: Isaac --- crates/pyspark-rs/src/catalog.rs | 78 ++- crates/pyspark-rs/src/datasource.rs | 119 +++- crates/pyspark-rs/src/gc.rs | 2 +- crates/pyspark-rs/src/readwriter.rs | 58 +- crates/pyspark-rs/src/session.rs | 885 +++++++++++++++++----------- crates/pyspark-rs/src/streaming.rs | 77 ++- dev/parity/known-differences.md | 103 ++-- docs/compatibility.md | 102 ++-- python/pyspark/sql/connect/plan.py | 28 +- python/tests/test_dropin_offline.py | 442 +++++++++++++- 10 files changed, 1422 insertions(+), 472 deletions(-) diff --git a/crates/pyspark-rs/src/catalog.rs b/crates/pyspark-rs/src/catalog.rs index 1eabc707..ed3d5b9f 100644 --- a/crates/pyspark-rs/src/catalog.rs +++ b/crates/pyspark-rs/src/catalog.rs @@ -200,10 +200,13 @@ fn named_tuples(py: Python<'_>, results: Vec) -> PyResult`. +/// defines none: ``. As PySpark's plain class, +/// it is subclassed: a subclass constructs of the arguments of its own `__init__`, and its copies +/// and pickles are of its class. #[pyclass( name = "Catalog", module = "pyspark.sql.connect.catalog", + subclass, weakref, dict )] @@ -229,32 +232,87 @@ impl PyCatalog { #[pymethods] impl PyCatalog { - /// `Catalog(sparkSession)`, as PySpark Connect's: a catalog of the session, another object - /// than `spark.catalog`, which the session keeps. + /// A catalog, as the `object.__new__` of PySpark's plain class: of any arguments, which + /// `__init__` takes, so that a subclass of another `__init__` is built as it is in PySpark. It + /// is of the first of them that is a session, or else of the session of no server until + /// `__init__` gives it one (see `session::constructor_session`). The class shows the signature + /// of `__init__`, as `inspect.signature` of PySpark's class does. #[new] + #[pyo3(signature = (*args, **kwargs), text_signature = "(sparkSession)")] + fn py_new( + args: &Bound<'_, pyo3::types::PyTuple>, + kwargs: Option<&Bound<'_, pyo3::types::PyDict>>, + ) -> PyResult { + let session = crate::session::constructor_session(args, kwargs, "sparkSession")?; + Ok(PyCatalog::new(session.catalog())) + } + + /// `Catalog(sparkSession)`, as PySpark Connect's `__init__`: a catalog of the session, another + /// object than `spark.catalog`, which the session keeps. #[allow(non_snake_case)] - fn py_new(py: Python<'_>, sparkSession: Py) -> Self { - PyCatalog::new(sparkSession.borrow(py).session.catalog()) + fn __init__(slf: &Bound<'_, Self>, sparkSession: PyRef<'_, crate::session::PySparkSession>) { + let catalog = sparkSession.session.catalog(); + // The catalog that it replaces is dropped once the object is not borrowed, as the drop of + // its session may run Python code. + let replaced = std::mem::replace(&mut slf.borrow_mut().catalog, catalog); + drop(replaced); } - /// The shallow copy of the catalog, as PySpark's `copy.copy`: of the same session. + /// The shallow copy of the catalog, as PySpark's `copy.copy`: of the same session, and of the + /// class of this one, as the copy of PySpark's of a subclass is. fn __copy__<'py>(slf: &Bound<'py, Self>) -> PyResult> { - let copy = Bound::new(slf.py(), PyCatalog::new(slf.borrow().catalog.clone()))?; + let catalog = PyCatalog::new(slf.borrow().catalog.clone()); + let copy = crate::reduce::new_of_class(&slf.get_type(), catalog)?; crate::group::copy_instance_state(copy.as_any(), slf.as_any())?; Ok(copy) } /// The reduce of the catalog, as PySpark's pickle of a plain object of its session: a new /// `Catalog` of its session, which unpickles with it, and not the catalog that the session - /// keeps (`spark.catalog`), whose attributes a copy or an unpickle would otherwise change. + /// keeps (`spark.catalog`), whose attributes a copy or an unpickle would otherwise change. A + /// catalog of a subclass is made of its class as an unpickle of PySpark's makes it, by + /// `copyreg.__newobj__`, without its `__init__`, which may take other arguments. fn __reduce__<'py>(slf: &Bound<'py, Self>) -> PyResult> { + use pyo3::types::PyTuple; let py = slf.py(); let session = crate::session::session_object(py, slf.borrow().catalog.spark_session())?; - let args = pyo3::types::PyTuple::new(py, [session.into_bound(py)])?; - let reduce = pyo3::types::PyTuple::new(py, [slf.get_type().into_any(), args.into_any()])?; + let session = session.into_bound(py).into_any(); + let cls = slf.get_type(); + let reduce = if cls.is(py.get_type::()) { + let args = PyTuple::new(py, [session])?; + PyTuple::new(py, [cls.into_any(), args.into_any()])? + } else { + let newobj = py.import("copyreg")?.getattr("__newobj__")?; + let args = PyTuple::new(py, [cls.into_any(), session])?; + PyTuple::new(py, [newobj, args.into_any()])? + }; crate::group::reduce_with_state(reduce.as_any(), slf.as_any()) } + /// The state of a pickle of a catalog, which pickle sets on the catalog that `__new__` made + /// of no arguments: of PySpark's pickle, the `__dict__` of its plain object, whose + /// `_sparkSession` is the session of the catalog, and the other entries its instance + /// attributes, which are all that the state of the drop-in's pickle holds. + fn __setstate__(slf: &Bound<'_, Self>, state: &Bound<'_, pyo3::types::PyDict>) -> PyResult<()> { + let py = slf.py(); + let state = state.copy()?; + if let Some(session) = state.get_item("_sparkSession")? { + if let Ok(session) = session.cast::() { + Self::__init__(slf, session.borrow()); + state.del_item("_sparkSession")?; + } + } + slf.getattr(pyo3::intern!(py, "__dict__"))? + .call_method1(pyo3::intern!(py, "update"), (state,))?; + Ok(()) + } + + /// The session of the catalog, as PySpark Connect's `Catalog._sparkSession`. + #[getter] + fn _sparkSession(&self, py: Python<'_>) -> PyResult> { + crate::session::session_object(py, self.catalog.spark_session()) + } + /// Get the current catalog. fn currentCatalog(&self, py: Python<'_>) -> PyResult { self.detached(py, |catalog| catalog.current_catalog().map_err(Box::new)) diff --git a/crates/pyspark-rs/src/datasource.rs b/crates/pyspark-rs/src/datasource.rs index ba763faf..6d460980 100644 --- a/crates/pyspark-rs/src/datasource.rs +++ b/crates/pyspark-rs/src/datasource.rs @@ -14,71 +14,131 @@ use crate::session::PySparkSession; /// Exposed as `SparkSession.dataSource` to register custom data sources. It is PySpark Connect's /// class, `pyspark.sql.connect.datasource.DataSourceRegistration`, which that module re-exports: /// `pyspark.sql.datasource.DataSourceRegistration` is PySpark's classic class, of which it is not. +/// As PySpark's plain class, it is subclassed: a subclass constructs of the arguments of its own +/// `__init__`, and its copies and pickles are of its class. #[pyclass( name = "DataSourceRegistration", module = "pyspark.sql.connect.datasource", + subclass, weakref, dict )] pub struct PyDataSourceRegistration { - pub(crate) session: Py, - /// Set once the value is written, as `__traverse__` reads `session`, which is not valid while - /// the object is created (see [`crate::gc::GcReady`]). - gc_ready: crate::gc::GcReady, + /// The session of the registration, as PySpark's `sparkSession`, which its `__init__` sets: + /// None before it, as the registration that `__new__` makes has none. + pub(crate) session: Option>, } impl PyDataSourceRegistration { pub fn new(session: Py) -> Self { PyDataSourceRegistration { - session, - gc_ready: crate::gc::GcReady::new(), + session: Some(session), + } + } + + /// The session of the registration, or the AttributeError of PySpark's registration whose + /// `__init__` did not run, which has no `sparkSession`. + fn session_of(slf: &Bound<'_, Self>) -> PyResult> { + let py = slf.py(); + match &slf.borrow().session { + Some(session) => Ok(session.clone_ref(py)), + None => Err(pyo3::exceptions::PyAttributeError::new_err(format!( + "'{}' object has no attribute 'sparkSession'", + slf.get_type().name()? + ))), } } } #[pymethods] impl PyDataSourceRegistration { - /// `DataSourceRegistration(sparkSession)`, as PySpark Connect's: the registration of the data - /// sources of the session. + /// A registration of no session, as the `object.__new__` of PySpark's plain class: of any + /// arguments, which `__init__` takes, so that a subclass of another `__init__` is built as it + /// is in PySpark, and an unpickle makes one of none. The class shows the signature of + /// `__init__`, as `inspect.signature` of PySpark's class does. #[new] + #[pyo3(signature = (*_args, **_kwargs), text_signature = "(sparkSession)")] + fn py_new( + _args: &Bound<'_, pyo3::types::PyTuple>, + _kwargs: Option<&Bound<'_, pyo3::types::PyDict>>, + ) -> Self { + PyDataSourceRegistration { session: None } + } + + /// `DataSourceRegistration(sparkSession)`, as PySpark Connect's `__init__`: the registration + /// of the data sources of the session. #[allow(non_snake_case)] - fn py_new(sparkSession: Py) -> Self { - PyDataSourceRegistration::new(sparkSession) + fn __init__(slf: &Bound<'_, Self>, sparkSession: Py) { + let replaced = slf.borrow_mut().session.replace(sparkSession); + drop(replaced); } /// The shallow copy of the data source registration, as PySpark's `copy.copy`: of the same - /// session, with the instance attributes of this one. - fn __copy__(slf: &Bound<'_, Self>) -> PyResult> { + /// session, and of the class of this one, with the instance attributes of this one. + fn __copy__<'py>(slf: &Bound<'py, Self>) -> PyResult> { let py = slf.py(); - let session = slf.borrow().session.clone_ref(py); - let copy = Py::new(py, PyDataSourceRegistration::new(session))?; - crate::reduce::copy_instance_dict(slf.as_any(), copy.bind(py).as_any())?; + let session = slf.borrow().session.as_ref().map(|s| s.clone_ref(py)); + let copy = + crate::reduce::new_of_class(&slf.get_type(), PyDataSourceRegistration { session })?; + crate::reduce::copy_instance_dict(slf.as_any(), copy.as_any())?; Ok(copy) } - /// The reduce of the data source registration, as PySpark's: `dataSource` of its session, - /// which unpickles with it, and its instance attributes. + /// The reduce of the data source registration, as PySpark's pickle of its plain object: + /// `copyreg.__newobj__` of its class, of the state of its `__dict__`, its `sparkSession`, which + /// unpickles with it, and its instance attributes (see `__setstate__`). So PySpark loads the + /// pickle as a registration of its own, and PySpark's pickle loads here. fn __reduce__<'py>(slf: &Bound<'py, Self>) -> PyResult> { + use pyo3::types::{PyDict, PyTuple}; + let py = slf.py(); + let state = PyDict::new(py); + if let Some(session) = &slf.borrow().session { + state.set_item("sparkSession", session.clone_ref(py))?; + } + state.call_method1("update", (slf.getattr(pyo3::intern!(py, "__dict__"))?,))?; + let newobj = py.import("copyreg")?.getattr("__newobj__")?; + let args = PyTuple::new(py, [slf.get_type()])?; + let reduced = match state.is_empty() { + true => PyTuple::new(py, [newobj, args.into_any()])?, + false => PyTuple::new(py, [newobj, args.into_any(), state.into_any()])?, + }; + Ok(reduced.into_any()) + } + + /// The state of the pickle of a registration, as pickle sets the `__dict__` of PySpark's: its + /// session, of `sparkSession`, and its instance attributes, of the other entries. + fn __setstate__(slf: &Bound<'_, Self>, state: &Bound<'_, pyo3::types::PyDict>) -> PyResult<()> { let py = slf.py(); - let session = slf.borrow().session.clone_ref(py).into_bound(py).into_any(); - let reduced = crate::reduce::getattr_reduce(py, session, "dataSource")?; - crate::reduce::reduce_with_instance_dict(slf.as_any(), reduced.into_any()) + let state = state.copy()?; + if let Some(session) = state.get_item("sparkSession")? { + Self::__init__(slf, session.extract()?); + state.del_item("sparkSession")?; + } + slf.getattr(pyo3::intern!(py, "__dict__"))? + .call_method1(pyo3::intern!(py, "update"), (state,))?; + Ok(()) } /// The session of the registration, as PySpark's `DataSourceRegistration.sparkSession`. #[getter(sparkSession)] - fn spark_session(&self, py: Python<'_>) -> Py { - self.session.clone_ref(py) + fn spark_session(slf: &Bound<'_, Self>) -> PyResult> { + Self::session_of(slf) } /// As PySpark's registration, a Python object that holds its session, which Python's garbage /// collector sees: the session is reported, so a reference cycle through it, such as a session - /// that keeps its `dataSource`, is freed. Nothing of a registration that is being created. + /// that keeps its `dataSource`, is freed. Only an `Option>` field is read, which is + /// `None` while the object is created, so no [`crate::gc::GcReady`] is needed. fn __traverse__(&self, visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { - if !self.gc_ready.get() { - return Ok(()); + if let Some(session) = &self.session { + visit.call(session)?; } - visit.call(&self.session) + Ok(()) + } + + /// Break a reference cycle that the garbage collector found unreachable. + fn __clear__(&mut self) { + self.session = None; } /// Register a custom data source class. @@ -86,7 +146,11 @@ impl PyDataSourceRegistration { /// Mirrors `pyspark.sql.SparkSession.dataSource.register(dataSource)`, of PySpark's parameter. /// The data source class is cloudpickled and sent to the server. #[allow(non_snake_case)] - fn register(&self, py: Python<'_>, dataSource: &Bound<'_, PyAny>) -> PyResult<()> { + fn register(slf: &Bound<'_, Self>, dataSource: &Bound<'_, PyAny>) -> PyResult<()> { + let py = slf.py(); + // As PySpark's `self.sparkSession._client.register_data_source(dataSource)`, which reads + // the session first. + let session = Self::session_of(slf)?.borrow(py).session.clone(); // The name of the classmethod `name()`, as PySpark Connect's: the AttributeError of a // class without one. Its `CommonInlineUserDefinedDataSource` sets the name in the string // field of its command, as protobuf converts it, before it pickles the class: bytes of @@ -102,7 +166,6 @@ impl PyDataSourceRegistration { let expr = CommonInlineUserDefinedDataSourceExpression::new(name, payload); // Register with the session (release GIL during the RPC) - let session = self.session.borrow(py).session.clone(); interruptible(py, &session, || { session.register_data_source(expr).map_err(Box::new) })?; diff --git a/crates/pyspark-rs/src/gc.rs b/crates/pyspark-rs/src/gc.rs index 0bc4e2ab..4c365c1f 100644 --- a/crates/pyspark-rs/src/gc.rs +++ b/crates/pyspark-rs/src/gc.rs @@ -33,7 +33,7 @@ impl GcReady { } /// The flag of a clone of a written value, which is written too: for a class whose value is -/// cloned, such as the session builder, of which PyO3 extracts a clone. +/// cloned. impl Clone for GcReady { fn clone(&self) -> Self { GcReady::new() diff --git a/crates/pyspark-rs/src/readwriter.rs b/crates/pyspark-rs/src/readwriter.rs index 27d5710f..976148f6 100644 --- a/crates/pyspark-rs/src/readwriter.rs +++ b/crates/pyspark-rs/src/readwriter.rs @@ -680,17 +680,16 @@ impl PyDataFrameReader { /// `DataFrameReader(client)`, as PySpark Connect's, of the session `client`: the reader that /// `spark.read` gives. As the `object.__new__` of PySpark's plain class, it takes the /// arguments of the `__init__` of a subclass too, of which the session is the first that is - /// a session (see `session::constructor_session`); `__init__` checks them. + /// a session, and none, as an unpickle gives none (see `session::constructor_session`); + /// `__init__` checks them. #[new] - #[classmethod] #[pyo3(signature = (*args, **kwargs), text_signature = "(client)")] fn py_new( - cls: &Bound<'_, pyo3::types::PyType>, args: &Bound<'_, pyo3::types::PyTuple>, kwargs: Option<&Bound<'_, pyo3::types::PyDict>>, ) -> PyResult { - let session = crate::session::constructor_session(cls, args, kwargs, "client")?; - Ok(PyDataFrameReader::new(session.borrow().session.read())) + let session = crate::session::constructor_session(args, kwargs, "client")?; + Ok(PyDataFrameReader::new(session.read())) } /// As PySpark's `__init__(client)`, which makes the reader of the session anew: of no @@ -755,6 +754,48 @@ impl PyDataFrameReader { crate::reduce::reduce_with_instance_dict(slf.as_any(), reduced) } + /// The state of a pickle of a reader, which pickle sets on the reader that `__new__` made of + /// no arguments. Of PySpark's pickle, the state is the `__dict__` of its plain object, which + /// holds `_options`: so the reader is of the session `_client`, and of the format, the schema + /// and the options of `_format`, `_schema` and `_options`, which it takes as `format`, + /// `schema` and `option` take them, and the other entries are its instance attributes. The + /// state of the drop-in's pickle holds the instance attributes alone (see `__reduce__`), as a + /// reader's `__dict__` cannot hold `_options`, a property of the reader. + fn __setstate__(slf: &Bound<'_, Self>, state: &Bound<'_, PyDict>) -> PyResult<()> { + let py = slf.py(); + let state = state.copy()?; + if state.contains("_options")? { + let take = |key: &str| -> PyResult>> { + let value = state.get_item(key)?; + if value.is_some() { + state.del_item(key)?; + } + Ok(value) + }; + if let Some(client) = take("_client")? { + let session = client.cast_into::()?; + let reader = PyDataFrameReader::new(session.borrow().session.read()); + let replaced = std::mem::replace(&mut *slf.borrow_mut(), reader); + drop(replaced); + } + if let Some(format) = take("_format")? { + let source = (!format.is_none()).then_some(&format); + Self::format(slf.borrow_mut(), source); + } + if let Some(schema) = take("_schema")? { + Self::schema(slf.borrow_mut(), &schema)?; + } + if let Some(options) = take("_options")? { + for (key, value) in options.cast_into::()?.iter() { + Self::option(slf.borrow_mut(), &key, &value)?; + } + } + } + slf.getattr(pyo3::intern!(py, "__dict__"))? + .call_method1(pyo3::intern!(py, "update"), (state,))?; + Ok(()) + } + /// A reader of its state, as `__reduce__` gives it, of the class `cls`: of the session /// `session`, the format, the schema, the options, the format that is kept (`unsent`), and /// the schema and the options that UTF-8 cannot encode (`unencodable`, as @@ -883,6 +924,13 @@ impl PyDataFrameReader { self.unsent_format = None; } + /// The session of the reader, as PySpark Connect's `DataFrameReader._client`, which its + /// pickle keeps. + #[getter] + fn _client(&self, py: Python<'_>) -> PyResult> { + crate::session::session_object(py, &self.inner.spark_session()) + } + /// The configured source format (PySpark Connect's `DataFrameReader._format`), which is the /// object itself when it is not a str that the core reader takes, as PySpark keeps it. #[getter] diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index 25c8afd2..3745552b 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -25,43 +25,97 @@ use crate::streaming::{PyDataStreamReader, PyStreamingQueryManager}; #[pyclass( name = "Builder", module = "pyspark.sql.connect.session", - from_py_object, weakref, dict )] -#[derive(Clone)] pub struct PySparkSessionBuilder { + /// The options of `remote`, `config` and `appName`, which the shallow copies of this builder + /// share, as PySpark's builder shares its `_options` dict with them: an option that one of + /// them sets is the option of each. + options: std::sync::Arc>, + /// The hook factories of `_registerHook`, which a new session calls with itself, as + /// PySpark's builder does: a list, which the shallow copies of this builder share, as they + /// share PySpark's `_hook_factories` list. None once the garbage collector cleared it. + hook_factories: Option>, + /// The channel builder of `channelBuilder`, which the session connects with, instead of + /// `remote()`: PySpark's `_channel_builder`, which is not an option, but an attribute of the + /// builder, which a shallow copy has too, and which a `channelBuilder` sets on its builder + /// alone. + channel_builder: Option>, +} + +/// The options of a session builder, which its shallow copies share (see +/// [`PySparkSessionBuilder::options`]). The lock of them is held only to read or replace them, +/// while no Python code runs, as the threads of the copies may wait for it. +#[derive(Default)] +struct BuilderOptions { /// The URL of `remote()` or of `config("spark.remote")`: None when it is not set, and - /// `Some(None)` when it is set to None, which PySpark's `create` asserts ([`Self::url`]). + /// `Some(None)` when it is set to None, which PySpark's `create` asserts ([`session_url`]). remote_url: Option>, /// Runtime configs set via `.config(...)`, applied after the session connects. configs: Vec<(String, Option)>, - /// The hook factories of `_registerHook`, which a new session calls with itself, as - /// PySpark's builder does. - hook_factories: Vec>>, - /// The channel builder of `channelBuilder`, which the session connects with, instead of - /// `remote()`: PySpark's `_channel_builder`, which is not an option. - channel_builder: Option>>, /// The configs that the session does not take, as PySpark's options dict keeps them: a key /// that is not a str or a str that UTF-8 cannot encode, or a value that cannot be encoded. /// `getOrCreate` fails with the first of them, as PySpark's `_set_all` does. unencodable: crate::options::UnencodableOptions, - /// Set once the value is written, as `__traverse__` reads `hook_factories`, which is not - /// valid while the builder object is created (see [`crate::gc::GcReady`]). - gc_ready: crate::gc::GcReady, +} + +impl BuilderOptions { + /// Set an option whose value may be None, which PySpark keeps in its options: such a key is + /// not set on the session, as `_set_all` sends it without a value. `spark.remote` is the URL + /// of the session, as PySpark Connect's `config` keeps it (the builder of + /// `pyspark.sql.SparkSession`, PySpark's classic one, validates the startup URLs). + fn set_value(&mut self, key: &str, value: Option) { + if key == "spark.remote" { + self.remote_url = Some(value); + } else { + self.configs.push((key.to_string(), value)); + } + } +} + +/// The URL of the session of a builder of the URL `remote_url` ([`BuilderOptions::remote_url`]): +/// that of `remote()`, as PySpark Connect's builder, which raises +/// `SESSION_NEED_CONN_STR_OR_BUILDER` without one. `SPARK_REMOTE` is read by the builder of +/// `pyspark.sql.SparkSession`, PySpark's classic one, which gives its URL to this builder. A URL +/// that is set to None fails the assertion of PySpark Connect's `create`. +fn session_url(py: Python<'_>, remote_url: Option>) -> PyResult { + match remote_url { + Some(None) => Err(pyo3::exceptions::PyAssertionError::new_err(())), + Some(Some(url)) => Ok(url), + None => Err(crate::column::raise_pyspark( + py, + "PySparkValueError", + "SESSION_NEED_CONN_STR_OR_BUILDER", + &[], + )), + } } impl PySparkSessionBuilder { - pub fn new() -> Self { + pub fn new(py: Python<'_>) -> Self { PySparkSessionBuilder { - remote_url: None, - configs: Vec::new(), - hook_factories: Vec::new(), + options: Default::default(), + hook_factories: Some(PyList::empty(py).unbind()), channel_builder: None, - unencodable: Default::default(), - gc_ready: crate::gc::GcReady::new(), } } + + /// The options of this builder and of its copies, locked. + fn options(&self) -> std::sync::MutexGuard<'_, BuilderOptions> { + self.options + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + /// The hook factories, in the order that `_registerHook` registered them. + fn factories(&self, py: Python<'_>) -> Vec> { + self.hook_factories + .iter() + .flat_map(|factories| factories.bind(py).iter()) + .map(Bound::unbind) + .collect() + } } /// Install `builder` as the class attribute `builder` of the native SparkSession: PySpark @@ -82,15 +136,36 @@ impl PySparkSessionBuilder { /// Construct a fresh builder, so `SparkSession.Builder()` works like pyspark /// (in addition to the `SparkSession.builder` class attribute). #[new] - fn py_new() -> Self { - PySparkSessionBuilder::new() + fn py_new(py: Python<'_>) -> Self { + PySparkSessionBuilder::new(py) + } + + /// The class attribute `_lock`, PySpark's `threading.RLock` of the builders, which their + /// `config`, `remote`, `appName`, `channelBuilder`, `_registerHook` and the options that + /// `create` and `getOrCreate` set hold, as PySpark's do (see `with_builder_lock`): one for + /// all of them, another than `SparkSession._lock`. + #[classattr] + fn _lock(py: Python<'_>) -> PyResult> { + Ok(py.import("threading")?.call_method0("RLock")?.unbind()) } - /// The shallow copy of a builder, as PySpark's `copy.copy`, of its options, its channel - /// builder, its hook factories and its instance attributes. + /// The shallow copy of a builder, as PySpark's `copy.copy` of its `__dict__`: the copy shares + /// the options and the hook factories of this builder, as PySpark's copy shares its + /// `_options` dict and its `_hook_factories` list, so that a `remote`, a `config` or a + /// `_registerHook` of either one is seen by both. It has the channel builder of this one, + /// which a `channelBuilder` of either one replaces on that one alone, as it rebinds the + /// attribute of PySpark's, and the instance attributes of this one. fn __copy__(slf: &Bound<'_, Self>) -> PyResult> { let py = slf.py(); - let copy = Py::new(py, slf.borrow().clone())?; + let copy = { + let this = slf.borrow(); + PySparkSessionBuilder { + options: std::sync::Arc::clone(&this.options), + hook_factories: this.hook_factories.as_ref().map(|list| list.clone_ref(py)), + channel_builder: this.channel_builder.as_ref().map(|cb| cb.clone_ref(py)), + } + }; + let copy = Py::new(py, copy)?; crate::reduce::copy_instance_dict(slf.as_any(), copy.bind(py).as_any())?; Ok(copy) } @@ -101,11 +176,15 @@ impl PySparkSessionBuilder { fn __reduce__<'py>(slf: &Bound<'py, Self>) -> PyResult> { let py = slf.py(); let this = slf.borrow(); - let mut options = Vec::with_capacity(this.configs.len() + 1); - if let Some(url) = &this.remote_url { - options.push(("spark.remote".to_string(), url.clone())); - } - options.extend(this.configs.iter().cloned()); + let options = { + let options = this.options(); + let mut pairs = Vec::with_capacity(options.configs.len() + 1); + if let Some(url) = &options.remote_url { + pairs.push(("spark.remote".to_string(), url.clone())); + } + pairs.extend(options.configs.iter().cloned()); + pairs + }; let options = PyList::new( py, options @@ -121,17 +200,12 @@ impl PySparkSessionBuilder { }) .collect::>>()?, )?; - let hook_factories = PyList::new( - py, - this.hook_factories - .iter() - .map(|factory| factory.clone_ref(py).into_bound(py)), - )?; + let hook_factories = PyList::new(py, this.factories(py))?; let args = PyTuple::new( py, [ options.into_any(), - crate::reduce::optional_object(py, this.channel_builder.as_deref()), + crate::reduce::optional_object(py, this.channel_builder.as_ref()), hook_factories.into_any(), ], )?; @@ -139,29 +213,31 @@ impl PySparkSessionBuilder { crate::reduce::reduce_with_instance_dict(slf.as_any(), reduced) } - /// As PySpark's builder, a Python object that holds its channel builder and its hook - /// factories (`_channel_builder` and `_hook_factories`), which Python's garbage collector - /// sees: they are reported, so a reference cycle through one, such as a channel builder or a - /// hook factory that keeps the builder, is freed. One that a copy of the builder shares is - /// reported by neither, so that its reference is not counted twice. Nothing of a builder - /// object that is being created. + /// As PySpark's builder, a Python object whose `__dict__` holds its channel builder and the + /// list of its hook factories (`_channel_builder` and `_hook_factories`), which Python's + /// garbage collector sees: they are reported, so a reference cycle through one, such as a + /// channel builder or a hook factory that keeps the builder, is freed. Each builder holds a + /// reference of its own to its channel builder and to the list, which the shallow copies of + /// the builder share and which reports the factories itself, as PySpark's list does: so no + /// reference is counted twice, and the collector frees a factory only with the last builder + /// that reaches it. Only `Option>` fields are read, which are `None` while the object + /// is created, so no [`crate::gc::GcReady`] is needed. fn __traverse__(&self, visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { - if !self.gc_ready.get() { - return Ok(()); + if let Some(channel_builder) = &self.channel_builder { + visit.call(channel_builder)?; } - for object in self.channel_builder.iter().chain(&self.hook_factories) { - if std::sync::Arc::strong_count(object) == 1 { - visit.call(&**object)?; - } + if let Some(hook_factories) = &self.hook_factories { + visit.call(hook_factories)?; } Ok(()) } /// Break a reference cycle that the garbage collector found unreachable: the builder drops - /// its channel builder and its hook factories. + /// its channel builder and its reference to the list of its hook factories, which a copy that + /// is reachable still holds. fn __clear__(&mut self) { self.channel_builder = None; - self.hook_factories.clear(); + self.hook_factories = None; } /// Set the remote Spark Connect server URL. Returns the builder (chainable). @@ -181,10 +257,10 @@ impl PySparkSessionBuilder { text_signature = "($self, location='sc://localhost')" )] fn remote<'py>( - mut slf: PyRefMut<'py, Self>, + slf: &Bound<'py, Self>, args: &Bound<'py, PyTuple>, kwargs: Option<&Bound<'py, PyDict>>, - ) -> PyResult> { + ) -> PyResult> { // The arguments are checked as CPython checks them, so that an omitted URL is told from // a None one, which is an argument of its own. crate::options::check_keyword_keys(kwargs)?; @@ -210,14 +286,18 @@ impl PySparkSessionBuilder { } given.push(value); } - let key = - crate::options::OptionKey::Text(crate::options::OptionText::literal("spark.remote")); - let value = match given.first() { - Some(location) => crate::options::reader_option_text(location)?, - None => Some(crate::options::OptionText::literal("sc://localhost")), - }; - slf.put_config(key, value); - Ok(slf) + // As PySpark's `config("spark.remote", location)`, which holds the lock of the builder. + with_builder_lock(slf, || { + let key = crate::options::OptionKey::Text(crate::options::OptionText::literal( + "spark.remote", + )); + let value = match given.first() { + Some(location) => crate::options::reader_option_text(location)?, + None => Some(crate::options::OptionText::literal("sc://localhost")), + }; + slf.borrow().put_config(key, value); + Ok(slf.clone()) + }) } /// Set a config option, or several via `map`. Mirrors @@ -226,79 +306,91 @@ impl PySparkSessionBuilder { /// on connect. `conf` accepts a `SparkConf` (its `getAll()` pairs are applied). #[pyo3(signature = (key=None, value=None, map=None, conf=None, **unexpected), text_signature = "($self, key=None, value=None, map=None, conf=None)")] fn config<'py>( - mut slf: PyRefMut<'py, Self>, + slf: &Bound<'py, Self>, key: Option<&Bound<'_, PyAny>>, value: Option<&Bound<'_, PyAny>>, map: Option<&Bound<'_, PyDict>>, conf: Option<&Bound<'_, PyAny>>, unexpected: Option<&Bound<'_, PyDict>>, - ) -> PyResult> { + ) -> PyResult> { crate::options::check_unexpected_keywords("SparkSession.Builder.config", unexpected)?; - // PySpark keeps the options in a dict, and sends each in a `KeyValue` of the key as it - // is, whose constructor leaves a key of None unset, the empty key. - if let Some(k) = key { - let k = crate::options::option_key(k)?; - let v = match value { - Some(v) => crate::options::reader_option_text(v)?, - None => None, - }; - slf.put_config(k, v); - } - if let Some(m) = map { - for (k, v) in m.iter() { - let ks = if k.is_none() { - crate::options::OptionKey::Text(crate::options::OptionText::literal("")) - } else { - crate::options::option_key(&k)? + // As PySpark's, which sets the options while it holds the lock of the builder. + with_builder_lock(slf, || { + let this = slf.borrow(); + // PySpark keeps the options in a dict, and sends each in a `KeyValue` of the key as + // it is, whose constructor leaves a key of None unset, the empty key. + if let Some(k) = key { + let k = crate::options::option_key(k)?; + let v = match value { + Some(v) => crate::options::reader_option_text(v)?, + None => None, }; - let vs = crate::options::reader_option_text(&v)?; - slf.put_config(ks, vs); + this.put_config(k, v); } - } - if let Some(c) = conf { - // A SparkConf: apply its (key, value) pairs via getAll(). - for pair in c.call_method0("getAll")?.try_iter()? { - let pair = pair?; - let k: String = crate::str_text(&pair.get_item(0)?)?; - let v: String = crate::str_text(&pair.get_item(1)?)?; - slf.set_option(&k, v); + if let Some(m) = map { + for (k, v) in m.iter() { + let ks = if k.is_none() { + crate::options::OptionKey::Text(crate::options::OptionText::literal("")) + } else { + crate::options::option_key(&k)? + }; + let vs = crate::options::reader_option_text(&v)?; + this.put_config(ks, vs); + } } - } - Ok(slf) + if let Some(c) = conf { + // A SparkConf: apply its (key, value) pairs via getAll(). + for pair in c.call_method0("getAll")?.try_iter()? { + let pair = pair?; + let k: String = crate::str_text(&pair.get_item(0)?)?; + let v: String = crate::str_text(&pair.get_item(1)?)?; + this.options().set_value(&k, Some(v)); + } + } + Ok(slf.clone()) + }) } /// `appName` - the option `spark.app.name`, as PySpark's builder sets. Like any option, /// it is set on the session silently, so it has no effect on a running server. #[pyo3(name = "appName")] fn app_name<'py>( - mut slf: PyRefMut<'py, Self>, + slf: &Bound<'py, Self>, name: &Bound<'_, PyAny>, - ) -> PyResult> { - let key = - crate::options::OptionKey::Text(crate::options::OptionText::literal("spark.app.name")); - let value = crate::options::reader_option_text(name)?; - slf.put_config(key, value); - Ok(slf) + ) -> PyResult> { + // As PySpark's `config("spark.app.name", name)`, which holds the lock of the builder. + with_builder_lock(slf, || { + let key = crate::options::OptionKey::Text(crate::options::OptionText::literal( + "spark.app.name", + )); + let value = crate::options::reader_option_text(name)?; + slf.borrow().put_config(key, value); + Ok(slf.clone()) + }) } /// `master`, which PySpark Connect's builder takes and ignores, of any value: a session of /// Spark Connect has no master. - fn master<'py>(slf: PyRefMut<'py, Self>, master: &Bound<'_, PyAny>) -> PyRefMut<'py, Self> { + fn master<'py>(slf: PyRef<'py, Self>, master: &Bound<'_, PyAny>) -> PyRef<'py, Self> { let _ = master; slf } /// `_registerHook` - register a factory of a `SparkSession.Hook`, as PySpark's builder: a /// session that the builder creates calls it with itself, and calls the `on_execute_plan` - /// of the hook with each `ExecutePlanRequest` of its executions before it sends it. + /// of the hook with each `ExecutePlanRequest` of its executions before it sends it. It is + /// appended to the list that the shallow copies of the builder share, as PySpark's. #[pyo3(name = "_registerHook")] fn register_hook<'py>( - mut slf: PyRefMut<'py, Self>, + slf: &Bound<'py, Self>, hook_factory: &Bound<'_, PyAny>, - ) -> PyRefMut<'py, Self> { - slf.hook_factories - .push(std::sync::Arc::new(hook_factory.clone().unbind())); - slf + ) -> PyResult> { + with_builder_lock(slf, || { + if let Some(factories) = &slf.borrow().hook_factories { + factories.bind(slf.py()).append(hook_factory)?; + } + Ok(slf.clone()) + }) } /// `channelBuilder` - accept a Spark Connect ChannelBuilder and reconstruct an @@ -309,14 +401,20 @@ impl PySparkSessionBuilder { #[pyo3(name = "channelBuilder")] #[allow(non_snake_case)] fn channel_builder<'py>( - mut slf: PyRefMut<'py, Self>, + slf: &Bound<'py, Self>, channelBuilder: &Bound<'_, PyAny>, - ) -> PyResult> { - // A builder without an endpoint is an error here, not when the session is created; the - // URL itself is read then, with the parameters of the builder as they are by then. - url_of_channel_builder(channelBuilder)?; - slf.channel_builder = Some(std::sync::Arc::new(channelBuilder.clone().unbind())); - Ok(slf) + ) -> PyResult> { + with_builder_lock(slf, || { + // A builder without an endpoint is an error here, not when the session is created; + // the URL itself is read then, with the parameters of the builder as they are by then. + url_of_channel_builder(channelBuilder)?; + let replaced = slf + .borrow_mut() + .channel_builder + .replace(channelBuilder.clone().unbind()); + drop(replaced); + Ok(slf.clone()) + }) } /// `enableHiveSupport` raises `PySparkNotImplementedError` `NOT_IMPLEMENTED`, as PySpark @@ -335,87 +433,78 @@ impl PySparkSessionBuilder { /// Get the active session of this thread, or else the default session, or create one /// (`getOrCreate`), and set the options of the builder on it. Mirrors PySpark Connect's: the - /// URL is that of `remote()`, and an existing session is returned whatever its URL. + /// URL is that of `remote()`, and an existing session is returned whatever its URL. As + /// PySpark's, it holds `SparkSession._lock` all the while, and the options are set on a + /// session that it creates twice, by `create` and again. #[pyo3(name = "getOrCreate")] - fn get_or_create(&self, py: Python<'_>) -> PyResult> { - let (session, created) = { - // As PySpark's getOrCreate, which holds `SparkSession._lock` for its lookup: a stop in - // progress, on any session, is waited for, so that the lookup does not return a session - // that it stops. A signal ends the wait, as it interrupts the acquire of that lock. - let _stopping = crate::interrupt::abortable_without_session( - py, - spark_connect::session::enter_stop_lock, - )?; - let _creating = session_lock(); - match active_session(py).or_else(|| default_session(py)) { - Some(session) => (session, false), + fn get_or_create(slf: &Bound<'_, Self>) -> PyResult> { + let py = slf.py(); + with_session_lock(py, || { + let existing = { + // A stop in progress of the Rust core, on any session, is waited for, so that the + // lookup does not return a session that it stops, as a stop of the Python client + // holds `SparkSession._lock`. A signal ends the wait, as it interrupts the acquire + // of that lock. + let _stopping = crate::interrupt::abortable_without_session( + py, + spark_connect::session::enter_stop_lock, + )?; + active_session(py).or_else(|| default_session(py)) + }; + let session = match existing { + Some(session) => session, // The URL is needed only to create a session, as PySpark's getOrCreate. - None => (new_session(py, &self.connection_url(py)?)?, true), - } - }; - if created { - self.install_hooks(py, &session)?; - } - self.apply_options(py, &session.borrow(py).session)?; - log_client_banner(py); - Ok(session) + None => Self::create(slf)?, + }; + Self::apply_options(slf, &session.borrow(py).session)?; + log_client_banner(py); + Ok(session) + }) } /// `create()` - a new session, which is the default session, and the active one of this - /// thread, only when they are not set. Mirrors PySpark's. - fn create(&self, py: Python<'_>) -> PyResult> { - let url = self.connection_url(py)?; - let session = { - let _creating = session_lock(); - new_session(py, &url)? + /// thread, only when they are not set. Mirrors PySpark's, which makes the session with the + /// hooks of its factories before it holds `SparkSession._lock` to make it the default. + fn create(slf: &Bound<'_, Self>) -> PyResult> { + let py = slf.py(); + let (url, factories) = { + let this = slf.borrow(); + (this.connection_url(py)?, this.factories(py)) }; - self.install_hooks(py, &session)?; - self.apply_options(py, &session.borrow(py).session)?; + let session = new_session(py, &url)?; + install_hook_factories(py, &factories, &session)?; + set_default_and_active(py, &session)?; + Self::apply_options(slf, &session.borrow(py).session)?; log_client_banner(py); Ok(session) } } impl PySparkSessionBuilder { - /// The hooks of the factories of `_registerHook` for `session`, a session that the builder - /// created, as PySpark's `SparkSession(hook_factories=...)` makes them. - fn install_hooks(&self, py: Python<'_>, session: &Py) -> PyResult<()> { - let factories: Vec> = self - .hook_factories - .iter() - .map(|factory| factory.clone_ref(py)) - .collect(); - install_hook_factories(py, &factories, session) - } - /// The connection URL of the session that the builder creates: that of the channel builder /// of `channelBuilder`, or else of `remote()`. As PySpark Connect's `create`, it is an error /// to give both a channel builder and a URL, or neither. fn connection_url(&self, py: Python<'_>) -> PyResult { + let remote_url = self.options().remote_url.clone(); match &self.channel_builder { - Some(_) if self.remote_url.is_some() => Err(crate::column::raise_pyspark( + Some(_) if remote_url.is_some() => Err(crate::column::raise_pyspark( py, "PySparkValueError", "SESSION_NEED_CONN_STR_OR_BUILDER", &[], )), Some(channel_builder) => url_of_channel_builder(channel_builder.bind(py)), - None => self.url(py), + None => session_url(py, remote_url), } } - /// Set an option, capturing `spark.remote` as the endpoint, as PySpark Connect's `config` - /// (the builder of `pyspark.sql.SparkSession`, PySpark's classic one, validates the startup - /// URLs). - fn set_option(&mut self, key: &str, value: String) { - self.set_option_value(key, Some(value)) - } - /// Set the config `key` to `value`, or to None, as PySpark's `config` keeps it in its options: /// a key and a value that the session takes are set, and the others are kept until - /// `getOrCreate` fails with them ([`PySparkSessionBuilder::unencodable`]). + /// `getOrCreate` fails with them ([`BuilderOptions::unencodable`]). The entries that are kept + /// are compared with the `==` of their keys, which is Python code, so they are changed apart + /// from the lock of the options. fn put_config( - &mut self, + &self, key: crate::options::OptionKey, value: Option, ) { @@ -424,64 +513,63 @@ impl PySparkSessionBuilder { None => Some(None), Some(value) => value.text.clone().map(Some), }; - match (name, text) { + let mut unencodable = self.options().unencodable.clone(); + let name = match (name, text) { (Some(name), Some(text)) => { - self.unencodable.remove(&key); - self.set_option_value(&name, text); + unencodable.remove(&key); + Some((name, text)) } _ => { - self.unencodable - .set(&key, value.map(crate::options::OptionKey::Text)); + unencodable.set(&key, value.map(crate::options::OptionKey::Text)); + None } - } - } - - /// Set an option whose value may be None, which PySpark keeps in its options: such a key is - /// not set on the session, as `_set_all` sends it without a value. - fn set_option_value(&mut self, key: &str, value: Option) { - if key == "spark.remote" { - self.remote_url = Some(value); - } else { - self.configs.push((key.to_string(), value)); - } - } - - /// The URL of the session: that of `remote()`, as PySpark Connect's builder, which raises - /// `SESSION_NEED_CONN_STR_OR_BUILDER` without one. `SPARK_REMOTE` is read by the builder of - /// `pyspark.sql.SparkSession`, PySpark's classic one, which gives its URL to this builder. - /// A URL that is set to None fails the assertion of PySpark Connect's `create`. - fn url(&self, py: Python<'_>) -> PyResult { - match &self.remote_url { - Some(None) => Err(pyo3::exceptions::PyAssertionError::new_err(())), - Some(Some(url)) => Ok(url.clone()), - None => Err(crate::column::raise_pyspark( - py, - "PySparkValueError", - "SESSION_NEED_CONN_STR_OR_BUILDER", - &[], - )), - } + }; + // The entries that are replaced are dropped once the lock is left, as the drop of their + // objects may run Python code. + let replaced = { + let mut options = self.options(); + if let Some((name, text)) = name { + options.set_value(&name, text); + } + std::mem::replace(&mut options.unencodable, unencodable) + }; + drop(replaced); } /// Set the options of the builder on `session`, as PySpark's `_apply_options`: in one /// request whose failures are ignored, such as those of static configurations, and - /// without `spark.master`, which a session does not take. + /// without `spark.master`, which a session does not take. As PySpark's, it holds the lock of + /// the builder while it reads the options and sends them. + fn apply_options(slf: &Bound<'_, Self>, session: &SparkSession) -> PyResult<()> { + with_builder_lock(slf, || slf.borrow().send_options(slf.py(), session)) + } + + /// Send the options of the builder to `session` (see [`Self::apply_options`]). // SparkError is large, as the core crate allows for its API. #[allow(clippy::result_large_err)] - fn apply_options(&self, py: Python<'_>, session: &SparkSession) -> PyResult<()> { + fn send_options(&self, py: Python<'_>, session: &SparkSession) -> PyResult<()> { + let mut options: Vec<(String, Option)> = Vec::new(); + let error = { + let builder_options = self.options(); + // A later value of a key replaces an earlier one, as the options of PySpark's builder + // do. + for (key, value) in builder_options + .configs + .iter() + .filter(|(key, _)| key != "spark.master") + { + match options.iter_mut().find(|(k, _)| k == key) { + Some(entry) => entry.1 = value.clone(), + None => options.push((key.clone(), value.clone())), + } + } + builder_options.unencodable.error() + }; // PySpark builds the request of every option first, so the first option that does not // encode fails it before any option is set. - if let Some(error) = self.unencodable.error() { + if let Some(error) = error { return Err(crate::errors::spark_error_to_py_exception(error)); } - // A later value of a key replaces an earlier one, as the options of PySpark's builder do. - let mut options: Vec<(String, Option)> = Vec::new(); - for (key, value) in self.configs.iter().filter(|(key, _)| key != "spark.master") { - match options.iter_mut().find(|(k, _)| k == key) { - Some(entry) => entry.1 = value.clone(), - None => options.push((key.clone(), value.clone())), - } - } if options.is_empty() { return Ok(()); } @@ -583,6 +671,23 @@ fn new_session_of_connection( configure: impl FnOnce(spark_connect::SparkSessionBuilder) -> spark_connect::SparkSessionBuilder + Send, ) -> PyResult> { + let session = connected_session(py, connection, user_id, configure)?; + let session = session_object(py, &session)?; + install_hook_factories(py, &hook_factories_of(hook_factories)?, &session)?; + log_client_banner(py); + Ok(session) +} + +/// The session of a new client of the URL or channel builder `connection`, of the user `user_id` +/// unless its URL has one, of the builder that `configure` configures, as PySpark's +/// `SparkSession(connection, userId)` makes its client. +fn connected_session( + py: Python<'_>, + connection: &Bound<'_, PyAny>, + user_id: Option, + configure: impl FnOnce(spark_connect::SparkSessionBuilder) -> spark_connect::SparkSessionBuilder + + Send, +) -> PyResult { let url = match connection.extract::() { Ok(url) => url, Err(_) => url_of_channel_builder(connection)?, @@ -591,24 +696,24 @@ fn new_session_of_connection( Some(user) => with_user_id(&url, &user)?, None => url, }; - let session = py - .detach(|| { - configure(SparkSession::builder().remote(&url)) - .get_or_create() - .map_err(Box::new) - }) - .to_pyerr()?; - let session = session_object(py, &session)?; - let factories = match hook_factories { + py.detach(|| { + configure(SparkSession::builder().remote(&url)) + .get_or_create() + .map_err(Box::new) + }) + .to_pyerr() +} + +/// The hook factories of the argument `hook_factories` of PySpark's `SparkSession(...)`, an +/// iterable of them, or None for none. +fn hook_factories_of(hook_factories: Option>) -> PyResult>> { + match hook_factories { Some(factories) => factories .try_iter()? .map(|factory| factory.map(Bound::unbind)) - .collect::>>()?, - None => Vec::new(), - }; - install_hook_factories(py, &factories, &session)?; - log_client_banner(py); - Ok(session) + .collect(), + None => Ok(Vec::new()), + } } /// Install on `session` the hooks that `factories` make of it, each called with the session, as @@ -706,11 +811,62 @@ fn call_execute_plan_hooks( Ok(request) } -/// Held while a session is got or created, as PySpark's `SparkSession._lock`, so that threads -/// get the same default session. Taken with the GIL, which is not released while it is held. -fn session_lock() -> std::sync::MutexGuard<'static, ()> { - static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); - LOCK.lock().unwrap_or_else(|e| e.into_inner()) +/// `f`, run while PySpark's `SparkSession._lock` is held, as its `with SparkSession._lock:` +/// holds it, where PySpark's `getOrCreate`, `stop` and `_set_default_and_active_session` take it: +/// so that threads get the same default session, a lookup waits for a stop in progress, and a +/// thread of the program that holds the lock holds them off. The lock is the class attribute, +/// read at each call, as PySpark reads it, so a lock that Python code set is the one taken, of +/// the `__enter__` and `__exit__` of its type, as `with` takes them. The acquire of an `RLock` +/// waits without the GIL, which a signal interrupts with the exception of its handler, and the +/// thread that holds it takes it again. `__exit__` is given the exception of `f`, if any. +fn with_session_lock(py: Python<'_>, f: impl FnOnce() -> PyResult) -> PyResult { + let lock = py + .get_type::() + .getattr(pyo3::intern!(py, "_lock"))?; + with_lock(&lock, f) +} + +/// `f`, run while the lock of the builder `builder` is held, as PySpark's builder holds +/// `self._lock`, the class attribute `Builder._lock` unless the builder has an attribute of its +/// own of the name, in its `config`, `channelBuilder`, `_registerHook` and `_apply_options`. +fn with_builder_lock( + builder: &Bound<'_, PySparkSessionBuilder>, + f: impl FnOnce() -> PyResult, +) -> PyResult { + let lock = builder.getattr(pyo3::intern!(builder.py(), "_lock"))?; + with_lock(&lock, f) +} + +/// `f`, run while `lock` is held, as the `with` statement holds it: of the `__enter__` and +/// `__exit__` of its type, which is given the exception of `f`, if any (see +/// [`with_session_lock`]). +fn with_lock(lock: &Bound<'_, PyAny>, f: impl FnOnce() -> PyResult) -> PyResult { + let py = lock.py(); + let manager = lock.get_type(); + let not_a_manager = |missed: &str| -> PyResult { + Ok(pyo3::exceptions::PyTypeError::new_err(format!( + "'{}' object does not support the context manager protocol{missed}", + manager.name()? + ))) + }; + let Ok(enter) = manager.getattr(pyo3::intern!(py, "__enter__")) else { + return Err(not_a_manager("")?); + }; + let Ok(exit) = manager.getattr(pyo3::intern!(py, "__exit__")) else { + return Err(not_a_manager(" (missed __exit__ method)")?); + }; + enter.call1((lock,))?; + let result = f(); + match &result { + Ok(_) => exit.call1((lock, py.None(), py.None(), py.None()))?, + Err(error) => exit.call1(( + lock, + error.get_type(py), + error.value(py), + error.traceback(py), + ))?, + }; + result } /// `session`, unless it stopped. @@ -733,17 +889,15 @@ fn default_session(py: Python<'_>) -> Option> { default_slot(py).and_then(|session| unless_stopped(py, &session)) } -/// A new session of `url`, which is the default session, and the active session of this -/// thread, when they are not set, as PySpark's `_set_default_and_active_session`. +/// The object of a new session of `url`, of a builder: neither the default nor the active +/// session, which `create` makes it then. fn new_session(py: Python<'_>, url: &str) -> PyResult> { // The channel connects lazily, so this does not wait on the server. let session = SparkSession::builder() .remote(url) .get_or_create() .to_pyerr()?; - let session = session_object(py, &session)?; - set_default_and_active(py, &session); - Ok(session) + session_object(py, &session) } /// The Python session objects by session ID, held weakly, so that the registry keeps none @@ -788,23 +942,22 @@ fn registered_session_object( /// a native class that takes the arguments of the `__init__` of a subclass too, as the /// `object.__new__` of PySpark's plain classes does: the first argument that is a session, of /// the keyword `name`, the positional arguments and the other keyword arguments, so that the -/// `__init__` of a subclass names and orders its arguments as it likes. Without a session, the -/// error of the keyword `name` or else of the first positional argument, which is not a session, -/// or without either the TypeError of a call without the argument, which CPython names after the -/// `__init__` of the class (`DataFrameReader.__init__() missing 1 required positional argument: -/// 'client'`). -pub(crate) fn constructor_session<'py>( - cls: &Bound<'py, pyo3::types::PyType>, - args: &Bound<'py, PyTuple>, - kwargs: Option<&Bound<'py, PyDict>>, +/// `__init__` of a subclass names and orders its arguments as it likes. Without one, the session +/// of no server ([`unconnected_session`]), until the `__init__` of the class gives the object a +/// session, as PySpark's `__init__` sets it: so an unpickle, which makes the object of its class +/// without arguments, makes one, and the `__init__` of the class, or that of a subclass, raises +/// the errors of the arguments, as PySpark's (`DataFrameReader.__init__() missing 1 required +/// positional argument: 'client'`). +pub(crate) fn constructor_session( + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, name: &str, -) -> PyResult> { - let py = cls.py(); +) -> PyResult { let keyword = match kwargs { Some(kwargs) => kwargs.get_item(name)?, None => None, }; - let mut given: Vec> = keyword.iter().cloned().collect(); + let mut given: Vec> = keyword.into_iter().collect(); given.extend(args.iter()); if let Some(kwargs) = kwargs { for (key, value) in kwargs.iter() { @@ -813,25 +966,34 @@ pub(crate) fn constructor_session<'py>( } } } - if let Some(session) = given + match given .into_iter() - .find(|value| value.is_instance_of::()) + .find_map(|value| value.cast_into::().ok()) { - return Ok(session.cast_into::()?); - } - match keyword.or_else(|| args.get_item(0).ok()) { - Some(first) => Ok(first.cast_into::()?), - None => { - let init = cls - .getattr(pyo3::intern!(py, "__init__"))? - .getattr(pyo3::intern!(py, "__qualname__"))?; - Err(pyo3::exceptions::PyTypeError::new_err(format!( - "{init}() missing 1 required positional argument: '{name}'" - ))) - } + Some(session) => Ok(session.borrow().session.clone()), + None => unconnected_session(), } } +/// The session of no server, of the objects that `__new__` makes before their `__init__` gives +/// them a session, as PySpark's plain objects have none before their `__init__` runs: of a client +/// that is closed, so that its requests fail with `NO_ACTIVE_SESSION`, as those of a stopped +/// session do, and nothing is sent of it, at the exit either. One for the process, made at its +/// first use. +pub(crate) fn unconnected_session() -> PyResult { + static UNCONNECTED: std::sync::OnceLock = std::sync::OnceLock::new(); + if let Some(session) = UNCONNECTED.get() { + return Ok(session.clone()); + } + // The client connects lazily, so it is made without a request, and closed before any. + let session = SparkSession::builder() + .remote("sc://localhost") + .get_or_create() + .to_pyerr()?; + session.close(); + Ok(UNCONNECTED.get_or_init(|| session).clone()) +} + /// The Python object of `session`, one per session as PySpark has them: its live object, or /// else a new one, which the registry holds weakly. So the DataFrames, the streams and the /// clients of a session give back the session's object, as PySpark's do. @@ -846,6 +1008,17 @@ pub(crate) fn session_object( } } let object = Py::new(py, PySparkSession::new(session.clone()))?; + register_session_object(py, object, session) +} + +/// Register `object`, an object of `session`, as the Python object of the session, which the +/// registry holds weakly, with the exit hook of the session: unless another thread registered one +/// meanwhile, which is returned then. +fn register_session_object( + py: Python<'_>, + object: Py, + session: &SparkSession, +) -> PyResult> { let weak = py .import("weakref")? .call_method1("ref", (&object,))? @@ -1060,17 +1233,22 @@ fn register_exit_hook(py: Python<'_>, session: &SparkSession) -> PyResult<()> { } /// Make `session` the default session, and the active session of this thread, when they are -/// not set, as PySpark's `SparkSession._set_default_and_active_session`. -fn set_default_and_active(py: Python<'_>, session: &Py) { - if default_slot(py).is_none() { - set_default_slot(py, Some(session.clone_ref(py))); - } +/// not set, as PySpark's `SparkSession._set_default_and_active_session`, which holds +/// `SparkSession._lock` to set the default session. +fn set_default_and_active(py: Python<'_>, session: &Py) -> PyResult<()> { + with_session_lock(py, || { + if default_slot(py).is_none() { + set_default_slot(py, Some(session.clone_ref(py))); + } + Ok(()) + })?; ACTIVE_SESSION.with(|active| { let mut active = active.borrow_mut(); if active.is_none() { *active = Some(session.clone_ref(py)); } }); + Ok(()) } /// Emit a one-time INFO log identifying this as the Rust-backed Spark Connect @@ -1195,7 +1373,6 @@ impl PyHook { } } -/// Python wrapper for a Spark session. /// A copy of the tags of a thread, which the threads that `_share_thread_tags` gives it share, /// for `pyspark.util.inheritable_thread_target`. #[pyclass(name = "_SharedTags", module = "pyspark.sql.connect.session", frozen)] @@ -1316,9 +1493,13 @@ fn interrupted_id(id: &Bound<'_, PyAny>) -> PyResult { crate::str_argument(id) } +/// Python wrapper for a Spark session, PySpark Connect's `SparkSession`. As PySpark's plain class, +/// it is subclassed: a subclass constructs of the arguments of its own `__init__`, which gives the +/// session its client through the `__init__` of the class, of PySpark's arguments. #[pyclass( name = "SparkSession", module = "pyspark.sql.connect.session", + subclass, weakref, dict )] @@ -1419,6 +1600,19 @@ impl PySparkSession { Ok(()) } + /// The object of `session`, of no client of its own: the value that `__new__` gives, which + /// `__init__` replaces, of the session of no server. It reports no progress, as no request of + /// it is sent. + fn of_no_client(session: SparkSession) -> Self { + PySparkSession { + session, + streams: std::sync::OnceLock::new(), + catalog: std::sync::OnceLock::new(), + release_session_on_close: std::sync::atomic::AtomicBool::new(true), + gc_ready: crate::gc::GcReady::new(), + } + } + pub fn new(session: SparkSession) -> Self { // The progress of the session, for the progress bars of its executions, and its // profiles, which the updates of its first execution update too. @@ -3099,63 +3293,67 @@ impl PySparkSession { /// this session. Listener callbacks are not joined; the current event's listener snapshot /// may finish afterward. fn stop(&self, py: Python<'_>) -> PyResult<()> { - let released = if self - .release_session_on_close - .load(std::sync::atomic::Ordering::SeqCst) - { - // As PySpark's stop, whose sleeps between the retries of the release a signal - // interrupts: the exception of the signal is raised at once, and leaves the session - // open, so nothing below runs. The core runs the stops of all sessions one at a time, - // as PySpark's `SparkSession._lock`, whose acquire a signal interrupts too; the resets - // below are idempotent, and run with the GIL, after the core's stop. - crate::interrupt::interruptible_outcome(py, &self.session, || { - self.session.stop().map_err(Box::new) - })? - } else { - // As PySpark's stop, which closes the client under `SparkSession._lock` all the same. - let _stopping = crate::interrupt::abortable_without_session( - py, - spark_connect::session::enter_stop_lock, - )?; - py.detach(|| self.session.close()); - Ok(()) - }; - crate::progress::forget_session(&self.session); - crate::profiler::forget_session(&self.session); - if let Err(error) = released { - // As PySpark's stop, whose `except Exception as e` logs the exception of the release - // through the logger of Spark Connect, which is disabled unless - // SPARK_CONNECT_LOG_LEVEL is set, with the `$` of its f-string. - let error = crate::errors::spark_error_to_py_exception(*error); - py.import("pyspark.sql.connect.logging")? - .getattr("logger")? - .call_method1( - "warning", - (format!( - "session.stop(): Session could not be released. Error: ${}", - error.value(py) - ),), + // As PySpark's stop, all of which holds `SparkSession._lock`. + with_session_lock(py, || { + let released = if self + .release_session_on_close + .load(std::sync::atomic::Ordering::SeqCst) + { + // As PySpark's stop, whose sleeps between the retries of the release a signal + // interrupts: the exception of the signal is raised at once, and leaves the + // session open, so nothing below runs. The core runs its stops of all sessions + // one at a time too, for the stops that do not hold `SparkSession._lock`; the + // resets below are idempotent, and run with the GIL, after the core's stop. + crate::interrupt::interruptible_outcome(py, &self.session, || { + self.session.stop().map_err(Box::new) + })? + } else { + // As PySpark's stop, which closes the client under its lock all the same. + let _stopping = crate::interrupt::abortable_without_session( + py, + spark_connect::session::enter_stop_lock, )?; - } - // As PySpark's, a stopped session is no longer the default session, or the active - // session of this thread. - let sid = self.session.session_id(); - let is_this = |s: &Py| s.borrow(py).session.session_id() == sid; - if default_slot(py).as_ref().is_some_and(is_this) { - set_default_slot(py, None); - } - ACTIVE_SESSION.with(|active| { - let mut active = active.borrow_mut(); - if active.as_ref().is_some_and(is_this) { - *active = None; + py.detach(|| self.session.close()); + Ok(()) + }; + crate::progress::forget_session(&self.session); + crate::profiler::forget_session(&self.session); + if let Err(error) = released { + // As PySpark's stop, whose `except Exception as e` logs the exception of the + // release through the logger of Spark Connect, which is disabled unless + // SPARK_CONNECT_LOG_LEVEL is set, with the `$` of its f-string. + let error = crate::errors::spark_error_to_py_exception(*error); + py.import("pyspark.sql.connect.logging")? + .getattr("logger")? + .call_method1( + "warning", + (format!( + "session.stop(): Session could not be released. Error: ${}", + error.value(py) + ),), + )?; } - }); - // Only touch the SQLContext cache if the module was ever imported (as pyspark does). - let modules = py.import("sys")?.getattr("modules")?; - if let Ok(ctx_mod) = modules.get_item("pyspark.sql.connect.context") { - ctx_mod.call_method1("_reset_instantiated_context", (sid,))?; - } - Ok(()) + // As PySpark's, a stopped session is no longer the default session, or the active + // session of this thread. + let sid = self.session.session_id(); + let is_this = |s: &Py| s.borrow(py).session.session_id() == sid; + if default_slot(py).as_ref().is_some_and(is_this) { + set_default_slot(py, None); + } + ACTIVE_SESSION.with(|active| { + let mut active = active.borrow_mut(); + if active.as_ref().is_some_and(is_this) { + *active = None; + } + }); + // Only touch the SQLContext cache if the module was ever imported (as pyspark + // does). + let modules = py.import("sys")?.getattr("modules")?; + if let Ok(ctx_mod) = modules.get_item("pyspark.sql.connect.context") { + ctx_mod.call_method1("_reset_instantiated_context", (sid,))?; + } + Ok(()) + }) } /// Enable `with SparkSession.builder.(...).getOrCreate() as session:` syntax. @@ -3446,19 +3644,45 @@ impl PySparkSession { self.session.is_stopped() } - /// `SparkSession(connection, userId=None, hook_factories=None)`, as PySpark Connect's: a - /// session of the URL or channel builder `connection`, which is neither the default nor the - /// active session, as PySpark's constructor does not make it either. + /// A session object of no client yet, as the `object.__new__` of PySpark's plain class: of any + /// arguments, which `__init__` takes, so that a subclass of another `__init__` is built as it + /// is in PySpark. Until `__init__` gives it a client, it is of the session of no server + /// ([`unconnected_session`]). The class shows the signature of `__init__`, as + /// `inspect.signature` of PySpark's class does. #[new] + #[pyo3( + signature = (*_args, **_kwargs), + text_signature = "(connection, userId=None, hook_factories=None)" + )] + fn py_new(_args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) -> PyResult { + Ok(PySparkSession::of_no_client(unconnected_session()?)) + } + + /// `SparkSession(connection, userId=None, hook_factories=None)`, as PySpark Connect's + /// `__init__`: the session of a new client of the URL or channel builder `connection`, which + /// is neither the default nor the active session, as PySpark's constructor does not make it + /// either, with the hooks that `hook_factories` make of it. So a subclass, whose `__init__` + /// calls it, is a session of its class, which its DataFrames give back. As PySpark's, an + /// `__init__` of a session that has a client already gives it a new one. #[pyo3(signature = (connection, userId=None, hook_factories=None))] #[allow(non_snake_case)] - fn py_new( - py: Python<'_>, + fn __init__( + slf: &Bound<'_, Self>, connection: &Bound<'_, PyAny>, userId: Option, hook_factories: Option>, - ) -> PyResult> { - new_session_of_connection(py, connection, userId, hook_factories, |builder| builder) + ) -> PyResult<()> { + let py = slf.py(); + let session = connected_session(py, connection, userId, |builder| builder)?; + let value = PySparkSession::new(session.clone()); + let replaced = std::mem::replace(&mut *slf.borrow_mut(), value); + // The objects of the session that it replaces, dropped once it is not borrowed, as their + // drop may run Python code. + drop(replaced); + let object = register_session_object(py, slf.clone().unbind(), &session)?; + install_hook_factories(py, &hook_factories_of(hook_factories)?, &object)?; + log_client_banner(py); + Ok(()) } /// The session of PySpark Connect's `SparkConnectClient(connection, ...)`, of the URL or @@ -3658,9 +3882,10 @@ impl PySparkSession { Py::new(py, PyActiveSession) } - /// The class attribute `_lock`, PySpark's `threading.RLock` of the session registry, for the - /// Python code that takes it. The drop-in's `getOrCreate` and `stop()` serialize through the - /// stop lock of the core and their own session lock, not through this object. + /// The class attribute `_lock`, PySpark's `threading.RLock` of the session registry, which + /// the builder's `getOrCreate`, `create` (to set the default session), `stop()` and + /// `_set_default_and_active_session` hold, as PySpark's do (see `with_session_lock`), as well + /// as the Python code that takes it. #[classattr] fn _lock(py: Python<'_>) -> PyResult> { Ok(py.import("threading")?.call_method0("RLock")?.unbind()) @@ -3974,8 +4199,8 @@ impl PySparkSession { /// are not set, as PySpark's, which `inheritable_thread_target` calls in a new thread. #[staticmethod] #[pyo3(name = "_set_default_and_active_session")] - fn set_default_and_active_session(py: Python<'_>, session: Py) { - set_default_and_active(py, &session); + fn set_default_and_active_session(py: Python<'_>, session: Py) -> PyResult<()> { + set_default_and_active(py, &session) } /// The DDL of the schema `struct`, as PySpark Connect's `_to_ddl`: the server converts diff --git a/crates/pyspark-rs/src/streaming.rs b/crates/pyspark-rs/src/streaming.rs index 089509c7..9652ad4c 100644 --- a/crates/pyspark-rs/src/streaming.rs +++ b/crates/pyspark-rs/src/streaming.rs @@ -448,19 +448,13 @@ impl PyDataStreamReader { /// `DataStreamReader(client)`, as PySpark Connect's, of the session `client`: the reader that /// `spark.readStream` gives. As the `object.__new__` of PySpark's plain class, it takes the /// arguments of the `__init__` of a subclass too, of which the session is the first that is - /// a session (see `session::constructor_session`); `__init__` checks them. + /// a session, and none, as an unpickle gives none (see `session::constructor_session`); + /// `__init__` checks them. #[new] - #[classmethod] #[pyo3(signature = (*args, **kwargs), text_signature = "(client)")] - fn py_new( - cls: &Bound<'_, pyo3::types::PyType>, - args: &Bound<'_, PyTuple>, - kwargs: Option<&Bound<'_, PyDict>>, - ) -> PyResult { - let session = crate::session::constructor_session(cls, args, kwargs, "client")?; - Ok(PyDataStreamReader::new( - session.borrow().session.read_stream(), - )) + fn py_new(args: &Bound<'_, PyTuple>, kwargs: Option<&Bound<'_, PyDict>>) -> PyResult { + let session = crate::session::constructor_session(args, kwargs, "client")?; + Ok(PyDataStreamReader::new(session.read_stream())) } /// As PySpark's `__init__(client)`, which makes the reader of the session anew: of no @@ -525,6 +519,60 @@ impl PyDataStreamReader { crate::reduce::reduce_with_instance_dict(slf.as_any(), reduced) } + /// The state of a pickle of a stream reader, which pickle sets on the reader that `__new__` + /// made of no arguments. Of PySpark's pickle, the state is the `__dict__` of its plain object, + /// which holds `_options`: so the reader is of the session `_client`, and of the format, the + /// schema, the options and the source name of `_format`, `_schema`, `_options` and + /// `_source_name`, which it takes as `format`, `schema`, `option` and `name` take them, and + /// the other entries are its instance attributes. The state of the drop-in's pickle holds the + /// instance attributes alone (see `__reduce__`), as a reader's `__dict__` cannot hold + /// `_options`, a property of the reader. + fn __setstate__(slf: &Bound<'_, Self>, state: &Bound<'_, PyDict>) -> PyResult<()> { + let py = slf.py(); + let state = state.copy()?; + if state.contains("_options")? { + let take = |key: &str| -> PyResult>> { + let value = state.get_item(key)?; + if value.is_some() { + state.del_item(key)?; + } + Ok(value) + }; + if let Some(client) = take("_client")? { + let session = client.cast_into::()?; + let reader = PyDataStreamReader::new(session.borrow().session.read_stream()); + let replaced = std::mem::replace(&mut *slf.borrow_mut(), reader); + drop(replaced); + } + if let Some(format) = take("_format")? { + let source = (!format.is_none()).then_some(&format); + Self::format(slf.borrow_mut(), source)?; + } + if let Some(schema) = take("_schema")? { + Self::schema(slf.borrow_mut(), &schema)?; + } + if let Some(options) = take("_options")? { + for (key, value) in options.cast_into::()?.iter() { + Self::option(slf.borrow_mut(), &key, &value)?; + } + } + if let Some(source_name) = take("_source_name")? { + if !source_name.is_none() { + Self::name(slf.borrow_mut(), &source_name)?; + } + } + } + slf.getattr(pyo3::intern!(py, "__dict__"))? + .call_method1(pyo3::intern!(py, "update"), (state,))?; + Ok(()) + } + + /// The source name of `name`, or None, as PySpark's `DataStreamReader._source_name`. + #[getter] + fn _source_name(&self) -> Option { + self.inner.get_source_name().map(str::to_string) + } + /// A stream reader of its state, as `__reduce__` gives it, of the class `cls`: of the session /// `session`, the format, the schema, the options, the source name, the format that is kept /// (`unsent`), and the schema and the options that UTF-8 cannot encode (`unencodable`). @@ -650,6 +698,13 @@ impl PyDataStreamReader { self.unsent_format = None; } + /// The session of the reader, as PySpark Connect's `DataStreamReader._client`, which its + /// pickle keeps. + #[getter] + fn _client(&self, py: Python<'_>) -> PyResult> { + crate::session::session_object(py, &self.inner.spark_session()) + } + /// The configured source format (PySpark Connect's `DataStreamReader._format`), which is the /// object itself when it is not a str that the core reader takes, as PySpark keeps it. #[getter] diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index 4f922d4b..576bdf65 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -209,6 +209,12 @@ difference is intentional or tracked for a fix. its line 1126, as PySpark's, in the file of the drop-in's module, which ends before the line, so it shows no source line. +- **A DataFrame of a plan that fails to build** (tracked). `DataFrame(plan, session)`, of the + class or of a subclass, builds the relation of `plan.plan(session.client)` when it is made, so a + plan that fails, such as the `_plan` of a DataFrame of a UDF that does not pickle or of a read of + an option that UTF-8 cannot encode, raises its error there, where PySpark's DataFrame keeps the + plan and raises the error at its first request. + - **Public base and native MRO** (intentional; a PyO3 class cannot have a Python class as its base, so the public base can only be a virtual one). `pyspark.sql.DataFrame` is implemented as a Python base/factory, allowing subclasses such as @@ -364,17 +370,17 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan a reference cycle through it, such as through a bound method of an object that keeps both the Column and the DataFrame, is not collected, where PySpark collects it. A stream writer holds its DataFrame, where PySpark's holds the plan, so a DataFrame that keeps its stream writer is freed by - the collector rather than at its last reference. The session builder reports its channel builder - and its hook factories, but not one that a copy of it shares, so a cycle through one of those is - not collected while the builder and its copy live, where PySpark collects it. + the collector rather than at its last reference. ## Session and client (Connect) - **Custom gRPC channels** (intentional; the transport is native). `ChannelBuilder` and `DefaultChannelBuilder` parse the connection string and hold its parameters, `host`, - `port`, `endpoint`, `get`, `set`, `userId`, `token`, `session_id`, `use_ssl`, `secure` - and `metadata()`. The transport reads them as a `sc://` URL, so they apply. The gRPC - channel members do not: `toChannel`, `add_interceptor`, `setChannelOption`. The keepalive + `endpoint`, `get`, `set`, `userId`, `token`, `session_id`, `use_ssl`, `secure` and + `metadata()`, and the drop-in's `DefaultChannelBuilder` has a `port` too, which PySpark's has + not (it keeps the port in `_port`). The transport reads the `host`, the `port` (15002 without + one) and the `_params` of a channel builder as a `sc://` URL, so they apply. The gRPC channel + members do not: `toChannel`, `add_interceptor`, `setChannelOption`. The keepalive properties and `userAgent` read the parameters as PySpark does, but `userAgent` reports PySpark's `python/` where the transport sends `python/rust` as the client type. The transport reads `SPARK_CONNECT_USER_AGENT` when a session is made of a connection string, and @@ -469,23 +475,27 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan an expression's extension is sent as it is, and so is one that the message of an extension changes (`plan.common.plan_id = ...`), whose change the core does not apply to the relation; the server rejects them with `CONNECT_INVALID_PLAN.NO_HANDLER_FOR_EXTENSION`. -- **Default port** (intentional). `DefaultChannelBuilder.default_port()` is 15002. PySpark - takes the port of a local JVM server in its testing mode; the drop-in has no JVM. -- **Missing URL error** (intentional). The one builder of the drop-in serves - `pyspark.sql.SparkSession.builder` and `pyspark.sql.connect.session.SparkSession.builder`. - Without a URL or a channel builder it raises `CONNECT_URL_NOT_SET`, which is both a - `PySparkRuntimeError` (the unified builder) and a `PySparkValueError` (the Connect - builder's `SESSION_NEED_CONN_STR_OR_BUILDER` family). Giving both a channel builder and - a URL raises `SESSION_NEED_CONN_STR_OR_BUILDER`. A channel builder without a host is - rejected at `channelBuilder(...)`, not when the session is created. -- **Plan logging of a deep DataFrame** (intentional; DESIGN). `DataFrame._plan` is a plan of the - relation of the Rust core, whose `plan` is a placeholder that holds the relation encoded, so - `_plan.to_proto` is not provided. - Python protobuf refuses to decode a relation nested beyond its limit (`too many levels - of nesting`), and the drop-in never decodes a deep plan in Python. - `SparkConnectClient._proto_to_string` renders any message, including a deep one built in - Python, as PySpark's does. Plan logging of a deep DataFrame therefore does not reach the - recursion handling that the test `test_recursion_handling_for_plan_logging` exercises. +- **A channel builder without a host** (intentional; the transport is native). The builders of + `pyspark.sql.SparkSession` and `pyspark.sql.connect.session.SparkSession` are PySpark's two + builders, with its errors: `CONNECT_URL_NOT_SET`, a `PySparkRuntimeError`, and + `SESSION_NEED_CONN_STR_OR_BUILDER`, a `PySparkValueError`, also for both a URL and a channel + builder. `channelBuilder(...)` rejects a channel builder without a `host` with a `ValueError`, + as the transport connects to the `sc://` URL of its `host`, `port` and `_params`; PySpark's + takes any object, and its session fails when it is made, of the error of the object (the + `TypeError` of a str's slicing, or the error of its `toChannel`). +- **Plan logging of a deep DataFrame** (intentional; DESIGN). `DataFrame._plan` is a + placeholder over the relation of the Rust core, whose `to_proto(client)` is the `Plan` of the + placeholder, `plan(client)`, as the root. The `to_proto` of a `LogicalPlan` is the `Plan` of + its relation as it is, as the transport compresses the plans that it sends, where PySpark's + compresses one of over `spark.connect.session.planCompression.threshold` bytes (10 MiB, which + its client asks the server for at its first plan) into a `compressed_operation`. Python + protobuf refuses to decode a relation nested beyond its limit (`too many levels of nesting`), + and the drop-in never decodes a deep plan. `SparkConnectClient._proto_to_string` renders a + message of the drop-in's protobuf runtime in Rust at any depth: past about 125 levels, PySpark's + `text_format` raises its `RecursionError`, and it returns ``, where the drop-in returns the full text; with `truncate=True`, both cut it at 8 + levels. Plan logging of a deep DataFrame therefore does not reach the recursion handling that + the test `test_recursion_handling_for_plan_logging` exercises. - **Client session ID** (intentional). `SparkConnectClient._session_id` is the session ID that the requests carry, which a set changes for them, as PySpark's, so the next response of another server-side session closes the session; `_server_session_id` is exposed and settable, @@ -513,7 +523,11 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan whatever the timing of the collections. The same goes for the other cycles that keep a PySpark session until a collection, such as that of an error that the `_handle_error` of its client raises again (`raise error`), whose traceback keeps the frame that holds the error and, - with it, the frames of the calls that led to it and their variables. + with it, the frames of the calls that led to it and their variables. And PySpark 4.3.0 does not + exit at all while a session other than its default one is left open to the end (of + `newSession()`, or `SparkSession(connection=...)`), as its `SparkSession.__del__` closes the + client in the teardown of the interpreter, whose grpcio `Channel.close()` waits forever; the + drop-in exits. - **A `conf` or a `client` kept apart from its session** (tracked). PySpark's `RuntimeConf` and the `SparkConnectClient` of a session hold the client, not the session object: once the program drops the session, its `__del__` closes the client, so the `conf` or `client` that the @@ -528,19 +542,28 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan which their copies and pickles keep as PySpark's do, but not their state, which PySpark's holds there (`vars(spark.streams)` is empty, where PySpark's has `_session` and `_sqlb`). - **Subclasses** (intentional; the state is native). PySpark's classes are plain classes that a - user subclasses. The drop-in's DataFrame, `Observation`, `DataFrameReader`, `DataStreamReader`, - `SparkSession.Hook`, `StorageLevel`, `PythonEvalType` and the data types are subclassed as - PySpark's: a subclass constructs of the arguments of its `__init__`, and its copies and pickles - are of the subclass. - The `Window` of `pyspark.sql.window` and of `pyspark.sql.connect.window` is subclassed, of its - static methods, and it and a subclass make objects of no state, as PySpark's. A reader is made - of a session, one of those arguments, so PySpark's pickle of a reader, which makes it of its - class without arguments, - does not load in the drop-in (`TypeError`). The other native classes are not subclassable - (`TypeError: type '...' is not an acceptable base type`), and of them only `SparkSession`, - `Catalog` and `DataSourceRegistration` construct, as PySpark's do; the others - raise `cannot create '...' instances`, as their constructors take the Python objects of - PySpark's client, which the drop-in's native state replaces (see `docs/compatibility.md`): + user subclasses. The drop-in's DataFrame, `Observation`, `SparkSession`, `Catalog`, + `DataSourceRegistration`, `DataFrameReader`, `DataStreamReader`, `SparkSession.Hook`, + `StorageLevel`, `PythonEvalType` and the data types are subclassed as PySpark's: a subclass + constructs of the arguments of its `__init__`, and its copies and pickles are of the subclass + (but those of a session, which are the active session of its ID, as PySpark's), and PySpark's + pickle of a reader, a catalog or a registration, which makes it of its class without + arguments, loads in the drop-in. The `Window` of `pyspark.sql.window` and of + `pyspark.sql.connect.window` is subclassed, of its static methods, and it and a subclass make + objects of no state, as PySpark's. The session, the readers and the catalog are of the native + state of a session, which the `__init__` of the class, of PySpark's arguments, gives them: one + whose `__init__` never ran, as of a subclass whose `__init__` does not call it, is of the + session of no server, whose requests raise `NO_ACTIVE_SESSION` and whose `is_stopped` is True, + where PySpark's raise the `AttributeError` of their `_client` or `_sparkSession` (a reader or a + catalog is of the first session among the arguments of the subclass, if any, which then + works). The `__init__` of a reader, a catalog or a registration takes a session, and raises the + `TypeError` of another object, where PySpark's keeps any object until it is used. + The other native classes are not subclassable (`TypeError: type '...' is not an acceptable + base type`), and do not construct (`cannot create '...' instances`): their objects are of the + native state of a plan, an expression or a query, which the arguments of PySpark's + constructors, the Python objects of its client, do not give, as the drop-in's native state + replaces those objects, so a subclass would have no `__init__` to call (see + `docs/compatibility.md`): - `Column`: an `Expression` of `pyspark.sql.connect.expressions`; - `DataFrameWriter`, `DataFrameWriterV2`, `DataStreamWriter`, `MergeIntoWriter`: the `LogicalPlan` of a DataFrame; @@ -593,12 +616,6 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan its JSON raise its error, but `createDataFrame` of local rows or of a pandas DataFrame of one raises the `NOT_IMPLEMENTED` of its `jsonValue()`, where PySpark's converters raise that of its `sqlType()` or `serialize` (see `docs/compatibility.md`). -- **Shallow copy of a session builder** (tracked). `copy.copy(SparkSession.builder)` has - options of its own, where PySpark's shares its `_options` dict and its hook factories - with the builder that it copies, so that a `config` of either one is seen by both. -- **Session registry lock** (intentional). `SparkSession._lock` is a `threading.RLock` - for the Python code that takes it. The drop-in serializes its own session creation with - a Rust lock, not with this one. - **Direct construction** (intentional). `SparkSession(connection=..., userId=..., hook_factories=...)` makes a session that is neither default nor active, as PySpark's constructor does. `userId` is added to the URL unless the URL has a `user_id`, which diff --git a/docs/compatibility.md b/docs/compatibility.md index 43dcbcc8..14c1bbe5 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -1779,7 +1779,11 @@ relation of the plan as a node of its own plans, which it numbers from the count IDs of its sessions, and the plans of `pyspark.sql.connect.plan` number theirs from `1 << 48`, past the core's, so that the two never give a relation the same ID. So the relation that it sends and the columns of its `df["name"]` have the client's plan ID, and its `_plan` is the plan of that -relation, not `plan`, as PySpark's is. +relation, not `plan`, as PySpark's is. It is made of the relation that `plan.plan(session.client)` +gives when it is made, so a plan that fails to build raises there, where PySpark's DataFrame keeps +the plan and builds it at its first request: `DataFrame(df._plan, spark)`, also of a subclass of +the DataFrame, of a DataFrame of a UDF that does not pickle, or of a read of an option that UTF-8 +cannot encode, raises the error of its build at once. ## SparkSession.client @@ -2389,7 +2393,10 @@ is gone the exit waits on that request's retries, as PySpark's does, so the exit long as those retries. A signal ends that wait, as it ends PySpark's, whose cleanup catches `Exception` only: the `KeyboardInterrupt` of Ctrl-C is raised out of the exit hook, which `atexit` reports before it runs the next one, and the session is then neither released nor -closed. A stopped session sends nothing at exit. +closed. A stopped session sends nothing at exit. PySpark 4.3.0 does not exit at all while a +session other than its default one is left open to the end, such as one of `newSession()` or of +`SparkSession(connection=...)`: its `SparkSession.__del__` closes the client in the teardown of the +interpreter, whose grpcio `Channel.close()` waits forever. The drop-in exits. Nor does a session that the program dropped, of which no object is left: neither the session nor a DataFrame, reader, writer, catalog, stream or ML object of it. PySpark's session object closes its client when it is freed, and the exit sends nothing for a closed client. The drop-in @@ -2677,13 +2684,13 @@ does, is reported by neither, so that no reference is counted twice: a cycle thr only when it is broken, where PySpark collects it. A stream writer holds its DataFrame, where PySpark's holds the plan, so a DataFrame that keeps its stream writer is freed by the collector, where PySpark frees it with its last reference. The functions of `foreachBatch` and `foreach` are -kept as their pickles, as PySpark's stream writer keeps them, so a function that keeps its writer is -freed with its last reference, and a copy or a pickle of the writer carries the pickle of the call. -The registration of `dataSource` reports its session, and the builder of -`pyspark.sql.connect.session.SparkSession` its channel builder and its hook factories, as PySpark's -plain objects hold them, so a session that keeps its `dataSource`, or a channel builder or a hook -factory that keeps its builder, is collected. One that a copy of the builder shares is reported by -neither, so a cycle through it is freed only when it is broken. +kept as their pickles, as PySpark's stream writer keeps them, so a function that keeps its writer +is freed with its last reference, and a copy or a pickle of the writer carries the pickle of the +call. The registration of `dataSource` reports its session, and the builder of +`pyspark.sql.connect.session.SparkSession` its channel builder and the list of its hook factories, +which its shallow copies share, as PySpark's plain objects hold them, so a session that keeps its +`dataSource`, or a channel builder or a hook factory that keeps its builder or a copy of it, is +collected. The condition and the assignments of a clause of `mergeInto` are pickled when the clause is added, which raises there, as PySpark's, and again when the merge runs, whose request carries the later @@ -2701,9 +2708,11 @@ SparkSession, its `conf`, `catalog`, `read`, `readStream`, `streams`, `udf`, `ud `dataSource`, `tvf` and `client`, and a DataFrame's `write`, `writeTo`, `writeStream`, `mergeInto` and its clauses, `na`, `stat`, `groupBy`, `cogroup`, `asTable` and the WindowSpec of its windows, and an `Observation`. So are `SparkSession.Hook`, nested in the session class, and -the classes of `pyspark.sql.connect.client` (its channel builders, `PlanObservedMetrics`, -`SparkConnectClient`, `RpcDeadlines`, `AnalyzeResult` and `ConfigResult`), which are of -`pyspark.sql.connect.client.core`. +the classes that `pyspark.sql.connect.client` names, which are of `pyspark.sql.connect.client.core`: +its channel builders, `SparkConnectClient` and `RpcDeadlines`, which PySpark's module names too, of +the `__all__` of its `core`, and `PlanObservedMetrics`, `AnalyzeResult` and `ConfigResult`, classes +of PySpark's `core` that only the drop-in's module names, with its own helpers and the names that +it imports, as the drop-in's client is that module. The functions of `pyspark.sql.connect.functions` are of its modules (`builtin` and `partitioning`), and so are those of `pyspark.sql.connect.avro.functions` and `pyspark.sql.connect.protobuf.functions`: they are not those of `pyspark.sql.functions`, @@ -2742,27 +2751,37 @@ by registration, so the MRO of a DataFrame is its native class and `object`, and `pyspark.sql.Column` names the concrete class. PySpark's classes are plain Python classes, which a user subclasses. So are these of the drop-in: -the DataFrame, `Observation`, `DataFrameReader` and `DataStreamReader`, which construct of their -PySpark arguments (`DataFrameReader(spark)`), also a subclass of an `__init__` of its own, and whose -setters, copies and pickles are of the subclass; `SparkSession.Hook`, `StorageLevel` and -`PythonEvalType`; and the `Window` of `pyspark.sql.window` and of `pyspark.sql.connect.window`, of -static methods and of PySpark's class attributes, `_PRECEDING_THRESHOLD` and `_FOLLOWING_THRESHOLD` -among them, and the `_spec` of the Connect class, which a subclass has too: `Window()`, also of a -subclass, makes an object of no state, as PySpark's, which takes attributes and weak references and -pickles as a plain object; and the data types, from `StringType` and `DecimalType` to `ArrayType`, -`StructField` and `StructType` (see above). A reader is made of a session, which one of the -arguments of its constructor must be, where PySpark's `__init__` keeps any `client`: without a -session the TypeError is of the missing `client`, or of the argument that is not a session, also -where PySpark's subclass raises the error of the other arguments of its `__init__`. So a reader that -PySpark pickled, which unpickles of its class without arguments, raises that TypeError in the -drop-in, where the drop-in's pickle of a reader loads in PySpark, of its class, as PySpark's own do: -where the session of its ID is active. +the DataFrame, `Observation`, `SparkSession`, `Catalog`, `DataSourceRegistration`, +`DataFrameReader` and `DataStreamReader`, which construct of their PySpark arguments +(`SparkSession(connection=...)`, `Catalog(spark)`, `DataFrameReader(spark)`), also a subclass of an +`__init__` of its own, and whose setters, copies and pickles are of the subclass (but those of a +session, which are the active session of its ID, as PySpark's); +`SparkSession.Hook`, `StorageLevel` and `PythonEvalType`; and the `Window` of `pyspark.sql.window` +and of `pyspark.sql.connect.window`, of static methods and of PySpark's class attributes, +`_PRECEDING_THRESHOLD` and `_FOLLOWING_THRESHOLD` among them, and the `_spec` of the Connect class, +which a subclass has too: `Window()`, also of a subclass, makes an object of no state, as PySpark's, +which takes attributes and weak references and pickles as a plain object; and the data types, from +`StringType` and `DecimalType` to `ArrayType`, `StructField` and `StructType` (see above). A +session of a subclass is the session that its DataFrames, readers and catalogs give back, as in +PySpark. The `__init__` of the class gives an object its session, as PySpark's `__init__` sets it: +the session of a new client, of `SparkSession(connection, userId=None, hook_factories=None)`, and +the session of its argument for the others, which must be a session, where PySpark's keeps any +object until it is used, which raises its `AttributeError` then. Until it runs, as for a subclass +whose `__init__` does not call it, a session, a reader or a catalog is of a session of no server, +whose requests raise `NO_ACTIVE_SESSION`, where PySpark's raise the `AttributeError` of `_client` +or `_sparkSession` (a reader or a catalog is of the first session among the arguments of a +subclass, if any, which then works), and a registration has no `sparkSession`, as PySpark's. So a +pickle of PySpark, which makes an object of its class without arguments and then sets its +`__dict__`, loads in the drop-in: a reader, of the `_client`, `_format`, `_schema` and `_options` +(and `_source_name`) of its state, which it sets as its setters do, a catalog of its +`_sparkSession` and a registration of its `sparkSession`; and the drop-in's pickles of them load in +PySpark, of their classes, as PySpark's own do: where the session of their ID is active. The other native classes cannot be subclassed -(`TypeError: type 'Catalog' is not an acceptable base type`), as their objects are of the state of -the Rust core. `SparkSession(connection=...)`, `Catalog(spark)` and `DataSourceRegistration(spark)` -construct as PySpark's; the others do not construct either -(`TypeError: cannot create '...' instances`), as PySpark's constructors take the Python objects of -its client, which the drop-in does not have: +(`TypeError: type 'Column' is not an acceptable base type`), and do not construct +(`TypeError: cannot create '...' instances`): their objects are made of the state of the Rust core, +a plan, an expression or a query, which the arguments of PySpark's constructors, the Python objects +of its client, do not give, as the drop-in does not have them, so a subclass would have no +`__init__` to call: - `Column` takes an `Expression` of `pyspark.sql.connect.expressions`; the drop-in's expressions are the Rust core's. - `DataFrameWriter`, `DataFrameWriterV2`, `DataStreamWriter` and `MergeIntoWriter` take the @@ -2824,10 +2843,12 @@ attributes that are set on them through copies and pickles, as PySpark's copy an they copy the object at all. Where PySpark's fail, the drop-in's fail with the same errors, such as the `AttributeError` of the creator of the session of a plain pickle of `spark.streams`, and the `TypeError` of its lock of a deep copy and of cloudpickle. Their `__dict__` holds only those -attributes, where PySpark's holds the state of the object too, as the `_options` of a builder: so a -shallow copy of a builder has options of its own, where PySpark's shares its `_options` dict. -`pyspark.sql.SparkSession.builder` is PySpark's own class, whose `__dict__` holds its `_options`, -which a shallow copy shares, as in PySpark. +attributes, where PySpark's holds the state of the object too, as the `_options` of a builder. A +shallow copy of the builder of `pyspark.sql.connect.session.SparkSession` shares its options and +its hook factories all the same, as PySpark's copy shares its `_options` dict and its +`_hook_factories` list: a `remote`, a `config` or a `_registerHook` of either one is seen by both, +while a `channelBuilder` sets the channel builder of the one that it is called on, as it rebinds an +attribute of PySpark's. A table name that is neither a str nor bytes, and bytes that are not UTF-8, give a DataFrame of `read.table`, `readStream.table` and `changes`, whose plan raises protobuf's `TypeError` or the @@ -2838,7 +2859,14 @@ rejects it as an invalid Spark Connect URL, as PySpark's `create` does for its ` `None`, given as `remote(None)`, `remote(location=None)` or `config("spark.remote", None)`, is kept as None: `create` and `getOrCreate` fail PySpark's assertion, unless a session exists, which `getOrCreate` returns. A later `remote` replaces it. A `remote()` without an argument is -`sc://localhost`, as in PySpark's Connect builder. +`sc://localhost`, as in PySpark's Connect builder. The locks are PySpark's: `getOrCreate` holds +`SparkSession._lock` from its lookup to the options that it sets, `create` to make its session the +default one, and `stop()` throughout, and the builder holds `SparkSession.Builder._lock` in +`config`, `remote`, `appName`, `channelBuilder` and `_registerHook`, and while it sets the options +of a session. Each is a class attribute, an `RLock`, read at each call, so a thread of the program +that holds one holds off the calls of the other threads that take it, and a signal ends the wait +for it. As PySpark's, `getOrCreate` sets the options of a session that it creates twice, in +`create` and again. The client honors the HTTP proxy of the environment (`grpc_proxy`, `https_proxy`, `http_proxy`, `no_grpc_proxy`, `no_proxy`) with grpcio's rules, see [HTTP Proxies](configuration.md#http-proxies), diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py index 39208de8..0a2059f6 100644 --- a/python/pyspark/sql/connect/plan.py +++ b/python/pyspark/sql/connect/plan.py @@ -19,9 +19,10 @@ The Rust core builds the plans of the DataFrame API. ``LogicalPlan`` is PySpark Connect's, from Apache Spark v4.3.0, the base of the plans that the modules of PySpark build themselves, such as those of ``pyspark.ml.connect.proto``: a plan of a child, of a ``plan(session)`` that is its -``spark.connect.Relation``, which ``DataFrame(plan, session)`` sends as it is. It has no -``to_proto``, ``_collect_references``, ``print`` or ``_repr_html_``, which are of the plans and the -Columns of PySpark Connect's DataFrame, which the core builds. ``_DataFramePlan`` is the plan of a +``spark.connect.Relation``, which ``DataFrame(plan, session)`` sends as it is, and of a +``to_proto(session)`` that is the ``spark.connect.Plan`` of it. It has no +``_collect_references``, ``print`` or ``_repr_html_``, which are of the plans and the Columns of +PySpark Connect's DataFrame, which the core builds. ``_DataFramePlan`` is the plan of a DataFrame of the core, its ``DataFrame._plan``. ``Read``, ``Range``, ``SQL``, ``SubqueryAlias``, ``CachedRemoteRelation`` and ``RemoveRemoteCachedRelation`` are PySpark Connect's, which ``pyspark.testing.connectutils`` and ``pyspark.sql.connect.conversion`` build DataFrames of. The @@ -89,6 +90,27 @@ def plan(self, session: Any) -> "proto.Relation": # type: ignore[empty-body] def command(self, session: Any) -> "proto.Command": # type: ignore[empty-body] ... + def to_proto(self, session: Any, debug: bool = False) -> "proto.Plan": + """ + Generates connect proto plan based on this LogicalPlan. + + Parameters + ---------- + session : :class:`SparkConnectClient`, optional. + a session that connects remote spark cluster. + debug: bool + if enabled, the proto plan will be printed. + """ + # As PySpark's, of its relation as the root of the plan. The transport compresses the + # plans that it sends itself, so the plan is the relation as it is. + plan = proto.Plan() + plan.root.CopyFrom(self.plan(session)) + + if debug: + print(plan) + + return plan + def _verify(self, session: Any) -> bool: """This method is used to verify that the current logical plan can be serialized to Proto and back and afterwards is identical.""" diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 21be7d6a..b12e990a 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -13030,7 +13030,9 @@ def test_cycles_through_the_sessions_and_builders_of_their_objects_are_collected # As PySpark's DataSourceRegistration, a plain object that holds its session, and its session # builder, which holds its channel builder and its hook factories: a session that keeps its # dataSource, or a copy or a new registration of it, is freed by the garbage collector, and so - # is a channel builder or a hook factory that keeps its builder. + # is a channel builder or a hook factory that keeps its builder, or the builder and its shallow + # copies, which share the list of its hook factories. A hook factory that a builder shares + # with a copy that is still reachable is kept, and makes the hook of the copy's session. import copy import gc import weakref @@ -13057,9 +13059,17 @@ def channel_builder(job): channel.job = job return ConnectSession.builder.channelBuilder(channel) + def with_copy(builder): + return builder, copy.copy(builder) + for make in (channel_builder, lambda job: ConnectSession.builder._registerHook(lambda session: job), - lambda job: ConnectSession.Builder()._registerHook(lambda session: job)): + lambda job: ConnectSession.Builder()._registerHook(lambda session: job), + lambda job: with_copy(channel_builder(job)), + lambda job: with_copy(ConnectSession.builder._registerHook(lambda session: job)), + lambda job: copy.copy(ConnectSession.builder._registerHook(lambda session: job)), + lambda job: with_copy(ConnectSession.builder)[1]._registerHook( + lambda session: job)): job = Job() job.held = make(job) freed = weakref.ref(job) @@ -13067,6 +13077,30 @@ def channel_builder(job): gc.collect() assert freed() is None + made = [] + + class Factory: + def __init__(self, job): + self.job = job + + def __call__(self, session): + made.append(session) + return ConnectSession.Hook() + + job = Job() + job.held = ConnectSession.builder.remote(service.remote)._registerHook(Factory(job)) + kept = copy.copy(job.held) + freed = weakref.ref(job) + del job + gc.collect() + assert freed() is not None + session = kept.create() + try: + assert made == [session] + finally: + session.release_session_on_close = False + session.stop() + def test_stream_writers_keep_the_pickles_of_their_functions_as_pyspark(mock_connect): # As PySpark's DataStreamWriter, whose WriteStreamOperationStart keeps the pickle of the @@ -22030,6 +22064,31 @@ def test_arrow_input_explicit_schema_matching_width(schema, values): assert names is None +def test_logical_plans_give_the_plans_of_their_relations_of_to_proto_as_pyspark( + mock_connect, capsys +): + # As PySpark's LogicalPlan.to_proto(session, debug=False): the Plan of the relation of + # plan(session) as its root, which debug prints. A DataFrame's _plan gives the Plan of the + # placeholder of its relation, which its plan(session) gives. + import inspect + + import pyspark.sql.connect.proto as pb2 + from pyspark.sql.connect.plan import LogicalPlan, Range + + spark, _ = mock_connect + assert list(inspect.signature(LogicalPlan.to_proto).parameters) == ["self", "session", "debug"] + assert inspect.signature(LogicalPlan.to_proto).parameters["debug"].default is False + plan = Range(0, 3, 1) + proto = plan.to_proto(spark.client) + assert isinstance(proto, pb2.Plan) and proto.WhichOneof("op_type") == "root" + assert proto.root == plan.plan(spark.client) + assert capsys.readouterr().out == "" + assert plan.to_proto(spark.client, debug=True) == proto + assert capsys.readouterr().out == f"{proto}\n" + df = spark.range(3) + assert df._plan.to_proto(spark.client).root == df._plan.plan(spark.client) + + def test_ml_plans_send_a_deep_dataframe_without_parsing_it_as_pyspark(mock_connect): # As PySpark's pyspark.ml, which builds its plans over the relation of a DataFrame without # parsing it: a DataFrame of 120 chained withColumn calls is nested deeper than a protobuf @@ -22201,17 +22260,30 @@ def test_objects_traversed_while_they_are_created_do_not_crash(mock_connect): # grouped data, the cogroups, the table arguments, the window specifications, the null and # statistics functions, the merge clauses, the stream writers, the data source registrations # and the session builders, and their copies, the rows of toLocalIterator, the expressions - # of Columns and table arguments, and their copies, and the array, map, field and struct - # types, which a schema of the server makes too. + # of Columns and table arguments, and their copies, the array, map, field and struct types, + # which a schema of the server makes too, and the sessions, the catalogs and the registrations + # of a Python subclass, which keeps its __dict__, and the objects that an unpickle makes + # without arguments. import copy import gc from pyspark.sql import Window from pyspark.sql import functions as F + from pyspark.sql.connect.catalog import Catalog from pyspark.sql.connect.client import DefaultChannelBuilder + from pyspark.sql.connect.datasource import DataSourceRegistration from pyspark.sql.connect.session import SparkSession as ConnectSession from pyspark.sql.observation import Observation + class SubSession(ConnectSession): + pass + + class SubCatalog(Catalog): + pass + + class SubRegistration(DataSourceRegistration): + pass + spark, _ = mock_connect df = spark.range(1) writer, writer_v2 = df.write, df.writeTo("t") @@ -22250,6 +22322,12 @@ def test_objects_traversed_while_they_are_created_do_not_crash(mock_connect): lambda: copy.deepcopy(kv), lambda: T.StructField("a", T.DecimalType(10, 2)), lambda: copy.deepcopy(field), lambda: T.StructType([T.StructField("a", T.LongType())]), lambda: copy.deepcopy(struct), lambda: struct.toNullable(), lambda: df.schema, + lambda: ConnectSession.__new__(ConnectSession), lambda: SubSession.__new__(SubSession), + lambda: Catalog(spark), lambda: SubCatalog(spark), lambda: copy.copy(SubCatalog(spark)), + lambda: SubRegistration(spark), lambda: copy.copy(SubRegistration(spark)), + lambda: DataSourceRegistration.__new__(DataSourceRegistration), + lambda: type(reader).__new__(type(reader)), + lambda: type(stream_reader).__new__(type(stream_reader)), ] thresholds = gc.get_threshold() gc.collect() @@ -29173,6 +29251,124 @@ def signal_then_release(): assert spark.is_stopped +def test_the_session_and_builder_locks_are_held_where_pyspark_holds_them(mock_connect): + # As PySpark's SparkSession._lock, an RLock that getOrCreate holds from its lookup to the + # options that it sets, _set_default_and_active_session (and so create) while it sets the + # default session, and stop all the while, and its SparkSession.Builder._lock, another RLock, + # that config, remote, appName, channelBuilder, _registerHook and the setting of the options + # of a session hold. They are class attributes, read at each call, so the lock that a program + # sets is the one held, as `with` holds it, of its __enter__ and its __exit__, which is given + # the exception that ends the block. A getOrCreate that creates a session holds both twice, + # as it sets the options in create and then again, which two requests send. + import threading + + from pyspark.sql.connect.client import DefaultChannelBuilder + from pyspark.sql.connect.session import SparkSession as ConnectSession + + spark, service = mock_connect + + class Recording: + def __init__(self): + self.lock = threading.RLock() + self.log = [] + + def __enter__(self): + self.log.append("enter") + return self.lock.__enter__() + + def __exit__(self, *exc): + self.log.append("exit" if exc[0] is None else exc[0].__name__) + return self.lock.__exit__(*exc) + + assert type(ConnectSession._lock) is type(threading.RLock()) + assert type(ConnectSession.Builder._lock) is type(threading.RLock()) + assert ConnectSession.Builder._lock is not ConnectSession._lock + assert ConnectSession.builder._lock is ConnectSession.Builder._lock + session_lock, builder_lock = Recording(), Recording() + + def held(call): + session_lock.log.clear() + builder_lock.log.clear() + try: + call() + except Exception: + pass + return session_lock.log[:], builder_lock.log[:] + + builder = ConnectSession.builder + sessions = [] + try: + ConnectSession._lock, ConnectSession.Builder._lock = session_lock, builder_lock + both = ["enter", "exit"] + for call in (lambda: builder.remote(service.remote), lambda: builder.config("k", "v"), + lambda: builder.config(map={"k": "w"}), lambda: builder.appName("a"), + lambda: builder._registerHook(lambda session: ConnectSession.Hook()), + lambda: ConnectSession.builder.channelBuilder( + DefaultChannelBuilder(service.remote))): + assert held(call) == ([], both) + assert held(lambda: builder.master("m")) == ([], []) + assert held(lambda: sessions.append(builder.getOrCreate())) == (both, both) + assert held(lambda: sessions.append(builder.create())) == (both, both) + assert held(sessions[-1].stop) == (both, []) + assert held(spark.stop) == (both, []) + sets = len(service.set_confs) + assert held(lambda: sessions.append(builder.getOrCreate())) == ( + ["enter", "enter", "exit", "exit"], both + both) + options = [("k", "w"), ("spark.app.name", "a")] + assert [pairs for pairs, _ in service.set_confs[sets:]] == [options, options] + assert held(lambda: ConnectSession._set_default_and_active_session(spark)) == (both, []) + failing = ConnectSession.builder.config("k\ud800", "v") + assert held(failing.getOrCreate) == ( + ["enter", "UnicodeEncodeError"], ["enter", "UnicodeEncodeError"]) + assert held(ConnectSession.builder.create) == ([], []) + assert held(lambda: sessions[-1].newSession()) == ([], []) + # A lock that is not a context manager fails as `with` does. + ConnectSession._lock = None + with pytest.raises(TypeError, match="^'NoneType' object does not support the context " + "manager protocol$"): + builder.getOrCreate() + finally: + ConnectSession._lock, ConnectSession.Builder._lock = session_lock.lock, builder_lock.lock + for session in sessions: + session.release_session_on_close = False + session.stop() + + +def test_a_thread_that_holds_the_session_lock_holds_off_the_builders_and_stops_as_pyspark( + mock_connect, +): + # As PySpark's SparkSession._lock, which a thread of the program may hold: getOrCreate, create + # and stop of another thread wait for it, without the GIL, and run once it is released. + import threading + + from pyspark.sql.connect.session import SparkSession as ConnectSession + + spark, service = mock_connect + other = ConnectSession.builder.remote(service.remote).create() + calls = { + "getOrCreate": lambda: ConnectSession.builder.remote(service.remote).getOrCreate(), + "create": lambda: ConnectSession.builder.remote(service.remote).create(), + "stop": other.stop, + } + made = [] + for name, call in calls.items(): + done = threading.Event() + + def run(call=call): + made.append(call()) + done.set() + + with ConnectSession._lock: + thread = threading.Thread(target=run, daemon=True) + thread.start() + assert not done.wait(0.5), f"{name} did not wait for the lock" + assert done.wait(5), f"{name} did not run once the lock was released" + thread.join(5) + assert made[0] is spark and made[2] is None and other.is_stopped + made[1].release_session_on_close = False + made[1].stop() + + def test_get_or_create_waits_for_a_stop_in_progress_and_creates_a_new_session_as_pyspark( mock_connect, ): @@ -34063,6 +34259,115 @@ def test_reader_and_stream_writer_attributes_survive_copy_and_pickle(mock_connec assert cloudpickle.loads(cloudpickle.dumps(obj)).foo == [1, 2] +def test_sessions_catalogs_and_registrations_subclass_as_pyspark(mock_connect): + # As PySpark Connect's SparkSession, Catalog and DataSourceRegistration, plain classes: a + # subclass constructs of the arguments of its own __init__, which calls that of the class of + # PySpark's arguments. A session of a subclass is of the user of its arguments, neither the + # default nor the active session, and the session that its DataFrames, readers and catalogs + # give back; a catalog and a registration of a subclass copy, deep-copy and pickle + # (cloudpickle, of the active session) of their class, with their attributes. A call without + # the arguments of the class raises CPython's TypeError, named after its __init__. + import copy + import gc + import inspect + import weakref + + import cloudpickle + import pyspark.sql.connect.proto as pb2 + from pyspark.errors.exceptions.connect import SparkConnectException + from pyspark.sql.connect.catalog import Catalog + from pyspark.sql.connect.datasource import DataSourceRegistration + from pyspark.sql.connect.session import SparkSession as ConnectSession + + spark, service = mock_connect + + class Tagged(ConnectSession): + def __init__(self, tag, connection): + super().__init__(connection, userId="tagged") + self.tag = tag + + class Plain(ConnectSession): + pass + + assert list(inspect.signature(ConnectSession).parameters) == [ + "connection", "userId", "hook_factories"] + for cls in (ConnectSession, Plain): + with pytest.raises(TypeError) as excinfo: + cls() + assert str(excinfo.value) == ( + "SparkSession.__init__() missing 1 required positional argument: 'connection'") + sessions = [Tagged("t", service.remote), Plain(connection=service.remote, userId="plain")] + try: + for session, cls, user in zip(sessions, (Tagged, Plain), ("tagged", "plain")): + assert type(session) is cls and session is not ConnectSession.getActiveSession() + assert ConnectSession._default_session is not session + df = session.range(1) + assert df.sparkSession is session and session.read._client is session + assert session.catalog._sparkSession is session + assert session.dataSource.sparkSession is session + df.collect() + sent = pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]) + assert sent.user_context.user_id == user + assert sessions[0].tag == "t" + finally: + for session in sessions: + session.release_session_on_close = False + session.stop() + + # A session whose __init__ never ran has no client: its requests raise NO_ACTIVE_SESSION, as + # those of a stopped session do, where PySpark's raise the AttributeError of its _client. + class Unconnected(ConnectSession): + def __init__(self, tag): + self.tag = tag + + unconnected = Unconnected("u") + assert unconnected.is_stopped and unconnected.tag == "u" + with pytest.raises(SparkConnectException) as raised: + unconnected.range(1).collect() + assert raised.value.getCondition() == "NO_ACTIVE_SESSION" + + class SubCatalog(Catalog): + def __init__(self, x, session): + super().__init__(session) + self.x = x + + class SubRegistration(DataSourceRegistration): + def __init__(self, session, *, x): + super().__init__(session) + self.x = x + + assert list(inspect.signature(Catalog).parameters) == ["sparkSession"] + assert list(inspect.signature(DataSourceRegistration).parameters) == ["sparkSession"] + for cls in (Catalog, DataSourceRegistration): + with pytest.raises(TypeError) as excinfo: + cls() + assert str(excinfo.value) == ( + f"{cls.__name__}.__init__() missing 1 required positional argument: 'sparkSession'") + for obj in (SubCatalog(1, spark), SubCatalog(x=1, session=spark), + SubRegistration(spark, x=1)): + for copied in (obj, copy.copy(obj), copy.deepcopy(obj), + pickle.loads(cloudpickle.dumps(obj))): + assert type(copied) is type(obj) and copied.x == 1 + if isinstance(copied, Catalog): + assert copied._sparkSession is spark + else: + assert copied.sparkSession is spark + # A registration whose __init__ never ran has no session, as PySpark's. + with pytest.raises(AttributeError, match="has no attribute 'sparkSession'"): + DataSourceRegistration.__new__(DataSourceRegistration).sparkSession + with pytest.raises(AttributeError, match="has no attribute 'sparkSession'"): + DataSourceRegistration.__new__(DataSourceRegistration).register(object) + # A cycle through the __dict__ of an object of a subclass is collected, as PySpark's. + for make in (lambda: Plain(service.remote), lambda: SubCatalog(1, spark), + lambda: SubRegistration(spark, x=1)): + obj = make() + obj.me = obj + freed = weakref.ref(obj) + del obj + gc.collect() + assert freed() is None + + from pyspark.sql.connect.readwriter import DataFrameReader as _ConnectReader # noqa: E402 from pyspark.sql.connect.streaming.readwriter import ( # noqa: E402 DataStreamReader as _ConnectStreamReader, @@ -34120,6 +34425,72 @@ def test_readers_construct_of_a_session_and_subclass_as_pyspark(mock_connect, su assert reader._format is None and reader._options == {} and reader.extra is None +def test_pyspark_pickles_of_readers_catalogs_and_registrations_load_as_pyspark(mock_connect): + # As PySpark's pickle of its plain objects, the class and the state of its __dict__, which an + # unpickle sets on the object that the class makes without arguments: a reader of PySpark is + # of its _client, _format, _schema and _options (and the _source_name of a stream reader), + # which it takes as its setters do, a catalog of its _sparkSession and a registration of its + # sparkSession, and the other entries are their attributes. Its session unpickles as the + # active session of its ID, as PySpark's does (cloudpickle, of the creator of the session). + import copyreg + + import cloudpickle + import pyspark.sql.connect.proto as pb2 + from pyspark.sql.connect.catalog import Catalog + from pyspark.sql.connect.datasource import DataSourceRegistration + from pyspark.sql.connect.readwriter import DataFrameReader + from pyspark.sql.connect.streaming.readwriter import DataStreamReader + + spark, service = mock_connect + + class PySparkObject: + # PySpark's object of `cls` of the __dict__ `state`, whose pickle is that of a plain + # object: `copyreg.__newobj__` of its class, which pickle checks against `__class__`, + # and its __dict__. + def __init__(self, cls, state): + self.cls, self.state = cls, state + + @property + def __class__(self): + return self.cls + + def __reduce__(self): + return copyreg.__newobj__, (self.cls,), self.state + + def loaded(cls, **state): + return pickle.loads(cloudpickle.dumps(PySparkObject(cls, state))) + + reader = loaded(DataFrameReader, _client=spark, _format="csv", _schema="a INT", + _options={"k": "v", "n": None}, extra=[1]) + assert type(reader) is DataFrameReader and reader._client is spark + assert (reader._format, reader._schema, reader._options) == ("csv", "a INT", {"k": "v"}) + assert reader.extra == [1] + reader.load("p").collect() + # ExecutePlanRequest.plan=3, Plan.root=1, Relation.read=4, Read.data_source=2: + # DataSource.format=1, schema=2, options=3 and paths=4. + read = pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]).plan.root.read + source = read.data_source + assert (source.format, source.schema, dict(source.options), list(source.paths)) == ( + "csv", "a INT", {"k": "v"}, ["p"]) + stream = loaded(DataStreamReader, _format="rate", _schema="", _client=spark, + _options={"rowsPerSecond": "5"}, _source_name="src_1") + assert type(stream) is DataStreamReader and stream._client is spark + assert (stream._format, stream._schema, stream._options, stream._source_name) == ( + "rate", "", {"rowsPerSecond": "5"}, "src_1") + catalog = loaded(Catalog, _sparkSession=spark, extra=2) + assert type(catalog) is Catalog and catalog._sparkSession is spark and catalog.extra == 2 + registration = loaded(DataSourceRegistration, sparkSession=spark) + assert type(registration) is DataSourceRegistration and registration.sparkSession is spark + assert vars(registration) == {} + # Without an active session, the session of the pickle raises NO_ACTIVE_SESSION, as + # PySpark's. + pickled = cloudpickle.dumps(PySparkObject(DataFrameReader, {"_client": spark, "_options": {}})) + spark.stop() + with pytest.raises(Exception) as raised: + pickle.loads(pickled) + assert raised.value.getCondition() == "NO_ACTIVE_SESSION" + + def test_a_reader_subclass_pickle_is_of_its_class_on_a_worker_as_pyspark(monkeypatch): # The pickle of a reader of a subclass names the subclass, which a worker without the # extension, official PySpark's, resolves to a subclass of PySpark's reader: the reader that @@ -34521,6 +34892,69 @@ def test_plain_pickles_name_the_session_creator_as_pyspark(mock_connect): pickle.dumps(obj) +def test_shallow_copies_of_a_builder_share_its_options_and_hooks_as_pyspark(mock_connect): + # As PySpark's copy.copy of its builder, a plain object whose __dict__ the copy shares: the + # copy shares the _options dict and the _hook_factories list, so a remote, a config or a + # _registerHook of either one is seen by both, and a copy of a copy shares them too. Its + # channel builder is an attribute, which a channelBuilder of either one rebinds on that one + # alone. A deep copy and a pickle have options of their own. + import copy + + import cloudpickle + from pyspark.errors import PySparkValueError + from pyspark.sql.connect.client import DefaultChannelBuilder + from pyspark.sql.connect.session import SparkSession as ConnectSession + + _, service = mock_connect + made = [] + + def factory(name): + def make(session): + made.append(name) + return ConnectSession.Hook() + return make + + def created(builder): + # The pairs that the session that the builder creates is given, and its hooks. + sets = len(service.set_confs) + made.clear() + session = builder.create() + session.release_session_on_close = False + session.stop() + return [pair for pairs, _ in service.set_confs[sets:] for pair in pairs], list(made) + + def assert_no_url(builder): + with pytest.raises(PySparkValueError) as error: + builder.create() + assert error.value.getCondition() == "SESSION_NEED_CONN_STR_OR_BUILDER" + + builder = ConnectSession.builder + copied = copy.copy(builder) + copied.remote(service.remote).config("k1", "v1")._registerHook(factory("copy")) + builder.config("k2", "v2")._registerHook(factory("builder")) + expected = ([("k1", "v1"), ("k2", "v2")], ["copy", "builder"]) + assert created(builder) == expected and created(copied) == expected + copy.copy(copied).config("k3", "v3") + assert created(builder)[0] == [("k1", "v1"), ("k2", "v2"), ("k3", "v3")] + for independent in (copy.deepcopy(builder), cloudpickle.loads(cloudpickle.dumps(builder))): + independent.config("k4", "v4") + assert ("k4", "v4") not in created(builder)[0] + + builder = ConnectSession.builder + copied = copy.copy(builder) + copied.channelBuilder(DefaultChannelBuilder(service.remote)) + assert_no_url(builder) + assert created(copied) == ([], []) + # A copy of a builder of a channel builder has it too, and a remote of either one is both + # builders' URL, which neither can take with a channel builder. + builder = ConnectSession.builder.channelBuilder(DefaultChannelBuilder(service.remote)) + copied = copy.copy(builder) + assert created(copied) == ([], []) + copied.remote(service.remote) + assert_no_url(builder) + assert_no_url(copied) + + def test_session_builders_copy_and_pickle_as_pyspark(mock_connect): # As PySpark's Builder, whose state is its options: a copy and a pickle of a builder keep # them, and the unpickled builder is a builder of the same class.