From fb0741927b0259082dc911c0eae03ece00140d33 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Wed, 18 Nov 2026 20:12:55 +0000 Subject: [PATCH] [SPARK-62023][CONNECT][PYTHON] Make SparkSession.client PySpark's SparkConnectClient: its state per session, a threading.local of live tags and extensions, PySpark's argument conversions and error order in its registrations and executions, and the members PySpark's tests read The four commits of the PR, squashed for the rebase: the per-session client state and thread_local (C1, C10), the registrations' argument conversions and order (C2, C3), PySpark's _session_id, _closed, _user_id, _use_reattachable_execute, _get_operation_statuses and _builder (C6, C7, N5), and the offline tests that keep the Python coverage above its floor. Co-authored-by: Isaac --- crates/pyspark-rs/src/client.rs | 4 +- crates/pyspark-rs/src/interrupt.rs | 16 + crates/pyspark-rs/src/session.rs | 406 ++++++++----- crates/spark-connect-core/src/channel.rs | 13 + crates/spark-connect-core/src/client.rs | 367 ++++++++++- .../tests/artifact_uploads.rs | 23 + .../tests/server_session.rs | 147 +++++ crates/spark-connect/src/session.rs | 142 ++++- dev/parity/known-differences.md | 32 +- docs/compatibility.md | 65 +- python/pyspark/sql/connect/client/__init__.py | 574 +++++++++++++++--- python/pyspark/sql/udf.py | 41 +- python/pyspark/util.py | 33 +- python/tests/_mock_connect_server.py | 18 + python/tests/test_dropin_e2e.py | 157 +++++ python/tests/test_dropin_offline.py | 553 ++++++++++++++++- python/tests/test_ml_offline.py | 7 +- 17 files changed, 2233 insertions(+), 365 deletions(-) diff --git a/crates/pyspark-rs/src/client.rs b/crates/pyspark-rs/src/client.rs index 6a0173d8..eb917418 100644 --- a/crates/pyspark-rs/src/client.rs +++ b/crates/pyspark-rs/src/client.rs @@ -318,8 +318,7 @@ pub(crate) fn set_retry_policy(session: &SparkSession, parameters: RetryParamete } /// The RPC deadlines of `deadlines`, the deadlines in seconds by the names of the fields of -/// PySpark's `RpcDeadlines`, None for no deadline. A field that the transport has no RPC of, as -/// `get_status`, is not read. +/// PySpark's `RpcDeadlines`, None for no deadline. pub(crate) fn rpc_deadlines(deadlines: &Bound<'_, PyDict>) -> PyResult { let of = |name: &str| -> PyResult> { let seconds: Option = match deadlines.get_item(name)? { @@ -344,6 +343,7 @@ pub(crate) fn rpc_deadlines(deadlines: &Bound<'_, PyDict>) -> PyResult>>( run_interruptible(py, session, f, true) } +/// Bring the user context extensions of `session` up to date before a call sends its requests, +/// on the calling thread, with the GIL: the function of `client` that the session keeps (see +/// `PySparkSession::_set_user_context_refresh`), which serializes the `google.protobuf.Any` +/// messages of its lists again, as PySpark serializes them when it builds each request, so that +/// a change of a message in place is sent. Its error is raised, as PySpark raises that of the +/// serialization from the call. +pub(crate) fn refresh_user_context(py: Python<'_>, session: &SparkSession) -> PyResult<()> { + if let Some(refresher) = session.user_context_refresher() { + if let Some(refresh) = refresher.downcast_ref::>() { + refresh.call0(py)?; + } + } + Ok(()) +} + /// The wait of [`interruptible_outcome`], of whose results a failed interrupt is the inner one /// when `catch_interrupt_failure`. fn run_interruptible>>( @@ -246,6 +261,7 @@ fn run_interruptible>>( f: impl FnOnce() -> Result + Send, catch_interrupt_failure: bool, ) -> PyResult>> { + refresh_user_context(py, session)?; let watch = Arc::new(SignalWatch { session: session.clone(), interrupted: AtomicBool::new(false), diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index c4a7a70e..25c8afd2 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -1211,6 +1211,101 @@ pub struct PySharedTags(spark_connect::session::SharedTags); )] pub struct PySharedUserContextExtensions(spark_connect_core::client::SharedUserContextExtensions); +#[pymethods] +impl PySharedTags { + /// Replace the tags of this set with `tags`, which the requests of the threads that have it + /// carry from now on, as a change of the set of PySpark's `thread_local.tags`. + fn _replace(&self, tags: Vec>) -> PyResult<()> { + self.0.replace(core_tags(&tags)?); + Ok(()) + } + + /// The tags of this set, as the strs that they are of. + fn _tags<'py>(&self, py: Python<'py>) -> PyResult>> { + self.0 + .tags() + .into_iter() + .map(|tag| match tag { + spark_connect::session::Tag::Text(text) => { + Ok(pyo3::types::PyString::new(py, &text).into_any()) + } + spark_connect::session::Tag::Unencodable(utf32le) => { + pyo3::types::PyBytes::new(py, &utf32le) + .call_method1("decode", ("utf-32-le", "surrogatepass")) + } + }) + .collect() + } +} + +#[pymethods] +impl PySharedUserContextExtensions { + /// Replace the extensions of this list with `extensions`, which the requests of the threads + /// that have it carry from now on, as a change of the list of PySpark's + /// `thread_local.user_context_extensions`. + fn _replace(&self, extensions: Vec<(String, Vec)>) -> PyResult<()> { + self.0.replace(decoded_extensions(extensions)?); + Ok(()) + } + + /// The extensions of this list, their IDs and the bytes of each `google.protobuf.Any`. + fn _extensions<'py>(&self, py: Python<'py>) -> Vec<(String, Bound<'py, pyo3::types::PyBytes>)> { + use prost::Message; + self.0 + .snapshot() + .into_iter() + .map(|(id, extension)| { + ( + id, + pyo3::types::PyBytes::new(py, &extension.encode_to_vec()), + ) + }) + .collect() + } +} + +/// The `google.protobuf.Any` of each of `extensions`, of its ID and its bytes. +fn decoded_extensions( + extensions: Vec<(String, Vec)>, +) -> PyResult> { + use prost::Message; + extensions + .into_iter() + .map(|(id, extension)| { + prost_types::Any::decode(extension.as_slice()) + .map(|extension| (id, extension)) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string())) + }) + .collect() +} + +/// The core tags of `tags`, the elements of the set of PySpark's `thread_local.tags`, as the +/// requests carry them: a str of its text, or of the UTF-32 bytes of a str that UTF-8 cannot +/// encode, with which each request fails, as PySpark's; bytes of their UTF-8; and the `TypeError` +/// of the protobuf string field of the tags of a request for another object, which PySpark raises +/// when it builds a request, and the drop-in when the set changes. +fn core_tags(tags: &[Bound<'_, PyAny>]) -> PyResult> { + tags.iter() + .map(|tag| { + if let Ok(text) = tag.cast::() { + return Ok(match text.to_cow() { + Ok(text) => spark_connect::session::Tag::Text(text.into_owned()), + Err(_) => { + let utf32le = + text.call_method1("encode", ("utf-32-le", "surrogatepass"))?; + spark_connect::session::Tag::Unencodable( + utf32le.cast::()?.as_bytes().to_vec(), + ) + } + }); + } + Ok(spark_connect::session::Tag::Text( + crate::options::str_argument(tag)?, + )) + }) + .collect() +} + /// The tag of `interruptTag` or the ID of `interruptOperation`, as PySpark's `_interrupt_request` /// takes it: it asserts that it is not None, and sets it in the string field of its request, as /// protobuf converts it, so bytes of UTF-8 are taken, and another value is a `TypeError`. @@ -1221,46 +1316,6 @@ fn interrupted_id(id: &Bound<'_, PyAny>) -> PyResult { crate::str_argument(id) } -/// A tag of `addTag` or `removeTag`: `CANNOT_BE_NONE` for None, as PySpark Connect's. -fn tag_of(py: Python<'_>, tag: &Bound<'_, PyAny>) -> PyResult { - if tag.is_none() { - return Err(crate::column::raise_pyspark( - py, - "PySparkValueError", - "CANNOT_BE_NONE", - &[("arg_name", "Spark Connect tag")], - )); - } - // As PySpark's `"," in tag`, which raises the `TypeError` of an object that is not a str, - // such as an int or bytes, before the checks of the tag; and of the set of the tags, which - // refuses one that is not hashable, such as a list. - tag.contains(",")?; - let tag = if let Ok(text) = tag.cast::() { - text - } else { - tag.hash()?; - return Ok(SessionTag::Text(tag.extract()?)); - }; - let Ok(text) = tag.to_cow() else { - // A str that UTF-8 cannot encode, which PySpark keeps, and which fails the requests that - // carry it. Its rendering has a comma if the str does, and is not empty. - let utf32le = tag.call_method1("encode", ("utf-32-le", "surrogatepass"))?; - let rendered = crate::plan_args::rendered(tag.as_any())?; - SparkSession::check_tag(&rendered).to_pyerr()?; - return Ok(SessionTag::Unencodable( - utf32le.cast::()?.as_bytes().to_vec(), - )); - }; - Ok(SessionTag::Text(text.into_owned())) -} - -/// A tag of `addTag` and `removeTag`: its text, or the UTF-32 bytes of a str that UTF-8 cannot -/// encode, which the core keeps (see `SparkSession::add_unencodable_tag`). -enum SessionTag { - Text(String), - Unencodable(Vec), -} - #[pyclass( name = "SparkSession", module = "pyspark.sql.connect.session", @@ -2907,25 +2962,28 @@ impl PySparkSession { } /// Copy a local file to the Spark-managed filesystem (`copyFromLocalToFs`), with the errors - /// of PySpark Connect's: `NO_SCHEMA_AND_DRIVER_DEFAULT_SCHEME` for a destination of a scheme, - /// an `AssertionError` for a local path that is not a file, and for a destination that is - /// not an absolute path. The local path is what PySpark's `Path(local_path).absolute()` takes, - /// a str or an `os.PathLike` such as a `pathlib.Path`, of its errors otherwise, and the file - /// of the str of that absolute path is uploaded, as PySpark's `LocalFile` reads it. The - /// destination is a str. + /// of PySpark Connect's: those of `urlparse` of the destination, such as the AttributeError + /// of a `pathlib.Path`, and `NO_SCHEMA_AND_DRIVER_DEFAULT_SCHEME` for a destination of a + /// scheme, and then those of its client's `copy_from_local_to_fs`: an `AssertionError` for a + /// local path that is not a file, and for a destination that is not an absolute path, and the + /// `TypeError` of a destination that is not a str. The local path is what PySpark's + /// `Path(local_path).absolute()` takes, a str or an `os.PathLike` such as a `pathlib.Path`, of + /// its errors otherwise, and the file of the str of that absolute path is uploaded, as + /// PySpark's `LocalFile` reads it. #[pyo3(name = "copyFromLocalToFs")] fn copy_from_local_to_fs( - &self, - py: Python<'_>, + slf: &Bound<'_, Self>, local_path: &Bound<'_, PyAny>, - dest_path: &str, + dest_path: &Bound<'_, PyAny>, ) -> PyResult<()> { - let scheme: String = py + let py = slf.py(); + // As PySpark's, which parses the destination first, of the errors of `urlparse`, such as + // the AttributeError of a `pathlib.Path`, and then copies the file with its client. + let scheme = py .import("urllib.parse")? .call_method1("urlparse", (dest_path,))? - .getattr("scheme")? - .extract()?; - if !scheme.is_empty() { + .getattr("scheme")?; + if scheme.is_truthy()? { return Err(crate::column::raise_pyspark( py, "PySparkValueError", @@ -2933,32 +2991,22 @@ impl PySparkSession { &[("arg_name", "dest_path")], )); } - let absolute = py - .import("pathlib")? - .getattr("Path")? - .call1((local_path,))? - .call_method0("absolute")?; - let is_file: bool = absolute.call_method0("is_file")?.extract()?; - if !is_file { - return Err(pyo3::exceptions::PyAssertionError::new_err( - "local path must be a file path.", - )); - } - let is_absolute: bool = py - .import("pathlib")? - .getattr("Path")? - .call1((dest_path,))? - .call_method0("is_absolute")? - .extract()?; - if !is_absolute { - return Err(pyo3::exceptions::PyAssertionError::new_err( - "destination FS path must be an absolute path.", - )); - } - let local_path: String = absolute.str()?.extract()?; + Self::client(slf)?.call_method1(py, "copy_from_local_to_fs", (local_path, dest_path))?; + Ok(()) + } + + /// Upload the local file of `local_path`, an absolute path, to `dest_path`, an absolute path + /// of the file system of the server, for the `copy_from_local_to_fs` of `client`, which + /// checks them as PySpark's ArtifactManager does. + fn _copy_from_local_to_fs( + &self, + py: Python<'_>, + local_path: String, + dest_path: String, + ) -> PyResult<()> { interruptible(py, &self.session, || { self.session - .copy_from_local_to_fs(&local_path, dest_path) + .copy_from_local_to_fs(&local_path, &dest_path) .map_err(Box::new) }) } @@ -3226,7 +3274,7 @@ impl PySparkSession { py: Python<'_>, name: &Bound<'_, PyAny>, return_type: Option<&Bound<'_, PyAny>>, - eval_type: i32, + eval_type: &Bound<'_, PyAny>, command: &Bound<'_, PyAny>, python_ver: String, deterministic: bool, @@ -3235,7 +3283,8 @@ impl PySparkSession { // first, as protobuf converts it: bytes of UTF-8 too. Its `PythonUDTF.to_plan` converts the // return type after it (see `udtf_return_type`), which raises the `UNSUPPORTED_OPERATION` // of a value that is not a DataType and of a type that `pyspark_types_to_proto_types` has - // no proto of, also nested, and then pickles the handler class, whose error comes last. + // no proto of, also nested, then sets the eval type in the int32 field of its PythonUDTF, + // of protobuf's errors, and then pickles the handler class, whose error comes last. let name = crate::str_argument(name)?; let return_type = match return_type { None => None, @@ -3244,6 +3293,12 @@ impl PySparkSession { .ok_or_else(|| crate::types::not_a_type_error(value))?, ), }; + let udtf = py + .import("pyspark.sql.connect.proto")? + .getattr("PythonUDTF")? + .call0()?; + udtf.setattr("eval_type", eval_type)?; + let eval_type: i32 = udtf.getattr("eval_type")?.extract()?; let command: Vec = if command.is_callable() { command.call0()?.extract()? } else { @@ -3261,24 +3316,9 @@ impl PySparkSession { }) } - /// Add the extension of the bytes of a `google.protobuf.Any`, `extension`, to the user context - /// of the requests of this session: of all the threads, if `is_global`, else of the current - /// thread. Its ID: for the user context extensions of `client`. - fn _add_user_context_extension(&self, extension: &[u8], is_global: bool) -> PyResult { - use prost::Message; - let extension = prost_types::Any::decode(extension) - .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?; - Ok(if is_global { - self.session.add_global_user_context_extension(extension) - } else { - self.session - .add_threadlocal_user_context_extension(extension) - }) - } - /// The user context extensions of this session, the global ones if `is_global`, else those - /// of the current thread, as their IDs and the bytes of each `google.protobuf.Any`: for - /// `client`. + /// of the current thread, as their IDs and the bytes of each `google.protobuf.Any`: for the + /// global list of `client` when it is made. fn _user_context_extensions<'py>( &self, py: Python<'py>, @@ -3308,30 +3348,11 @@ impl PySparkSession { &self, extensions: Vec<(String, Vec)>, ) -> PyResult<()> { - use prost::Message; - let extensions = extensions - .into_iter() - .map(|(id, extension)| { - prost_types::Any::decode(extension.as_slice()) - .map(|extension| (id, extension)) - .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string())) - }) - .collect::>>()?; - self.session.set_global_user_context_extensions(extensions); + self.session + .set_global_user_context_extensions(decoded_extensions(extensions)?); Ok(()) } - /// Remove the user context extension of `extension_id`, for `client`. - fn _remove_user_context_extension(&self, extension_id: &str) { - self.session.remove_user_context_extension(extension_id); - } - - /// Remove the user context extensions of the current thread and the global ones, for - /// `client`. - fn _clear_user_context_extensions(&self) { - self.session.clear_user_context_extensions(); - } - /// The responses of the server to `command`, the bytes of a `spark.connect.Command` built /// outside the client, such as the ML commands of `pyspark.ml`, as the bytes of each /// `ExecutePlanResponse`: for the `execute_command` of `client`. The command is decoded, and @@ -3669,6 +3690,64 @@ impl PySparkSession { self.session.set_server_session_id(id); } + /// The session ID that the requests of the session carry, as PySpark's + /// `SparkConnectClient._session_id`: the session's, unless `_set_client_session_id` set + /// another. + fn _client_session_id(&self) -> String { + self.session.client_session_id() + } + + /// Make the requests of the session carry `session_id`, as a set of PySpark's + /// `SparkConnectClient._session_id`, which its requests read, of the error of the string + /// field of their session ID for a value that is not a str: the session's `session_id` stays, + /// and the requests of the artifacts carry it, as those of PySpark's ArtifactManager. + fn _set_client_session_id(&self, session_id: &Bound<'_, PyAny>) -> PyResult<()> { + self.session + .set_client_session_id(crate::str_argument(session_id)?); + Ok(()) + } + + /// The connection string of the channel builder of the session, as PySpark's + /// `SparkConnectClient._builder` is of: `sc://` and its endpoint, of its parameters, each + /// value percent-encoded, as `DefaultChannelBuilder` decodes them. + fn _connection_string(&self) -> String { + let builder = self.session.channel_builder(); + let mut url = format!("sc://{}/", builder.endpoint()); + for (key, value) in builder.params() { + url.push(';'); + url.push_str(key); + url.push('='); + url.push_str("e_param(value)); + } + url + } + + /// The user of the requests of the session, as PySpark's `SparkConnectClient._user_id`. + fn _user_id(&self) -> Option { + self.session.user_id().map(str::to_string) + } + + /// Open the session again, as PySpark's `SparkConnectClient._closed = False`. + fn _reopen(&self) { + self.session.reopen() + } + + /// The bytes of the response of the server to `request`, the bytes of a `GetStatusRequest` + /// that `client` builds, as PySpark's `_get_operation_statuses` sends it. + fn _get_status<'py>( + &self, + py: Python<'py>, + request: &[u8], + ) -> PyResult> { + use prost::Message; + let request = spark_connect_proto::GetStatusRequest::decode(request) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?; + let response = interruptible(py, &self.session, || { + self.session.get_status_proto(request).map_err(Box::new) + })?; + Ok(pyo3::types::PyBytes::new(py, &response.encode_to_vec())) + } + /// The execution hook base class, reached as `SparkSession.Hook` (mirrors pyspark). #[classattr] #[allow(non_snake_case)] @@ -3875,29 +3954,19 @@ impl PySparkSession { }) } - /// Add a tag of the current thread, with the errors of PySpark Connect's tags: `CANNOT_BE_NONE` - /// for None, and those of [`SparkSession::check_tag`]. + /// Add a tag of the current thread, as PySpark Connect's `addTag`, which its client adds to + /// the set of its `thread_local.tags` (`client.add_tag`), of the errors of PySpark's. #[pyo3(name = "addTag")] - fn add_tag(&self, py: Python<'_>, tag: &Bound<'_, PyAny>) -> PyResult<()> { - match tag_of(py, tag)? { - SessionTag::Text(tag) => self.session.add_tag(&tag).to_pyerr(), - SessionTag::Unencodable(utf32le) => { - self.session.add_unencodable_tag(utf32le); - Ok(()) - } - } + fn add_tag(slf: &Bound<'_, Self>, tag: &Bound<'_, PyAny>) -> PyResult<()> { + Self::client(slf)?.call_method1(slf.py(), "add_tag", (tag,))?; + Ok(()) } - /// Remove a tag of the current thread, with the errors of `addTag`, as PySpark Connect's. + /// Remove a tag of the current thread, as PySpark Connect's `removeTag`, of its client + /// (`client.remove_tag`). #[pyo3(name = "removeTag")] - fn remove_tag(&self, py: Python<'_>, tag: &Bound<'_, PyAny>) -> PyResult<()> { - match tag_of(py, tag)? { - SessionTag::Text(tag) => { - SparkSession::check_tag(&tag).to_pyerr()?; - self.session.remove_tag(&tag); - } - SessionTag::Unencodable(utf32le) => self.session.remove_unencodable_tag(&utf32le), - } + fn remove_tag(slf: &Bound<'_, Self>, tag: &Bound<'_, PyAny>) -> PyResult<()> { + Self::client(slf)?.call_method1(slf.py(), "remove_tag", (tag,))?; Ok(()) } @@ -3920,23 +3989,20 @@ impl PySparkSession { }) } - /// The tags of the current thread, a set, as PySpark's `getTags`. + /// The tags of the current thread, as PySpark Connect's `getTags`: the set of the + /// `thread_local.tags` of its client (`client.get_tags`), which the later changes of the tags + /// change too. #[pyo3(name = "getTags")] - fn get_tags<'py>(&self, py: Python<'py>) -> PyResult> { - let tags = pyo3::types::PySet::new(py, self.session.get_tags())?; - // A tag that UTF-8 cannot encode, as it was given. - for utf32le in self.session.unencodable_tags() { - tags.add( - pyo3::types::PyBytes::new(py, &utf32le) - .call_method1("decode", ("utf-32-le", "surrogatepass"))?, - )?; - } - Ok(tags) + fn get_tags(slf: &Bound<'_, Self>) -> PyResult> { + Self::client(slf)?.call_method0(slf.py(), "get_tags") } + /// Remove the tags of the current thread, as PySpark Connect's `clearTags`, of its client + /// (`client.clear_tags`), which gives the thread a new empty set. #[pyo3(name = "clearTags")] - fn clear_tags(&self) { - self.session.clear_tags() + fn clear_tags(slf: &Bound<'_, Self>) -> PyResult<()> { + Self::client(slf)?.call_method0(slf.py(), "clear_tags")?; + Ok(()) } /// A copy of the tags of the current thread, which `pyspark.util.inheritable_thread_target` @@ -3968,6 +4034,48 @@ impl PySparkSession { .share_threadlocal_user_context_extensions(&extensions.0) } + /// A set of the tags `tags`, which no thread has yet, for a set of the `thread_local.tags` of + /// `client`. + fn _new_tags(&self, tags: Vec>) -> PyResult { + Ok(PySharedTags(spark_connect::session::SharedTags::new( + core_tags(&tags)?, + ))) + } + + /// A list of `extensions`, their IDs and the bytes of each `google.protobuf.Any`, which no + /// thread has yet, for a list of the `thread_local.user_context_extensions` of `client`. + fn _new_user_context_extensions( + &self, + extensions: Vec<(String, Vec)>, + ) -> PyResult { + Ok(PySharedUserContextExtensions( + spark_connect_core::client::SharedUserContextExtensions::new(decoded_extensions( + extensions, + )?), + )) + } + + /// Keep `refresh`, a function of no arguments, which a call of this session calls before it + /// sends requests (see `crate::interrupt::refresh_user_context`): for `client`, which + /// serializes the `google.protobuf.Any` messages of its lists of user context extensions + /// again then, as PySpark serializes them when it builds each request. + fn _set_user_context_refresh(&self, refresh: Py) { + self.session + .set_user_context_refresher(Some(std::sync::Arc::new(refresh))); + } + + /// Forget the tags of the current thread, which has none from now on, for a + /// `thread_local.tags` of `client` that is deleted. + fn _clear_thread_tags(&self) { + self.session.clear_tags() + } + + /// Forget the user context extensions of the current thread, for a + /// `thread_local.user_context_extensions` of `client` that is deleted. + fn _clear_thread_user_context_extensions(&self) { + self.session.clear_threadlocal_user_context_extensions() + } + /// Add local files, or `file:` URIs, as artifacts of the session, as PySpark's /// `addArtifacts(*path, pyfile=False, archive=False, file=False)`: each path is named by /// the flags and its extension, and the errors for a path that is not supported are diff --git a/crates/spark-connect-core/src/channel.rs b/crates/spark-connect-core/src/channel.rs index f39e3377..a4e672b4 100644 --- a/crates/spark-connect-core/src/channel.rs +++ b/crates/spark-connect-core/src/channel.rs @@ -170,6 +170,19 @@ impl ChannelBuilder { self.params.insert(key.to_string(), value.to_string()); } + /// Remove the parameter `key`, as PySpark's `newSession` pops the session ID of the channel + /// builder of its client. + pub fn remove(&mut self, key: &str) { + self.params.remove(key); + } + + /// The parameters of the connection, by their keys, in their order. + pub fn params(&self) -> impl Iterator { + self.params + .iter() + .map(|(key, value)| (key.as_str(), value.as_str())) + } + /// `host:port` endpoint used to dial the gRPC channel. pub fn endpoint(&self) -> String { format!("{}:{}", self.host, self.port) diff --git a/crates/spark-connect-core/src/client.rs b/crates/spark-connect-core/src/client.rs index 6676e872..61f00aed 100644 --- a/crates/spark-connect-core/src/client.rs +++ b/crates/spark-connect-core/src/client.rs @@ -20,8 +20,9 @@ use spark_connect_proto::{ analyze_plan_request::Analyze, AnalyzePlanRequest, AnalyzePlanResponse, ArtifactStatusesRequest, CloneSessionRequest, CompressionCodec, ConfigRequest, ConfigResponse, ExecutePlanRequest, ExecutePlanResponse, FetchErrorDetailsRequest, FetchErrorDetailsResponse, - InterruptRequest, InterruptResponse, Plan, ReattachExecuteRequest, ReleaseExecuteRequest, - ReleaseExecuteResponse, ReleaseSessionRequest, UserContext, + GetStatusRequest, GetStatusResponse, InterruptRequest, InterruptResponse, Plan, + ReattachExecuteRequest, ReleaseExecuteRequest, ReleaseExecuteResponse, ReleaseSessionRequest, + UserContext, }; use crate::artifact::{ @@ -502,6 +503,7 @@ pub struct RpcDeadlines { pub artifact_status: Option, pub fetch_error_details: Option, pub clone_session: Option, + pub get_status: Option, } impl Default for RpcDeadlines { @@ -519,6 +521,7 @@ impl Default for RpcDeadlines { artifact_status: minutes(10), fetch_error_details: minutes(10), clone_session: minutes(10), + get_status: minutes(10), } } } @@ -537,6 +540,7 @@ impl RpcDeadlines { artifact_status: None, fetch_error_details: None, clone_session: None, + get_status: None, } } } @@ -760,6 +764,15 @@ pub struct SparkConnectClient { stub: SparkConnectServiceClient, /// Session ID (UUID v4) identifying this session on the server. session_id: String, + /// The session ID that the requests carry and the responses are checked against, shared by + /// the clones of the client: `session_id` until a caller sets another, as PySpark's + /// `client._session_id` is an attribute that its requests read (see + /// [`SparkConnectClient::set_client_session_id`]). Those of the artifacts carry + /// `session_id`, as PySpark's ArtifactManager keeps the ID of the client when it was built. + client_session_id: std::sync::Arc>, + /// The channel builder of the connection, as PySpark's `client._builder`: of no session ID + /// for the client of a new session, and of its own for that of a clone. + builder: crate::channel::ChannelBuilder, /// User context (user_id from the channel builder). user_id: Option, /// The metadata of the connection string, and the `authorization` of its token. @@ -799,9 +812,10 @@ pub struct SparkConnectClient { /// client, as PySpark's `_server_session_id`. Each request sends it as the observed ID, and /// each response is checked against it. server_session_id: std::sync::Arc>>, - /// Whether the client tracks its server-side session: the observed ID of its requests, the - /// check of the IDs of its responses, and its close when the server reports that the session - /// changed. See [`SparkConnectClient::set_session_tracking`]. + /// Whether the client tracks its session: the session ID and the observed server-side session + /// ID of the requests that its callers build, the check of the IDs of its responses, and its + /// close when the server reports that the session changed. See + /// [`SparkConnectClient::set_session_tracking`]. session_tracking: bool, /// The executions in flight, shared by the clones of the client, in the order they started, /// each with the thread that waits for it now, if any. See @@ -956,6 +970,19 @@ pub fn is_wait_aborted(error: &SparkError) -> bool { struct UserContextExtensions { global: Vec<(String, prost_types::Any)>, by_thread: std::collections::HashMap, + /// What a binding keeps to bring the extensions up to date before the calls of a thread send + /// requests (see [`SparkConnectClient::set_user_context_refresher`]). + refresher: Option, +} + +/// The opaque refresher of the user context extensions of a binding. +#[derive(Clone)] +struct UserContextRefresher(std::sync::Arc); + +impl std::fmt::Debug for UserContextRefresher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("UserContextRefresher") + } } /// The user context extensions of a thread, shared with the threads that are given them by @@ -969,13 +996,27 @@ pub struct SharedUserContextExtensions( ); impl SharedUserContextExtensions { - fn new(extensions: Vec<(String, prost_types::Any)>) -> Self { + /// A list of the extensions `extensions`, of their IDs, which no thread has yet (see + /// [`SparkConnectClient::share_threadlocal_user_context_extensions`]), as the list that a + /// binding keeps of PySpark's `thread_local.user_context_extensions`. Not a stable API. + #[doc(hidden)] + pub fn new(extensions: Vec<(String, prost_types::Any)>) -> Self { Self(std::sync::Arc::new(std::sync::Mutex::new(extensions))) } - fn snapshot(&self) -> Vec<(String, prost_types::Any)> { + /// The extensions of this list, of their IDs, in their order. Not a stable API. + #[doc(hidden)] + pub fn snapshot(&self) -> Vec<(String, prost_types::Any)> { self.0.lock().unwrap_or_else(|e| e.into_inner()).clone() } + + /// Replace the extensions of this list with `extensions`, which the requests of the threads + /// that have it carry from now on, as a change of the list of PySpark's + /// `thread_local.user_context_extensions`. Not a stable API. + #[doc(hidden)] + pub fn replace(&self, extensions: Vec<(String, prost_types::Any)>) { + *self.0.lock().unwrap_or_else(|e| e.into_inner()) = extensions; + } } /// Forget the extensions of a thread that ends, as PySpark's `thread_local` is freed with its @@ -1078,7 +1119,9 @@ impl SparkConnectClient { channel, host: builder.host().to_string(), stub, + client_session_id: std::sync::Arc::new(std::sync::RwLock::new(session_id.clone())), session_id, + builder: builder.clone(), user_id: builder.user_id().map(String::from).or_else(default_user_id), metadata, headers, @@ -1103,6 +1146,42 @@ impl SparkConnectClient { &self.session_id } + /// The session ID that the requests of this client and of its clones carry, as PySpark's + /// `client._session_id`: the session ID, unless + /// [`SparkConnectClient::set_client_session_id`] set another. Not a stable API. + #[doc(hidden)] + pub fn client_session_id(&self) -> String { + self.client_session_id + .read() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } + + /// Make the requests of this client and of its clones carry `session_id`, against which + /// their responses are checked, as a set of PySpark's `client._session_id` does: the server + /// then answers of the session of that ID, and a response of another server-side session + /// than the one that the client observed closes the client. [`SparkConnectClient::session_id`] + /// stays the ID of the session, as the `session_id` of PySpark's session does, and the + /// requests of the artifacts carry it, as PySpark's ArtifactManager keeps the ID of the + /// client when it was built. A client that does not track its session (see + /// [`SparkConnectClient::set_session_tracking`]) sends the requests that its callers build + /// with their own session ID. Not a stable API. + #[doc(hidden)] + pub fn set_client_session_id(&self, session_id: String) { + *self + .client_session_id + .write() + .unwrap_or_else(|e| e.into_inner()) = session_id; + } + + /// The channel builder of the connection of this client, as PySpark's `client._builder`: of + /// no session ID for the client of a new session, and of its own for that of a clone. Not a + /// stable API. + #[doc(hidden)] + pub fn channel_builder(&self) -> &crate::channel::ChannelBuilder { + &self.builder + } + /// Create a new client that reuses this client's channel/stub/metadata but /// starts a brand-new server-side session (fresh session id). Mirrors the /// reference client's `newSession()`, which opens a new session over the same @@ -1110,11 +1189,17 @@ impl SparkConnectClient { /// reattachable, and has the default retry policy and result chunking, as PySpark's /// `newSession()` and `clone()` build their client of these only. pub fn with_new_session_id(&self) -> Self { + let session_id = Uuid::new_v4().to_string(); + // As PySpark's `newSession`, which pops the session ID of a copy of the builder. + let mut builder = self.builder.clone(); + builder.remove(crate::channel::PARAM_SESSION_ID); SparkConnectClient { channel: self.channel.clone(), host: self.host.clone(), stub: self.stub.clone(), - session_id: Uuid::new_v4().to_string(), + client_session_id: std::sync::Arc::new(std::sync::RwLock::new(session_id.clone())), + session_id, + builder, user_id: self.user_id.clone(), metadata: self.metadata.clone(), headers: self.headers.clone(), @@ -1144,6 +1229,14 @@ impl SparkConnectClient { self.closed.store(true, std::sync::atomic::Ordering::SeqCst); } + /// Open this client again, which [`SparkConnectClient::close`] closed, as PySpark's + /// `client._closed = False` lets its RPCs be sent again. Not a stable API. + #[doc(hidden)] + pub fn reopen(&self) { + self.closed + .store(false, std::sync::atomic::Ordering::SeqCst); + } + /// Whether the client is closed. See [`SparkConnectClient::close`]. pub fn is_closed(&self) -> bool { self.closed.load(std::sync::atomic::Ordering::SeqCst) @@ -1199,6 +1292,16 @@ impl SparkConnectClient { &self.host } + /// Sets the session ID of a request that the caller built to the client's, as PySpark's + /// requests read `client._session_id`. The requests of a client that does not track its + /// session, those of the transport-injection stub, are of the reference client's session and + /// keep its ID, as they keep their observed one (see `observe_on`). + fn session_on(&self, session_id: &mut String) { + if self.session_tracking { + *session_id = self.client_session_id(); + } + } + /// Sets the observed server-side session ID of a request, once a response has set it, as /// PySpark's requests do. fn observe_on(&self, observed: &mut Option) { @@ -1222,10 +1325,10 @@ impl SparkConnectClient { } // An empty session ID is none: the server leaves both IDs unset in some responses, such // as those of FetchErrorDetails, so only another session's ID is a mismatch. - if !session_id.is_empty() && session_id != self.session_id { + let expected = self.client_session_id(); + if !session_id.is_empty() && session_id != expected { return Err(assertion_error(format!( - "Received incorrect session identifier for request:{session_id} != {}", - self.session_id + "Received incorrect session identifier for request:{session_id} != {expected}" ))); } let mut observed = self @@ -1370,6 +1473,47 @@ impl SparkConnectClient { all.by_thread.insert(thread, extensions.clone()); } + /// Keep `refresher`, or none, for this client and its clones: what a binding keeps to bring + /// the user context extensions up to date before a call of a thread sends requests, as the + /// Python client re-serializes the `google.protobuf.Any` messages of its lists, which PySpark + /// serializes when it builds each request. The core never calls it. Not a stable API. + #[doc(hidden)] + pub fn set_user_context_refresher( + &self, + refresher: Option>, + ) { + self.user_context_extensions + .lock() + .unwrap_or_else(|e| e.into_inner()) + .refresher = refresher.map(UserContextRefresher); + } + + /// The refresher of [`SparkConnectClient::set_user_context_refresher`], if any. Not a stable + /// API. + #[doc(hidden)] + pub fn user_context_refresher( + &self, + ) -> Option> { + self.user_context_extensions + .lock() + .unwrap_or_else(|e| e.into_inner()) + .refresher + .clone() + .map(|refresher| refresher.0) + } + + /// Forget the user context extensions of the current thread, which has none from now on, + /// as PySpark's `thread_local` once its `user_context_extensions` is deleted. Not a stable + /// API. + #[doc(hidden)] + pub fn clear_threadlocal_user_context_extensions(&self) { + self.user_context_extensions + .lock() + .unwrap_or_else(|e| e.into_inner()) + .by_thread + .remove(&std::thread::current().id()); + } + /// Add `extension` to the user context of the requests of all the threads, and return its /// ID, as PySpark's `add_global_user_context_extension`. pub fn add_global_user_context_extension(&self, extension: prost_types::Any) -> String { @@ -1542,15 +1686,16 @@ impl SparkConnectClient { self.reattachable.load(std::sync::atomic::Ordering::SeqCst) } - /// Turn the tracking of the server-side session on (the default) or off. With it on, the - /// requests carry the server-side session ID observed in the responses, each response must - /// be of this session and of that server-side session, and an + /// Turn the tracking of the session on (the default) or off. With it on, the requests that + /// the callers build carry the session ID of the client (see + /// [`SparkConnectClient::client_session_id`]) and the server-side session ID observed in the + /// responses, each response must be of that session and of that server-side session, and an /// `INVALID_HANDLE.SESSION_CHANGED` error closes the client, as PySpark's /// `SparkConnectClient` does. /// /// The transport-injection stub turns it off. The requests that it sends are of the - /// reference client's session, not of its own, and that client tracks its session itself, - /// over the grpcio stub that does no tracking. + /// reference client's session, not of its own, and keep their IDs, and that client tracks + /// its session itself, over the grpcio stub that does no tracking. pub fn set_session_tracking(&mut self, enabled: bool) { self.session_tracking = enabled; } @@ -1646,6 +1791,7 @@ impl SparkConnectClient { self.fill_client_type(&mut request.client_type)?; self.ensure_open()?; add_result_chunking_option(&mut request, self.result_chunking); + self.session_on(&mut request.session_id); // The caller has extended the user context with the extensions of its thread. self.fill_user_context(&mut request.user_context); let of_a_relation = matches!( @@ -1819,6 +1965,7 @@ impl SparkConnectClient { &self, mut request: AnalyzePlanRequest, ) -> Result { + self.session_on(&mut request.session_id); self.fill_client_type(&mut request.client_type)?; self.ensure_open()?; self.extend_user_context(&mut request.user_context); @@ -2151,6 +2298,7 @@ impl SparkConnectClient { &self, mut request: FetchErrorDetailsRequest, ) -> Result { + self.session_on(&mut request.session_id); self.fill_client_type(&mut request.client_type)?; self.ensure_open()?; self.extend_user_context(&mut request.user_context); @@ -2226,7 +2374,7 @@ impl SparkConnectClient { client_type: String, ) -> Option { let request = FetchErrorDetailsRequest { - session_id: self.session_id.clone(), + session_id: self.client_session_id(), error_id: error_id.to_string(), client_type: Some(client_type), user_context: self.user_id.as_ref().map(|user_id| UserContext { @@ -2282,7 +2430,7 @@ impl SparkConnectClient { return None; } let mut request = ConfigRequest { - session_id: self.session_id.clone(), + session_id: self.client_session_id(), user_context: Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2335,6 +2483,7 @@ impl SparkConnectClient { /// /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.config`. pub async fn config(&self, mut request: ConfigRequest) -> Result { + self.session_on(&mut request.session_id); self.fill_client_type(&mut request.client_type)?; self.ensure_open()?; self.extend_user_context(&mut request.user_context); @@ -2362,6 +2511,7 @@ impl SparkConnectClient { /// /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.interrupt`. pub async fn interrupt(&self, mut request: InterruptRequest) -> Result { + self.session_on(&mut request.session_id); self.fill_client_type(&mut request.client_type)?; self.ensure_open()?; self.extend_user_context(&mut request.user_context); @@ -2390,7 +2540,7 @@ impl SparkConnectClient { /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.get_configs`. pub async fn get_configs(&self, keys: &[&str]) -> Result>> { let mut request = ConfigRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2463,7 +2613,7 @@ impl SparkConnectClient { silent, }; let request = ConfigRequest { - session_id: self.session_id.clone(), + session_id: self.client_session_id(), user_context: Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2486,7 +2636,7 @@ impl SparkConnectClient { pairs: &[(&str, Option<&str>)], ) -> Result>> { let mut request = ConfigRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2530,7 +2680,7 @@ impl SparkConnectClient { /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.unset_config`. pub async fn unset_config(&self, key: &str) -> Result> { let mut request = ConfigRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2552,7 +2702,7 @@ impl SparkConnectClient { /// Returns a HashMap of all configuration key-value pairs. pub async fn get_configs_all(&self) -> Result> { let mut request = ConfigRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2581,7 +2731,7 @@ impl SparkConnectClient { /// Returns true if the configuration can be changed at runtime. pub async fn is_config_modifiable(&self, key: &str) -> Result { let mut request = ConfigRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2612,7 +2762,7 @@ impl SparkConnectClient { /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.interrupt_all`. pub async fn interrupt_all(&self) -> Result> { let mut request = InterruptRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2629,7 +2779,7 @@ impl SparkConnectClient { /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.interrupt_tag`. pub async fn interrupt_tag(&self, tag: &str) -> Result> { let mut request = InterruptRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2648,7 +2798,7 @@ impl SparkConnectClient { /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.interrupt_operation`. pub async fn interrupt_operation(&self, operation_id: &str) -> Result> { let mut request = InterruptRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2678,7 +2828,7 @@ impl SparkConnectClient { // As PySpark's request: of the client type "python", not of the user agent, and with // no observed server-side session ID. let request = CloneSessionRequest { - session_id: self.session_id.clone(), + session_id: self.client_session_id(), user_context: Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2710,6 +2860,12 @@ impl SparkConnectClient { } } let mut cloned = self.with_new_session_id(); + // As PySpark's `clone`, which sets the new session ID in a copy of the builder. + cloned + .builder + .set(crate::channel::PARAM_SESSION_ID, &response.new_session_id); + cloned.client_session_id = + std::sync::Arc::new(std::sync::RwLock::new(response.new_session_id.clone())); cloned.session_id = response.new_session_id; Ok(cloned) } @@ -2721,7 +2877,7 @@ impl SparkConnectClient { /// session on the server until its timeout. pub async fn release_session(&self) -> Result<()> { let mut request = ReleaseSessionRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2745,6 +2901,44 @@ impl SparkConnectClient { self.verify_response(&response.session_id, &response.server_side_session_id) } + /// The statuses of operations of this session, as PySpark's `_get_operation_statuses`: the + /// GetStatus RPC of `request`, of the operations and the extensions that the caller asks of, + /// which the client gives the IDs of its session, its user agent as the client type, its user + /// and the server-side session ID that it observed, as PySpark builds it, under the retry + /// policy and the deadline of `get_status`. Its response is checked as the others are. + pub async fn get_status_proto( + &self, + mut request: GetStatusRequest, + ) -> Result { + self.ensure_open()?; + request.session_id = self.client_session_id(); + request.client_type = Some(self.client_type()?); + if let Some(user_id) = self.user_id.as_ref().filter(|user_id| !user_id.is_empty()) { + request.user_context = Some(UserContext { + user_id: user_id.clone(), + ..Default::default() + }); + } + if let Some(observed) = self.observed_server_session_id() { + request.client_observed_server_side_session_id = Some(observed); + } + let response = self + .with_retry(|| { + let deadline = self.rpc_deadlines.get_status; + let mut req = with_deadline(Request::new(request.clone()), deadline); + self._attach_metadata(&mut req); + let mut stub = self.stub.clone(); + async move { + within_deadline(deadline, stub.get_status(req)) + .await + .map(Response::into_inner) + } + }) + .await?; + self.verify_response(&response.session_id, &response.server_side_session_id)?; + Ok(response) + } + /// Add artifacts to this session. /// /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.add_artifacts`, which @@ -2915,7 +3109,9 @@ impl SparkConnectClient { /// The AddArtifacts call of `requests`, once. Stamps user_context on every frame as it is /// sent; all upload paths must go through here. As PySpark's ArtifactManager, the frames - /// carry no client type and no observed server-side session ID. + /// carry no client type and no observed server-side session ID, and the ID of the session, + /// which a set of the client's session ID does not change, as the manager keeps the ID of + /// the client when it was built. async fn add_artifacts_rpc( &self, requests: impl Iterator + Send + 'static, @@ -2955,7 +3151,7 @@ impl SparkConnectClient { /// The ArtifactStatus call of `names`, once: the response, which says whether each artifact /// exists. As PySpark's, the request carries no client type and no observed server-side - /// session ID. + /// session ID, and the ID of the session, as the frames of `add_artifacts_rpc`. async fn artifact_status_rpc( &self, names: &[String], @@ -4867,6 +5063,63 @@ mod tests { .collect() } + #[tokio::test] + async fn a_refresher_is_kept_for_the_client_and_its_clones_and_not_a_new_session() { + // What a binding keeps to bring its extensions up to date, which the client gives back, + // as its clones do, while the client of a new session has none. + let builder = ChannelBuilder::parse("sc://localhost").unwrap(); + let client = SparkConnectClient::connect(&builder).await.unwrap(); + assert!(client.user_context_refresher().is_none()); + client.set_user_context_refresher(Some(std::sync::Arc::new(7u8))); + let kept = client.clone().user_context_refresher().unwrap(); + assert_eq!(kept.downcast_ref::(), Some(&7)); + assert!(client + .with_new_session_id() + .user_context_refresher() + .is_none()); + client.set_user_context_refresher(None); + assert!(client.user_context_refresher().is_none()); + } + + #[tokio::test] + async fn a_list_given_to_threads_is_theirs_and_its_changes_are_theirs() { + // As the list of PySpark's thread_local.user_context_extensions, which a binding keeps and + // changes: a new list that a thread is given is its extensions, whose changes are the + // extensions of the threads that have it, and a cleared thread has none. + let builder = ChannelBuilder::parse("sc://localhost").unwrap(); + let client = std::sync::Arc::new(SparkConnectClient::connect(&builder).await.unwrap()); + client.add_threadlocal_user_context_extension(any_of("own")); + let list = + SharedUserContextExtensions::new(vec![("threadlocal_a".to_string(), any_of("a"))]); + client.share_threadlocal_user_context_extensions(&list); + assert_eq!(extension_ids(&client), vec!["threadlocal_a"]); + list.replace(vec![("threadlocal_b".to_string(), any_of("b"))]); + assert_eq!(extension_ids(&client), vec!["threadlocal_b"]); + assert_eq!(list.snapshot()[0].0, "threadlocal_b"); + + let seen = { + let client = client.clone(); + let list = list.clone(); + std::thread::spawn(move || { + let before = extension_ids(&client); + client.share_threadlocal_user_context_extensions(&list); + list.replace(vec![("threadlocal_c".to_string(), any_of("c"))]); + (before, extension_ids(&client)) + }) + .join() + .unwrap() + }; + assert_eq!( + seen, + (Vec::::new(), vec!["threadlocal_c".to_string()]) + ); + assert_eq!(extension_ids(&client), vec!["threadlocal_c"]); + + client.clear_threadlocal_user_context_extensions(); + assert!(extension_ids(&client).is_empty()); + assert_eq!(list.snapshot()[0].0, "threadlocal_c"); + } + #[tokio::test] async fn threadlocal_user_context_extensions_of_a_copy_are_shared_as_pyspark() { // As PySpark's inheritable_thread_target: the threads given a copy of the extensions of a @@ -4912,6 +5165,58 @@ mod tests { assert_eq!(extension_ids(&client), vec![main, late]); } + #[test] + fn a_client_session_id_is_sent_and_checked_while_the_session_id_stays() { + // As PySpark's `client._session_id`, which its requests read: a set changes the ID that + // the clones of the client send and check the responses against, while `session_id` + // stays the session's, as the `session_id` of PySpark's session; a client of a new + // session has its own, and its builder no session ID, as PySpark's `newSession` pops it. + let client = + connect_blocking("sc://localhost/;session_id=7c4e2a0e-0f6b-4b9a-9d1e-0123456789ab"); + let original = client.session_id().to_string(); + assert_eq!(client.client_session_id(), original); + let clone = client.clone(); + clone.set_client_session_id("changed".to_string()); + assert_eq!(client.client_session_id(), "changed"); + assert_eq!(client.session_id(), original); + assert!(client.verify_response("changed", "").is_ok()); + let err = client.verify_response(&original, "").unwrap_err(); + assert!(err.message().contains(&format!("{original} != changed"))); + assert_eq!( + client + .channel_builder() + .get(crate::channel::PARAM_SESSION_ID), + Some(original.as_str()) + ); + let new = client.with_new_session_id(); + assert_eq!(new.client_session_id(), new.session_id()); + assert_ne!(new.session_id(), original); + assert_eq!( + new.channel_builder().get(crate::channel::PARAM_SESSION_ID), + None + ); + assert_eq!( + client.channel_builder().port(), + new.channel_builder().port() + ); + } + + #[test] + fn a_closed_client_reopens_and_the_status_rpc_has_a_deadline() { + // As PySpark's `client._closed = False`, which sends the requests again, and its + // `RpcDeadlines.get_status`, of 10 minutes, and none when the deadlines are disabled. + let client = connect_blocking("sc://localhost"); + client.close(); + assert!(client.is_closed() && client.ensure_open().is_err()); + client.clone().reopen(); + assert!(!client.is_closed() && client.ensure_open().is_ok()); + assert_eq!( + RpcDeadlines::default().get_status, + Some(Duration::from_secs(600)) + ); + assert_eq!(RpcDeadlines::disabled().get_status, None); + } + #[test] fn host_and_observed_server_session_id_of_the_client() { let client = connect_blocking("sc://Other:114"); diff --git a/crates/spark-connect-core/tests/artifact_uploads.rs b/crates/spark-connect-core/tests/artifact_uploads.rs index becb24b2..3a775d77 100644 --- a/crates/spark-connect-core/tests/artifact_uploads.rs +++ b/crates/spark-connect-core/tests/artifact_uploads.rs @@ -229,6 +229,29 @@ async fn a_small_file_is_one_batch_named_by_its_extension() { assert_eq!(chunk.crc, compute_crc32(b"cafebabe") as i64); } +#[tokio::test] +async fn the_frames_keep_the_session_id_of_the_session_that_the_client_sets_another_of() { + // As PySpark's ArtifactManager, which keeps the session ID of the client when it was built: a + // set of `client._session_id` changes the session of the other requests, not of the uploads. + let fixture = Fixture::start([]).await; + let client = fixture.client().await; + client.set_client_session_id("another-session".to_string()); + let dir = tempfile::tempdir().unwrap(); + let class = file_with(&dir, "A.class", b"cafebabe"); + + client + .add_artifacts(&[class.as_str()], false, false, false) + .await + .unwrap(); + + let attempts = fixture.attempts(); + assert_eq!(attempts.len(), 1); + assert!(!attempts[0].is_empty()); + for frame in &attempts[0] { + assert_eq!(frame.session_id, client.session_id()); + } +} + #[tokio::test] async fn a_transient_failure_retries_the_whole_upload_from_its_start() { // The first attempt receives two frames and then fails; the second receives the upload. diff --git a/crates/spark-connect-core/tests/server_session.rs b/crates/spark-connect-core/tests/server_session.rs index c29a6200..403b5015 100644 --- a/crates/spark-connect-core/tests/server_session.rs +++ b/crates/spark-connect-core/tests/server_session.rs @@ -647,6 +647,153 @@ async fn a_client_without_session_tracking_passes_the_ids_of_its_requests_throug } } +/// The session ID of each request of `method` that the server received, in order. +fn sessions_of( + state: &State, + method: &str, + session_id: impl Fn(T) -> String, +) -> Vec { + state + .requests_of::(method) + .into_iter() + .map(session_id) + .collect() +} + +#[tokio::test] +async fn a_client_without_session_tracking_passes_the_session_id_of_each_call_through() { + // Every call that the transport-injection stub forwards decoded keeps the session ID of the + // reference client's request, against which that client checks the response itself. + use spark_connect_proto::{AnalyzePlanRequest, FetchErrorDetailsRequest, InterruptRequest}; + + let fixture = Fixture::start().await; + let mut client = fixture.client().await; + client.set_session_tracking(false); + let reference = "the-reference-session".to_string(); + + client + .config(ConfigRequest { + session_id: reference.clone(), + ..Default::default() + }) + .await + .unwrap(); + client + .interrupt(InterruptRequest { + session_id: reference.clone(), + ..Default::default() + }) + .await + .unwrap(); + client + .analyze_plan(AnalyzePlanRequest { + session_id: reference.clone(), + ..Default::default() + }) + .await + .unwrap(); + client + .fetch_error_details(FetchErrorDetailsRequest { + session_id: reference.clone(), + ..Default::default() + }) + .await + .unwrap(); + let mut stream = client + .execute_plan(ExecutePlanRequest { + session_id: reference.clone(), + ..Default::default() + }) + .await + .unwrap(); + while stream.message().await.unwrap().is_some() {} + + let state = &fixture.state; + let sessions = [ + sessions_of(state, "Config", |r: ConfigRequest| r.session_id), + sessions_of(state, "Interrupt", |r: InterruptRequest| r.session_id), + sessions_of(state, "AnalyzePlan", |r: AnalyzePlanRequest| r.session_id), + sessions_of(state, "FetchErrorDetails", |r: FetchErrorDetailsRequest| { + r.session_id + }), + sessions_of(state, "ExecutePlan", |r: ExecutePlanRequest| r.session_id), + ]; + for of_a_method in sessions { + assert_eq!(of_a_method, [reference.as_str()]); + } +} + +#[tokio::test] +async fn a_set_client_session_id_is_that_of_the_calls_and_not_of_the_artifacts_as_pyspark() { + // As a set of PySpark's `client._session_id`, which its requests read, while its + // ArtifactManager keeps the session ID of the client when it was built. + use spark_connect_proto::{ + AnalyzePlanRequest, ArtifactStatusesRequest, FetchErrorDetailsRequest, InterruptRequest, + }; + + let fixture = Fixture::start().await; + let client = fixture.client().await; + let session_id = client.session_id().to_string(); + client.set_client_session_id("the-client-session".to_string()); + + // The callers build their requests of the session's ID. + client.config(config_request(&client)).await.unwrap(); + client + .interrupt(InterruptRequest { + session_id: session_id.clone(), + ..Default::default() + }) + .await + .unwrap(); + client + .analyze_plan(AnalyzePlanRequest { + session_id: session_id.clone(), + ..Default::default() + }) + .await + .unwrap(); + client + .fetch_error_details(FetchErrorDetailsRequest { + session_id: session_id.clone(), + ..Default::default() + }) + .await + .unwrap(); + // An execution that drops after its first response, which the client reattaches. + *fixture.state.drop_next_execute.lock().unwrap() = true; + run_execution(&client).await; + client.get_configs(&["k"]).await.unwrap(); + client + .cache_artifacts(vec![b"blob".to_vec()]) + .await + .unwrap(); + + let state = &fixture.state; + let calls = [ + sessions_of(state, "Config", |r: ConfigRequest| r.session_id), + sessions_of(state, "Interrupt", |r: InterruptRequest| r.session_id), + sessions_of(state, "AnalyzePlan", |r: AnalyzePlanRequest| r.session_id), + sessions_of(state, "FetchErrorDetails", |r: FetchErrorDetailsRequest| { + r.session_id + }), + sessions_of(state, "ExecutePlan", |r: ExecutePlanRequest| r.session_id), + // As PySpark's, of the session ID of the request of the execution. + sessions_of(state, "ReattachExecute", |r: ReattachExecuteRequest| { + r.session_id + }), + ]; + for of_a_method in calls { + assert!(!of_a_method.is_empty()); + assert!(of_a_method.iter().all(|id| id == "the-client-session")); + } + assert_eq!( + sessions_of(state, "ArtifactStatus", |r: ArtifactStatusesRequest| { + r.session_id + }), + [session_id] + ); +} + #[tokio::test] async fn executions_ask_the_server_to_chunk_arrow_batches() { use spark_connect_proto::execute_plan_request::request_option::RequestOption; diff --git a/crates/spark-connect/src/session.rs b/crates/spark-connect/src/session.rs index 75a83faf..6f733236 100644 --- a/crates/spark-connect/src/session.rs +++ b/crates/spark-connect/src/session.rs @@ -82,13 +82,39 @@ fn forget_thread_tags( #[derive(Clone, Default)] pub struct SharedTags(Arc>>); +impl SharedTags { + /// A set of the tags `tags`, which no thread has yet (see [`SparkSession::share_thread_tags`]), + /// as the set that a binding keeps of the tags of PySpark's `thread_local`. Not a stable API. + #[doc(hidden)] + pub fn new(tags: Vec) -> Self { + SharedTags(Arc::new(Mutex::new(tags))) + } + + /// Replace the tags of this set with `tags`, which the requests of the threads that have it + /// carry from now on, as a change of the set of PySpark's `thread_local.tags`. Not a stable + /// API. + #[doc(hidden)] + pub fn replace(&self, tags: Vec) { + *self.0.lock().unwrap_or_else(|e| e.into_inner()) = tags; + } + + /// The tags of this set. Not a stable API. + #[doc(hidden)] + pub fn tags(&self) -> Vec { + self.0.lock().unwrap_or_else(|e| e.into_inner()).clone() + } +} + /// An operation tag: its text, or the UTF-32 little-endian bytes of a str of Python that UTF-8 /// cannot encode, such as one of a lone surrogate, which PySpark's client keeps as a tag, and /// with which each request that carries the tags fails (see -/// [`SparkSession::add_unencodable_tag`]). +/// [`SparkSession::add_unencodable_tag`]). Not a stable API. +#[doc(hidden)] #[derive(Clone, Debug, PartialEq, Eq)] -enum Tag { +pub enum Tag { + /// A tag of its text. Text(String), + /// The UTF-32 little-endian bytes of a tag that UTF-8 cannot encode. Unencodable(Vec), } @@ -1464,6 +1490,78 @@ impl SparkSession { .share_threadlocal_user_context_extensions(extensions); } + /// The session ID that the requests of this session carry, as PySpark's `client._session_id` + /// (see [`SparkConnectClient::client_session_id`]). Not a stable API. + #[doc(hidden)] + pub fn client_session_id(&self) -> String { + self.client.client_session_id() + } + + /// Make the requests of this session carry `session_id`, as a set of PySpark's + /// `client._session_id` (see [`SparkConnectClient::set_client_session_id`]): + /// [`SparkSession::session_id`] stays the ID of the session. Not a stable API. + #[doc(hidden)] + pub fn set_client_session_id(&self, session_id: String) { + self.client.set_client_session_id(session_id); + } + + /// The channel builder of the connection of this session, as PySpark's `client._builder` + /// (see [`SparkConnectClient::channel_builder`]). Not a stable API. + #[doc(hidden)] + pub fn channel_builder(&self) -> &ChannelBuilder { + self.client.channel_builder() + } + + /// The user of the requests of this session, as PySpark's `client._user_id`: of the + /// connection string, else of the environment, if any. Not a stable API. + #[doc(hidden)] + pub fn user_id(&self) -> Option<&str> { + self.client.user_id() + } + + /// Open this session again, which `close()` closed, as PySpark's `client._closed = False` + /// lets the requests of its client be sent again. Not a stable API. + #[doc(hidden)] + pub fn reopen(&self) { + self.stopped.store(false, Ordering::SeqCst); + self.client.reopen(); + } + + /// The statuses of the operations of this session of `request`, a GetStatus request built + /// outside the core, as PySpark's `_get_operation_statuses` (see + /// [`SparkConnectClient::get_status_proto`]). Not a stable API. + #[doc(hidden)] + pub fn get_status_proto( + &self, + request: proto::GetStatusRequest, + ) -> Result { + block_on(self.client.get_status_proto(request)) + } + + /// Keep `refresher`, or none, for this session: what a binding keeps to bring the user + /// context extensions up to date before a call of a thread sends requests (see + /// [`SparkConnectClient::set_user_context_refresher`]). Not a stable API. + #[doc(hidden)] + pub fn set_user_context_refresher( + &self, + refresher: Option>, + ) { + self.client.set_user_context_refresher(refresher); + } + + /// The refresher of [`SparkSession::set_user_context_refresher`], if any. Not a stable API. + #[doc(hidden)] + pub fn user_context_refresher(&self) -> Option> { + self.client.user_context_refresher() + } + + /// Forget the user context extensions of the current thread, as PySpark's + /// `thread_local` once its `user_context_extensions` is deleted. Not a stable API. + #[doc(hidden)] + pub fn clear_threadlocal_user_context_extensions(&self) { + self.client.clear_threadlocal_user_context_extensions(); + } + /// Return the Spark version of the connected server. /// /// Mirrors `pyspark.sql.SparkSession.version`. @@ -3763,6 +3861,46 @@ mod tests { assert_eq!(run(copy, "e"), vec!["a", "b", "e"]); } + #[test] + fn a_set_of_tags_given_to_threads_is_theirs_and_its_changes_are_theirs() { + // As the set of PySpark's thread_local.tags, which a binding keeps and changes: a new set + // that a thread is given is its tags, whose changes are the tags of the threads that have + // it, of their text or of the bytes of a str that UTF-8 cannot encode, and a thread that is + // not given it keeps its own. + let spark = session(); + spark.add_tag("own").unwrap(); + let set = SharedTags::new(vec![Tag::Text("a".to_string())]); + spark.share_thread_tags(&set); + assert_eq!(spark.get_tags(), vec!["a"]); + let surrogate = vec![0x00, 0xd8, 0x00, 0x00]; + set.replace(vec![ + Tag::Text("b".to_string()), + Tag::Unencodable(surrogate.clone()), + ]); + assert_eq!(spark.get_tags(), vec!["b"]); + assert_eq!(spark.unencodable_tags(), vec![surrogate.clone()]); + assert!(spark.tags().is_err()); + assert_eq!(set.tags().len(), 2); + + let seen = { + let spark = spark.clone(); + let set = set.clone(); + std::thread::spawn(move || { + let before = spark.get_tags(); + spark.share_thread_tags(&set); + set.replace(vec![Tag::Text("c".to_string())]); + (before, spark.get_tags()) + }) + .join() + .unwrap() + }; + assert_eq!(seen, (Vec::::new(), vec!["c".to_string()])); + assert_eq!(spark.get_tags(), vec!["c"]); + spark.clear_tags(); + assert!(spark.get_tags().is_empty()); + assert_eq!(set.tags(), vec![Tag::Text("c".to_string())]); + } + #[test] fn finished_threads_leave_no_tags() { // The threads that share a copy of the tags are forgotten when they end, as PySpark's diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index f34eb4ba..4f922d4b 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -392,7 +392,7 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan read when they are set. The transport classifies the failures as `DefaultPolicy.can_retry` does, so a policy of its own `can_retry` or `to_state`, and a backoff, jitter or delay that is not a whole number of milliseconds, raise `NOT_IMPLEMENTED`. `RpcDeadlines` sets the - deadlines of the transport, which has no GetStatus RPC (`get_status`). `client.reattach` + deadlines of the transport, `get_status` that of `_get_operation_statuses`. `client.reattach` (`ExecutePlanResponseReattachableIterator`) is not provided: `disable_reattachable_execute` and `enable_reattachable_execute` set the executions of the transport not reattachable or reattachable. @@ -442,10 +442,13 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan ...`), the failure that ends its retries (`Given up on retrying. error: ...`) and each gRPC failure that it converts (`GRPC Error received`), which the transport, whose retries and errors are its own, does not. -- **`SparkConnectClient.get_tags`** (intentional). A copy of the tags of the thread, where - PySpark returns the set that its client keeps, which its later changes change too, as - `SparkSession.getTags` does, of which the drop-in returns a copy too: the tags are those of the - core session, which its requests read, and not a Python set. +- **`SparkConnectClient.thread_local`** (intentional). A subclass of `threading.local`, of + subclasses of `set` and `list` for its tags and its user context extensions, which keep the Rust + transport's lists of them in step, where PySpark's is a `threading.local` of a plain set and + list. The global list of the extensions is a subclass of `list` too. A set of tags refuses an + object that is neither a str nor bytes when it is added, where PySpark's raises the `TypeError` + of the field of the tags when it builds a request. A list that the attribute of a `copy.copy` of + a client is set to is not one that the requests of the session carry. - **Plans of a DataFrame's relation** (intentional; DESIGN). `df._plan.plan(client)` is a placeholder that holds the DataFrame's relation, which the core builds at the call, as PySpark's `plan` does, with its pickles, warnings and errors: the core builds the relation, and @@ -483,12 +486,19 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan `SparkConnectClient._proto_to_string` renders any message, including a deep one built in Python, as PySpark's does. Plan logging of a deep DataFrame therefore does not reach the recursion handling that the test `test_recursion_handling_for_plan_logging` exercises. -- **Client session ID** (not provided). `SparkConnectClient._session_id` is not exposed and - is not settable; the session ID belongs to the Rust core. `_server_session_id` is - exposed and settable, as PySpark's, and a changed value closes the session. -- **`_closed`** (tracked). Reading `_closed` gives the session's stopped state. Setting it - to True closes the session, as a test's stand-in for the client's close. Setting it to - False does not reopen a closed session. +- **Client session ID** (intentional). `SparkConnectClient._session_id` is the session ID that + the requests carry, which a set changes for them, as PySpark's, so the next response of + another server-side session closes the session; `_server_session_id` is exposed and settable, + as PySpark's, and a changed value closes the session. A value that is not a str raises the + `TypeError` of the string field of the session ID of the requests when it is set, where + PySpark raises it when it builds the next request. +- **`_closed`** (intentional). Reading `_closed` gives the session's stopped state. Setting it + to True closes the session, and setting it to False opens it again, as PySpark's. After + `close()` or `stop()`, which close PySpark's channel too, PySpark's requests still fail once + `_closed` is False, with grpc's `ValueError: Cannot invoke RPC: Channel closed!`, where the + drop-in's are sent again. +- **`_user_id`** (intentional). It is read-only, where PySpark's is an attribute that a set + changes for the requests. - **The exit of a dropped session in a reference cycle** (intentional). PySpark's `SparkSession.__del__` closes the client of a session object that is freed, and the exit hook of a closed client sends nothing; the drop-in's exit hook holds its session weakly, so a diff --git a/docs/compatibility.md b/docs/compatibility.md index fa1f238a..21775875 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -1646,20 +1646,40 @@ client does: messages that the server sent, of their `repr`, which `pairs` and `flows` convert as `LiteralExpression._to_value` does, raising its `UNSUPPORTED_LITERAL` when they are read; - the registrations of `register_udf`, `register_udtf`, `register_java` and - `register_data_source`, the artifacts of `add_artifacts`, `copy_from_local_to_fs`, - `cache_artifact` and `cache_artifacts`, with the checks of PySpark's client rather than those of - `SparkSession.addArtifacts` and `copyFromLocalToFs`, the tags, the interrupts, `token`, `host`, - and the clients of `newSession` and `clone`. + `register_data_source`, which set their arguments in the fields of their commands, convert the + return type and pickle the function in the order of PySpark's, of protobuf's errors for a value + that a field does not take (an int is a bool, a bool is not an int), the artifacts of + `add_artifacts`, `copy_from_local_to_fs`, `cache_artifact` and `cache_artifacts`, with the + checks of PySpark's client rather than those of `SparkSession.addArtifacts` and + `copyFromLocalToFs`, the tags, the interrupts, `token`, `host`, and the clients of `newSession` + and `clone`. `spark.udf.register`, `registerJavaFunction` and `registerJavaUDAF` register + through the client, as PySpark's do. The executions read their `observations` as PySpark's: of + an object that is not a dict, such as None or an empty list, they set none, and of one that is + not a container, such as None, they raise the `TypeError` of `in` when observed metrics arrive. `SparkConnectClient(connection, user_id=None, ...)` takes the arguments of PySpark's: it makes a client of a session of its own of the connection string or the `ChannelBuilder` `connection`, which is neither the default nor the active session. Its `retry_policy`, `use_reattachable_execute`, `session_hooks`, `allow_arrow_batch_chunking`, `preferred_arrow_chunk_size`, `rpc_deadlines` -(`pyspark.sql.connect.client.RpcDeadlines`, of which the transport has no GetStatus RPC) and +(`pyspark.sql.connect.client.RpcDeadlines`) and `max_retry_exception_elapsed_time` are those of the transport. A `channel_options` of a connection string raises `NOT_IMPLEMENTED`, as `DefaultChannelBuilder` does: the Rust transport opens the channel; as PySpark's, the client of a `ChannelBuilder` does not read it. +The members of PySpark's client that its tests and some libraries read are there too. `_builder` +is the channel builder of the connection, a `DefaultChannelBuilder` of the session's connection +string for `spark.client` (of its `endpoint`, `userAgent` and parameters), and the one given for a +client made of one. `_session_id` is the session ID that the requests carry, which a set changes +for them, as PySpark's, so the next response of another server-side session than the one that the +client observed closes the session, while the session's `session_id` stays, and so does that of the +requests of the artifacts, as PySpark's ArtifactManager keeps the ID that the client was built +with; `_server_session_id` is the server-side one. `_closed` closes the session when it is set to True and opens it again +when it is set to False. `_user_id` is the user of the requests, `_use_reattachable_execute` +whether the executions are reattachable, and `_get_operation_statuses` the statuses of the +operations of the session, of a GetStatus RPC under the `get_status` deadline. The differences +that remain are in `dev/parity/known-differences.md` ("Client session ID", "`_closed`" and +"`_user_id`"). + The retry policies of `get_retry_policies` and `set_retry_policies` are PySpark's own (`pyspark.sql.connect.client.retries`, vendored from v4.3.0, which needs `grpcio`, `grpcio-status` and `googleapis-common-protos`). The transport retries a failure with the @@ -1736,17 +1756,30 @@ The errors of the RPCs that PySpark's client leaves to grpc, those of `clone` (a `SparkSession.cloneSession`) and of `copy_from_local_to_fs` (and `SparkSession.copyFromLocalToFs`), which raise grpc's `RpcError` there (`_InactiveRpcError`), are converted as those of the other RPCs: they are the exception of their error class, such as `UnknownException` or -`UnsupportedOperationException`, as the drop-in has no grpc. `global_user_context_extensions` is -the list attribute of PySpark's client, whose changes, and a list that it is set to, the requests -of all the threads carry. - -`get_tags`, as `SparkSession.getTags`, returns a copy of the tags of the thread, where PySpark -returns the set that its client keeps, which the later changes of the tags change too: the tags -are those of the core session, which its requests read, and not a Python set. The official tests of -the client (`sql/tests/connect/client/test_client*.py`) mock the gRPC stub and the reattachable -iterator of PySpark's client, which the drop-in does not have; those of its retries -(`test_connect_retry.py`) and of `pyspark.errors.exceptions.connect.convert_exception` -(`test_connect_errors_conversion.py`) pass. +`UnsupportedOperationException`, as the drop-in has no grpc. + +The clients of a session share their attributes, as PySpark's session has one client: an +attribute that is set on `spark.client` is there the next time, and so are its lock and its +`global_user_context_extensions`, the list attribute of PySpark's client, whose changes, and a list +that it is set to, the requests of all the threads carry, while a list that it is no longer set to +is not read. The `thread_local` of the client is a `threading.local`, as PySpark's: its `tags` and +`user_context_extensions` exist once a thread uses them, and are the set and the list that the +requests of the thread carry, as they change or are set, as the tags of `SparkSession.addTag` and +`getTags` are; another attribute that a thread sets is its own. A thread of +`inheritable_thread_target` shares the copies of the attributes of the thread that decorated its +function, as PySpark's does. The extensions are serialized again before each call that sends +requests, so a `google.protobuf.Any` of the lists that is changed in place is sent as it is, as +PySpark serializes them when it builds each request. The classes of these differ: +`thread_local` is a subclass of `threading.local`, and the set of its tags, the list of its +extensions and the global list are subclasses of `set` and `list`, of a plain `repr`, whose copies +and pickles are a plain set and plain lists; a set of tags refuses an object that is neither a str +nor bytes when it is added, of the `TypeError` that PySpark raises when it builds a request of it. +A `copy.copy` of the client has the attributes of its session, but a list that the copy's +attribute is set to is not one that the requests carry. The official tests of the client +(`sql/tests/connect/client/test_client*.py`) mock the gRPC stub and the reattachable iterator of +PySpark's client, which the drop-in does not have; those of its retries (`test_connect_retry.py`) +and of `pyspark.errors.exceptions.connect.convert_exception` (`test_connect_errors_conversion.py`) +pass. ## Spark Declarative Pipelines CLI diff --git a/python/pyspark/sql/connect/client/__init__.py b/python/pyspark/sql/connect/client/__init__.py index 15096e37..4f65a97a 100644 --- a/python/pyspark/sql/connect/client/__init__.py +++ b/python/pyspark/sql/connect/client/__init__.py @@ -34,6 +34,7 @@ import os import platform import sys +import threading import urllib.parse import uuid import warnings @@ -82,8 +83,8 @@ class RpcDeadlines: stream to resume receiving results. Non-reattachable ExecutePlan has no deadline because a timeout there would kill the execution with no recovery path. - The Rust transport applies the deadline of each of its RPCs. It sends no GetStatus, the RPC of - ``get_status``. + The Rust transport applies the deadline of each of its RPCs, ``get_status`` that of the + GetStatus RPC of ``SparkConnectClient._get_operation_statuses``. """ # As PySpark Connect's, of pyspark.sql.connect.client.core, whose name this class has too. @@ -715,6 +716,12 @@ class SparkConnectClient: # As PySpark Connect's, of pyspark.sql.connect.client.core, whose name this class has too. __module__ = "pyspark.sql.connect.client.core" + # The session of the client is kept apart from its attributes, which are those of its session: + # the clients of a session share them (see `_client_state`), as PySpark's session has one + # client, of one `thread_local`, one list of global user context extensions, its lock, and the + # attributes that a caller sets. + __slots__ = ("_session", "__dict__", "__weakref__") + # The thread of an ML cache RPC in flight, as SparkConnectClient's re-entrancy guard of a # RemoteModelRef finalizer that deletes a model. _ml_cache_rpc_thread: Optional[int] = None @@ -793,34 +800,29 @@ def _of_session( return client def _init_of_session(self, session: Any, builder: Optional[ChannelBuilder]) -> None: - import threading - self._session = session - # The channel builder of the connection of a client made of one, of its host and token, - # as PySpark's `_builder`; the client of a session reads them of the session. - self._builder = builder - # The state of the current thread of the session, as SparkConnectClient's thread_local. - self.thread_local = _ThreadLocalOfClient(session) - # The global user context extensions, as SparkConnectClient's list of them, of those that - # the session has: its changes, and a list that the attribute is set to, are the - # session's (see __setattr__). - extensions = _extensions_of(session._user_context_extensions(True)) - extensions = _GlobalUserContextExtensions(session, extensions) - object.__setattr__(self, "global_user_context_extensions", extensions) - self.global_user_context_extensions_lock = threading.Lock() + # The attributes of the client are those of its session, which its other clients have + # too: its channel builder, its thread_local, its global user context extensions and their + # lock, and the attributes that a caller set. + object.__setattr__(self, "__dict__", _client_state(session, builder)) def __setattr__(self, name: str, value: Any) -> None: # As PySpark's client, whose requests read its `global_user_context_extensions`, a list # that its methods and a caller may change or rebind: the session takes the extensions of # the list that the attribute is set to. - if name == "global_user_context_extensions" and "_session" in self.__dict__: + if name == "global_user_context_extensions" and hasattr(self, "_session"): value = _GlobalUserContextExtensions(self._session, value) + object.__setattr__(self, name, value) value._sync() + return object.__setattr__(self, name, value) def __copy__(self) -> "SparkConnectClient": - # As copy.copy of PySpark's client, a plain object: a client of the same attributes. + # As copy.copy of PySpark's client, a plain object: a client of the same session and of a + # copy of its attributes, whose rebinding of a list is its own. The requests of the + # session carry the extensions of the list of the session's own client. copied = object.__new__(type(self)) + object.__setattr__(copied, "_session", self._session) copied.__dict__.update(self.__dict__) return copied @@ -866,14 +868,72 @@ def _server_session_id(self, value: Optional[str]) -> None: @property def _closed(self) -> bool: """Whether the client is closed, as SparkConnectClient's ``_closed``. Setting it to True - closes the session, which the session registry then no longer returns; a closed client - stays closed.""" + closes the session, which the session registry then no longer returns, and setting it + to False opens it again, as PySpark's requests are sent again then.""" return self._session.is_stopped @_closed.setter def _closed(self, value: bool) -> None: if value: self._session._close() + else: + self._session._reopen() + + @property + def _session_id(self) -> str: + """The session ID that the requests of the client carry, as SparkConnectClient's + ``_session_id``: the session's, until it is set to another, which the requests carry from + then on, as PySpark's do; the server answers them of the session of that ID, and a + response of another server-side session than the one that the client observed closes + the client. The session's ``session_id`` stays, as PySpark's does, and so does that of + the requests of the artifacts, as PySpark's ArtifactManager keeps the ID that the client + was built with.""" + return self._session._client_session_id() + + @_session_id.setter + def _session_id(self, value: str) -> None: + self._session._set_client_session_id(value) + + @property + def _user_id(self) -> Optional[str]: + """The user of the requests of the client, as SparkConnectClient's ``_user_id``: of the + connection string, else of ``user_id``, else of ``SPARK_USER`` or ``USER``.""" + return self._session._user_id() + + @property + def _use_reattachable_execute(self) -> bool: + """Whether the executions of the client are reattachable, as SparkConnectClient's + ``_use_reattachable_execute``, which ``enable_reattachable_execute`` and + ``disable_reattachable_execute`` set, and which a caller may set.""" + return self._session._reattachable_execute() + + @_use_reattachable_execute.setter + def _use_reattachable_execute(self, value: bool) -> None: + self._session._set_reattachable_execute(bool(value)) + + def _get_operation_statuses( + self, + operation_ids: Optional[list[str]] = None, + operation_extensions: Optional[list[Any]] = None, + request_extensions: Optional[list[Any]] = None, + ) -> "pb2.GetStatusResponse": + """The statuses of the operations of the session, as SparkConnectClient's: of those of + ``operation_ids``, or of all of them if None or empty, of the per-operation + ``operation_extensions`` and the request-level ``request_extensions`` (each a + ``google.protobuf.Any``) that the request carries, as the full ``GetStatusResponse``. The + Rust transport sets the IDs of the session, the client type and the user of the request, + as PySpark's does.""" + import pyspark.sql.connect.proto as pb2 + + req = pb2.GetStatusRequest() + req.operation_status.SetInParent() + if operation_ids: + req.operation_status.operation_ids.extend(operation_ids) + if operation_extensions: + req.operation_status.extensions.extend(operation_extensions) + if request_extensions: + req.extensions.extend(request_extensions) + return pb2.GetStatusResponse.FromString(self._session._get_status(req.SerializeToString())) @property def is_closed(self) -> bool: @@ -1091,13 +1151,14 @@ def semantic_hash(self, plan: "pb2.Plan") -> int: def _execute_plan(self, plan: "pb2.Plan", observations: Any, reported: bool) -> Any: """Start the execution of ``plan``, a ``spark.connect.Plan`` that the client sends as it is, as SparkConnectClient's request of a copy of it: its responses, whose observed metrics - set the Observations of ``observations`` by their names, and whose progress reaches the - progress handlers when ``reported``.""" - return self._session._execute_plan( - plan.SerializeToString(), list(observations.items()), reported - ) - - def _results(self, responses: Any) -> Iterator[Any]: + set the Observations of ``observations``, a dict, by their names, and whose progress + reaches the progress handlers when ``reported``. Of an ``observations`` that is not a + dict, such as None or an empty list, none is set: PySpark's client reads it only when + observed metrics arrive (see ``_results``).""" + items = list(observations.items()) if isinstance(observations, Mapping) else [] + return self._session._execute_plan(plan.SerializeToString(), items, reported) + + def _results(self, responses: Any, observations: Any) -> Iterator[Any]: """The results of ``responses``, as SparkConnectClient's ``_execute_and_fetch_as_iterator`` yields them of each response: its plan metrics, its observed metrics, the schema of its result, the properties of the result of a command by @@ -1110,7 +1171,13 @@ def _results(self, responses: Any) -> Iterator[Any]: try: for data, metrics, observed_metrics in responses: yield from metrics - yield from observed_metrics + for observed in observed_metrics: + # As PySpark's client, which looks the name of the observed metrics up in + # `observations` when they arrive: the TypeError of `in` of an object that is + # not a container, such as None. + if observed.name != "__python_accumulator__": + _ = observed.name in observations + yield observed b = pb2.ExecutePlanResponse.FromString(data) if b.HasField("schema"): dt = types.proto_schema_to_pyspark_data_type(b.schema) @@ -1180,7 +1247,7 @@ def _execute_and_fetch( properties: dict[str, Any] = {} responses = self._execute_plan(plan, observations, True) - for response in self._results(responses): + for response in self._results(responses, observations): if isinstance(response, StructType): schema = response elif isinstance(response, pa.RecordBatch): @@ -1238,7 +1305,7 @@ def to_table_as_iterator( if logger.isEnabledFor(logging.DEBUG): logger.debug(f"Executing plan {self._proto_to_string(plan, True)}") sent = _plan_of(plan) - for response in self._results(self._execute_plan(sent, observations, True)): + for response in self._results(self._execute_plan(sent, observations, True), observations): if isinstance(response, StructType): yield response elif isinstance(response, pa.RecordBatch): @@ -1336,7 +1403,8 @@ def execute_command_as_iterator( f"Execute command as iterator for command {self._proto_to_string(command, True)}" ) plan = _plan_of_command(command) - for response in self._results(self._execute_plan(plan, observations or {}, False)): + observations = observations or {} + for response in self._results(self._execute_plan(plan, observations, False), observations): if isinstance(response, dict): yield response else: @@ -1349,8 +1417,13 @@ def execute_command_as_iterator( def add_threadlocal_user_context_extension(self, extension: Any) -> str: """Add ``extension``, a ``google.protobuf.Any``, to the user context of the requests that - the current thread sends, as SparkConnectClient's: its ID.""" - return self._session._add_user_context_extension(extension.SerializeToString(), False) + the current thread sends, as SparkConnectClient's: to the list of + ``thread_local.user_context_extensions``, of its ID.""" + if not hasattr(self.thread_local, "user_context_extensions"): + self.thread_local.user_context_extensions = list() + extension_id = "threadlocal_" + str(uuid.uuid4()) + self.thread_local.user_context_extensions.append((extension_id, extension)) + return extension_id def add_global_user_context_extension(self, extension: Any) -> str: """Add ``extension``, a ``google.protobuf.Any``, to the user context of the requests of @@ -1362,10 +1435,15 @@ def add_global_user_context_extension(self, extension: Any) -> str: return extension_id def remove_user_context_extension(self, extension_id: str) -> None: - """Remove the user context extension of ``extension_id``, as SparkConnectClient's: a - global one rebinds ``global_user_context_extensions`` to a list without it.""" + """Remove the user context extension of ``extension_id``, as SparkConnectClient's: of the + current thread, which rebinds ``thread_local.user_context_extensions`` to a list without + it, or a global one, which rebinds ``global_user_context_extensions``.""" if extension_id.find("threadlocal_") == 0: - self._session._remove_user_context_extension(extension_id) + if not hasattr(self.thread_local, "user_context_extensions"): + return + self.thread_local.user_context_extensions = list( + filter(lambda ex: ex[0] != extension_id, self.thread_local.user_context_extensions) + ) elif extension_id.find("global_") == 0: with self.global_user_context_extensions_lock: self.global_user_context_extensions = list( @@ -1374,30 +1452,66 @@ def remove_user_context_extension(self, extension_id: str) -> None: def clear_user_context_extensions(self) -> None: """Remove the user context extensions of the current thread and the global ones, as - SparkConnectClient's: ``global_user_context_extensions`` is a new, empty list.""" - self._session._clear_user_context_extensions() + SparkConnectClient's: each is a new, empty list, that of the thread if it has one.""" + if hasattr(self.thread_local, "user_context_extensions"): + self.thread_local.user_context_extensions = list() with self.global_user_context_extensions_lock: self.global_user_context_extensions = list() def add_tag(self, tag: str) -> None: """Add ``tag`` to the tags of the current thread, which the executions that it starts - carry, as SparkConnectClient's: ``CANNOT_BE_NONE`` for None, ``VALUE_ALLOWED`` for a tag - of a comma and ``VALUE_NOT_NON_EMPTY_STR`` for an empty one.""" - self._session.addTag(tag) + carry, as SparkConnectClient's: to the set of ``thread_local.tags``, of the errors of + ``_throw_if_invalid_tag``.""" + self._throw_if_invalid_tag(tag) + if not hasattr(self.thread_local, "tags"): + self.thread_local.tags = set() + self.thread_local.tags.add(tag) def remove_tag(self, tag: str) -> None: """Remove ``tag`` from the tags of the current thread, as SparkConnectClient's: of the errors of ``add_tag``, and nothing for a tag that the thread does not have.""" - self._session.removeTag(tag) + self._throw_if_invalid_tag(tag) + if not hasattr(self.thread_local, "tags"): + self.thread_local.tags = set() + # Use discard, not remove: removing an absent tag is a documented no-op + # (see SparkSession.removeTag), matching the Classic behavior. + self.thread_local.tags.discard(tag) def get_tags(self) -> set[str]: - """The tags of the current thread, as SparkConnectClient's: a copy of them, which the - Rust transport keeps, where PySpark's client gives the set that it keeps.""" - return self._session.getTags() + """The tags of the current thread, as SparkConnectClient's: the set of + ``thread_local.tags``, which the later changes of the tags change too.""" + if not hasattr(self.thread_local, "tags"): + self.thread_local.tags = set() + return self.thread_local.tags def clear_tags(self) -> None: - """Remove the tags of the current thread, as SparkConnectClient's.""" - self._session.clearTags() + """Remove the tags of the current thread, as SparkConnectClient's: ``thread_local.tags`` + is a new, empty set.""" + self.thread_local.tags = set() + + def _throw_if_invalid_tag(self, tag: str) -> None: + """ + Validate if a tag for ExecutePlanRequest.tags is valid. Throw ``ValueError`` if + not. + """ + spark_job_tags_sep = "," + if tag is None: + raise PySparkValueError( + errorClass="CANNOT_BE_NONE", messageParameters={"arg_name": "Spark Connect tag"} + ) + if spark_job_tags_sep in tag: + raise PySparkValueError( + errorClass="VALUE_ALLOWED", + messageParameters={ + "arg_name": "Spark Connect tag", + "disallowed_value": spark_job_tags_sep, + }, + ) + if len(tag) == 0: + raise PySparkValueError( + errorClass="VALUE_NOT_NON_EMPTY_STR", + messageParameters={"arg_name": "Spark Connect tag", "arg_value": tag}, + ) def interrupt_all(self) -> Optional[list[str]]: """Interrupt the executions of the session, as SparkConnectClient's: the IDs of the @@ -1429,24 +1543,38 @@ def register_udf( """Register the Python function ``function`` as a temporary function of the session of the name ``name``, else of a name that the client generates, as SparkConnectClient's ``register_udf``: the name. A DDL ``return_type`` is parsed by the server, and the - function is pickled with the type, as PySpark's ``PythonUDF`` does.""" + function is pickled with the type, as PySpark's ``PythonUDF`` does. As PySpark's, the + name and ``deterministic`` are set in the fields of the command first, then the return + type is parsed and converted, then ``eval_type`` is set, and the function is pickled + last, so each raises the error of its protobuf field or of its conversion in that + order.""" + import pyspark.sql.connect.proto as pb2 + from pyspark.serializers import CloudPickleSerializer + if name is None: name = f"fun_{uuid.uuid4().hex}" + # As PySpark's CommonInlineUserDefinedFunction.to_plan_udf, which sets these first. + fun = pb2.CommonInlineUserDefinedFunction() + fun.function_name = name + fun.deterministic = deterministic + # And then its PythonUDF.to_plan: the parse of a DDL string, the proto of the return type, + # the eval type, and the pickle. output_type = return_type if isinstance(return_type, str): output_type = self._session._parse_ddl(return_type) assert isinstance(output_type, DataType) else: _check_data_type(return_type) - from pyspark.serializers import CloudPickleSerializer - + udf = pb2.PythonUDF() + udf.eval_type = eval_type + command = CloudPickleSerializer().dumps((function, output_type)) self._session._registerPythonUdf( - name, + fun.function_name, output_type, - eval_type, - CloudPickleSerializer().dumps((function, output_type)), + udf.eval_type, + command, f"{sys.version_info.major}.{sys.version_info.minor}", - deterministic, + fun.deterministic, ) return name @@ -1460,24 +1588,30 @@ def register_udtf( ) -> str: """Register the user-defined table function class ``function`` as the temporary function ``name`` of the session, as SparkConnectClient's ``register_udtf``: the name. A - DDL ``return_type`` is left to the server to parse. As PySpark's, the name and then the - return type are converted before the handler is pickled, so a handler that does not - pickle raises ``UDTF_SERIALIZATION_ERROR`` only of a valid name and return type.""" + DDL ``return_type`` is left to the server to parse. As PySpark's, the ``__name__`` of the + handler class is read, the name and ``deterministic`` are set in the fields of the + command, the return type is converted and ``eval_type`` set before the handler is + pickled, so a handler that does not pickle raises ``UDTF_SERIALIZATION_ERROR`` only of + valid arguments.""" + import pyspark.sql.connect.proto as pb2 from pyspark.sql.connect.utils import get_python_ver from pyspark.sql.udtf import _pickled_handler # As PySpark's PythonUDTF, which reads the name of the handler class when it is made. _ = function.__name__ - # As PySpark's register_udtf, which sets the name in its command and converts the return - # type (PythonUDTF.to_plan) before it pickles the handler: `_registerPythonUdtf` calls the - # pickling after them. + # As PySpark's CommonInlineUserDefinedTableFunction.udtf_plan, which sets these first, and + # then converts the return type and sets the eval type (PythonUDTF.to_plan) before it + # pickles the handler: `_registerPythonUdtf` converts them, and then calls the pickling. + fun = pb2.CommonInlineUserDefinedTableFunction() + fun.function_name = name + fun.deterministic = deterministic self._session._registerPythonUdtf( - name, + fun.function_name, return_type, eval_type, functools.partial(_pickled_handler, function), get_python_ver(), - deterministic, + fun.deterministic, ) return name @@ -1497,9 +1631,21 @@ def register_java( ``name`` of the session, as SparkConnectClient's ``register_java``: an aggregate one for ``aggregate``, of the class's own type, and of a ``return_type``, a DataType or a DDL string that the server parses, a function that is not an aggregate one, as PySpark's - ``JavaUDF`` of an output type is.""" + ``JavaUDF`` of an output type is. As PySpark's, the name, the class name, the return + type and then ``aggregate``, unless there is a return type, are set in the fields of the + command in that order, so each raises the error of its protobuf field in that order.""" + import pyspark.sql.connect.proto as pb2 + + # As PySpark's CommonInlineUserDefinedFunction.to_plan_judf and JavaUDF.to_plan. + fun = pb2.CommonInlineUserDefinedFunction() + fun.function_name = name + java = pb2.JavaUDF() + java.class_name = javaClassName + if return_type is not None and not isinstance(return_type, str): + _check_data_type(return_type) + java.aggregate = aggregate if return_type is None else False self._session._registerJavaFunction( - name, javaClassName, return_type, aggregate if return_type is None else False + fun.function_name, java.class_name, return_type, java.aggregate ) def add_artifacts(self, *paths: str, pyfile: bool, archive: bool, file: bool) -> None: @@ -1512,11 +1658,15 @@ def add_artifacts(self, *paths: str, pyfile: bool, archive: bool, file: bool) -> def copy_from_local_to_fs(self, local_path: str, dest_path: str) -> None: """Copy the local file ``local_path`` to the absolute path ``dest_path`` of the file system of the server, as SparkConnectClient's ``copy_from_local_to_fs``: of the - assertions of its ArtifactManager, without the check of a scheme of + assertions and the errors of its ArtifactManager, without the check of a scheme of ``SparkSession.copyFromLocalToFs``.""" - assert Path(local_path).absolute().is_file(), "local path must be a file path." + abs_path = Path(local_path).absolute() + assert abs_path.is_file(), "local path must be a file path." assert Path(dest_path).is_absolute(), "destination FS path must be an absolute path." - self._session.copyFromLocalToFs(local_path, dest_path) + # As the name of the artifact of PySpark's ArtifactManager, which raises the TypeError of + # the concatenation for a destination that is not a str. + _ = "forward_to_fs" + dest_path + self._session._copy_from_local_to_fs(str(abs_path), dest_path) def cache_artifact(self, blob: bytes) -> str: """Cache ``blob``, a bytes-like object, as an artifact of the session, as @@ -1744,17 +1894,23 @@ class _GlobalUserContextExtensions(list): """The global user context extensions of a client, as the list of PySpark's ``SparkConnectClient.global_user_context_extensions``, of their IDs and their ``google.protobuf.Any``: a list whose changes the session of the client takes, so that the - requests of all its threads carry them, as PySpark's requests read the list. Its copies and its - pickles are plain lists.""" + requests of all its threads carry them, as PySpark's requests read the list, while it is the + list of the clients of the session. Its copies and its pickles are plain lists.""" def __init__(self, session: Any, extensions: Iterable[Any] = ()) -> None: super().__init__(extensions) - self._session = session + self._session_ref = weakref.ref(session) def _sync(self) -> None: - self._session._set_global_user_context_extensions( - [(extension_id, extension.SerializeToString()) for extension_id, extension in self] - ) + session = self._session_ref() + if session is None: + return + state = _CLIENT_STATES.get(session) + # A list that the attribute of the session's clients is no longer set to is not the one + # that the requests read, as in PySpark, and neither is a copy's. + if state is None or state.get("global_user_context_extensions") is not self: + return + session._set_global_user_context_extensions(_serialized_extensions(self)) def __reduce_ex__(self, protocol: Any) -> Any: return (list, (list(self),)) @@ -1766,35 +1922,164 @@ def __deepcopy__(self, memo: Any) -> list[Any]: return copy.deepcopy(list(self), memo) -def _synced(name: str) -> Any: - # A method of a list that changes it, after which the session takes the extensions. - method = getattr(list, name) +class _ThreadExtensions(list): + """The user context extensions of the threads that have it, as the list of PySpark's + ``thread_local.user_context_extensions``, of their IDs and their ``google.protobuf.Any``: the + list of a thread, which the requests of the thread carry, as PySpark's read it, and of the + threads that are given it, as those of ``inheritable_thread_target``. Its changes are those of + the list of the Rust transport that it holds, which the threads that have it share. Its copies + and its pickles are plain lists.""" + + def __init__(self, session: Any, extensions: Iterable[Any]) -> None: + super().__init__(extensions) + self._session_ref = weakref.ref(session) + self._handle = session._new_user_context_extensions(_serialized_extensions(self)) - def synced(self: _GlobalUserContextExtensions, *args: Any, **kwargs: Any) -> Any: + @classmethod + def _of(cls, thread_local: Any, extensions: Any) -> Any: + # The list of the current thread once `thread_local.user_context_extensions` is set to + # `extensions`: a list of the session, which the thread shares, or a new one of its + # extensions, which the thread has alone. + session = thread_local._session_ref() + if session is None: + return extensions + if not (isinstance(extensions, cls) and extensions._session_ref() is session): + extensions = cls(session, extensions) + session._share_thread_user_context_extensions(extensions._handle) + return extensions + + def _sync(self) -> None: + self._handle._replace(_serialized_extensions(self)) + + def _copy(self) -> "_ThreadExtensions": + # A list of deep copies of the extensions, which no thread has yet. + session = self._session_ref() + assert session is not None + return type(self)(session, copy.deepcopy(list(self))) + + def __reduce_ex__(self, protocol: Any) -> Any: + return (list, (list(self),)) + + def __copy__(self) -> list[Any]: + return list(self) + + def __deepcopy__(self, memo: Any) -> list[Any]: + return copy.deepcopy(list(self), memo) + + +class _ThreadTags(set): + """The operation tags of the threads that have it, as the set of PySpark's + ``thread_local.tags``: the set of a thread, which its executions carry, as PySpark's read it, + and of the threads that are given it, as those of ``inheritable_thread_target``. Its changes + are those of the set of the Rust transport that it holds, which the threads that have it + share: a str that UTF-8 cannot encode fails each request, as in PySpark, and an object that is + neither a str nor bytes raises the ``TypeError`` of the protobuf field of the tags, which + PySpark raises when it builds a request, when it is added. Its copies and its pickles are plain + sets.""" + + def __init__(self, session: Any, tags: Iterable[Any]) -> None: + super().__init__(tags) + self._session_ref = weakref.ref(session) + self._handle = session._new_tags(list(self)) + + @classmethod + def _of(cls, thread_local: Any, tags: Any) -> Any: + # The set of the current thread once `thread_local.tags` is set to `tags`: a set of the + # session, which the thread shares, or a new one of its tags, which the thread has alone. + session = thread_local._session_ref() + if session is None: + return tags + if not (isinstance(tags, cls) and tags._session_ref() is session): + tags = cls(session, tags) + session._share_thread_tags(tags._handle) + return tags + + def _sync(self) -> None: + self._handle._replace(list(self)) + + def _copy(self) -> "_ThreadTags": + # A set of the same tags, which no thread has yet. + session = self._session_ref() + assert session is not None + return type(self)(session, copy.deepcopy(set(self))) + + def __repr__(self) -> str: + # As the repr of PySpark's plain set. + return repr(set(self)) + + def __reduce_ex__(self, protocol: Any) -> Any: + return (set, (list(self),)) + + def __copy__(self) -> set[Any]: + return set(self) + + def __deepcopy__(self, memo: Any) -> set[Any]: + return copy.deepcopy(set(self), memo) + + +def _serialized_extensions(extensions: Iterable[Any]) -> list[tuple[Any, bytes]]: + # The IDs and the bytes of the google.protobuf.Any of `extensions`, as the Rust transport takes + # them for the user context of its requests. + return [(extension_id, extension.SerializeToString()) for extension_id, extension in extensions] + + +def _synced(base: type, name: str) -> Any: + # A method of `base`, a list or a set, that changes it, after which the Rust transport takes + # the extensions or the tags: a change that it does not take, of an object that it cannot + # send, is undone, and its error raised. + method = getattr(base, name) + + def synced(self: Any, *args: Any, **kwargs: Any) -> Any: + before = base(self) result = method(self, *args, **kwargs) - self._sync() + try: + self._sync() + except BaseException: + base.clear(self) + if base is list: + base.extend(self, before) + else: + base.update(self, before) + raise return result synced.__name__ = synced.__qualname__ = name return synced +for _list in (_GlobalUserContextExtensions, _ThreadExtensions): + for _name in ( + "append", + "extend", + "insert", + "remove", + "pop", + "clear", + "sort", + "reverse", + "__setitem__", + "__delitem__", + "__iadd__", + "__imul__", + ): + setattr(_list, _name, _synced(list, _name)) for _name in ( - "append", - "extend", - "insert", + "add", + "discard", "remove", "pop", "clear", - "sort", - "reverse", - "__setitem__", - "__delitem__", - "__iadd__", - "__imul__", + "update", + "difference_update", + "intersection_update", + "symmetric_difference_update", + "__ior__", + "__iand__", + "__isub__", + "__ixor__", ): - setattr(_GlobalUserContextExtensions, _name, _synced(_name)) -del _name + setattr(_ThreadTags, _name, _synced(set, _name)) +del _list, _name def _extensions_of(extensions: Any) -> list[Any]: @@ -1805,21 +2090,108 @@ def _extensions_of(extensions: Any) -> list[Any]: return [(extension_id, ProtoAny.FromString(data)) for extension_id, data in extensions] -class _ThreadLocalOfClient: - """The state of the current thread of the session of a client, as SparkConnectClient's - ``thread_local``: its user context extensions and its tags, which the Rust transport keeps - for each thread.""" +class _ThreadLocalOfClient(threading.local): + """The state of each thread of the session of a client, as SparkConnectClient's + ``thread_local``, a ``threading.local``: the attributes that a thread sets are its own, and + PySpark's ``tags`` and ``user_context_extensions`` exist once a thread uses them. The set of + the tags and the list of the extensions that a thread sets are those that its requests carry + (see ``_ThreadTags`` and ``_ThreadExtensions``), and so are those of the threads that are given + them, as by ``inheritable_thread_target``. Deleting them leaves the thread with none.""" - def __init__(self, session: Any) -> None: - self._session = session + # The session of the clients, held weakly, which all threads see. + __slots__ = ("_session_ref",) - @property - def user_context_extensions(self) -> list[Any]: - return _extensions_of(self._session._user_context_extensions(False)) + def __setattr__(self, name: str, value: Any) -> None: + if name == "tags": + value = _ThreadTags._of(self, value) + elif name == "user_context_extensions": + value = _ThreadExtensions._of(self, value) + super().__setattr__(name, value) + + def __delattr__(self, name: str) -> None: + super().__delattr__(name) + session = self._session_ref() + if session is None: + return + if name == "tags": + session._clear_thread_tags() + elif name == "user_context_extensions": + session._clear_thread_user_context_extensions() + + +def _thread_local_copies(thread_local: Any) -> list[tuple[str, Any]]: + """The attributes of the current thread of ``thread_local``, a client's, as PySpark's + ``inheritable_thread_target`` copies them for the threads of a function, of ``copy.deepcopy`` + of each: the copies of the set of the tags and of the list of the extensions are a set and a + list that the threads that are given them share, as the threads share PySpark's copies.""" + copies = [] + for attr, value in thread_local.__dict__.items(): + if isinstance(value, (_ThreadTags, _ThreadExtensions)): + value = value._copy() + else: + value = copy.deepcopy(value) + copies.append((attr, value)) + return copies + + +# The attributes of the clients of each session, which they share, as PySpark's session has one +# client (see `SparkConnectClient.__slots__`): held weakly by their session, which they hold +# weakly too. +_CLIENT_STATES: "weakref.WeakKeyDictionary[Any, dict[str, Any]]" = weakref.WeakKeyDictionary() +_CLIENT_STATES_LOCK = threading.Lock() + + +def _client_state(session: Any, builder: Optional[ChannelBuilder]) -> dict[str, Any]: + """The attributes of the clients of ``session``, made with the first of them: its channel + builder ``builder``, as PySpark's ``_builder``, else a ``DefaultChannelBuilder`` of the + connection of the session, of the host and the token of the client; its ``thread_local``; its + ``global_user_context_extensions``, those that the session has, whose changes, and a list that + the attribute is set to, are the session's (see ``SparkConnectClient.__setattr__``); and their + lock.""" + with _CLIENT_STATES_LOCK: + state = _CLIENT_STATES.get(session) + if state is None: + connection_string = getattr(session, "_connection_string", None) + if builder is None and connection_string is not None: + # As PySpark's client of a session, of the channel builder of its connection. + builder = DefaultChannelBuilder(connection_string()) + thread_local = _ThreadLocalOfClient() + thread_local._session_ref = weakref.ref(session) + extensions = _extensions_of(session._user_context_extensions(True)) + state = { + "_builder": builder, + "thread_local": thread_local, + "global_user_context_extensions": _GlobalUserContextExtensions( + session, extensions + ), + "global_user_context_extensions_lock": threading.Lock(), + } + _CLIENT_STATES[session] = state + # As PySpark's requests, which serialize the extensions when they are built: the calls + # of the session serialize them again before they send requests. + session._set_user_context_refresh( + functools.partial(_refresh_user_context_extensions, weakref.ref(session)) + ) + return state - @property - def tags(self) -> Any: - return set(self._session.getTags()) + +def _refresh_user_context_extensions(session_ref: Any) -> None: + """Serialize the user context extensions of the session of ``session_ref`` again for the + Rust transport, of its clients' global list and of the list of the current thread, before a + call of the session sends requests, as PySpark serializes them when it builds each request: + so a ``google.protobuf.Any`` of the lists that was changed in place is sent as it is now.""" + session = session_ref() + if session is None: + return + state = _CLIENT_STATES.get(session) + if state is None: + return + extensions = state.get("global_user_context_extensions") + if isinstance(extensions, _GlobalUserContextExtensions) and extensions: + extensions._sync() + own = getattr(state.get("thread_local"), "user_context_extensions", None) + if isinstance(own, _ThreadExtensions) and own: + own._sync() # The client of each session, by its ID, while it is in use. diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py index 8d0a7250..9ad10cb8 100644 --- a/python/pyspark/sql/udf.py +++ b/python/pyspark/sql/udf.py @@ -22,14 +22,14 @@ import functools import sys -import uuid import warnings -from typing import Callable, Optional, Any, Union +from typing import Callable, Optional, Any from pyspark.errors import PySparkNotImplementedError, PySparkRuntimeError, PySparkTypeError from pyspark.sql.pandas.types import to_arrow_type from pyspark.sql.types import DataType, StringType, StructType -from pyspark.serializers import CloudPickleSerializer +# As PySpark's module, of whose namespace it is (see the stubs). +from pyspark.serializers import CloudPickleSerializer # noqa: F401 from pyspark.util import PythonEvalType __all__ = ["UDFRegistration"] @@ -590,7 +590,9 @@ def register( "SQL_GROUPED_AGG_PANDAS_ITER_UDF or SQL_GROUPED_AGG_ARROW_ITER_UDF" }, ) - self._register_udf(f.func, f.returnType, name, f.evalType, f.deterministic) + self.sparkSession._client.register_udf( + f.func, f.returnType, name, f.evalType, f.deterministic + ) return f else: if returnType is None: @@ -601,34 +603,9 @@ def register( # As PySpark's: the registration takes the return type as given, not the one that # the wrapper parsed, so a DDL string is parsed again for the command. - self._register_udf(py_udf.func, returnType, name) + self.sparkSession._client.register_udf(py_udf.func, returnType, name) return py_udf - def _register_udf( - self, - function: Callable[..., Any], - return_type: Union[DataType, str], - name: str, - eval_type: int = PythonEvalType.SQL_BATCHED_UDF, - deterministic: bool = True, - ) -> None: - # As PySpark's SparkConnectClient.register_udf: a function of no name is registered as - # `fun_` and a random hex, and the PythonUDF command is the pickled (function, return - # type), of a DDL string as the server of the session parses it, as its PythonUDF parses - # the str that it is given when it builds the command. - if name is None: - name = f"fun_{uuid.uuid4().hex}" - if isinstance(return_type, str): - return_type = self.sparkSession._parse_ddl(return_type) - self.sparkSession._registerPythonUdf( - name, - return_type, - eval_type, - CloudPickleSerializer().dumps((function, return_type)), - "%d.%d" % sys.version_info[:2], - deterministic, - ) - def registerJavaFunction( self, name: str, @@ -648,10 +625,10 @@ def registerJavaFunction( Return type: a DataType, or a DDL string, which the server parses, as PySpark sends it. Defaults to the class's own type. """ - self.sparkSession._registerJavaFunction(name, javaClassName, returnType, False) + self.sparkSession._client.register_java(name, javaClassName, returnType) def registerJavaUDAF(self, name: str, javaClassName: str) -> None: """Register a Java user-defined aggregate function (UDAF) by class name. Mirrors ``UDFRegistration.registerJavaUDAF``. """ - self.sparkSession._registerJavaFunction(name, javaClassName, None, True) + self.sparkSession._client.register_java(name, javaClassName, aggregate=True) diff --git a/python/pyspark/util.py b/python/pyspark/util.py index eebb89ea..98a8536e 100644 --- a/python/pyspark/util.py +++ b/python/pyspark/util.py @@ -176,10 +176,11 @@ def inheritable_thread_target(f: Union[Callable, "SparkSession"]) -> Callable: >>> Thread(target=inheritable_thread_target(session)(target_func)).start() # doctest: +SKIP """ # As PySpark's of Spark Connect, the only mode of this client: `f` is the session, whose - # tags and user context extensions, those of this thread when the function is decorated, the - # threads of the function inherit, with the session as their active one. As PySpark's, which - # gives each thread the one copy of the `thread_local` of the client, the threads share that - # copy of the tags and of the extensions. + # `thread_local` of its client, of this thread when the function is decorated, the threads of + # the function inherit, with the session as their active one. As PySpark's, which gives each + # thread the one copy of each attribute of the `thread_local`, the threads share that copy of + # the tags and of the user context extensions. + from pyspark.sql.connect.client import _thread_local_copies from pyspark.sql.connect.session import SparkSession session = f @@ -188,8 +189,7 @@ def outer(ff: Callable) -> Callable: assert isinstance(session, SparkSession), ( "f is expected to be SparkSession for spark connect" ) - tags = session._copy_thread_tags() - extensions = session._copy_thread_user_context_extensions() + session_client_thread_local_attrs = _thread_local_copies(session.client.thread_local) @functools.wraps(ff) def inner(*args: Any, **kwargs: Any) -> Any: @@ -199,9 +199,13 @@ def inner(*args: Any, **kwargs: Any) -> Any: ) SparkSession._set_default_and_active_session(session) - # Set the tags and the user context extensions of the session in the child thread. - session._share_thread_tags(tags) - session._share_thread_user_context_extensions(extensions) + # Set thread locals in child thread. + for attr, value in session_client_thread_local_attrs: + setattr( + session.client.thread_local, + attr, + value, + ) return ff(*args, **kwargs) return inner @@ -242,18 +246,21 @@ def __init__( self._session = session def copy_local_properties(*a: Any, **k: Any) -> Any: + # Set tags in child thread. assert hasattr(self, "_tags") assert session is not None - session.clearTags() - for tag in self._tags: - session.addTag(tag) + thread_local = session.client.thread_local + thread_local.tags = self._tags return target(*a, **k) super().__init__(target=copy_local_properties, *args, **kwargs) # type: ignore[misc] def start(self) -> None: assert hasattr(self, "_session") - self._tags = set(self._session.getTags()) + thread_local = self._session.client.thread_local + if not hasattr(thread_local, "tags"): + thread_local.tags = set() + self._tags = set(thread_local.tags) return super().start() diff --git a/python/tests/_mock_connect_server.py b/python/tests/_mock_connect_server.py index 8aa7b7af..62f2441e 100644 --- a/python/tests/_mock_connect_server.py +++ b/python/tests/_mock_connect_server.py @@ -387,6 +387,8 @@ def reset(self): self.rpc_events = [] self.set_confs = [] self.clone_requests = [] + # The GetStatusRequests of each GetStatus. + self.get_status_requests = [] # The DDL of the JSON of each JsonToDDL. self.ddl_string = "" with self.listener_condition: @@ -912,6 +914,21 @@ def clone_session(self, request, context): new_id = fields.get(4, [self.cloned_session_id.encode()])[0] return message_field(1, fields[1][0]) + message_field(3, new_id) + def get_status(self, request, context): + # GetStatusRequest.session_id=1, operation_status=5, OperationStatusRequest + # .operation_ids=1; GetStatusResponse.session_id=1, operation_statuses=3, + # OperationStatus.operation_id=1, state=2. Each operation that the request names is + # OPERATION_STATE_UNKNOWN=1, as the server answers of an operation that it does not + # know. + if self.is_recorded(request, "get_status"): + self.get_status_requests.append(request) + fields = parse_message(request) + operation_status = parse_message(fields.get(5, [b""])[0]) + return message_field(1, fields[1][0]) + b"".join( + message_field(3, message_field(1, operation_id) + varint_field(2, 1)) + for operation_id in operation_status.get(1, []) + ) + def config(self, request, context): # ConfigRequest.session_id=1, operation=3, Operation.get=2, Get.keys=1; # ConfigResponse.session_id=1, pairs=2, KeyValue.key=1, value=2. @@ -1015,6 +1032,7 @@ def config(self, request, context): "Config": grpc.unary_unary_rpc_method_handler(service.config), "CloneSession": grpc.unary_unary_rpc_method_handler(service.clone_session), "Interrupt": grpc.unary_unary_rpc_method_handler(service.interrupt), + "GetStatus": grpc.unary_unary_rpc_method_handler(service.get_status), "AddArtifacts": grpc.stream_unary_rpc_method_handler(service.add_artifacts), } server.add_generic_rpc_handlers( diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py index ce99b9f1..3661a671 100644 --- a/python/tests/test_dropin_e2e.py +++ b/python/tests/test_dropin_e2e.py @@ -426,6 +426,163 @@ class Mine(Observation): assert unused._result is None +def test_client_state_is_one_per_session_and_its_thread_local_a_threading_local_as_pyspark( + session, +): + # As PySpark Connect's SparkConnectClient: the attributes of the client of a session are its + # own, and its thread_local is a threading.local whose tags and user context extensions exist + # once a thread uses them, are the set and the list that the requests of the thread carry, + # also as they change, and are of that thread only. The extensions are serialized for each + # request, so one changed in place is sent as it is. + import threading + + import pyspark.sql.connect.proto as pb2 + from pyspark.sql.connect.client import SparkConnectClient + + try: + from google.protobuf.any_pb2 import Any + except ImportError: + from pyspark.sql.connect.proto._rust import Any + + session.client.state_of_the_test = 1 + assert session.client.state_of_the_test == 1 + held = session.client.global_user_context_extensions + assert session.client.global_user_context_extensions_lock is ( + session.client.global_user_context_extensions_lock + ) + + seen = [] + + class Hook: + def on_execute_plan(self, request): + seen.append( + ( + sorted(request.tags), + [(e.type_url, e.value) for e in request.user_context.extensions], + ) + ) + return request + + client = SparkConnectClient(os.environ["SPARK_REMOTE"], session_hooks=[Hook()]) + + def run(): + plan = pb2.Plan() + plan.root.range.end = 1 + plan.root.range.step = 1 + client.to_table(plan, {}) + return seen[-1] + + local = client.thread_local + assert not hasattr(local, "tags") and not hasattr(local, "user_context_extensions") + first = Any(type_url="type.googleapis.com/t1", value=b"1") + client.add_threadlocal_user_context_extension(first) + extensions = local.user_context_extensions + assert extensions[0][1] is first and local.user_context_extensions is extensions + extensions.append(("threadlocal_x", Any(type_url="type.googleapis.com/t2", value=b"2"))) + assert run() == ([], [("type.googleapis.com/t1", b"1"), ("type.googleapis.com/t2", b"2")]) + first.value = b"changed" + assert run()[1][0] == ("type.googleapis.com/t1", b"changed") + client.add_tag("a") + assert local.tags is client.get_tags() + local.tags.add("b") + assert run()[0] == ["a", "b"] + found = {} + + def other(): + found["tags"] = hasattr(local, "tags") + found["extensions"] = hasattr(local, "user_context_extensions") + + thread = threading.Thread(target=other) + thread.start() + thread.join() + assert found == {"tags": False, "extensions": False} + + shared = Any(type_url="type.googleapis.com/g1", value=b"g") + client.add_global_user_context_extension(shared) + shared.value = b"changed" + assert run()[1][0] == ("type.googleapis.com/g1", b"changed") + client.close() + del session.client.state_of_the_test + assert held is session.client.global_user_context_extensions + + +def test_client_registrations_and_executions_take_their_arguments_as_pyspark(session): + # As PySpark Connect's register_udf, which sets the name and `deterministic` in its command + # before the server parses a DDL return type, and the eval type after it, and its executions, + # which read their `observations` only when observed metrics arrive. + import threading + + import pyspark.sql.connect.proto as pb2 + from pyspark.errors import ParseException + from pyspark.sql.connect.observation import Observation + + client = session.client + lock = threading.Lock() + + def unpicklable(x): + return lock + + with pytest.raises(TypeError, match="cannot be interpreted as an integer"): + client.register_udf(unpicklable, "not a type ###", deterministic="x") + with pytest.raises(ParseException): + client.register_udf(unpicklable, "not a type ###", eval_type="x") + with pytest.raises(TypeError, match="cannot be interpreted as an integer"): + client.register_udf(unpicklable, "int", eval_type="x") + + observation = Observation("client_observations") + observed = session.range(3).observe(observation, F.count(F.lit(1)).alias("n")) + plan = pb2.Plan() + plan.root.CopyFrom(observed._plan.plan(client)) + assert client.to_table(plan, [])[0].num_rows == 3 + for call in ( + lambda: client.to_table(plan, None), + lambda: client.to_pandas(plan, None), + lambda: list(client.to_table_as_iterator(plan, None)), + ): + with pytest.raises(TypeError, match="argument of type 'NoneType' is not iterable"): + call() + plain_frame = session.range(2) + plain = pb2.Plan() + plain.root.CopyFrom(plain_frame._plan.plan(client)) + assert client.to_table(plain, None)[0].num_rows == 2 + assert len(client.to_pandas(plain, None)[0]) == 2 + + +def test_client_members_that_pysparks_tests_read_as_pyspark(session): + # As PySpark Connect's SparkConnectClient: _get_operation_statuses is the GetStatus RPC of the + # session, of the operations asked of; a set of _session_id makes the requests carry another + # session ID, so the response of another server-side session closes the session, as + # test_connect_session's test_reset_when_server_and_client_sessionids_mismatch relies on; and + # _closed reopens a session that it closed. + import uuid + + from pyspark.errors.exceptions.connect import SparkConnectException + from pyspark.sql.connect.session import SparkSession as ConnectSession + + remote = os.environ["SPARK_REMOTE"] + other = ConnectSession.builder.remote(remote).create() + client = other.client + other.range(3).collect() + statuses = client._get_operation_statuses() + assert type(statuses).__name__ == "GetStatusResponse" + assert statuses.session_id == other.session_id + unknown = "00000000-0000-4000-8000-000000000000" + (status,) = client._get_operation_statuses([unknown]).operation_statuses + assert status.operation_id == unknown + assert client._builder.endpoint == remote[len("sc://") :].split("/")[0] + + client._closed = True + with pytest.raises(SparkConnectException): + other.range(1).count() + client._closed = False + assert other.range(1).count() == 1 and not other.is_stopped + + client._session_id = str(uuid.uuid4()) + with pytest.raises(SparkConnectException): + other.range(3).collect() + assert other.is_stopped + + def test_client_of_a_connection_string_as_pyspark(session): # As PySpark Connect's SparkConnectClient(connection, ...): a client of a session of its own, # of the plans that it is given, built without a DataFrame, whose executions may be not diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 6e006077..21be7d6a 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -2088,7 +2088,8 @@ def batch(index, num_chunks, data, start_offset=None): def test_tags_are_those_of_each_thread_as_pyspark(mock_connect): # As PySpark's: the tags of addTag are those of the current thread, of a set of getTags, - # which the operations that the thread starts carry; a new thread starts without tags. + # which the operations that the thread starts carry; a new thread starts without tags. The + # set of getTags is the live one, which the later tags change, as PySpark's. import threading spark, service = mock_connect @@ -2101,9 +2102,11 @@ def sent_tags(): seen = {} def child(): - seen["before"] = spark.getTags() + seen["before"] = set(spark.getTags()) + live = spark.getTags() spark.addTag("child") seen["after"] = spark.getTags() + assert seen["after"] is live spark.range(1).collect() seen["sent"] = sent_tags() @@ -4274,6 +4277,9 @@ class Session: def _user_context_extensions(self, is_global): return [] + def _set_user_context_refresh(self, refresh): + pass + client = SparkConnectClient._of_session(Session()) relation = pb2.Relation() relation.sql.query = "q" * 2000 @@ -7523,14 +7529,26 @@ def test_udf_defaults_and_registration(): assert pandas_udf(lambda s: s, rt, functionType=ft).evalType == ft with pytest.raises(PySparkTypeError, match="INVALID_RETURN_TYPE_FOR_PANDAS_UDF"): pandas_udf(lambda s: s, T.IntegerType(), functionType=PandasUDFType.GROUPED_MAP) - # UDFRegistration.register wires to the session's _registerPythonUdf (so the name - # resolves in SQL) and returns the UDF. + # UDFRegistration.register registers through the client of the session, as PySpark's, whose + # register_udf wires to the session's _registerPythonUdf (so the name resolves in SQL), and + # returns the UDF. + from pyspark.sql.connect.client import SparkConnectClient from pyspark.sql.udf import UDFRegistration class _StubSession: def __init__(self): self.calls = [] + @property + def _client(self): + return SparkConnectClient._of_session(self) + + def _user_context_extensions(self, is_global): + return [] + + def _set_user_context_refresh(self, refresh): + pass + def _registerPythonUdf(self, name, returnType, evalType, command, python_ver, deterministic): self.calls.append((name, evalType, isinstance(command, (bytes, bytearray)))) @@ -24995,6 +25013,9 @@ class Session: def _user_context_extensions(self, is_global): return [] + def _set_user_context_refresh(self, refresh): + pass + def _analyze_plan(self, data): Session.sent = pb2.AnalyzePlanRequest.FromString(data) return Session.response.SerializeToString() @@ -25458,7 +25479,8 @@ def test_client_raises_the_argument_errors_of_pyspark(mock_connect): # TypeError of hashlib for a blob that is not bytes-like, which its ArtifactManager hashes # before it sends anything (test_dropin_e2e.py caches bytes-like ones), and register_udf and # register_udtf raise the UNSUPPORTED_OPERATION of pyspark_types_to_proto_types for a return - # type that is neither a DataType nor a DDL string, before they pickle the function. + # type that is neither a DataType nor a DDL string, after the name and before they pickle the + # function (see test_client_registrations_convert_their_arguments_in_the_order_of_pyspark). from pyspark.errors import PySparkValueError spark, service = mock_connect @@ -25493,6 +25515,116 @@ def eval(self): assert (len(service.execute_requests), len(service.analyze_requests)) == sent +def test_client_registrations_convert_their_arguments_in_the_order_of_pyspark(mock_connect): + # As PySpark Connect's register_udf (CommonInlineUserDefinedFunction.to_plan_udf, then + # PythonUDF.to_plan): the name and `deterministic` are set in the fields of the command, the + # return type is converted, the eval type set, and the function pickled last, so each raises + # the error of its protobuf field or of its conversion in that order, and an int as + # `deterministic` is a bool. register_java sets the name, the class name, the return type and + # then `aggregate`, of no return type only; register_udtf reads the __name__ of the handler, + # sets the name and `deterministic`, converts the return type and sets the eval type before + # it pickles the handler. spark.udf.register and registerJavaFunction register through them. + import threading + + import pyspark.sql.connect.proto as pb2 + from pyspark.errors import PySparkPicklingError, PySparkValueError + from pyspark.sql.functions import udf + from pyspark.sql.types import IntegerType + + spark, service = mock_connect + client = spark.client + lock = threading.Lock() + + def unpicklable(x): + return lock + + class Good: + def eval(self): + yield (1,) + + class Bad: + def eval(self): + yield (lock,) + + def raised(call, kind, message): + with pytest.raises(kind) as error: + call() + assert str(error.value).startswith(message), str(error.value) + return error.value + + bad_name = "bad argument type for built-in operation" + not_an_int = "'str' object cannot be interpreted as an integer" + sent = len(service.execute_requests), len(service.analyze_requests) + raised(lambda: client.register_udf(unpicklable, "int", name=5), TypeError, bad_name) + raised(lambda: client.register_udf(unpicklable, None, name=5), TypeError, bad_name) + raised(lambda: client.register_udf(unpicklable, "int", deterministic="x"), TypeError, not_an_int) + raised(lambda: client.register_udf(unpicklable, 5, deterministic="x"), TypeError, not_an_int) + raised(lambda: client.register_udf(unpicklable, IntegerType(), eval_type="x"), TypeError, not_an_int) + raised( + lambda: client.register_udf(unpicklable, IntegerType(), eval_type=True), + TypeError, + "Field spark.connect.PythonUDF.eval_type: Expected an int, got a boolean.", + ) + raised( + lambda: client.register_udf(unpicklable, IntegerType(), eval_type=2**40), + ValueError, + "Value out of range: 1099511627776", + ) + raised( + lambda: client.register_udf(unpicklable, IntegerType(), eval_type=100.0), + TypeError, + "'float' object cannot be interpreted as an integer", + ) + raised(lambda: client.register_java(5, "x.Y", aggregate="x"), TypeError, bad_name) + raised(lambda: client.register_java("j", 5, return_type=5), TypeError, bad_name) + error = raised(lambda: client.register_java("j", "x.Y", return_type=5), PySparkValueError, "") + assert error.getCondition() == "UNSUPPORTED_OPERATION" + raised(lambda: client.register_java("j", "x.Y", aggregate="x"), TypeError, not_an_int) + raised(lambda: client.register_udtf(object(), "x int", 5), AttributeError, "'object' object") + raised(lambda: client.register_udtf(Bad, "x int", 5, deterministic="x"), TypeError, bad_name) + raised(lambda: client.register_udtf(Bad, "x int", "t", deterministic="x"), TypeError, not_an_int) + error = raised(lambda: client.register_udtf(Bad, 5, "t", eval_type=True), PySparkValueError, "") + assert error.getCondition() == "UNSUPPORTED_OPERATION" + raised( + lambda: client.register_udtf(Bad, "x int", "t", eval_type=True), + TypeError, + "Field spark.connect.PythonUDTF.eval_type: Expected an int, got a boolean.", + ) + raised( + lambda: client.register_udtf(Good, "x int", "t", eval_type=2**40), + ValueError, + "Value out of range: 1099511627776", + ) + error = raised( + lambda: client.register_udtf(Bad, "x int", "t", deterministic=1), PySparkPicklingError, "" + ) + assert error.getCondition() == "UDTF_SERIALIZATION_ERROR" + raised(lambda: spark.udf.register(5, udf(unpicklable)), TypeError, bad_name) + raised(lambda: spark.udf.register("\udc80", udf(unpicklable)), UnicodeEncodeError, "'utf-8'") + raised(lambda: spark.udf.registerJavaFunction(5, "x.Y"), TypeError, bad_name) + raised(lambda: spark.udf.registerJavaUDAF("j", 5), TypeError, bad_name) + not_none = "'NoneType' object cannot be interpreted as an integer" + raised(lambda: client.register_udf(unpicklable, 5, deterministic=None), TypeError, not_none) + raised(lambda: client.register_udtf(Bad, 5, "t", deterministic=None), TypeError, not_none) + raised(lambda: client.register_java("j", "x.Y", aggregate=None), TypeError, not_none) + assert (len(service.execute_requests), len(service.analyze_requests)) == sent + + def command(): + return pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]).plan.command + + client.register_udf(lambda x: x, IntegerType(), name="d1", deterministic=1) + assert command().register_function.deterministic is True + client.register_udf(lambda x: x, IntegerType(), name="d0", deterministic=0) + assert command().register_function.deterministic is False + client.register_java("j1", "x.Y", aggregate=1) + assert command().register_function.java_udf.aggregate is True + client.register_java("j2", "x.Y", return_type="int", aggregate="ignored") + java = command().register_function.java_udf + assert java.aggregate is False and java.output_type.unparsed.data_type_string == "int" + client.register_udtf(Good, "x int", "t1", deterministic=1) + assert command().register_table_function.deterministic is True + + def test_client_registers_a_udtf_in_the_order_of_pyspark(mock_connect): # As PySpark Connect's register_udtf: its PythonUDTF reads the name of the handler class when # it is made, and its udtf_plan sets the name in the string field of its command and then @@ -25608,6 +25740,345 @@ def name(cls): client.copy_from_local_to_fs(str(path), "/dest/a.py") +def test_the_attributes_of_the_clients_of_a_session_are_its_own_as_pyspark(mock_connect): + # As PySpark Connect's session, of one client: the clients of a session that SparkSession.client + # gives share its attributes, also once the program dropped them, so an attribute set on one, + # the lock and the list of the global user context extensions are those of the next. A list + # read before an extension is added is the one that it is added to, and the one that the + # requests carry; a list that the attribute is no longer set to is not. + import gc + import threading + + import pyspark.sql.connect.proto as pb2 + from pyspark.sql.connect.proto._rust import Any + + spark, service = mock_connect + + def sent(): + request = pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]) + return [extension.type_url for extension in request.user_context.extensions] + + def run(): + spark.range(1).collect() + return sent() + + client = spark.client + client.attribute = 1 + lock = client.global_user_context_extensions_lock + assert isinstance(lock, type(threading.Lock())) + held = client.global_user_context_extensions + del client + gc.collect() + client = spark.client + assert client.attribute == 1 and client.global_user_context_extensions_lock is lock + assert client.global_user_context_extensions is held + first = client.add_global_user_context_extension(Any(type_url="type.googleapis.com/e1")) + held.append(("global_2", Any(type_url="type.googleapis.com/e2"))) + assert [e.type_url for _, e in spark.client.global_user_context_extensions] == [ + "type.googleapis.com/e1", + "type.googleapis.com/e2", + ] + assert run() == ["type.googleapis.com/e1", "type.googleapis.com/e2"] + # Rebinding the attribute detaches the list that was held, as in PySpark. + client.remove_user_context_extension(first) + assert client.global_user_context_extensions is not held + held.append(("global_3", Any(type_url="type.googleapis.com/e3"))) + assert run() == ["type.googleapis.com/e2"] + # A change of an extension in place is sent, as PySpark serializes them for each request. + client.global_user_context_extensions[0][1].type_url = "type.googleapis.com/changed" + assert run() == ["type.googleapis.com/changed"] + client.clear_user_context_extensions() + assert run() == [] + del client.attribute + + +def test_the_thread_local_of_a_client_is_a_threading_local_of_live_tags_and_extensions_as_pyspark( + mock_connect, +): + # As PySpark Connect's thread_local, a threading.local: its tags and user_context_extensions + # exist once a thread uses them, are the set and the list that the requests of the thread + # carry, also as they are changed or set, and are the thread's own, as are its other + # attributes. The tags of SparkSession.addTag are those of the client, and getTags gives their + # set. A thread of inheritable_thread_target shares the copies of the attributes of the thread + # that decorated its function. Deleting the attributes leaves the thread with none. + import threading + + import pyspark.sql.connect.proto as pb2 + from pyspark.sql.connect.proto._rust import Any + from pyspark.util import inheritable_thread_target + + spark, service = mock_connect + client = spark.client + local = client.thread_local + + def run(): + spark.range(1).collect() + request = pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]) + return sorted(request.tags), [e.type_url for e in request.user_context.extensions] + + assert isinstance(local, threading.local) + assert not hasattr(local, "tags") and not hasattr(local, "user_context_extensions") + first = Any(type_url="type.googleapis.com/t1") + client.add_threadlocal_user_context_extension(first) + extensions = local.user_context_extensions + assert extensions[0][1] is first and local.user_context_extensions is extensions + extensions.append(("threadlocal_x", Any(type_url="type.googleapis.com/t2"))) + assert run() == ([], ["type.googleapis.com/t1", "type.googleapis.com/t2"]) + local.user_context_extensions = [("threadlocal_y", Any(type_url="type.googleapis.com/t3"))] + assert run() == ([], ["type.googleapis.com/t3"]) + local.user_context_extensions[0][1].type_url = "type.googleapis.com/t4" + assert run() == ([], ["type.googleapis.com/t4"]) + + spark.addTag("a") + tags = local.tags + assert tags is client.get_tags() is spark.getTags() and tags == {"a"} + assert repr(tags) == "{'a'}" + tags.add("b") + assert run() == (["a", "b"], ["type.googleapis.com/t4"]) + local.tags = {"z"} + assert run() == (["z"], ["type.googleapis.com/t4"]) and spark.getTags() == {"z"} + # A tag that the field of the tags of a request cannot take is refused, and the set is kept. + with pytest.raises(TypeError, match="bad argument type for built-in operation"): + local.tags.add(("not", "a", "str")) + assert local.tags == {"z"} and run()[0] == ["z"] + + seen = {} + + def other(): + seen["tags"] = hasattr(local, "tags") + seen["extensions"] = hasattr(local, "user_context_extensions") + local.other = 1 + seen["run"] = run() + + thread = threading.Thread(target=other) + thread.start() + thread.join() + assert seen == {"tags": False, "extensions": False, "run": ([], [])} + assert not hasattr(local, "other") + assert sorted(vars(local)) == ["tags", "user_context_extensions"] + + @inheritable_thread_target(spark) + def inherited(): + local.tags.add(threading.current_thread().name) + return sorted(spark.getTags()), run() + + results = {} + threads = [ + threading.Thread(target=lambda n=n: results.setdefault(n, inherited()), name=f"t{n}") + for n in range(2) + ] + for thread in threads: + thread.start() + thread.join() + # The threads share the copy of the set of tags, as PySpark's, and not the parent's set. + assert results[1][0] == ["t0", "t1", "z"] + assert results[1][1] == (["t0", "t1", "z"], ["type.googleapis.com/t4"]) + assert spark.getTags() == {"z"} + + del local.tags + del local.user_context_extensions + assert not hasattr(local, "tags") and run() == ([], []) + spark.clearTags() + assert spark.getTags() == set() and run() == ([], []) + + +def test_client_members_that_pysparks_tests_read_are_pysparks(mock_connect): + # As PySpark Connect's SparkConnectClient, whose tests and libraries read these members: + # _builder is the DefaultChannelBuilder of the connection; _session_id the session ID of the + # requests, which a set changes for them while the session's session_id stays; _user_id the + # user of the requests; _use_reattachable_execute whether the executions are reattachable, + # of enable_reattachable_execute and of a set; and _closed closes the session, and opens it + # again when it is set to False. + import uuid + + import pyspark.sql.connect.proto as pb2 + from pyspark.errors.exceptions.connect import SparkConnectException + from pyspark.sql.connect.client import DefaultChannelBuilder + + spark, service = mock_connect + client = spark.client + assert isinstance(client._builder, DefaultChannelBuilder) + assert client._builder.endpoint == service.remote[len("sc://"):] + assert client._builder.userAgent.startswith("_SPARK_CONNECT_PYTHON spark/") + assert client.host == client._builder.host + assert client._session_id == spark.session_id + assert client._user_id == os.environ.get("SPARK_USER", os.environ.get("USER")) + assert client._use_reattachable_execute is True + client._use_reattachable_execute = False + assert client._use_reattachable_execute is False + client.enable_reattachable_execute() + assert client._use_reattachable_execute is True + + original = spark.session_id + changed = str(uuid.uuid4()) + client._session_id = changed + spark.range(1).collect() + request = pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]) + assert request.session_id == changed and spark.session_id == original + assert client._session_id == changed + with pytest.raises(TypeError, match="bad argument type for built-in operation"): + client._session_id = 5 + client._session_id = original + + assert client._closed is False + client._closed = True + with pytest.raises(SparkConnectException) as error: + spark.range(1).collect() + assert error.value.getCondition() == "NO_ACTIVE_SESSION" + assert client._closed is True and client.is_closed and spark.is_stopped + client._closed = False + assert client._closed is False and not client.is_closed and not spark.is_stopped + spark.range(1).collect() + + +def test_client_get_operation_statuses_sends_pysparks_get_status_request(mock_connect): + # As PySpark Connect's SparkConnectClient._get_operation_statuses (client/core.py:2183-2237): + # a GetStatusRequest of the session ID, of the client type and of the user of the client's + # requests, of an operation_status that is set even of no operation ID, which asks of all of + # them, of the operation IDs, and of the extensions of each level; the GetStatusResponse is + # what it gives. + import pyspark.sql.connect.proto as pb2 + from pyspark.sql.connect.proto._rust import Any + + spark, service = mock_connect + client = spark.client + spark.range(1).collect() + execute = pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]) + every = client._get_operation_statuses() + assert type(every).__name__ == "GetStatusResponse" + assert every.session_id == spark.session_id and len(every.operation_statuses) == 0 + asked = client._get_operation_statuses( + ["op-1", "op-2"], + [Any(type_url="type.googleapis.com/operation")], + [Any(type_url="type.googleapis.com/request")], + ) + unknown = pb2.GetStatusResponse.OperationStatus.OPERATION_STATE_UNKNOWN + assert [(status.operation_id, status.state) for status in asked.operation_statuses] == [ + ("op-1", unknown), + ("op-2", unknown), + ] + first, second = (pb2.GetStatusRequest.FromString(r) for r in service.get_status_requests) + for request in (first, second): + assert request.session_id == spark.session_id + assert request.client_type == execute.client_type + assert request.user_context.user_id == execute.user_context.user_id == client._user_id + assert request.HasField("operation_status") + assert list(first.operation_status.operation_ids) == [] + assert len(first.operation_status.extensions) == 0 and len(first.extensions) == 0 + assert list(second.operation_status.operation_ids) == ["op-1", "op-2"] + assert [e.type_url for e in second.operation_status.extensions] == [ + "type.googleapis.com/operation" + ] + assert [e.type_url for e in second.extensions] == ["type.googleapis.com/request"] + + +def test_the_tags_and_extensions_of_a_thread_copy_and_pickle_as_pysparks_set_and_list( + mock_connect, +): + # As PySpark Connect's thread_local.tags and user_context_extensions, a plain set and a plain + # list: their copies and deep copies are a set and a list of the same members, and their + # pickles a plain set and a plain list. + import copy + import pickle + + from pyspark.sql.connect.proto._rust import Any + + spark, _ = mock_connect + client = spark.client + local = client.thread_local + client.add_tag("a") + client.add_threadlocal_user_context_extension(Any(type_url="type.googleapis.com/t")) + for value, kind in ((local.tags, set), (local.user_context_extensions, list)): + for copied in (copy.copy(value), copy.deepcopy(value)): + assert type(copied) is kind and copied == value + pickled = pickle.loads(pickle.dumps(local.tags)) + assert type(pickled) is set and pickled == {"a"} + local.user_context_extensions = [] + pickled = pickle.loads(pickle.dumps(local.user_context_extensions)) + assert type(pickled) is list and pickled == [] + + +def test_the_tags_and_extensions_of_a_thread_that_has_none_are_pysparks(mock_connect): + # As PySpark Connect's client in a thread whose thread_local has neither: the removal of a + # thread-local extension and the clear of the extensions leave the thread without a list of + # them, while the removal of a tag and get_tags give it an empty set of tags. + import threading + + spark, _ = mock_connect + client = spark.client + seen = {} + + def other(): + local = client.thread_local + client.remove_user_context_extension("threadlocal_missing") + client.clear_user_context_extensions() + seen["extensions"] = hasattr(local, "user_context_extensions") + client.remove_tag("missing") + seen["removed"] = set(local.tags) + del local.tags + seen["got"] = set(client.get_tags()) + + thread = threading.Thread(target=other) + thread.start() + thread.join() + assert seen == {"extensions": False, "removed": set(), "got": set()} + + +def test_the_other_attributes_of_a_thread_local_are_copied_and_deleted_as_pysparks(mock_connect): + # As PySpark Connect's thread_local, each attribute of which inheritable_thread_target copies + # with copy.deepcopy, when it decorates a function, to the threads that run the function, and + # an attribute of which a thread deletes is gone. + import threading + + from pyspark.util import inheritable_thread_target + + spark, _ = mock_connect + local = spark.client.thread_local + local.marker = ["parent"] + + @inheritable_thread_target(spark) + def child(): + return local.marker + + seen = [] + thread = threading.Thread(target=lambda: seen.append(child())) + thread.start() + thread.join() + assert seen == [["parent"]] and seen[0] is not local.marker + del local.marker + assert not hasattr(local, "marker") + + +def test_the_thread_local_of_a_client_outlives_its_session_as_pysparks(mock_connect): + # As PySpark Connect's thread_local, a threading.local, and its global list of extensions, + # which a caller may keep: once the session is gone, the tags and the extensions that a thread + # sets and deletes, and a change of the list, are those of plain ones. + import gc + import weakref + + from pyspark.sql.connect.proto._rust import Any + + spark, _ = mock_connect + other = spark.newSession() + local = other.client.thread_local + extensions = other.client.global_user_context_extensions + local.tags = {"a"} + gone = weakref.ref(other) + other.stop() + del other + gc.collect() + assert gone() is None + local.tags = {"b"} + assert type(local.tags) is set and local.tags == {"b"} + del local.tags + assert not hasattr(local, "tags") + local.user_context_extensions = [("threadlocal_a", Any(type_url="type.googleapis.com/a"))] + assert type(local.user_context_extensions) is list + del local.user_context_extensions + extensions.append(("global_a", Any(type_url="type.googleapis.com/a"))) + assert [extension_id for extension_id, _ in extensions] == ["global_a"] + + def test_client_tags_interrupts_and_sessions_are_those_of_its_session_as_pyspark(mock_connect): # As PySpark Connect's SparkConnectClient, whose tags, interrupts and sessions are those of # SparkSession.addTag, interruptOperation, newSession and cloneSession. @@ -30059,6 +30530,78 @@ def test_copy_from_local_to_fs_requires_an_absolute_destination(mock_connect, tm spark.copyFromLocalToFs(str(local), "relative/out.txt") +def test_copy_from_local_to_fs_raises_pysparks_errors_of_a_destination(mock_connect, tmp_path): + # As PySpark Connect's copyFromLocalToFs, which parses the destination with urlparse before + # its client's copy_from_local_to_fs: of an object that is not a str, the AttributeError of + # urlparse, or of bytes, the TypeError of the Path of the destination. The client's, which + # parses nothing, raises the TypeError of the name of the artifact, the concatenation of a + # prefix and the destination, for a Path. + import pathlib + import re + + spark, _ = mock_connect + local = tmp_path / "local.txt" + local.write_text("x") + destination = pathlib.Path("/dest/out.txt") + with pytest.raises(AttributeError, match="'PosixPath' object has no attribute 'decode'"): + spark.copyFromLocalToFs(str(local), destination) + with pytest.raises(AttributeError, match="'int' object has no attribute 'decode'"): + spark.copyFromLocalToFs(str(local), 5) + with pytest.raises(TypeError, match="argument should be a str"): + spark.copyFromLocalToFs(str(local), b"/dest/out.txt") + with pytest.raises( + TypeError, match=re.escape('can only concatenate str (not "PosixPath") to str') + ): + spark.client.copy_from_local_to_fs(str(local), destination) + with pytest.raises(TypeError, match="argument should be a str"): + spark.client.copy_from_local_to_fs(str(local), b"/dest/out.txt") + + +def test_client_executions_read_their_observations_as_pyspark(mock_connect): + # As PySpark Connect's to_table and to_table_as_iterator, which look the names of observed + # metrics up in `observations` when they arrive: None and an empty list execute a plan whose + # results have none, an empty list one whose results have some, and None and an int raise + # the TypeError of `in` when they arrive. The Observations of other names are not set. (The + # e2e test covers to_pandas, whose configurations the mock server does not have.) + import pyspark.sql.connect.proto as pb2 + from pyspark.sql import Observation + from pyspark.sql import functions as F + + spark, service = mock_connect + client = spark.client + observation = Observation("o") + df = spark.range(3).observe(observation, F.count(F.lit(1)).alias("c")) + batch, metrics = _count_batch_and_observed_metrics(service, df) + plan = pb2.Plan() + plan.root.CopyFrom(df._plan.plan(client)) + service.execute_payload = batch + assert client.to_table(plan, None)[0].num_rows == 1 + assert len(list(client.to_table_as_iterator(plan, None))) >= 1 + service.execute_payload = batch + metrics("o", varint_field(7, 3), "c") + assert client.to_table(plan, [])[0].num_rows == 1 + for observations, kind in ((None, "NoneType"), (5, "int")): + with pytest.raises(TypeError, match=f"argument of type '{kind}' is not iterable"): + client.to_table(plan, observations) + with pytest.raises(TypeError, match="argument of type 'NoneType' is not iterable"): + list(client.to_table_as_iterator(plan, None)) + assert observation.get == {} + client.to_table(plan, df._plan.observations) + assert observation.get == {"c": 3} + # The observed metrics of the accumulators of the Python workers are not looked up, as + # PySpark's client reads the updates of its UDF profiler of them instead; those of another + # accumulator than the profiler's are ignored. + from pyspark.accumulators import specialAccumulatorSer + + # ExecutePlanResponse.observed_metrics=6 -> ObservedMetrics name=1, values=2 (Expression + # .Literal binary=2). + update = message_field(2, message_field(2, specialAccumulatorSer.dumps((999, None)))) + service.execute_payload = batch + message_field( + 6, message_field(1, "__python_accumulator__") + update + ) + assert client.to_table(plan, None)[0].num_rows == 1 + assert len(list(client.to_table_as_iterator(plan, None))) >= 1 + + def test_requests_of_a_stopped_session_raise_no_active_session_as_pyspark(): # As PySpark's closed client: stop() releases the session and closes the client, so each # request of the stopped session, and of its DataFrames, raises NO_ACTIVE_SESSION. diff --git a/python/tests/test_ml_offline.py b/python/tests/test_ml_offline.py index 4cf09781..f905ccd9 100644 --- a/python/tests/test_ml_offline.py +++ b/python/tests/test_ml_offline.py @@ -2942,17 +2942,18 @@ def packed(graph_id): analysis = pb2.AnalyzePlanRequest.FromString(service.analyze_requests[-1]) assert list(analysis.user_context.extensions) == [packed("global"), packed("local")] - # Another thread has the global extensions only. + # Another thread has the global extensions only, and no list of its own until it adds one: + # PySpark's thread_local, a threading.local, has no user_context_extensions there. seen = [] def other(): - seen.append(client.thread_local.user_context_extensions) + seen.append(hasattr(client.thread_local, "user_context_extensions")) client.execute_command(command) thread = threading.Thread(target=other) thread.start() thread.join() - assert seen == [[]] + assert seen == [False] request = _last_request_but_deletes(service) assert list(request.user_context.extensions) == [packed("global")]