diff --git a/crates/pyspark-rs/src/client.rs b/crates/pyspark-rs/src/client.rs index 6a0173d8..eb917418 100644 --- a/crates/pyspark-rs/src/client.rs +++ b/crates/pyspark-rs/src/client.rs @@ -318,8 +318,7 @@ pub(crate) fn set_retry_policy(session: &SparkSession, parameters: RetryParamete } /// The RPC deadlines of `deadlines`, the deadlines in seconds by the names of the fields of -/// PySpark's `RpcDeadlines`, None for no deadline. A field that the transport has no RPC of, as -/// `get_status`, is not read. +/// PySpark's `RpcDeadlines`, None for no deadline. pub(crate) fn rpc_deadlines(deadlines: &Bound<'_, PyDict>) -> PyResult { let of = |name: &str| -> PyResult> { let seconds: Option = match deadlines.get_item(name)? { @@ -344,6 +343,7 @@ pub(crate) fn rpc_deadlines(deadlines: &Bound<'_, PyDict>) -> PyResult>>( run_interruptible(py, session, f, true) } +/// Bring the user context extensions of `session` up to date before a call sends its requests, +/// on the calling thread, with the GIL: the function of `client` that the session keeps (see +/// `PySparkSession::_set_user_context_refresh`), which serializes the `google.protobuf.Any` +/// messages of its lists again, as PySpark serializes them when it builds each request, so that +/// a change of a message in place is sent. Its error is raised, as PySpark raises that of the +/// serialization from the call. +pub(crate) fn refresh_user_context(py: Python<'_>, session: &SparkSession) -> PyResult<()> { + if let Some(refresher) = session.user_context_refresher() { + if let Some(refresh) = refresher.downcast_ref::>() { + refresh.call0(py)?; + } + } + Ok(()) +} + /// The wait of [`interruptible_outcome`], of whose results a failed interrupt is the inner one /// when `catch_interrupt_failure`. fn run_interruptible>>( @@ -246,6 +261,7 @@ fn run_interruptible>>( f: impl FnOnce() -> Result + Send, catch_interrupt_failure: bool, ) -> PyResult>> { + refresh_user_context(py, session)?; let watch = Arc::new(SignalWatch { session: session.clone(), interrupted: AtomicBool::new(false), diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index c4a7a70e..25c8afd2 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -1211,6 +1211,101 @@ pub struct PySharedTags(spark_connect::session::SharedTags); )] pub struct PySharedUserContextExtensions(spark_connect_core::client::SharedUserContextExtensions); +#[pymethods] +impl PySharedTags { + /// Replace the tags of this set with `tags`, which the requests of the threads that have it + /// carry from now on, as a change of the set of PySpark's `thread_local.tags`. + fn _replace(&self, tags: Vec>) -> PyResult<()> { + self.0.replace(core_tags(&tags)?); + Ok(()) + } + + /// The tags of this set, as the strs that they are of. + fn _tags<'py>(&self, py: Python<'py>) -> PyResult>> { + self.0 + .tags() + .into_iter() + .map(|tag| match tag { + spark_connect::session::Tag::Text(text) => { + Ok(pyo3::types::PyString::new(py, &text).into_any()) + } + spark_connect::session::Tag::Unencodable(utf32le) => { + pyo3::types::PyBytes::new(py, &utf32le) + .call_method1("decode", ("utf-32-le", "surrogatepass")) + } + }) + .collect() + } +} + +#[pymethods] +impl PySharedUserContextExtensions { + /// Replace the extensions of this list with `extensions`, which the requests of the threads + /// that have it carry from now on, as a change of the list of PySpark's + /// `thread_local.user_context_extensions`. + fn _replace(&self, extensions: Vec<(String, Vec)>) -> PyResult<()> { + self.0.replace(decoded_extensions(extensions)?); + Ok(()) + } + + /// The extensions of this list, their IDs and the bytes of each `google.protobuf.Any`. + fn _extensions<'py>(&self, py: Python<'py>) -> Vec<(String, Bound<'py, pyo3::types::PyBytes>)> { + use prost::Message; + self.0 + .snapshot() + .into_iter() + .map(|(id, extension)| { + ( + id, + pyo3::types::PyBytes::new(py, &extension.encode_to_vec()), + ) + }) + .collect() + } +} + +/// The `google.protobuf.Any` of each of `extensions`, of its ID and its bytes. +fn decoded_extensions( + extensions: Vec<(String, Vec)>, +) -> PyResult> { + use prost::Message; + extensions + .into_iter() + .map(|(id, extension)| { + prost_types::Any::decode(extension.as_slice()) + .map(|extension| (id, extension)) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string())) + }) + .collect() +} + +/// The core tags of `tags`, the elements of the set of PySpark's `thread_local.tags`, as the +/// requests carry them: a str of its text, or of the UTF-32 bytes of a str that UTF-8 cannot +/// encode, with which each request fails, as PySpark's; bytes of their UTF-8; and the `TypeError` +/// of the protobuf string field of the tags of a request for another object, which PySpark raises +/// when it builds a request, and the drop-in when the set changes. +fn core_tags(tags: &[Bound<'_, PyAny>]) -> PyResult> { + tags.iter() + .map(|tag| { + if let Ok(text) = tag.cast::() { + return Ok(match text.to_cow() { + Ok(text) => spark_connect::session::Tag::Text(text.into_owned()), + Err(_) => { + let utf32le = + text.call_method1("encode", ("utf-32-le", "surrogatepass"))?; + spark_connect::session::Tag::Unencodable( + utf32le.cast::()?.as_bytes().to_vec(), + ) + } + }); + } + Ok(spark_connect::session::Tag::Text( + crate::options::str_argument(tag)?, + )) + }) + .collect() +} + /// The tag of `interruptTag` or the ID of `interruptOperation`, as PySpark's `_interrupt_request` /// takes it: it asserts that it is not None, and sets it in the string field of its request, as /// protobuf converts it, so bytes of UTF-8 are taken, and another value is a `TypeError`. @@ -1221,46 +1316,6 @@ fn interrupted_id(id: &Bound<'_, PyAny>) -> PyResult { crate::str_argument(id) } -/// A tag of `addTag` or `removeTag`: `CANNOT_BE_NONE` for None, as PySpark Connect's. -fn tag_of(py: Python<'_>, tag: &Bound<'_, PyAny>) -> PyResult { - if tag.is_none() { - return Err(crate::column::raise_pyspark( - py, - "PySparkValueError", - "CANNOT_BE_NONE", - &[("arg_name", "Spark Connect tag")], - )); - } - // As PySpark's `"," in tag`, which raises the `TypeError` of an object that is not a str, - // such as an int or bytes, before the checks of the tag; and of the set of the tags, which - // refuses one that is not hashable, such as a list. - tag.contains(",")?; - let tag = if let Ok(text) = tag.cast::() { - text - } else { - tag.hash()?; - return Ok(SessionTag::Text(tag.extract()?)); - }; - let Ok(text) = tag.to_cow() else { - // A str that UTF-8 cannot encode, which PySpark keeps, and which fails the requests that - // carry it. Its rendering has a comma if the str does, and is not empty. - let utf32le = tag.call_method1("encode", ("utf-32-le", "surrogatepass"))?; - let rendered = crate::plan_args::rendered(tag.as_any())?; - SparkSession::check_tag(&rendered).to_pyerr()?; - return Ok(SessionTag::Unencodable( - utf32le.cast::()?.as_bytes().to_vec(), - )); - }; - Ok(SessionTag::Text(text.into_owned())) -} - -/// A tag of `addTag` and `removeTag`: its text, or the UTF-32 bytes of a str that UTF-8 cannot -/// encode, which the core keeps (see `SparkSession::add_unencodable_tag`). -enum SessionTag { - Text(String), - Unencodable(Vec), -} - #[pyclass( name = "SparkSession", module = "pyspark.sql.connect.session", @@ -2907,25 +2962,28 @@ impl PySparkSession { } /// Copy a local file to the Spark-managed filesystem (`copyFromLocalToFs`), with the errors - /// of PySpark Connect's: `NO_SCHEMA_AND_DRIVER_DEFAULT_SCHEME` for a destination of a scheme, - /// an `AssertionError` for a local path that is not a file, and for a destination that is - /// not an absolute path. The local path is what PySpark's `Path(local_path).absolute()` takes, - /// a str or an `os.PathLike` such as a `pathlib.Path`, of its errors otherwise, and the file - /// of the str of that absolute path is uploaded, as PySpark's `LocalFile` reads it. The - /// destination is a str. + /// of PySpark Connect's: those of `urlparse` of the destination, such as the AttributeError + /// of a `pathlib.Path`, and `NO_SCHEMA_AND_DRIVER_DEFAULT_SCHEME` for a destination of a + /// scheme, and then those of its client's `copy_from_local_to_fs`: an `AssertionError` for a + /// local path that is not a file, and for a destination that is not an absolute path, and the + /// `TypeError` of a destination that is not a str. The local path is what PySpark's + /// `Path(local_path).absolute()` takes, a str or an `os.PathLike` such as a `pathlib.Path`, of + /// its errors otherwise, and the file of the str of that absolute path is uploaded, as + /// PySpark's `LocalFile` reads it. #[pyo3(name = "copyFromLocalToFs")] fn copy_from_local_to_fs( - &self, - py: Python<'_>, + slf: &Bound<'_, Self>, local_path: &Bound<'_, PyAny>, - dest_path: &str, + dest_path: &Bound<'_, PyAny>, ) -> PyResult<()> { - let scheme: String = py + let py = slf.py(); + // As PySpark's, which parses the destination first, of the errors of `urlparse`, such as + // the AttributeError of a `pathlib.Path`, and then copies the file with its client. + let scheme = py .import("urllib.parse")? .call_method1("urlparse", (dest_path,))? - .getattr("scheme")? - .extract()?; - if !scheme.is_empty() { + .getattr("scheme")?; + if scheme.is_truthy()? { return Err(crate::column::raise_pyspark( py, "PySparkValueError", @@ -2933,32 +2991,22 @@ impl PySparkSession { &[("arg_name", "dest_path")], )); } - let absolute = py - .import("pathlib")? - .getattr("Path")? - .call1((local_path,))? - .call_method0("absolute")?; - let is_file: bool = absolute.call_method0("is_file")?.extract()?; - if !is_file { - return Err(pyo3::exceptions::PyAssertionError::new_err( - "local path must be a file path.", - )); - } - let is_absolute: bool = py - .import("pathlib")? - .getattr("Path")? - .call1((dest_path,))? - .call_method0("is_absolute")? - .extract()?; - if !is_absolute { - return Err(pyo3::exceptions::PyAssertionError::new_err( - "destination FS path must be an absolute path.", - )); - } - let local_path: String = absolute.str()?.extract()?; + Self::client(slf)?.call_method1(py, "copy_from_local_to_fs", (local_path, dest_path))?; + Ok(()) + } + + /// Upload the local file of `local_path`, an absolute path, to `dest_path`, an absolute path + /// of the file system of the server, for the `copy_from_local_to_fs` of `client`, which + /// checks them as PySpark's ArtifactManager does. + fn _copy_from_local_to_fs( + &self, + py: Python<'_>, + local_path: String, + dest_path: String, + ) -> PyResult<()> { interruptible(py, &self.session, || { self.session - .copy_from_local_to_fs(&local_path, dest_path) + .copy_from_local_to_fs(&local_path, &dest_path) .map_err(Box::new) }) } @@ -3226,7 +3274,7 @@ impl PySparkSession { py: Python<'_>, name: &Bound<'_, PyAny>, return_type: Option<&Bound<'_, PyAny>>, - eval_type: i32, + eval_type: &Bound<'_, PyAny>, command: &Bound<'_, PyAny>, python_ver: String, deterministic: bool, @@ -3235,7 +3283,8 @@ impl PySparkSession { // first, as protobuf converts it: bytes of UTF-8 too. Its `PythonUDTF.to_plan` converts the // return type after it (see `udtf_return_type`), which raises the `UNSUPPORTED_OPERATION` // of a value that is not a DataType and of a type that `pyspark_types_to_proto_types` has - // no proto of, also nested, and then pickles the handler class, whose error comes last. + // no proto of, also nested, then sets the eval type in the int32 field of its PythonUDTF, + // of protobuf's errors, and then pickles the handler class, whose error comes last. let name = crate::str_argument(name)?; let return_type = match return_type { None => None, @@ -3244,6 +3293,12 @@ impl PySparkSession { .ok_or_else(|| crate::types::not_a_type_error(value))?, ), }; + let udtf = py + .import("pyspark.sql.connect.proto")? + .getattr("PythonUDTF")? + .call0()?; + udtf.setattr("eval_type", eval_type)?; + let eval_type: i32 = udtf.getattr("eval_type")?.extract()?; let command: Vec = if command.is_callable() { command.call0()?.extract()? } else { @@ -3261,24 +3316,9 @@ impl PySparkSession { }) } - /// Add the extension of the bytes of a `google.protobuf.Any`, `extension`, to the user context - /// of the requests of this session: of all the threads, if `is_global`, else of the current - /// thread. Its ID: for the user context extensions of `client`. - fn _add_user_context_extension(&self, extension: &[u8], is_global: bool) -> PyResult { - use prost::Message; - let extension = prost_types::Any::decode(extension) - .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?; - Ok(if is_global { - self.session.add_global_user_context_extension(extension) - } else { - self.session - .add_threadlocal_user_context_extension(extension) - }) - } - /// The user context extensions of this session, the global ones if `is_global`, else those - /// of the current thread, as their IDs and the bytes of each `google.protobuf.Any`: for - /// `client`. + /// of the current thread, as their IDs and the bytes of each `google.protobuf.Any`: for the + /// global list of `client` when it is made. fn _user_context_extensions<'py>( &self, py: Python<'py>, @@ -3308,30 +3348,11 @@ impl PySparkSession { &self, extensions: Vec<(String, Vec)>, ) -> PyResult<()> { - use prost::Message; - let extensions = extensions - .into_iter() - .map(|(id, extension)| { - prost_types::Any::decode(extension.as_slice()) - .map(|extension| (id, extension)) - .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string())) - }) - .collect::>>()?; - self.session.set_global_user_context_extensions(extensions); + self.session + .set_global_user_context_extensions(decoded_extensions(extensions)?); Ok(()) } - /// Remove the user context extension of `extension_id`, for `client`. - fn _remove_user_context_extension(&self, extension_id: &str) { - self.session.remove_user_context_extension(extension_id); - } - - /// Remove the user context extensions of the current thread and the global ones, for - /// `client`. - fn _clear_user_context_extensions(&self) { - self.session.clear_user_context_extensions(); - } - /// The responses of the server to `command`, the bytes of a `spark.connect.Command` built /// outside the client, such as the ML commands of `pyspark.ml`, as the bytes of each /// `ExecutePlanResponse`: for the `execute_command` of `client`. The command is decoded, and @@ -3669,6 +3690,64 @@ impl PySparkSession { self.session.set_server_session_id(id); } + /// The session ID that the requests of the session carry, as PySpark's + /// `SparkConnectClient._session_id`: the session's, unless `_set_client_session_id` set + /// another. + fn _client_session_id(&self) -> String { + self.session.client_session_id() + } + + /// Make the requests of the session carry `session_id`, as a set of PySpark's + /// `SparkConnectClient._session_id`, which its requests read, of the error of the string + /// field of their session ID for a value that is not a str: the session's `session_id` stays, + /// and the requests of the artifacts carry it, as those of PySpark's ArtifactManager. + fn _set_client_session_id(&self, session_id: &Bound<'_, PyAny>) -> PyResult<()> { + self.session + .set_client_session_id(crate::str_argument(session_id)?); + Ok(()) + } + + /// The connection string of the channel builder of the session, as PySpark's + /// `SparkConnectClient._builder` is of: `sc://` and its endpoint, of its parameters, each + /// value percent-encoded, as `DefaultChannelBuilder` decodes them. + fn _connection_string(&self) -> String { + let builder = self.session.channel_builder(); + let mut url = format!("sc://{}/", builder.endpoint()); + for (key, value) in builder.params() { + url.push(';'); + url.push_str(key); + url.push('='); + url.push_str("e_param(value)); + } + url + } + + /// The user of the requests of the session, as PySpark's `SparkConnectClient._user_id`. + fn _user_id(&self) -> Option { + self.session.user_id().map(str::to_string) + } + + /// Open the session again, as PySpark's `SparkConnectClient._closed = False`. + fn _reopen(&self) { + self.session.reopen() + } + + /// The bytes of the response of the server to `request`, the bytes of a `GetStatusRequest` + /// that `client` builds, as PySpark's `_get_operation_statuses` sends it. + fn _get_status<'py>( + &self, + py: Python<'py>, + request: &[u8], + ) -> PyResult> { + use prost::Message; + let request = spark_connect_proto::GetStatusRequest::decode(request) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?; + let response = interruptible(py, &self.session, || { + self.session.get_status_proto(request).map_err(Box::new) + })?; + Ok(pyo3::types::PyBytes::new(py, &response.encode_to_vec())) + } + /// The execution hook base class, reached as `SparkSession.Hook` (mirrors pyspark). #[classattr] #[allow(non_snake_case)] @@ -3875,29 +3954,19 @@ impl PySparkSession { }) } - /// Add a tag of the current thread, with the errors of PySpark Connect's tags: `CANNOT_BE_NONE` - /// for None, and those of [`SparkSession::check_tag`]. + /// Add a tag of the current thread, as PySpark Connect's `addTag`, which its client adds to + /// the set of its `thread_local.tags` (`client.add_tag`), of the errors of PySpark's. #[pyo3(name = "addTag")] - fn add_tag(&self, py: Python<'_>, tag: &Bound<'_, PyAny>) -> PyResult<()> { - match tag_of(py, tag)? { - SessionTag::Text(tag) => self.session.add_tag(&tag).to_pyerr(), - SessionTag::Unencodable(utf32le) => { - self.session.add_unencodable_tag(utf32le); - Ok(()) - } - } + fn add_tag(slf: &Bound<'_, Self>, tag: &Bound<'_, PyAny>) -> PyResult<()> { + Self::client(slf)?.call_method1(slf.py(), "add_tag", (tag,))?; + Ok(()) } - /// Remove a tag of the current thread, with the errors of `addTag`, as PySpark Connect's. + /// Remove a tag of the current thread, as PySpark Connect's `removeTag`, of its client + /// (`client.remove_tag`). #[pyo3(name = "removeTag")] - fn remove_tag(&self, py: Python<'_>, tag: &Bound<'_, PyAny>) -> PyResult<()> { - match tag_of(py, tag)? { - SessionTag::Text(tag) => { - SparkSession::check_tag(&tag).to_pyerr()?; - self.session.remove_tag(&tag); - } - SessionTag::Unencodable(utf32le) => self.session.remove_unencodable_tag(&utf32le), - } + fn remove_tag(slf: &Bound<'_, Self>, tag: &Bound<'_, PyAny>) -> PyResult<()> { + Self::client(slf)?.call_method1(slf.py(), "remove_tag", (tag,))?; Ok(()) } @@ -3920,23 +3989,20 @@ impl PySparkSession { }) } - /// The tags of the current thread, a set, as PySpark's `getTags`. + /// The tags of the current thread, as PySpark Connect's `getTags`: the set of the + /// `thread_local.tags` of its client (`client.get_tags`), which the later changes of the tags + /// change too. #[pyo3(name = "getTags")] - fn get_tags<'py>(&self, py: Python<'py>) -> PyResult> { - let tags = pyo3::types::PySet::new(py, self.session.get_tags())?; - // A tag that UTF-8 cannot encode, as it was given. - for utf32le in self.session.unencodable_tags() { - tags.add( - pyo3::types::PyBytes::new(py, &utf32le) - .call_method1("decode", ("utf-32-le", "surrogatepass"))?, - )?; - } - Ok(tags) + fn get_tags(slf: &Bound<'_, Self>) -> PyResult> { + Self::client(slf)?.call_method0(slf.py(), "get_tags") } + /// Remove the tags of the current thread, as PySpark Connect's `clearTags`, of its client + /// (`client.clear_tags`), which gives the thread a new empty set. #[pyo3(name = "clearTags")] - fn clear_tags(&self) { - self.session.clear_tags() + fn clear_tags(slf: &Bound<'_, Self>) -> PyResult<()> { + Self::client(slf)?.call_method0(slf.py(), "clear_tags")?; + Ok(()) } /// A copy of the tags of the current thread, which `pyspark.util.inheritable_thread_target` @@ -3968,6 +4034,48 @@ impl PySparkSession { .share_threadlocal_user_context_extensions(&extensions.0) } + /// A set of the tags `tags`, which no thread has yet, for a set of the `thread_local.tags` of + /// `client`. + fn _new_tags(&self, tags: Vec>) -> PyResult { + Ok(PySharedTags(spark_connect::session::SharedTags::new( + core_tags(&tags)?, + ))) + } + + /// A list of `extensions`, their IDs and the bytes of each `google.protobuf.Any`, which no + /// thread has yet, for a list of the `thread_local.user_context_extensions` of `client`. + fn _new_user_context_extensions( + &self, + extensions: Vec<(String, Vec)>, + ) -> PyResult { + Ok(PySharedUserContextExtensions( + spark_connect_core::client::SharedUserContextExtensions::new(decoded_extensions( + extensions, + )?), + )) + } + + /// Keep `refresh`, a function of no arguments, which a call of this session calls before it + /// sends requests (see `crate::interrupt::refresh_user_context`): for `client`, which + /// serializes the `google.protobuf.Any` messages of its lists of user context extensions + /// again then, as PySpark serializes them when it builds each request. + fn _set_user_context_refresh(&self, refresh: Py) { + self.session + .set_user_context_refresher(Some(std::sync::Arc::new(refresh))); + } + + /// Forget the tags of the current thread, which has none from now on, for a + /// `thread_local.tags` of `client` that is deleted. + fn _clear_thread_tags(&self) { + self.session.clear_tags() + } + + /// Forget the user context extensions of the current thread, for a + /// `thread_local.user_context_extensions` of `client` that is deleted. + fn _clear_thread_user_context_extensions(&self) { + self.session.clear_threadlocal_user_context_extensions() + } + /// Add local files, or `file:` URIs, as artifacts of the session, as PySpark's /// `addArtifacts(*path, pyfile=False, archive=False, file=False)`: each path is named by /// the flags and its extension, and the errors for a path that is not supported are diff --git a/crates/spark-connect-core/src/channel.rs b/crates/spark-connect-core/src/channel.rs index f39e3377..a4e672b4 100644 --- a/crates/spark-connect-core/src/channel.rs +++ b/crates/spark-connect-core/src/channel.rs @@ -170,6 +170,19 @@ impl ChannelBuilder { self.params.insert(key.to_string(), value.to_string()); } + /// Remove the parameter `key`, as PySpark's `newSession` pops the session ID of the channel + /// builder of its client. + pub fn remove(&mut self, key: &str) { + self.params.remove(key); + } + + /// The parameters of the connection, by their keys, in their order. + pub fn params(&self) -> impl Iterator { + self.params + .iter() + .map(|(key, value)| (key.as_str(), value.as_str())) + } + /// `host:port` endpoint used to dial the gRPC channel. pub fn endpoint(&self) -> String { format!("{}:{}", self.host, self.port) diff --git a/crates/spark-connect-core/src/client.rs b/crates/spark-connect-core/src/client.rs index 6676e872..61f00aed 100644 --- a/crates/spark-connect-core/src/client.rs +++ b/crates/spark-connect-core/src/client.rs @@ -20,8 +20,9 @@ use spark_connect_proto::{ analyze_plan_request::Analyze, AnalyzePlanRequest, AnalyzePlanResponse, ArtifactStatusesRequest, CloneSessionRequest, CompressionCodec, ConfigRequest, ConfigResponse, ExecutePlanRequest, ExecutePlanResponse, FetchErrorDetailsRequest, FetchErrorDetailsResponse, - InterruptRequest, InterruptResponse, Plan, ReattachExecuteRequest, ReleaseExecuteRequest, - ReleaseExecuteResponse, ReleaseSessionRequest, UserContext, + GetStatusRequest, GetStatusResponse, InterruptRequest, InterruptResponse, Plan, + ReattachExecuteRequest, ReleaseExecuteRequest, ReleaseExecuteResponse, ReleaseSessionRequest, + UserContext, }; use crate::artifact::{ @@ -502,6 +503,7 @@ pub struct RpcDeadlines { pub artifact_status: Option, pub fetch_error_details: Option, pub clone_session: Option, + pub get_status: Option, } impl Default for RpcDeadlines { @@ -519,6 +521,7 @@ impl Default for RpcDeadlines { artifact_status: minutes(10), fetch_error_details: minutes(10), clone_session: minutes(10), + get_status: minutes(10), } } } @@ -537,6 +540,7 @@ impl RpcDeadlines { artifact_status: None, fetch_error_details: None, clone_session: None, + get_status: None, } } } @@ -760,6 +764,15 @@ pub struct SparkConnectClient { stub: SparkConnectServiceClient, /// Session ID (UUID v4) identifying this session on the server. session_id: String, + /// The session ID that the requests carry and the responses are checked against, shared by + /// the clones of the client: `session_id` until a caller sets another, as PySpark's + /// `client._session_id` is an attribute that its requests read (see + /// [`SparkConnectClient::set_client_session_id`]). Those of the artifacts carry + /// `session_id`, as PySpark's ArtifactManager keeps the ID of the client when it was built. + client_session_id: std::sync::Arc>, + /// The channel builder of the connection, as PySpark's `client._builder`: of no session ID + /// for the client of a new session, and of its own for that of a clone. + builder: crate::channel::ChannelBuilder, /// User context (user_id from the channel builder). user_id: Option, /// The metadata of the connection string, and the `authorization` of its token. @@ -799,9 +812,10 @@ pub struct SparkConnectClient { /// client, as PySpark's `_server_session_id`. Each request sends it as the observed ID, and /// each response is checked against it. server_session_id: std::sync::Arc>>, - /// Whether the client tracks its server-side session: the observed ID of its requests, the - /// check of the IDs of its responses, and its close when the server reports that the session - /// changed. See [`SparkConnectClient::set_session_tracking`]. + /// Whether the client tracks its session: the session ID and the observed server-side session + /// ID of the requests that its callers build, the check of the IDs of its responses, and its + /// close when the server reports that the session changed. See + /// [`SparkConnectClient::set_session_tracking`]. session_tracking: bool, /// The executions in flight, shared by the clones of the client, in the order they started, /// each with the thread that waits for it now, if any. See @@ -956,6 +970,19 @@ pub fn is_wait_aborted(error: &SparkError) -> bool { struct UserContextExtensions { global: Vec<(String, prost_types::Any)>, by_thread: std::collections::HashMap, + /// What a binding keeps to bring the extensions up to date before the calls of a thread send + /// requests (see [`SparkConnectClient::set_user_context_refresher`]). + refresher: Option, +} + +/// The opaque refresher of the user context extensions of a binding. +#[derive(Clone)] +struct UserContextRefresher(std::sync::Arc); + +impl std::fmt::Debug for UserContextRefresher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("UserContextRefresher") + } } /// The user context extensions of a thread, shared with the threads that are given them by @@ -969,13 +996,27 @@ pub struct SharedUserContextExtensions( ); impl SharedUserContextExtensions { - fn new(extensions: Vec<(String, prost_types::Any)>) -> Self { + /// A list of the extensions `extensions`, of their IDs, which no thread has yet (see + /// [`SparkConnectClient::share_threadlocal_user_context_extensions`]), as the list that a + /// binding keeps of PySpark's `thread_local.user_context_extensions`. Not a stable API. + #[doc(hidden)] + pub fn new(extensions: Vec<(String, prost_types::Any)>) -> Self { Self(std::sync::Arc::new(std::sync::Mutex::new(extensions))) } - fn snapshot(&self) -> Vec<(String, prost_types::Any)> { + /// The extensions of this list, of their IDs, in their order. Not a stable API. + #[doc(hidden)] + pub fn snapshot(&self) -> Vec<(String, prost_types::Any)> { self.0.lock().unwrap_or_else(|e| e.into_inner()).clone() } + + /// Replace the extensions of this list with `extensions`, which the requests of the threads + /// that have it carry from now on, as a change of the list of PySpark's + /// `thread_local.user_context_extensions`. Not a stable API. + #[doc(hidden)] + pub fn replace(&self, extensions: Vec<(String, prost_types::Any)>) { + *self.0.lock().unwrap_or_else(|e| e.into_inner()) = extensions; + } } /// Forget the extensions of a thread that ends, as PySpark's `thread_local` is freed with its @@ -1078,7 +1119,9 @@ impl SparkConnectClient { channel, host: builder.host().to_string(), stub, + client_session_id: std::sync::Arc::new(std::sync::RwLock::new(session_id.clone())), session_id, + builder: builder.clone(), user_id: builder.user_id().map(String::from).or_else(default_user_id), metadata, headers, @@ -1103,6 +1146,42 @@ impl SparkConnectClient { &self.session_id } + /// The session ID that the requests of this client and of its clones carry, as PySpark's + /// `client._session_id`: the session ID, unless + /// [`SparkConnectClient::set_client_session_id`] set another. Not a stable API. + #[doc(hidden)] + pub fn client_session_id(&self) -> String { + self.client_session_id + .read() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } + + /// Make the requests of this client and of its clones carry `session_id`, against which + /// their responses are checked, as a set of PySpark's `client._session_id` does: the server + /// then answers of the session of that ID, and a response of another server-side session + /// than the one that the client observed closes the client. [`SparkConnectClient::session_id`] + /// stays the ID of the session, as the `session_id` of PySpark's session does, and the + /// requests of the artifacts carry it, as PySpark's ArtifactManager keeps the ID of the + /// client when it was built. A client that does not track its session (see + /// [`SparkConnectClient::set_session_tracking`]) sends the requests that its callers build + /// with their own session ID. Not a stable API. + #[doc(hidden)] + pub fn set_client_session_id(&self, session_id: String) { + *self + .client_session_id + .write() + .unwrap_or_else(|e| e.into_inner()) = session_id; + } + + /// The channel builder of the connection of this client, as PySpark's `client._builder`: of + /// no session ID for the client of a new session, and of its own for that of a clone. Not a + /// stable API. + #[doc(hidden)] + pub fn channel_builder(&self) -> &crate::channel::ChannelBuilder { + &self.builder + } + /// Create a new client that reuses this client's channel/stub/metadata but /// starts a brand-new server-side session (fresh session id). Mirrors the /// reference client's `newSession()`, which opens a new session over the same @@ -1110,11 +1189,17 @@ impl SparkConnectClient { /// reattachable, and has the default retry policy and result chunking, as PySpark's /// `newSession()` and `clone()` build their client of these only. pub fn with_new_session_id(&self) -> Self { + let session_id = Uuid::new_v4().to_string(); + // As PySpark's `newSession`, which pops the session ID of a copy of the builder. + let mut builder = self.builder.clone(); + builder.remove(crate::channel::PARAM_SESSION_ID); SparkConnectClient { channel: self.channel.clone(), host: self.host.clone(), stub: self.stub.clone(), - session_id: Uuid::new_v4().to_string(), + client_session_id: std::sync::Arc::new(std::sync::RwLock::new(session_id.clone())), + session_id, + builder, user_id: self.user_id.clone(), metadata: self.metadata.clone(), headers: self.headers.clone(), @@ -1144,6 +1229,14 @@ impl SparkConnectClient { self.closed.store(true, std::sync::atomic::Ordering::SeqCst); } + /// Open this client again, which [`SparkConnectClient::close`] closed, as PySpark's + /// `client._closed = False` lets its RPCs be sent again. Not a stable API. + #[doc(hidden)] + pub fn reopen(&self) { + self.closed + .store(false, std::sync::atomic::Ordering::SeqCst); + } + /// Whether the client is closed. See [`SparkConnectClient::close`]. pub fn is_closed(&self) -> bool { self.closed.load(std::sync::atomic::Ordering::SeqCst) @@ -1199,6 +1292,16 @@ impl SparkConnectClient { &self.host } + /// Sets the session ID of a request that the caller built to the client's, as PySpark's + /// requests read `client._session_id`. The requests of a client that does not track its + /// session, those of the transport-injection stub, are of the reference client's session and + /// keep its ID, as they keep their observed one (see `observe_on`). + fn session_on(&self, session_id: &mut String) { + if self.session_tracking { + *session_id = self.client_session_id(); + } + } + /// Sets the observed server-side session ID of a request, once a response has set it, as /// PySpark's requests do. fn observe_on(&self, observed: &mut Option) { @@ -1222,10 +1325,10 @@ impl SparkConnectClient { } // An empty session ID is none: the server leaves both IDs unset in some responses, such // as those of FetchErrorDetails, so only another session's ID is a mismatch. - if !session_id.is_empty() && session_id != self.session_id { + let expected = self.client_session_id(); + if !session_id.is_empty() && session_id != expected { return Err(assertion_error(format!( - "Received incorrect session identifier for request:{session_id} != {}", - self.session_id + "Received incorrect session identifier for request:{session_id} != {expected}" ))); } let mut observed = self @@ -1370,6 +1473,47 @@ impl SparkConnectClient { all.by_thread.insert(thread, extensions.clone()); } + /// Keep `refresher`, or none, for this client and its clones: what a binding keeps to bring + /// the user context extensions up to date before a call of a thread sends requests, as the + /// Python client re-serializes the `google.protobuf.Any` messages of its lists, which PySpark + /// serializes when it builds each request. The core never calls it. Not a stable API. + #[doc(hidden)] + pub fn set_user_context_refresher( + &self, + refresher: Option>, + ) { + self.user_context_extensions + .lock() + .unwrap_or_else(|e| e.into_inner()) + .refresher = refresher.map(UserContextRefresher); + } + + /// The refresher of [`SparkConnectClient::set_user_context_refresher`], if any. Not a stable + /// API. + #[doc(hidden)] + pub fn user_context_refresher( + &self, + ) -> Option> { + self.user_context_extensions + .lock() + .unwrap_or_else(|e| e.into_inner()) + .refresher + .clone() + .map(|refresher| refresher.0) + } + + /// Forget the user context extensions of the current thread, which has none from now on, + /// as PySpark's `thread_local` once its `user_context_extensions` is deleted. Not a stable + /// API. + #[doc(hidden)] + pub fn clear_threadlocal_user_context_extensions(&self) { + self.user_context_extensions + .lock() + .unwrap_or_else(|e| e.into_inner()) + .by_thread + .remove(&std::thread::current().id()); + } + /// Add `extension` to the user context of the requests of all the threads, and return its /// ID, as PySpark's `add_global_user_context_extension`. pub fn add_global_user_context_extension(&self, extension: prost_types::Any) -> String { @@ -1542,15 +1686,16 @@ impl SparkConnectClient { self.reattachable.load(std::sync::atomic::Ordering::SeqCst) } - /// Turn the tracking of the server-side session on (the default) or off. With it on, the - /// requests carry the server-side session ID observed in the responses, each response must - /// be of this session and of that server-side session, and an + /// Turn the tracking of the session on (the default) or off. With it on, the requests that + /// the callers build carry the session ID of the client (see + /// [`SparkConnectClient::client_session_id`]) and the server-side session ID observed in the + /// responses, each response must be of that session and of that server-side session, and an /// `INVALID_HANDLE.SESSION_CHANGED` error closes the client, as PySpark's /// `SparkConnectClient` does. /// /// The transport-injection stub turns it off. The requests that it sends are of the - /// reference client's session, not of its own, and that client tracks its session itself, - /// over the grpcio stub that does no tracking. + /// reference client's session, not of its own, and keep their IDs, and that client tracks + /// its session itself, over the grpcio stub that does no tracking. pub fn set_session_tracking(&mut self, enabled: bool) { self.session_tracking = enabled; } @@ -1646,6 +1791,7 @@ impl SparkConnectClient { self.fill_client_type(&mut request.client_type)?; self.ensure_open()?; add_result_chunking_option(&mut request, self.result_chunking); + self.session_on(&mut request.session_id); // The caller has extended the user context with the extensions of its thread. self.fill_user_context(&mut request.user_context); let of_a_relation = matches!( @@ -1819,6 +1965,7 @@ impl SparkConnectClient { &self, mut request: AnalyzePlanRequest, ) -> Result { + self.session_on(&mut request.session_id); self.fill_client_type(&mut request.client_type)?; self.ensure_open()?; self.extend_user_context(&mut request.user_context); @@ -2151,6 +2298,7 @@ impl SparkConnectClient { &self, mut request: FetchErrorDetailsRequest, ) -> Result { + self.session_on(&mut request.session_id); self.fill_client_type(&mut request.client_type)?; self.ensure_open()?; self.extend_user_context(&mut request.user_context); @@ -2226,7 +2374,7 @@ impl SparkConnectClient { client_type: String, ) -> Option { let request = FetchErrorDetailsRequest { - session_id: self.session_id.clone(), + session_id: self.client_session_id(), error_id: error_id.to_string(), client_type: Some(client_type), user_context: self.user_id.as_ref().map(|user_id| UserContext { @@ -2282,7 +2430,7 @@ impl SparkConnectClient { return None; } let mut request = ConfigRequest { - session_id: self.session_id.clone(), + session_id: self.client_session_id(), user_context: Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2335,6 +2483,7 @@ impl SparkConnectClient { /// /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.config`. pub async fn config(&self, mut request: ConfigRequest) -> Result { + self.session_on(&mut request.session_id); self.fill_client_type(&mut request.client_type)?; self.ensure_open()?; self.extend_user_context(&mut request.user_context); @@ -2362,6 +2511,7 @@ impl SparkConnectClient { /// /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.interrupt`. pub async fn interrupt(&self, mut request: InterruptRequest) -> Result { + self.session_on(&mut request.session_id); self.fill_client_type(&mut request.client_type)?; self.ensure_open()?; self.extend_user_context(&mut request.user_context); @@ -2390,7 +2540,7 @@ impl SparkConnectClient { /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.get_configs`. pub async fn get_configs(&self, keys: &[&str]) -> Result>> { let mut request = ConfigRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2463,7 +2613,7 @@ impl SparkConnectClient { silent, }; let request = ConfigRequest { - session_id: self.session_id.clone(), + session_id: self.client_session_id(), user_context: Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2486,7 +2636,7 @@ impl SparkConnectClient { pairs: &[(&str, Option<&str>)], ) -> Result>> { let mut request = ConfigRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2530,7 +2680,7 @@ impl SparkConnectClient { /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.unset_config`. pub async fn unset_config(&self, key: &str) -> Result> { let mut request = ConfigRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2552,7 +2702,7 @@ impl SparkConnectClient { /// Returns a HashMap of all configuration key-value pairs. pub async fn get_configs_all(&self) -> Result> { let mut request = ConfigRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2581,7 +2731,7 @@ impl SparkConnectClient { /// Returns true if the configuration can be changed at runtime. pub async fn is_config_modifiable(&self, key: &str) -> Result { let mut request = ConfigRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2612,7 +2762,7 @@ impl SparkConnectClient { /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.interrupt_all`. pub async fn interrupt_all(&self) -> Result> { let mut request = InterruptRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2629,7 +2779,7 @@ impl SparkConnectClient { /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.interrupt_tag`. pub async fn interrupt_tag(&self, tag: &str) -> Result> { let mut request = InterruptRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2648,7 +2798,7 @@ impl SparkConnectClient { /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.interrupt_operation`. pub async fn interrupt_operation(&self, operation_id: &str) -> Result> { let mut request = InterruptRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2678,7 +2828,7 @@ impl SparkConnectClient { // As PySpark's request: of the client type "python", not of the user agent, and with // no observed server-side session ID. let request = CloneSessionRequest { - session_id: self.session_id.clone(), + session_id: self.client_session_id(), user_context: Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2710,6 +2860,12 @@ impl SparkConnectClient { } } let mut cloned = self.with_new_session_id(); + // As PySpark's `clone`, which sets the new session ID in a copy of the builder. + cloned + .builder + .set(crate::channel::PARAM_SESSION_ID, &response.new_session_id); + cloned.client_session_id = + std::sync::Arc::new(std::sync::RwLock::new(response.new_session_id.clone())); cloned.session_id = response.new_session_id; Ok(cloned) } @@ -2721,7 +2877,7 @@ impl SparkConnectClient { /// session on the server until its timeout. pub async fn release_session(&self) -> Result<()> { let mut request = ReleaseSessionRequest::default(); - request.session_id = self.session_id.clone(); + request.session_id = self.client_session_id(); request.user_context = Some(UserContext { user_id: self.user_id.clone().unwrap_or_default(), ..Default::default() @@ -2745,6 +2901,44 @@ impl SparkConnectClient { self.verify_response(&response.session_id, &response.server_side_session_id) } + /// The statuses of operations of this session, as PySpark's `_get_operation_statuses`: the + /// GetStatus RPC of `request`, of the operations and the extensions that the caller asks of, + /// which the client gives the IDs of its session, its user agent as the client type, its user + /// and the server-side session ID that it observed, as PySpark builds it, under the retry + /// policy and the deadline of `get_status`. Its response is checked as the others are. + pub async fn get_status_proto( + &self, + mut request: GetStatusRequest, + ) -> Result { + self.ensure_open()?; + request.session_id = self.client_session_id(); + request.client_type = Some(self.client_type()?); + if let Some(user_id) = self.user_id.as_ref().filter(|user_id| !user_id.is_empty()) { + request.user_context = Some(UserContext { + user_id: user_id.clone(), + ..Default::default() + }); + } + if let Some(observed) = self.observed_server_session_id() { + request.client_observed_server_side_session_id = Some(observed); + } + let response = self + .with_retry(|| { + let deadline = self.rpc_deadlines.get_status; + let mut req = with_deadline(Request::new(request.clone()), deadline); + self._attach_metadata(&mut req); + let mut stub = self.stub.clone(); + async move { + within_deadline(deadline, stub.get_status(req)) + .await + .map(Response::into_inner) + } + }) + .await?; + self.verify_response(&response.session_id, &response.server_side_session_id)?; + Ok(response) + } + /// Add artifacts to this session. /// /// Mirrors `pyspark.sql.connect.client.core.SparkConnectClient.add_artifacts`, which @@ -2915,7 +3109,9 @@ impl SparkConnectClient { /// The AddArtifacts call of `requests`, once. Stamps user_context on every frame as it is /// sent; all upload paths must go through here. As PySpark's ArtifactManager, the frames - /// carry no client type and no observed server-side session ID. + /// carry no client type and no observed server-side session ID, and the ID of the session, + /// which a set of the client's session ID does not change, as the manager keeps the ID of + /// the client when it was built. async fn add_artifacts_rpc( &self, requests: impl Iterator + Send + 'static, @@ -2955,7 +3151,7 @@ impl SparkConnectClient { /// The ArtifactStatus call of `names`, once: the response, which says whether each artifact /// exists. As PySpark's, the request carries no client type and no observed server-side - /// session ID. + /// session ID, and the ID of the session, as the frames of `add_artifacts_rpc`. async fn artifact_status_rpc( &self, names: &[String], @@ -4867,6 +5063,63 @@ mod tests { .collect() } + #[tokio::test] + async fn a_refresher_is_kept_for_the_client_and_its_clones_and_not_a_new_session() { + // What a binding keeps to bring its extensions up to date, which the client gives back, + // as its clones do, while the client of a new session has none. + let builder = ChannelBuilder::parse("sc://localhost").unwrap(); + let client = SparkConnectClient::connect(&builder).await.unwrap(); + assert!(client.user_context_refresher().is_none()); + client.set_user_context_refresher(Some(std::sync::Arc::new(7u8))); + let kept = client.clone().user_context_refresher().unwrap(); + assert_eq!(kept.downcast_ref::(), Some(&7)); + assert!(client + .with_new_session_id() + .user_context_refresher() + .is_none()); + client.set_user_context_refresher(None); + assert!(client.user_context_refresher().is_none()); + } + + #[tokio::test] + async fn a_list_given_to_threads_is_theirs_and_its_changes_are_theirs() { + // As the list of PySpark's thread_local.user_context_extensions, which a binding keeps and + // changes: a new list that a thread is given is its extensions, whose changes are the + // extensions of the threads that have it, and a cleared thread has none. + let builder = ChannelBuilder::parse("sc://localhost").unwrap(); + let client = std::sync::Arc::new(SparkConnectClient::connect(&builder).await.unwrap()); + client.add_threadlocal_user_context_extension(any_of("own")); + let list = + SharedUserContextExtensions::new(vec![("threadlocal_a".to_string(), any_of("a"))]); + client.share_threadlocal_user_context_extensions(&list); + assert_eq!(extension_ids(&client), vec!["threadlocal_a"]); + list.replace(vec![("threadlocal_b".to_string(), any_of("b"))]); + assert_eq!(extension_ids(&client), vec!["threadlocal_b"]); + assert_eq!(list.snapshot()[0].0, "threadlocal_b"); + + let seen = { + let client = client.clone(); + let list = list.clone(); + std::thread::spawn(move || { + let before = extension_ids(&client); + client.share_threadlocal_user_context_extensions(&list); + list.replace(vec![("threadlocal_c".to_string(), any_of("c"))]); + (before, extension_ids(&client)) + }) + .join() + .unwrap() + }; + assert_eq!( + seen, + (Vec::::new(), vec!["threadlocal_c".to_string()]) + ); + assert_eq!(extension_ids(&client), vec!["threadlocal_c"]); + + client.clear_threadlocal_user_context_extensions(); + assert!(extension_ids(&client).is_empty()); + assert_eq!(list.snapshot()[0].0, "threadlocal_c"); + } + #[tokio::test] async fn threadlocal_user_context_extensions_of_a_copy_are_shared_as_pyspark() { // As PySpark's inheritable_thread_target: the threads given a copy of the extensions of a @@ -4912,6 +5165,58 @@ mod tests { assert_eq!(extension_ids(&client), vec![main, late]); } + #[test] + fn a_client_session_id_is_sent_and_checked_while_the_session_id_stays() { + // As PySpark's `client._session_id`, which its requests read: a set changes the ID that + // the clones of the client send and check the responses against, while `session_id` + // stays the session's, as the `session_id` of PySpark's session; a client of a new + // session has its own, and its builder no session ID, as PySpark's `newSession` pops it. + let client = + connect_blocking("sc://localhost/;session_id=7c4e2a0e-0f6b-4b9a-9d1e-0123456789ab"); + let original = client.session_id().to_string(); + assert_eq!(client.client_session_id(), original); + let clone = client.clone(); + clone.set_client_session_id("changed".to_string()); + assert_eq!(client.client_session_id(), "changed"); + assert_eq!(client.session_id(), original); + assert!(client.verify_response("changed", "").is_ok()); + let err = client.verify_response(&original, "").unwrap_err(); + assert!(err.message().contains(&format!("{original} != changed"))); + assert_eq!( + client + .channel_builder() + .get(crate::channel::PARAM_SESSION_ID), + Some(original.as_str()) + ); + let new = client.with_new_session_id(); + assert_eq!(new.client_session_id(), new.session_id()); + assert_ne!(new.session_id(), original); + assert_eq!( + new.channel_builder().get(crate::channel::PARAM_SESSION_ID), + None + ); + assert_eq!( + client.channel_builder().port(), + new.channel_builder().port() + ); + } + + #[test] + fn a_closed_client_reopens_and_the_status_rpc_has_a_deadline() { + // As PySpark's `client._closed = False`, which sends the requests again, and its + // `RpcDeadlines.get_status`, of 10 minutes, and none when the deadlines are disabled. + let client = connect_blocking("sc://localhost"); + client.close(); + assert!(client.is_closed() && client.ensure_open().is_err()); + client.clone().reopen(); + assert!(!client.is_closed() && client.ensure_open().is_ok()); + assert_eq!( + RpcDeadlines::default().get_status, + Some(Duration::from_secs(600)) + ); + assert_eq!(RpcDeadlines::disabled().get_status, None); + } + #[test] fn host_and_observed_server_session_id_of_the_client() { let client = connect_blocking("sc://Other:114"); diff --git a/crates/spark-connect-core/tests/artifact_uploads.rs b/crates/spark-connect-core/tests/artifact_uploads.rs index becb24b2..3a775d77 100644 --- a/crates/spark-connect-core/tests/artifact_uploads.rs +++ b/crates/spark-connect-core/tests/artifact_uploads.rs @@ -229,6 +229,29 @@ async fn a_small_file_is_one_batch_named_by_its_extension() { assert_eq!(chunk.crc, compute_crc32(b"cafebabe") as i64); } +#[tokio::test] +async fn the_frames_keep_the_session_id_of_the_session_that_the_client_sets_another_of() { + // As PySpark's ArtifactManager, which keeps the session ID of the client when it was built: a + // set of `client._session_id` changes the session of the other requests, not of the uploads. + let fixture = Fixture::start([]).await; + let client = fixture.client().await; + client.set_client_session_id("another-session".to_string()); + let dir = tempfile::tempdir().unwrap(); + let class = file_with(&dir, "A.class", b"cafebabe"); + + client + .add_artifacts(&[class.as_str()], false, false, false) + .await + .unwrap(); + + let attempts = fixture.attempts(); + assert_eq!(attempts.len(), 1); + assert!(!attempts[0].is_empty()); + for frame in &attempts[0] { + assert_eq!(frame.session_id, client.session_id()); + } +} + #[tokio::test] async fn a_transient_failure_retries_the_whole_upload_from_its_start() { // The first attempt receives two frames and then fails; the second receives the upload. diff --git a/crates/spark-connect-core/tests/server_session.rs b/crates/spark-connect-core/tests/server_session.rs index c29a6200..403b5015 100644 --- a/crates/spark-connect-core/tests/server_session.rs +++ b/crates/spark-connect-core/tests/server_session.rs @@ -647,6 +647,153 @@ async fn a_client_without_session_tracking_passes_the_ids_of_its_requests_throug } } +/// The session ID of each request of `method` that the server received, in order. +fn sessions_of( + state: &State, + method: &str, + session_id: impl Fn(T) -> String, +) -> Vec { + state + .requests_of::(method) + .into_iter() + .map(session_id) + .collect() +} + +#[tokio::test] +async fn a_client_without_session_tracking_passes_the_session_id_of_each_call_through() { + // Every call that the transport-injection stub forwards decoded keeps the session ID of the + // reference client's request, against which that client checks the response itself. + use spark_connect_proto::{AnalyzePlanRequest, FetchErrorDetailsRequest, InterruptRequest}; + + let fixture = Fixture::start().await; + let mut client = fixture.client().await; + client.set_session_tracking(false); + let reference = "the-reference-session".to_string(); + + client + .config(ConfigRequest { + session_id: reference.clone(), + ..Default::default() + }) + .await + .unwrap(); + client + .interrupt(InterruptRequest { + session_id: reference.clone(), + ..Default::default() + }) + .await + .unwrap(); + client + .analyze_plan(AnalyzePlanRequest { + session_id: reference.clone(), + ..Default::default() + }) + .await + .unwrap(); + client + .fetch_error_details(FetchErrorDetailsRequest { + session_id: reference.clone(), + ..Default::default() + }) + .await + .unwrap(); + let mut stream = client + .execute_plan(ExecutePlanRequest { + session_id: reference.clone(), + ..Default::default() + }) + .await + .unwrap(); + while stream.message().await.unwrap().is_some() {} + + let state = &fixture.state; + let sessions = [ + sessions_of(state, "Config", |r: ConfigRequest| r.session_id), + sessions_of(state, "Interrupt", |r: InterruptRequest| r.session_id), + sessions_of(state, "AnalyzePlan", |r: AnalyzePlanRequest| r.session_id), + sessions_of(state, "FetchErrorDetails", |r: FetchErrorDetailsRequest| { + r.session_id + }), + sessions_of(state, "ExecutePlan", |r: ExecutePlanRequest| r.session_id), + ]; + for of_a_method in sessions { + assert_eq!(of_a_method, [reference.as_str()]); + } +} + +#[tokio::test] +async fn a_set_client_session_id_is_that_of_the_calls_and_not_of_the_artifacts_as_pyspark() { + // As a set of PySpark's `client._session_id`, which its requests read, while its + // ArtifactManager keeps the session ID of the client when it was built. + use spark_connect_proto::{ + AnalyzePlanRequest, ArtifactStatusesRequest, FetchErrorDetailsRequest, InterruptRequest, + }; + + let fixture = Fixture::start().await; + let client = fixture.client().await; + let session_id = client.session_id().to_string(); + client.set_client_session_id("the-client-session".to_string()); + + // The callers build their requests of the session's ID. + client.config(config_request(&client)).await.unwrap(); + client + .interrupt(InterruptRequest { + session_id: session_id.clone(), + ..Default::default() + }) + .await + .unwrap(); + client + .analyze_plan(AnalyzePlanRequest { + session_id: session_id.clone(), + ..Default::default() + }) + .await + .unwrap(); + client + .fetch_error_details(FetchErrorDetailsRequest { + session_id: session_id.clone(), + ..Default::default() + }) + .await + .unwrap(); + // An execution that drops after its first response, which the client reattaches. + *fixture.state.drop_next_execute.lock().unwrap() = true; + run_execution(&client).await; + client.get_configs(&["k"]).await.unwrap(); + client + .cache_artifacts(vec![b"blob".to_vec()]) + .await + .unwrap(); + + let state = &fixture.state; + let calls = [ + sessions_of(state, "Config", |r: ConfigRequest| r.session_id), + sessions_of(state, "Interrupt", |r: InterruptRequest| r.session_id), + sessions_of(state, "AnalyzePlan", |r: AnalyzePlanRequest| r.session_id), + sessions_of(state, "FetchErrorDetails", |r: FetchErrorDetailsRequest| { + r.session_id + }), + sessions_of(state, "ExecutePlan", |r: ExecutePlanRequest| r.session_id), + // As PySpark's, of the session ID of the request of the execution. + sessions_of(state, "ReattachExecute", |r: ReattachExecuteRequest| { + r.session_id + }), + ]; + for of_a_method in calls { + assert!(!of_a_method.is_empty()); + assert!(of_a_method.iter().all(|id| id == "the-client-session")); + } + assert_eq!( + sessions_of(state, "ArtifactStatus", |r: ArtifactStatusesRequest| { + r.session_id + }), + [session_id] + ); +} + #[tokio::test] async fn executions_ask_the_server_to_chunk_arrow_batches() { use spark_connect_proto::execute_plan_request::request_option::RequestOption; diff --git a/crates/spark-connect/src/session.rs b/crates/spark-connect/src/session.rs index 75a83faf..6f733236 100644 --- a/crates/spark-connect/src/session.rs +++ b/crates/spark-connect/src/session.rs @@ -82,13 +82,39 @@ fn forget_thread_tags( #[derive(Clone, Default)] pub struct SharedTags(Arc>>); +impl SharedTags { + /// A set of the tags `tags`, which no thread has yet (see [`SparkSession::share_thread_tags`]), + /// as the set that a binding keeps of the tags of PySpark's `thread_local`. Not a stable API. + #[doc(hidden)] + pub fn new(tags: Vec) -> Self { + SharedTags(Arc::new(Mutex::new(tags))) + } + + /// Replace the tags of this set with `tags`, which the requests of the threads that have it + /// carry from now on, as a change of the set of PySpark's `thread_local.tags`. Not a stable + /// API. + #[doc(hidden)] + pub fn replace(&self, tags: Vec) { + *self.0.lock().unwrap_or_else(|e| e.into_inner()) = tags; + } + + /// The tags of this set. Not a stable API. + #[doc(hidden)] + pub fn tags(&self) -> Vec { + self.0.lock().unwrap_or_else(|e| e.into_inner()).clone() + } +} + /// An operation tag: its text, or the UTF-32 little-endian bytes of a str of Python that UTF-8 /// cannot encode, such as one of a lone surrogate, which PySpark's client keeps as a tag, and /// with which each request that carries the tags fails (see -/// [`SparkSession::add_unencodable_tag`]). +/// [`SparkSession::add_unencodable_tag`]). Not a stable API. +#[doc(hidden)] #[derive(Clone, Debug, PartialEq, Eq)] -enum Tag { +pub enum Tag { + /// A tag of its text. Text(String), + /// The UTF-32 little-endian bytes of a tag that UTF-8 cannot encode. Unencodable(Vec), } @@ -1464,6 +1490,78 @@ impl SparkSession { .share_threadlocal_user_context_extensions(extensions); } + /// The session ID that the requests of this session carry, as PySpark's `client._session_id` + /// (see [`SparkConnectClient::client_session_id`]). Not a stable API. + #[doc(hidden)] + pub fn client_session_id(&self) -> String { + self.client.client_session_id() + } + + /// Make the requests of this session carry `session_id`, as a set of PySpark's + /// `client._session_id` (see [`SparkConnectClient::set_client_session_id`]): + /// [`SparkSession::session_id`] stays the ID of the session. Not a stable API. + #[doc(hidden)] + pub fn set_client_session_id(&self, session_id: String) { + self.client.set_client_session_id(session_id); + } + + /// The channel builder of the connection of this session, as PySpark's `client._builder` + /// (see [`SparkConnectClient::channel_builder`]). Not a stable API. + #[doc(hidden)] + pub fn channel_builder(&self) -> &ChannelBuilder { + self.client.channel_builder() + } + + /// The user of the requests of this session, as PySpark's `client._user_id`: of the + /// connection string, else of the environment, if any. Not a stable API. + #[doc(hidden)] + pub fn user_id(&self) -> Option<&str> { + self.client.user_id() + } + + /// Open this session again, which `close()` closed, as PySpark's `client._closed = False` + /// lets the requests of its client be sent again. Not a stable API. + #[doc(hidden)] + pub fn reopen(&self) { + self.stopped.store(false, Ordering::SeqCst); + self.client.reopen(); + } + + /// The statuses of the operations of this session of `request`, a GetStatus request built + /// outside the core, as PySpark's `_get_operation_statuses` (see + /// [`SparkConnectClient::get_status_proto`]). Not a stable API. + #[doc(hidden)] + pub fn get_status_proto( + &self, + request: proto::GetStatusRequest, + ) -> Result { + block_on(self.client.get_status_proto(request)) + } + + /// Keep `refresher`, or none, for this session: what a binding keeps to bring the user + /// context extensions up to date before a call of a thread sends requests (see + /// [`SparkConnectClient::set_user_context_refresher`]). Not a stable API. + #[doc(hidden)] + pub fn set_user_context_refresher( + &self, + refresher: Option>, + ) { + self.client.set_user_context_refresher(refresher); + } + + /// The refresher of [`SparkSession::set_user_context_refresher`], if any. Not a stable API. + #[doc(hidden)] + pub fn user_context_refresher(&self) -> Option> { + self.client.user_context_refresher() + } + + /// Forget the user context extensions of the current thread, as PySpark's + /// `thread_local` once its `user_context_extensions` is deleted. Not a stable API. + #[doc(hidden)] + pub fn clear_threadlocal_user_context_extensions(&self) { + self.client.clear_threadlocal_user_context_extensions(); + } + /// Return the Spark version of the connected server. /// /// Mirrors `pyspark.sql.SparkSession.version`. @@ -3763,6 +3861,46 @@ mod tests { assert_eq!(run(copy, "e"), vec!["a", "b", "e"]); } + #[test] + fn a_set_of_tags_given_to_threads_is_theirs_and_its_changes_are_theirs() { + // As the set of PySpark's thread_local.tags, which a binding keeps and changes: a new set + // that a thread is given is its tags, whose changes are the tags of the threads that have + // it, of their text or of the bytes of a str that UTF-8 cannot encode, and a thread that is + // not given it keeps its own. + let spark = session(); + spark.add_tag("own").unwrap(); + let set = SharedTags::new(vec![Tag::Text("a".to_string())]); + spark.share_thread_tags(&set); + assert_eq!(spark.get_tags(), vec!["a"]); + let surrogate = vec![0x00, 0xd8, 0x00, 0x00]; + set.replace(vec![ + Tag::Text("b".to_string()), + Tag::Unencodable(surrogate.clone()), + ]); + assert_eq!(spark.get_tags(), vec!["b"]); + assert_eq!(spark.unencodable_tags(), vec![surrogate.clone()]); + assert!(spark.tags().is_err()); + assert_eq!(set.tags().len(), 2); + + let seen = { + let spark = spark.clone(); + let set = set.clone(); + std::thread::spawn(move || { + let before = spark.get_tags(); + spark.share_thread_tags(&set); + set.replace(vec![Tag::Text("c".to_string())]); + (before, spark.get_tags()) + }) + .join() + .unwrap() + }; + assert_eq!(seen, (Vec::::new(), vec!["c".to_string()])); + assert_eq!(spark.get_tags(), vec!["c"]); + spark.clear_tags(); + assert!(spark.get_tags().is_empty()); + assert_eq!(set.tags(), vec![Tag::Text("c".to_string())]); + } + #[test] fn finished_threads_leave_no_tags() { // The threads that share a copy of the tags are forgotten when they end, as PySpark's diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index f34eb4ba..4f922d4b 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -392,7 +392,7 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan read when they are set. The transport classifies the failures as `DefaultPolicy.can_retry` does, so a policy of its own `can_retry` or `to_state`, and a backoff, jitter or delay that is not a whole number of milliseconds, raise `NOT_IMPLEMENTED`. `RpcDeadlines` sets the - deadlines of the transport, which has no GetStatus RPC (`get_status`). `client.reattach` + deadlines of the transport, `get_status` that of `_get_operation_statuses`. `client.reattach` (`ExecutePlanResponseReattachableIterator`) is not provided: `disable_reattachable_execute` and `enable_reattachable_execute` set the executions of the transport not reattachable or reattachable. @@ -442,10 +442,13 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan ...`), the failure that ends its retries (`Given up on retrying. error: ...`) and each gRPC failure that it converts (`GRPC Error received`), which the transport, whose retries and errors are its own, does not. -- **`SparkConnectClient.get_tags`** (intentional). A copy of the tags of the thread, where - PySpark returns the set that its client keeps, which its later changes change too, as - `SparkSession.getTags` does, of which the drop-in returns a copy too: the tags are those of the - core session, which its requests read, and not a Python set. +- **`SparkConnectClient.thread_local`** (intentional). A subclass of `threading.local`, of + subclasses of `set` and `list` for its tags and its user context extensions, which keep the Rust + transport's lists of them in step, where PySpark's is a `threading.local` of a plain set and + list. The global list of the extensions is a subclass of `list` too. A set of tags refuses an + object that is neither a str nor bytes when it is added, where PySpark's raises the `TypeError` + of the field of the tags when it builds a request. A list that the attribute of a `copy.copy` of + a client is set to is not one that the requests of the session carry. - **Plans of a DataFrame's relation** (intentional; DESIGN). `df._plan.plan(client)` is a placeholder that holds the DataFrame's relation, which the core builds at the call, as PySpark's `plan` does, with its pickles, warnings and errors: the core builds the relation, and @@ -483,12 +486,19 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan `SparkConnectClient._proto_to_string` renders any message, including a deep one built in Python, as PySpark's does. Plan logging of a deep DataFrame therefore does not reach the recursion handling that the test `test_recursion_handling_for_plan_logging` exercises. -- **Client session ID** (not provided). `SparkConnectClient._session_id` is not exposed and - is not settable; the session ID belongs to the Rust core. `_server_session_id` is - exposed and settable, as PySpark's, and a changed value closes the session. -- **`_closed`** (tracked). Reading `_closed` gives the session's stopped state. Setting it - to True closes the session, as a test's stand-in for the client's close. Setting it to - False does not reopen a closed session. +- **Client session ID** (intentional). `SparkConnectClient._session_id` is the session ID that + the requests carry, which a set changes for them, as PySpark's, so the next response of + another server-side session closes the session; `_server_session_id` is exposed and settable, + as PySpark's, and a changed value closes the session. A value that is not a str raises the + `TypeError` of the string field of the session ID of the requests when it is set, where + PySpark raises it when it builds the next request. +- **`_closed`** (intentional). Reading `_closed` gives the session's stopped state. Setting it + to True closes the session, and setting it to False opens it again, as PySpark's. After + `close()` or `stop()`, which close PySpark's channel too, PySpark's requests still fail once + `_closed` is False, with grpc's `ValueError: Cannot invoke RPC: Channel closed!`, where the + drop-in's are sent again. +- **`_user_id`** (intentional). It is read-only, where PySpark's is an attribute that a set + changes for the requests. - **The exit of a dropped session in a reference cycle** (intentional). PySpark's `SparkSession.__del__` closes the client of a session object that is freed, and the exit hook of a closed client sends nothing; the drop-in's exit hook holds its session weakly, so a diff --git a/docs/compatibility.md b/docs/compatibility.md index fa1f238a..21775875 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -1646,20 +1646,40 @@ client does: messages that the server sent, of their `repr`, which `pairs` and `flows` convert as `LiteralExpression._to_value` does, raising its `UNSUPPORTED_LITERAL` when they are read; - the registrations of `register_udf`, `register_udtf`, `register_java` and - `register_data_source`, the artifacts of `add_artifacts`, `copy_from_local_to_fs`, - `cache_artifact` and `cache_artifacts`, with the checks of PySpark's client rather than those of - `SparkSession.addArtifacts` and `copyFromLocalToFs`, the tags, the interrupts, `token`, `host`, - and the clients of `newSession` and `clone`. + `register_data_source`, which set their arguments in the fields of their commands, convert the + return type and pickle the function in the order of PySpark's, of protobuf's errors for a value + that a field does not take (an int is a bool, a bool is not an int), the artifacts of + `add_artifacts`, `copy_from_local_to_fs`, `cache_artifact` and `cache_artifacts`, with the + checks of PySpark's client rather than those of `SparkSession.addArtifacts` and + `copyFromLocalToFs`, the tags, the interrupts, `token`, `host`, and the clients of `newSession` + and `clone`. `spark.udf.register`, `registerJavaFunction` and `registerJavaUDAF` register + through the client, as PySpark's do. The executions read their `observations` as PySpark's: of + an object that is not a dict, such as None or an empty list, they set none, and of one that is + not a container, such as None, they raise the `TypeError` of `in` when observed metrics arrive. `SparkConnectClient(connection, user_id=None, ...)` takes the arguments of PySpark's: it makes a client of a session of its own of the connection string or the `ChannelBuilder` `connection`, which is neither the default nor the active session. Its `retry_policy`, `use_reattachable_execute`, `session_hooks`, `allow_arrow_batch_chunking`, `preferred_arrow_chunk_size`, `rpc_deadlines` -(`pyspark.sql.connect.client.RpcDeadlines`, of which the transport has no GetStatus RPC) and +(`pyspark.sql.connect.client.RpcDeadlines`) and `max_retry_exception_elapsed_time` are those of the transport. A `channel_options` of a connection string raises `NOT_IMPLEMENTED`, as `DefaultChannelBuilder` does: the Rust transport opens the channel; as PySpark's, the client of a `ChannelBuilder` does not read it. +The members of PySpark's client that its tests and some libraries read are there too. `_builder` +is the channel builder of the connection, a `DefaultChannelBuilder` of the session's connection +string for `spark.client` (of its `endpoint`, `userAgent` and parameters), and the one given for a +client made of one. `_session_id` is the session ID that the requests carry, which a set changes +for them, as PySpark's, so the next response of another server-side session than the one that the +client observed closes the session, while the session's `session_id` stays, and so does that of the +requests of the artifacts, as PySpark's ArtifactManager keeps the ID that the client was built +with; `_server_session_id` is the server-side one. `_closed` closes the session when it is set to True and opens it again +when it is set to False. `_user_id` is the user of the requests, `_use_reattachable_execute` +whether the executions are reattachable, and `_get_operation_statuses` the statuses of the +operations of the session, of a GetStatus RPC under the `get_status` deadline. The differences +that remain are in `dev/parity/known-differences.md` ("Client session ID", "`_closed`" and +"`_user_id`"). + The retry policies of `get_retry_policies` and `set_retry_policies` are PySpark's own (`pyspark.sql.connect.client.retries`, vendored from v4.3.0, which needs `grpcio`, `grpcio-status` and `googleapis-common-protos`). The transport retries a failure with the @@ -1736,17 +1756,30 @@ The errors of the RPCs that PySpark's client leaves to grpc, those of `clone` (a `SparkSession.cloneSession`) and of `copy_from_local_to_fs` (and `SparkSession.copyFromLocalToFs`), which raise grpc's `RpcError` there (`_InactiveRpcError`), are converted as those of the other RPCs: they are the exception of their error class, such as `UnknownException` or -`UnsupportedOperationException`, as the drop-in has no grpc. `global_user_context_extensions` is -the list attribute of PySpark's client, whose changes, and a list that it is set to, the requests -of all the threads carry. - -`get_tags`, as `SparkSession.getTags`, returns a copy of the tags of the thread, where PySpark -returns the set that its client keeps, which the later changes of the tags change too: the tags -are those of the core session, which its requests read, and not a Python set. The official tests of -the client (`sql/tests/connect/client/test_client*.py`) mock the gRPC stub and the reattachable -iterator of PySpark's client, which the drop-in does not have; those of its retries -(`test_connect_retry.py`) and of `pyspark.errors.exceptions.connect.convert_exception` -(`test_connect_errors_conversion.py`) pass. +`UnsupportedOperationException`, as the drop-in has no grpc. + +The clients of a session share their attributes, as PySpark's session has one client: an +attribute that is set on `spark.client` is there the next time, and so are its lock and its +`global_user_context_extensions`, the list attribute of PySpark's client, whose changes, and a list +that it is set to, the requests of all the threads carry, while a list that it is no longer set to +is not read. The `thread_local` of the client is a `threading.local`, as PySpark's: its `tags` and +`user_context_extensions` exist once a thread uses them, and are the set and the list that the +requests of the thread carry, as they change or are set, as the tags of `SparkSession.addTag` and +`getTags` are; another attribute that a thread sets is its own. A thread of +`inheritable_thread_target` shares the copies of the attributes of the thread that decorated its +function, as PySpark's does. The extensions are serialized again before each call that sends +requests, so a `google.protobuf.Any` of the lists that is changed in place is sent as it is, as +PySpark serializes them when it builds each request. The classes of these differ: +`thread_local` is a subclass of `threading.local`, and the set of its tags, the list of its +extensions and the global list are subclasses of `set` and `list`, of a plain `repr`, whose copies +and pickles are a plain set and plain lists; a set of tags refuses an object that is neither a str +nor bytes when it is added, of the `TypeError` that PySpark raises when it builds a request of it. +A `copy.copy` of the client has the attributes of its session, but a list that the copy's +attribute is set to is not one that the requests carry. The official tests of the client +(`sql/tests/connect/client/test_client*.py`) mock the gRPC stub and the reattachable iterator of +PySpark's client, which the drop-in does not have; those of its retries (`test_connect_retry.py`) +and of `pyspark.errors.exceptions.connect.convert_exception` (`test_connect_errors_conversion.py`) +pass. ## Spark Declarative Pipelines CLI diff --git a/python/pyspark/sql/connect/client/__init__.py b/python/pyspark/sql/connect/client/__init__.py index 15096e37..4f65a97a 100644 --- a/python/pyspark/sql/connect/client/__init__.py +++ b/python/pyspark/sql/connect/client/__init__.py @@ -34,6 +34,7 @@ import os import platform import sys +import threading import urllib.parse import uuid import warnings @@ -82,8 +83,8 @@ class RpcDeadlines: stream to resume receiving results. Non-reattachable ExecutePlan has no deadline because a timeout there would kill the execution with no recovery path. - The Rust transport applies the deadline of each of its RPCs. It sends no GetStatus, the RPC of - ``get_status``. + The Rust transport applies the deadline of each of its RPCs, ``get_status`` that of the + GetStatus RPC of ``SparkConnectClient._get_operation_statuses``. """ # As PySpark Connect's, of pyspark.sql.connect.client.core, whose name this class has too. @@ -715,6 +716,12 @@ class SparkConnectClient: # As PySpark Connect's, of pyspark.sql.connect.client.core, whose name this class has too. __module__ = "pyspark.sql.connect.client.core" + # The session of the client is kept apart from its attributes, which are those of its session: + # the clients of a session share them (see `_client_state`), as PySpark's session has one + # client, of one `thread_local`, one list of global user context extensions, its lock, and the + # attributes that a caller sets. + __slots__ = ("_session", "__dict__", "__weakref__") + # The thread of an ML cache RPC in flight, as SparkConnectClient's re-entrancy guard of a # RemoteModelRef finalizer that deletes a model. _ml_cache_rpc_thread: Optional[int] = None @@ -793,34 +800,29 @@ def _of_session( return client def _init_of_session(self, session: Any, builder: Optional[ChannelBuilder]) -> None: - import threading - self._session = session - # The channel builder of the connection of a client made of one, of its host and token, - # as PySpark's `_builder`; the client of a session reads them of the session. - self._builder = builder - # The state of the current thread of the session, as SparkConnectClient's thread_local. - self.thread_local = _ThreadLocalOfClient(session) - # The global user context extensions, as SparkConnectClient's list of them, of those that - # the session has: its changes, and a list that the attribute is set to, are the - # session's (see __setattr__). - extensions = _extensions_of(session._user_context_extensions(True)) - extensions = _GlobalUserContextExtensions(session, extensions) - object.__setattr__(self, "global_user_context_extensions", extensions) - self.global_user_context_extensions_lock = threading.Lock() + # The attributes of the client are those of its session, which its other clients have + # too: its channel builder, its thread_local, its global user context extensions and their + # lock, and the attributes that a caller set. + object.__setattr__(self, "__dict__", _client_state(session, builder)) def __setattr__(self, name: str, value: Any) -> None: # As PySpark's client, whose requests read its `global_user_context_extensions`, a list # that its methods and a caller may change or rebind: the session takes the extensions of # the list that the attribute is set to. - if name == "global_user_context_extensions" and "_session" in self.__dict__: + if name == "global_user_context_extensions" and hasattr(self, "_session"): value = _GlobalUserContextExtensions(self._session, value) + object.__setattr__(self, name, value) value._sync() + return object.__setattr__(self, name, value) def __copy__(self) -> "SparkConnectClient": - # As copy.copy of PySpark's client, a plain object: a client of the same attributes. + # As copy.copy of PySpark's client, a plain object: a client of the same session and of a + # copy of its attributes, whose rebinding of a list is its own. The requests of the + # session carry the extensions of the list of the session's own client. copied = object.__new__(type(self)) + object.__setattr__(copied, "_session", self._session) copied.__dict__.update(self.__dict__) return copied @@ -866,14 +868,72 @@ def _server_session_id(self, value: Optional[str]) -> None: @property def _closed(self) -> bool: """Whether the client is closed, as SparkConnectClient's ``_closed``. Setting it to True - closes the session, which the session registry then no longer returns; a closed client - stays closed.""" + closes the session, which the session registry then no longer returns, and setting it + to False opens it again, as PySpark's requests are sent again then.""" return self._session.is_stopped @_closed.setter def _closed(self, value: bool) -> None: if value: self._session._close() + else: + self._session._reopen() + + @property + def _session_id(self) -> str: + """The session ID that the requests of the client carry, as SparkConnectClient's + ``_session_id``: the session's, until it is set to another, which the requests carry from + then on, as PySpark's do; the server answers them of the session of that ID, and a + response of another server-side session than the one that the client observed closes + the client. The session's ``session_id`` stays, as PySpark's does, and so does that of + the requests of the artifacts, as PySpark's ArtifactManager keeps the ID that the client + was built with.""" + return self._session._client_session_id() + + @_session_id.setter + def _session_id(self, value: str) -> None: + self._session._set_client_session_id(value) + + @property + def _user_id(self) -> Optional[str]: + """The user of the requests of the client, as SparkConnectClient's ``_user_id``: of the + connection string, else of ``user_id``, else of ``SPARK_USER`` or ``USER``.""" + return self._session._user_id() + + @property + def _use_reattachable_execute(self) -> bool: + """Whether the executions of the client are reattachable, as SparkConnectClient's + ``_use_reattachable_execute``, which ``enable_reattachable_execute`` and + ``disable_reattachable_execute`` set, and which a caller may set.""" + return self._session._reattachable_execute() + + @_use_reattachable_execute.setter + def _use_reattachable_execute(self, value: bool) -> None: + self._session._set_reattachable_execute(bool(value)) + + def _get_operation_statuses( + self, + operation_ids: Optional[list[str]] = None, + operation_extensions: Optional[list[Any]] = None, + request_extensions: Optional[list[Any]] = None, + ) -> "pb2.GetStatusResponse": + """The statuses of the operations of the session, as SparkConnectClient's: of those of + ``operation_ids``, or of all of them if None or empty, of the per-operation + ``operation_extensions`` and the request-level ``request_extensions`` (each a + ``google.protobuf.Any``) that the request carries, as the full ``GetStatusResponse``. The + Rust transport sets the IDs of the session, the client type and the user of the request, + as PySpark's does.""" + import pyspark.sql.connect.proto as pb2 + + req = pb2.GetStatusRequest() + req.operation_status.SetInParent() + if operation_ids: + req.operation_status.operation_ids.extend(operation_ids) + if operation_extensions: + req.operation_status.extensions.extend(operation_extensions) + if request_extensions: + req.extensions.extend(request_extensions) + return pb2.GetStatusResponse.FromString(self._session._get_status(req.SerializeToString())) @property def is_closed(self) -> bool: @@ -1091,13 +1151,14 @@ def semantic_hash(self, plan: "pb2.Plan") -> int: def _execute_plan(self, plan: "pb2.Plan", observations: Any, reported: bool) -> Any: """Start the execution of ``plan``, a ``spark.connect.Plan`` that the client sends as it is, as SparkConnectClient's request of a copy of it: its responses, whose observed metrics - set the Observations of ``observations`` by their names, and whose progress reaches the - progress handlers when ``reported``.""" - return self._session._execute_plan( - plan.SerializeToString(), list(observations.items()), reported - ) - - def _results(self, responses: Any) -> Iterator[Any]: + set the Observations of ``observations``, a dict, by their names, and whose progress + reaches the progress handlers when ``reported``. Of an ``observations`` that is not a + dict, such as None or an empty list, none is set: PySpark's client reads it only when + observed metrics arrive (see ``_results``).""" + items = list(observations.items()) if isinstance(observations, Mapping) else [] + return self._session._execute_plan(plan.SerializeToString(), items, reported) + + def _results(self, responses: Any, observations: Any) -> Iterator[Any]: """The results of ``responses``, as SparkConnectClient's ``_execute_and_fetch_as_iterator`` yields them of each response: its plan metrics, its observed metrics, the schema of its result, the properties of the result of a command by @@ -1110,7 +1171,13 @@ def _results(self, responses: Any) -> Iterator[Any]: try: for data, metrics, observed_metrics in responses: yield from metrics - yield from observed_metrics + for observed in observed_metrics: + # As PySpark's client, which looks the name of the observed metrics up in + # `observations` when they arrive: the TypeError of `in` of an object that is + # not a container, such as None. + if observed.name != "__python_accumulator__": + _ = observed.name in observations + yield observed b = pb2.ExecutePlanResponse.FromString(data) if b.HasField("schema"): dt = types.proto_schema_to_pyspark_data_type(b.schema) @@ -1180,7 +1247,7 @@ def _execute_and_fetch( properties: dict[str, Any] = {} responses = self._execute_plan(plan, observations, True) - for response in self._results(responses): + for response in self._results(responses, observations): if isinstance(response, StructType): schema = response elif isinstance(response, pa.RecordBatch): @@ -1238,7 +1305,7 @@ def to_table_as_iterator( if logger.isEnabledFor(logging.DEBUG): logger.debug(f"Executing plan {self._proto_to_string(plan, True)}") sent = _plan_of(plan) - for response in self._results(self._execute_plan(sent, observations, True)): + for response in self._results(self._execute_plan(sent, observations, True), observations): if isinstance(response, StructType): yield response elif isinstance(response, pa.RecordBatch): @@ -1336,7 +1403,8 @@ def execute_command_as_iterator( f"Execute command as iterator for command {self._proto_to_string(command, True)}" ) plan = _plan_of_command(command) - for response in self._results(self._execute_plan(plan, observations or {}, False)): + observations = observations or {} + for response in self._results(self._execute_plan(plan, observations, False), observations): if isinstance(response, dict): yield response else: @@ -1349,8 +1417,13 @@ def execute_command_as_iterator( def add_threadlocal_user_context_extension(self, extension: Any) -> str: """Add ``extension``, a ``google.protobuf.Any``, to the user context of the requests that - the current thread sends, as SparkConnectClient's: its ID.""" - return self._session._add_user_context_extension(extension.SerializeToString(), False) + the current thread sends, as SparkConnectClient's: to the list of + ``thread_local.user_context_extensions``, of its ID.""" + if not hasattr(self.thread_local, "user_context_extensions"): + self.thread_local.user_context_extensions = list() + extension_id = "threadlocal_" + str(uuid.uuid4()) + self.thread_local.user_context_extensions.append((extension_id, extension)) + return extension_id def add_global_user_context_extension(self, extension: Any) -> str: """Add ``extension``, a ``google.protobuf.Any``, to the user context of the requests of @@ -1362,10 +1435,15 @@ def add_global_user_context_extension(self, extension: Any) -> str: return extension_id def remove_user_context_extension(self, extension_id: str) -> None: - """Remove the user context extension of ``extension_id``, as SparkConnectClient's: a - global one rebinds ``global_user_context_extensions`` to a list without it.""" + """Remove the user context extension of ``extension_id``, as SparkConnectClient's: of the + current thread, which rebinds ``thread_local.user_context_extensions`` to a list without + it, or a global one, which rebinds ``global_user_context_extensions``.""" if extension_id.find("threadlocal_") == 0: - self._session._remove_user_context_extension(extension_id) + if not hasattr(self.thread_local, "user_context_extensions"): + return + self.thread_local.user_context_extensions = list( + filter(lambda ex: ex[0] != extension_id, self.thread_local.user_context_extensions) + ) elif extension_id.find("global_") == 0: with self.global_user_context_extensions_lock: self.global_user_context_extensions = list( @@ -1374,30 +1452,66 @@ def remove_user_context_extension(self, extension_id: str) -> None: def clear_user_context_extensions(self) -> None: """Remove the user context extensions of the current thread and the global ones, as - SparkConnectClient's: ``global_user_context_extensions`` is a new, empty list.""" - self._session._clear_user_context_extensions() + SparkConnectClient's: each is a new, empty list, that of the thread if it has one.""" + if hasattr(self.thread_local, "user_context_extensions"): + self.thread_local.user_context_extensions = list() with self.global_user_context_extensions_lock: self.global_user_context_extensions = list() def add_tag(self, tag: str) -> None: """Add ``tag`` to the tags of the current thread, which the executions that it starts - carry, as SparkConnectClient's: ``CANNOT_BE_NONE`` for None, ``VALUE_ALLOWED`` for a tag - of a comma and ``VALUE_NOT_NON_EMPTY_STR`` for an empty one.""" - self._session.addTag(tag) + carry, as SparkConnectClient's: to the set of ``thread_local.tags``, of the errors of + ``_throw_if_invalid_tag``.""" + self._throw_if_invalid_tag(tag) + if not hasattr(self.thread_local, "tags"): + self.thread_local.tags = set() + self.thread_local.tags.add(tag) def remove_tag(self, tag: str) -> None: """Remove ``tag`` from the tags of the current thread, as SparkConnectClient's: of the errors of ``add_tag``, and nothing for a tag that the thread does not have.""" - self._session.removeTag(tag) + self._throw_if_invalid_tag(tag) + if not hasattr(self.thread_local, "tags"): + self.thread_local.tags = set() + # Use discard, not remove: removing an absent tag is a documented no-op + # (see SparkSession.removeTag), matching the Classic behavior. + self.thread_local.tags.discard(tag) def get_tags(self) -> set[str]: - """The tags of the current thread, as SparkConnectClient's: a copy of them, which the - Rust transport keeps, where PySpark's client gives the set that it keeps.""" - return self._session.getTags() + """The tags of the current thread, as SparkConnectClient's: the set of + ``thread_local.tags``, which the later changes of the tags change too.""" + if not hasattr(self.thread_local, "tags"): + self.thread_local.tags = set() + return self.thread_local.tags def clear_tags(self) -> None: - """Remove the tags of the current thread, as SparkConnectClient's.""" - self._session.clearTags() + """Remove the tags of the current thread, as SparkConnectClient's: ``thread_local.tags`` + is a new, empty set.""" + self.thread_local.tags = set() + + def _throw_if_invalid_tag(self, tag: str) -> None: + """ + Validate if a tag for ExecutePlanRequest.tags is valid. Throw ``ValueError`` if + not. + """ + spark_job_tags_sep = "," + if tag is None: + raise PySparkValueError( + errorClass="CANNOT_BE_NONE", messageParameters={"arg_name": "Spark Connect tag"} + ) + if spark_job_tags_sep in tag: + raise PySparkValueError( + errorClass="VALUE_ALLOWED", + messageParameters={ + "arg_name": "Spark Connect tag", + "disallowed_value": spark_job_tags_sep, + }, + ) + if len(tag) == 0: + raise PySparkValueError( + errorClass="VALUE_NOT_NON_EMPTY_STR", + messageParameters={"arg_name": "Spark Connect tag", "arg_value": tag}, + ) def interrupt_all(self) -> Optional[list[str]]: """Interrupt the executions of the session, as SparkConnectClient's: the IDs of the @@ -1429,24 +1543,38 @@ def register_udf( """Register the Python function ``function`` as a temporary function of the session of the name ``name``, else of a name that the client generates, as SparkConnectClient's ``register_udf``: the name. A DDL ``return_type`` is parsed by the server, and the - function is pickled with the type, as PySpark's ``PythonUDF`` does.""" + function is pickled with the type, as PySpark's ``PythonUDF`` does. As PySpark's, the + name and ``deterministic`` are set in the fields of the command first, then the return + type is parsed and converted, then ``eval_type`` is set, and the function is pickled + last, so each raises the error of its protobuf field or of its conversion in that + order.""" + import pyspark.sql.connect.proto as pb2 + from pyspark.serializers import CloudPickleSerializer + if name is None: name = f"fun_{uuid.uuid4().hex}" + # As PySpark's CommonInlineUserDefinedFunction.to_plan_udf, which sets these first. + fun = pb2.CommonInlineUserDefinedFunction() + fun.function_name = name + fun.deterministic = deterministic + # And then its PythonUDF.to_plan: the parse of a DDL string, the proto of the return type, + # the eval type, and the pickle. output_type = return_type if isinstance(return_type, str): output_type = self._session._parse_ddl(return_type) assert isinstance(output_type, DataType) else: _check_data_type(return_type) - from pyspark.serializers import CloudPickleSerializer - + udf = pb2.PythonUDF() + udf.eval_type = eval_type + command = CloudPickleSerializer().dumps((function, output_type)) self._session._registerPythonUdf( - name, + fun.function_name, output_type, - eval_type, - CloudPickleSerializer().dumps((function, output_type)), + udf.eval_type, + command, f"{sys.version_info.major}.{sys.version_info.minor}", - deterministic, + fun.deterministic, ) return name @@ -1460,24 +1588,30 @@ def register_udtf( ) -> str: """Register the user-defined table function class ``function`` as the temporary function ``name`` of the session, as SparkConnectClient's ``register_udtf``: the name. A - DDL ``return_type`` is left to the server to parse. As PySpark's, the name and then the - return type are converted before the handler is pickled, so a handler that does not - pickle raises ``UDTF_SERIALIZATION_ERROR`` only of a valid name and return type.""" + DDL ``return_type`` is left to the server to parse. As PySpark's, the ``__name__`` of the + handler class is read, the name and ``deterministic`` are set in the fields of the + command, the return type is converted and ``eval_type`` set before the handler is + pickled, so a handler that does not pickle raises ``UDTF_SERIALIZATION_ERROR`` only of + valid arguments.""" + import pyspark.sql.connect.proto as pb2 from pyspark.sql.connect.utils import get_python_ver from pyspark.sql.udtf import _pickled_handler # As PySpark's PythonUDTF, which reads the name of the handler class when it is made. _ = function.__name__ - # As PySpark's register_udtf, which sets the name in its command and converts the return - # type (PythonUDTF.to_plan) before it pickles the handler: `_registerPythonUdtf` calls the - # pickling after them. + # As PySpark's CommonInlineUserDefinedTableFunction.udtf_plan, which sets these first, and + # then converts the return type and sets the eval type (PythonUDTF.to_plan) before it + # pickles the handler: `_registerPythonUdtf` converts them, and then calls the pickling. + fun = pb2.CommonInlineUserDefinedTableFunction() + fun.function_name = name + fun.deterministic = deterministic self._session._registerPythonUdtf( - name, + fun.function_name, return_type, eval_type, functools.partial(_pickled_handler, function), get_python_ver(), - deterministic, + fun.deterministic, ) return name @@ -1497,9 +1631,21 @@ def register_java( ``name`` of the session, as SparkConnectClient's ``register_java``: an aggregate one for ``aggregate``, of the class's own type, and of a ``return_type``, a DataType or a DDL string that the server parses, a function that is not an aggregate one, as PySpark's - ``JavaUDF`` of an output type is.""" + ``JavaUDF`` of an output type is. As PySpark's, the name, the class name, the return + type and then ``aggregate``, unless there is a return type, are set in the fields of the + command in that order, so each raises the error of its protobuf field in that order.""" + import pyspark.sql.connect.proto as pb2 + + # As PySpark's CommonInlineUserDefinedFunction.to_plan_judf and JavaUDF.to_plan. + fun = pb2.CommonInlineUserDefinedFunction() + fun.function_name = name + java = pb2.JavaUDF() + java.class_name = javaClassName + if return_type is not None and not isinstance(return_type, str): + _check_data_type(return_type) + java.aggregate = aggregate if return_type is None else False self._session._registerJavaFunction( - name, javaClassName, return_type, aggregate if return_type is None else False + fun.function_name, java.class_name, return_type, java.aggregate ) def add_artifacts(self, *paths: str, pyfile: bool, archive: bool, file: bool) -> None: @@ -1512,11 +1658,15 @@ def add_artifacts(self, *paths: str, pyfile: bool, archive: bool, file: bool) -> def copy_from_local_to_fs(self, local_path: str, dest_path: str) -> None: """Copy the local file ``local_path`` to the absolute path ``dest_path`` of the file system of the server, as SparkConnectClient's ``copy_from_local_to_fs``: of the - assertions of its ArtifactManager, without the check of a scheme of + assertions and the errors of its ArtifactManager, without the check of a scheme of ``SparkSession.copyFromLocalToFs``.""" - assert Path(local_path).absolute().is_file(), "local path must be a file path." + abs_path = Path(local_path).absolute() + assert abs_path.is_file(), "local path must be a file path." assert Path(dest_path).is_absolute(), "destination FS path must be an absolute path." - self._session.copyFromLocalToFs(local_path, dest_path) + # As the name of the artifact of PySpark's ArtifactManager, which raises the TypeError of + # the concatenation for a destination that is not a str. + _ = "forward_to_fs" + dest_path + self._session._copy_from_local_to_fs(str(abs_path), dest_path) def cache_artifact(self, blob: bytes) -> str: """Cache ``blob``, a bytes-like object, as an artifact of the session, as @@ -1744,17 +1894,23 @@ class _GlobalUserContextExtensions(list): """The global user context extensions of a client, as the list of PySpark's ``SparkConnectClient.global_user_context_extensions``, of their IDs and their ``google.protobuf.Any``: a list whose changes the session of the client takes, so that the - requests of all its threads carry them, as PySpark's requests read the list. Its copies and its - pickles are plain lists.""" + requests of all its threads carry them, as PySpark's requests read the list, while it is the + list of the clients of the session. Its copies and its pickles are plain lists.""" def __init__(self, session: Any, extensions: Iterable[Any] = ()) -> None: super().__init__(extensions) - self._session = session + self._session_ref = weakref.ref(session) def _sync(self) -> None: - self._session._set_global_user_context_extensions( - [(extension_id, extension.SerializeToString()) for extension_id, extension in self] - ) + session = self._session_ref() + if session is None: + return + state = _CLIENT_STATES.get(session) + # A list that the attribute of the session's clients is no longer set to is not the one + # that the requests read, as in PySpark, and neither is a copy's. + if state is None or state.get("global_user_context_extensions") is not self: + return + session._set_global_user_context_extensions(_serialized_extensions(self)) def __reduce_ex__(self, protocol: Any) -> Any: return (list, (list(self),)) @@ -1766,35 +1922,164 @@ def __deepcopy__(self, memo: Any) -> list[Any]: return copy.deepcopy(list(self), memo) -def _synced(name: str) -> Any: - # A method of a list that changes it, after which the session takes the extensions. - method = getattr(list, name) +class _ThreadExtensions(list): + """The user context extensions of the threads that have it, as the list of PySpark's + ``thread_local.user_context_extensions``, of their IDs and their ``google.protobuf.Any``: the + list of a thread, which the requests of the thread carry, as PySpark's read it, and of the + threads that are given it, as those of ``inheritable_thread_target``. Its changes are those of + the list of the Rust transport that it holds, which the threads that have it share. Its copies + and its pickles are plain lists.""" + + def __init__(self, session: Any, extensions: Iterable[Any]) -> None: + super().__init__(extensions) + self._session_ref = weakref.ref(session) + self._handle = session._new_user_context_extensions(_serialized_extensions(self)) - def synced(self: _GlobalUserContextExtensions, *args: Any, **kwargs: Any) -> Any: + @classmethod + def _of(cls, thread_local: Any, extensions: Any) -> Any: + # The list of the current thread once `thread_local.user_context_extensions` is set to + # `extensions`: a list of the session, which the thread shares, or a new one of its + # extensions, which the thread has alone. + session = thread_local._session_ref() + if session is None: + return extensions + if not (isinstance(extensions, cls) and extensions._session_ref() is session): + extensions = cls(session, extensions) + session._share_thread_user_context_extensions(extensions._handle) + return extensions + + def _sync(self) -> None: + self._handle._replace(_serialized_extensions(self)) + + def _copy(self) -> "_ThreadExtensions": + # A list of deep copies of the extensions, which no thread has yet. + session = self._session_ref() + assert session is not None + return type(self)(session, copy.deepcopy(list(self))) + + def __reduce_ex__(self, protocol: Any) -> Any: + return (list, (list(self),)) + + def __copy__(self) -> list[Any]: + return list(self) + + def __deepcopy__(self, memo: Any) -> list[Any]: + return copy.deepcopy(list(self), memo) + + +class _ThreadTags(set): + """The operation tags of the threads that have it, as the set of PySpark's + ``thread_local.tags``: the set of a thread, which its executions carry, as PySpark's read it, + and of the threads that are given it, as those of ``inheritable_thread_target``. Its changes + are those of the set of the Rust transport that it holds, which the threads that have it + share: a str that UTF-8 cannot encode fails each request, as in PySpark, and an object that is + neither a str nor bytes raises the ``TypeError`` of the protobuf field of the tags, which + PySpark raises when it builds a request, when it is added. Its copies and its pickles are plain + sets.""" + + def __init__(self, session: Any, tags: Iterable[Any]) -> None: + super().__init__(tags) + self._session_ref = weakref.ref(session) + self._handle = session._new_tags(list(self)) + + @classmethod + def _of(cls, thread_local: Any, tags: Any) -> Any: + # The set of the current thread once `thread_local.tags` is set to `tags`: a set of the + # session, which the thread shares, or a new one of its tags, which the thread has alone. + session = thread_local._session_ref() + if session is None: + return tags + if not (isinstance(tags, cls) and tags._session_ref() is session): + tags = cls(session, tags) + session._share_thread_tags(tags._handle) + return tags + + def _sync(self) -> None: + self._handle._replace(list(self)) + + def _copy(self) -> "_ThreadTags": + # A set of the same tags, which no thread has yet. + session = self._session_ref() + assert session is not None + return type(self)(session, copy.deepcopy(set(self))) + + def __repr__(self) -> str: + # As the repr of PySpark's plain set. + return repr(set(self)) + + def __reduce_ex__(self, protocol: Any) -> Any: + return (set, (list(self),)) + + def __copy__(self) -> set[Any]: + return set(self) + + def __deepcopy__(self, memo: Any) -> set[Any]: + return copy.deepcopy(set(self), memo) + + +def _serialized_extensions(extensions: Iterable[Any]) -> list[tuple[Any, bytes]]: + # The IDs and the bytes of the google.protobuf.Any of `extensions`, as the Rust transport takes + # them for the user context of its requests. + return [(extension_id, extension.SerializeToString()) for extension_id, extension in extensions] + + +def _synced(base: type, name: str) -> Any: + # A method of `base`, a list or a set, that changes it, after which the Rust transport takes + # the extensions or the tags: a change that it does not take, of an object that it cannot + # send, is undone, and its error raised. + method = getattr(base, name) + + def synced(self: Any, *args: Any, **kwargs: Any) -> Any: + before = base(self) result = method(self, *args, **kwargs) - self._sync() + try: + self._sync() + except BaseException: + base.clear(self) + if base is list: + base.extend(self, before) + else: + base.update(self, before) + raise return result synced.__name__ = synced.__qualname__ = name return synced +for _list in (_GlobalUserContextExtensions, _ThreadExtensions): + for _name in ( + "append", + "extend", + "insert", + "remove", + "pop", + "clear", + "sort", + "reverse", + "__setitem__", + "__delitem__", + "__iadd__", + "__imul__", + ): + setattr(_list, _name, _synced(list, _name)) for _name in ( - "append", - "extend", - "insert", + "add", + "discard", "remove", "pop", "clear", - "sort", - "reverse", - "__setitem__", - "__delitem__", - "__iadd__", - "__imul__", + "update", + "difference_update", + "intersection_update", + "symmetric_difference_update", + "__ior__", + "__iand__", + "__isub__", + "__ixor__", ): - setattr(_GlobalUserContextExtensions, _name, _synced(_name)) -del _name + setattr(_ThreadTags, _name, _synced(set, _name)) +del _list, _name def _extensions_of(extensions: Any) -> list[Any]: @@ -1805,21 +2090,108 @@ def _extensions_of(extensions: Any) -> list[Any]: return [(extension_id, ProtoAny.FromString(data)) for extension_id, data in extensions] -class _ThreadLocalOfClient: - """The state of the current thread of the session of a client, as SparkConnectClient's - ``thread_local``: its user context extensions and its tags, which the Rust transport keeps - for each thread.""" +class _ThreadLocalOfClient(threading.local): + """The state of each thread of the session of a client, as SparkConnectClient's + ``thread_local``, a ``threading.local``: the attributes that a thread sets are its own, and + PySpark's ``tags`` and ``user_context_extensions`` exist once a thread uses them. The set of + the tags and the list of the extensions that a thread sets are those that its requests carry + (see ``_ThreadTags`` and ``_ThreadExtensions``), and so are those of the threads that are given + them, as by ``inheritable_thread_target``. Deleting them leaves the thread with none.""" - def __init__(self, session: Any) -> None: - self._session = session + # The session of the clients, held weakly, which all threads see. + __slots__ = ("_session_ref",) - @property - def user_context_extensions(self) -> list[Any]: - return _extensions_of(self._session._user_context_extensions(False)) + def __setattr__(self, name: str, value: Any) -> None: + if name == "tags": + value = _ThreadTags._of(self, value) + elif name == "user_context_extensions": + value = _ThreadExtensions._of(self, value) + super().__setattr__(name, value) + + def __delattr__(self, name: str) -> None: + super().__delattr__(name) + session = self._session_ref() + if session is None: + return + if name == "tags": + session._clear_thread_tags() + elif name == "user_context_extensions": + session._clear_thread_user_context_extensions() + + +def _thread_local_copies(thread_local: Any) -> list[tuple[str, Any]]: + """The attributes of the current thread of ``thread_local``, a client's, as PySpark's + ``inheritable_thread_target`` copies them for the threads of a function, of ``copy.deepcopy`` + of each: the copies of the set of the tags and of the list of the extensions are a set and a + list that the threads that are given them share, as the threads share PySpark's copies.""" + copies = [] + for attr, value in thread_local.__dict__.items(): + if isinstance(value, (_ThreadTags, _ThreadExtensions)): + value = value._copy() + else: + value = copy.deepcopy(value) + copies.append((attr, value)) + return copies + + +# The attributes of the clients of each session, which they share, as PySpark's session has one +# client (see `SparkConnectClient.__slots__`): held weakly by their session, which they hold +# weakly too. +_CLIENT_STATES: "weakref.WeakKeyDictionary[Any, dict[str, Any]]" = weakref.WeakKeyDictionary() +_CLIENT_STATES_LOCK = threading.Lock() + + +def _client_state(session: Any, builder: Optional[ChannelBuilder]) -> dict[str, Any]: + """The attributes of the clients of ``session``, made with the first of them: its channel + builder ``builder``, as PySpark's ``_builder``, else a ``DefaultChannelBuilder`` of the + connection of the session, of the host and the token of the client; its ``thread_local``; its + ``global_user_context_extensions``, those that the session has, whose changes, and a list that + the attribute is set to, are the session's (see ``SparkConnectClient.__setattr__``); and their + lock.""" + with _CLIENT_STATES_LOCK: + state = _CLIENT_STATES.get(session) + if state is None: + connection_string = getattr(session, "_connection_string", None) + if builder is None and connection_string is not None: + # As PySpark's client of a session, of the channel builder of its connection. + builder = DefaultChannelBuilder(connection_string()) + thread_local = _ThreadLocalOfClient() + thread_local._session_ref = weakref.ref(session) + extensions = _extensions_of(session._user_context_extensions(True)) + state = { + "_builder": builder, + "thread_local": thread_local, + "global_user_context_extensions": _GlobalUserContextExtensions( + session, extensions + ), + "global_user_context_extensions_lock": threading.Lock(), + } + _CLIENT_STATES[session] = state + # As PySpark's requests, which serialize the extensions when they are built: the calls + # of the session serialize them again before they send requests. + session._set_user_context_refresh( + functools.partial(_refresh_user_context_extensions, weakref.ref(session)) + ) + return state - @property - def tags(self) -> Any: - return set(self._session.getTags()) + +def _refresh_user_context_extensions(session_ref: Any) -> None: + """Serialize the user context extensions of the session of ``session_ref`` again for the + Rust transport, of its clients' global list and of the list of the current thread, before a + call of the session sends requests, as PySpark serializes them when it builds each request: + so a ``google.protobuf.Any`` of the lists that was changed in place is sent as it is now.""" + session = session_ref() + if session is None: + return + state = _CLIENT_STATES.get(session) + if state is None: + return + extensions = state.get("global_user_context_extensions") + if isinstance(extensions, _GlobalUserContextExtensions) and extensions: + extensions._sync() + own = getattr(state.get("thread_local"), "user_context_extensions", None) + if isinstance(own, _ThreadExtensions) and own: + own._sync() # The client of each session, by its ID, while it is in use. diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py index 8d0a7250..9ad10cb8 100644 --- a/python/pyspark/sql/udf.py +++ b/python/pyspark/sql/udf.py @@ -22,14 +22,14 @@ import functools import sys -import uuid import warnings -from typing import Callable, Optional, Any, Union +from typing import Callable, Optional, Any from pyspark.errors import PySparkNotImplementedError, PySparkRuntimeError, PySparkTypeError from pyspark.sql.pandas.types import to_arrow_type from pyspark.sql.types import DataType, StringType, StructType -from pyspark.serializers import CloudPickleSerializer +# As PySpark's module, of whose namespace it is (see the stubs). +from pyspark.serializers import CloudPickleSerializer # noqa: F401 from pyspark.util import PythonEvalType __all__ = ["UDFRegistration"] @@ -590,7 +590,9 @@ def register( "SQL_GROUPED_AGG_PANDAS_ITER_UDF or SQL_GROUPED_AGG_ARROW_ITER_UDF" }, ) - self._register_udf(f.func, f.returnType, name, f.evalType, f.deterministic) + self.sparkSession._client.register_udf( + f.func, f.returnType, name, f.evalType, f.deterministic + ) return f else: if returnType is None: @@ -601,34 +603,9 @@ def register( # As PySpark's: the registration takes the return type as given, not the one that # the wrapper parsed, so a DDL string is parsed again for the command. - self._register_udf(py_udf.func, returnType, name) + self.sparkSession._client.register_udf(py_udf.func, returnType, name) return py_udf - def _register_udf( - self, - function: Callable[..., Any], - return_type: Union[DataType, str], - name: str, - eval_type: int = PythonEvalType.SQL_BATCHED_UDF, - deterministic: bool = True, - ) -> None: - # As PySpark's SparkConnectClient.register_udf: a function of no name is registered as - # `fun_` and a random hex, and the PythonUDF command is the pickled (function, return - # type), of a DDL string as the server of the session parses it, as its PythonUDF parses - # the str that it is given when it builds the command. - if name is None: - name = f"fun_{uuid.uuid4().hex}" - if isinstance(return_type, str): - return_type = self.sparkSession._parse_ddl(return_type) - self.sparkSession._registerPythonUdf( - name, - return_type, - eval_type, - CloudPickleSerializer().dumps((function, return_type)), - "%d.%d" % sys.version_info[:2], - deterministic, - ) - def registerJavaFunction( self, name: str, @@ -648,10 +625,10 @@ def registerJavaFunction( Return type: a DataType, or a DDL string, which the server parses, as PySpark sends it. Defaults to the class's own type. """ - self.sparkSession._registerJavaFunction(name, javaClassName, returnType, False) + self.sparkSession._client.register_java(name, javaClassName, returnType) def registerJavaUDAF(self, name: str, javaClassName: str) -> None: """Register a Java user-defined aggregate function (UDAF) by class name. Mirrors ``UDFRegistration.registerJavaUDAF``. """ - self.sparkSession._registerJavaFunction(name, javaClassName, None, True) + self.sparkSession._client.register_java(name, javaClassName, aggregate=True) diff --git a/python/pyspark/util.py b/python/pyspark/util.py index eebb89ea..98a8536e 100644 --- a/python/pyspark/util.py +++ b/python/pyspark/util.py @@ -176,10 +176,11 @@ def inheritable_thread_target(f: Union[Callable, "SparkSession"]) -> Callable: >>> Thread(target=inheritable_thread_target(session)(target_func)).start() # doctest: +SKIP """ # As PySpark's of Spark Connect, the only mode of this client: `f` is the session, whose - # tags and user context extensions, those of this thread when the function is decorated, the - # threads of the function inherit, with the session as their active one. As PySpark's, which - # gives each thread the one copy of the `thread_local` of the client, the threads share that - # copy of the tags and of the extensions. + # `thread_local` of its client, of this thread when the function is decorated, the threads of + # the function inherit, with the session as their active one. As PySpark's, which gives each + # thread the one copy of each attribute of the `thread_local`, the threads share that copy of + # the tags and of the user context extensions. + from pyspark.sql.connect.client import _thread_local_copies from pyspark.sql.connect.session import SparkSession session = f @@ -188,8 +189,7 @@ def outer(ff: Callable) -> Callable: assert isinstance(session, SparkSession), ( "f is expected to be SparkSession for spark connect" ) - tags = session._copy_thread_tags() - extensions = session._copy_thread_user_context_extensions() + session_client_thread_local_attrs = _thread_local_copies(session.client.thread_local) @functools.wraps(ff) def inner(*args: Any, **kwargs: Any) -> Any: @@ -199,9 +199,13 @@ def inner(*args: Any, **kwargs: Any) -> Any: ) SparkSession._set_default_and_active_session(session) - # Set the tags and the user context extensions of the session in the child thread. - session._share_thread_tags(tags) - session._share_thread_user_context_extensions(extensions) + # Set thread locals in child thread. + for attr, value in session_client_thread_local_attrs: + setattr( + session.client.thread_local, + attr, + value, + ) return ff(*args, **kwargs) return inner @@ -242,18 +246,21 @@ def __init__( self._session = session def copy_local_properties(*a: Any, **k: Any) -> Any: + # Set tags in child thread. assert hasattr(self, "_tags") assert session is not None - session.clearTags() - for tag in self._tags: - session.addTag(tag) + thread_local = session.client.thread_local + thread_local.tags = self._tags return target(*a, **k) super().__init__(target=copy_local_properties, *args, **kwargs) # type: ignore[misc] def start(self) -> None: assert hasattr(self, "_session") - self._tags = set(self._session.getTags()) + thread_local = self._session.client.thread_local + if not hasattr(thread_local, "tags"): + thread_local.tags = set() + self._tags = set(thread_local.tags) return super().start() diff --git a/python/tests/_mock_connect_server.py b/python/tests/_mock_connect_server.py index 8aa7b7af..62f2441e 100644 --- a/python/tests/_mock_connect_server.py +++ b/python/tests/_mock_connect_server.py @@ -387,6 +387,8 @@ def reset(self): self.rpc_events = [] self.set_confs = [] self.clone_requests = [] + # The GetStatusRequests of each GetStatus. + self.get_status_requests = [] # The DDL of the JSON of each JsonToDDL. self.ddl_string = "" with self.listener_condition: @@ -912,6 +914,21 @@ def clone_session(self, request, context): new_id = fields.get(4, [self.cloned_session_id.encode()])[0] return message_field(1, fields[1][0]) + message_field(3, new_id) + def get_status(self, request, context): + # GetStatusRequest.session_id=1, operation_status=5, OperationStatusRequest + # .operation_ids=1; GetStatusResponse.session_id=1, operation_statuses=3, + # OperationStatus.operation_id=1, state=2. Each operation that the request names is + # OPERATION_STATE_UNKNOWN=1, as the server answers of an operation that it does not + # know. + if self.is_recorded(request, "get_status"): + self.get_status_requests.append(request) + fields = parse_message(request) + operation_status = parse_message(fields.get(5, [b""])[0]) + return message_field(1, fields[1][0]) + b"".join( + message_field(3, message_field(1, operation_id) + varint_field(2, 1)) + for operation_id in operation_status.get(1, []) + ) + def config(self, request, context): # ConfigRequest.session_id=1, operation=3, Operation.get=2, Get.keys=1; # ConfigResponse.session_id=1, pairs=2, KeyValue.key=1, value=2. @@ -1015,6 +1032,7 @@ def config(self, request, context): "Config": grpc.unary_unary_rpc_method_handler(service.config), "CloneSession": grpc.unary_unary_rpc_method_handler(service.clone_session), "Interrupt": grpc.unary_unary_rpc_method_handler(service.interrupt), + "GetStatus": grpc.unary_unary_rpc_method_handler(service.get_status), "AddArtifacts": grpc.stream_unary_rpc_method_handler(service.add_artifacts), } server.add_generic_rpc_handlers( diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py index ce99b9f1..3661a671 100644 --- a/python/tests/test_dropin_e2e.py +++ b/python/tests/test_dropin_e2e.py @@ -426,6 +426,163 @@ class Mine(Observation): assert unused._result is None +def test_client_state_is_one_per_session_and_its_thread_local_a_threading_local_as_pyspark( + session, +): + # As PySpark Connect's SparkConnectClient: the attributes of the client of a session are its + # own, and its thread_local is a threading.local whose tags and user context extensions exist + # once a thread uses them, are the set and the list that the requests of the thread carry, + # also as they change, and are of that thread only. The extensions are serialized for each + # request, so one changed in place is sent as it is. + import threading + + import pyspark.sql.connect.proto as pb2 + from pyspark.sql.connect.client import SparkConnectClient + + try: + from google.protobuf.any_pb2 import Any + except ImportError: + from pyspark.sql.connect.proto._rust import Any + + session.client.state_of_the_test = 1 + assert session.client.state_of_the_test == 1 + held = session.client.global_user_context_extensions + assert session.client.global_user_context_extensions_lock is ( + session.client.global_user_context_extensions_lock + ) + + seen = [] + + class Hook: + def on_execute_plan(self, request): + seen.append( + ( + sorted(request.tags), + [(e.type_url, e.value) for e in request.user_context.extensions], + ) + ) + return request + + client = SparkConnectClient(os.environ["SPARK_REMOTE"], session_hooks=[Hook()]) + + def run(): + plan = pb2.Plan() + plan.root.range.end = 1 + plan.root.range.step = 1 + client.to_table(plan, {}) + return seen[-1] + + local = client.thread_local + assert not hasattr(local, "tags") and not hasattr(local, "user_context_extensions") + first = Any(type_url="type.googleapis.com/t1", value=b"1") + client.add_threadlocal_user_context_extension(first) + extensions = local.user_context_extensions + assert extensions[0][1] is first and local.user_context_extensions is extensions + extensions.append(("threadlocal_x", Any(type_url="type.googleapis.com/t2", value=b"2"))) + assert run() == ([], [("type.googleapis.com/t1", b"1"), ("type.googleapis.com/t2", b"2")]) + first.value = b"changed" + assert run()[1][0] == ("type.googleapis.com/t1", b"changed") + client.add_tag("a") + assert local.tags is client.get_tags() + local.tags.add("b") + assert run()[0] == ["a", "b"] + found = {} + + def other(): + found["tags"] = hasattr(local, "tags") + found["extensions"] = hasattr(local, "user_context_extensions") + + thread = threading.Thread(target=other) + thread.start() + thread.join() + assert found == {"tags": False, "extensions": False} + + shared = Any(type_url="type.googleapis.com/g1", value=b"g") + client.add_global_user_context_extension(shared) + shared.value = b"changed" + assert run()[1][0] == ("type.googleapis.com/g1", b"changed") + client.close() + del session.client.state_of_the_test + assert held is session.client.global_user_context_extensions + + +def test_client_registrations_and_executions_take_their_arguments_as_pyspark(session): + # As PySpark Connect's register_udf, which sets the name and `deterministic` in its command + # before the server parses a DDL return type, and the eval type after it, and its executions, + # which read their `observations` only when observed metrics arrive. + import threading + + import pyspark.sql.connect.proto as pb2 + from pyspark.errors import ParseException + from pyspark.sql.connect.observation import Observation + + client = session.client + lock = threading.Lock() + + def unpicklable(x): + return lock + + with pytest.raises(TypeError, match="cannot be interpreted as an integer"): + client.register_udf(unpicklable, "not a type ###", deterministic="x") + with pytest.raises(ParseException): + client.register_udf(unpicklable, "not a type ###", eval_type="x") + with pytest.raises(TypeError, match="cannot be interpreted as an integer"): + client.register_udf(unpicklable, "int", eval_type="x") + + observation = Observation("client_observations") + observed = session.range(3).observe(observation, F.count(F.lit(1)).alias("n")) + plan = pb2.Plan() + plan.root.CopyFrom(observed._plan.plan(client)) + assert client.to_table(plan, [])[0].num_rows == 3 + for call in ( + lambda: client.to_table(plan, None), + lambda: client.to_pandas(plan, None), + lambda: list(client.to_table_as_iterator(plan, None)), + ): + with pytest.raises(TypeError, match="argument of type 'NoneType' is not iterable"): + call() + plain_frame = session.range(2) + plain = pb2.Plan() + plain.root.CopyFrom(plain_frame._plan.plan(client)) + assert client.to_table(plain, None)[0].num_rows == 2 + assert len(client.to_pandas(plain, None)[0]) == 2 + + +def test_client_members_that_pysparks_tests_read_as_pyspark(session): + # As PySpark Connect's SparkConnectClient: _get_operation_statuses is the GetStatus RPC of the + # session, of the operations asked of; a set of _session_id makes the requests carry another + # session ID, so the response of another server-side session closes the session, as + # test_connect_session's test_reset_when_server_and_client_sessionids_mismatch relies on; and + # _closed reopens a session that it closed. + import uuid + + from pyspark.errors.exceptions.connect import SparkConnectException + from pyspark.sql.connect.session import SparkSession as ConnectSession + + remote = os.environ["SPARK_REMOTE"] + other = ConnectSession.builder.remote(remote).create() + client = other.client + other.range(3).collect() + statuses = client._get_operation_statuses() + assert type(statuses).__name__ == "GetStatusResponse" + assert statuses.session_id == other.session_id + unknown = "00000000-0000-4000-8000-000000000000" + (status,) = client._get_operation_statuses([unknown]).operation_statuses + assert status.operation_id == unknown + assert client._builder.endpoint == remote[len("sc://") :].split("/")[0] + + client._closed = True + with pytest.raises(SparkConnectException): + other.range(1).count() + client._closed = False + assert other.range(1).count() == 1 and not other.is_stopped + + client._session_id = str(uuid.uuid4()) + with pytest.raises(SparkConnectException): + other.range(3).collect() + assert other.is_stopped + + def test_client_of_a_connection_string_as_pyspark(session): # As PySpark Connect's SparkConnectClient(connection, ...): a client of a session of its own, # of the plans that it is given, built without a DataFrame, whose executions may be not diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 6e006077..21be7d6a 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -2088,7 +2088,8 @@ def batch(index, num_chunks, data, start_offset=None): def test_tags_are_those_of_each_thread_as_pyspark(mock_connect): # As PySpark's: the tags of addTag are those of the current thread, of a set of getTags, - # which the operations that the thread starts carry; a new thread starts without tags. + # which the operations that the thread starts carry; a new thread starts without tags. The + # set of getTags is the live one, which the later tags change, as PySpark's. import threading spark, service = mock_connect @@ -2101,9 +2102,11 @@ def sent_tags(): seen = {} def child(): - seen["before"] = spark.getTags() + seen["before"] = set(spark.getTags()) + live = spark.getTags() spark.addTag("child") seen["after"] = spark.getTags() + assert seen["after"] is live spark.range(1).collect() seen["sent"] = sent_tags() @@ -4274,6 +4277,9 @@ class Session: def _user_context_extensions(self, is_global): return [] + def _set_user_context_refresh(self, refresh): + pass + client = SparkConnectClient._of_session(Session()) relation = pb2.Relation() relation.sql.query = "q" * 2000 @@ -7523,14 +7529,26 @@ def test_udf_defaults_and_registration(): assert pandas_udf(lambda s: s, rt, functionType=ft).evalType == ft with pytest.raises(PySparkTypeError, match="INVALID_RETURN_TYPE_FOR_PANDAS_UDF"): pandas_udf(lambda s: s, T.IntegerType(), functionType=PandasUDFType.GROUPED_MAP) - # UDFRegistration.register wires to the session's _registerPythonUdf (so the name - # resolves in SQL) and returns the UDF. + # UDFRegistration.register registers through the client of the session, as PySpark's, whose + # register_udf wires to the session's _registerPythonUdf (so the name resolves in SQL), and + # returns the UDF. + from pyspark.sql.connect.client import SparkConnectClient from pyspark.sql.udf import UDFRegistration class _StubSession: def __init__(self): self.calls = [] + @property + def _client(self): + return SparkConnectClient._of_session(self) + + def _user_context_extensions(self, is_global): + return [] + + def _set_user_context_refresh(self, refresh): + pass + def _registerPythonUdf(self, name, returnType, evalType, command, python_ver, deterministic): self.calls.append((name, evalType, isinstance(command, (bytes, bytearray)))) @@ -24995,6 +25013,9 @@ class Session: def _user_context_extensions(self, is_global): return [] + def _set_user_context_refresh(self, refresh): + pass + def _analyze_plan(self, data): Session.sent = pb2.AnalyzePlanRequest.FromString(data) return Session.response.SerializeToString() @@ -25458,7 +25479,8 @@ def test_client_raises_the_argument_errors_of_pyspark(mock_connect): # TypeError of hashlib for a blob that is not bytes-like, which its ArtifactManager hashes # before it sends anything (test_dropin_e2e.py caches bytes-like ones), and register_udf and # register_udtf raise the UNSUPPORTED_OPERATION of pyspark_types_to_proto_types for a return - # type that is neither a DataType nor a DDL string, before they pickle the function. + # type that is neither a DataType nor a DDL string, after the name and before they pickle the + # function (see test_client_registrations_convert_their_arguments_in_the_order_of_pyspark). from pyspark.errors import PySparkValueError spark, service = mock_connect @@ -25493,6 +25515,116 @@ def eval(self): assert (len(service.execute_requests), len(service.analyze_requests)) == sent +def test_client_registrations_convert_their_arguments_in_the_order_of_pyspark(mock_connect): + # As PySpark Connect's register_udf (CommonInlineUserDefinedFunction.to_plan_udf, then + # PythonUDF.to_plan): the name and `deterministic` are set in the fields of the command, the + # return type is converted, the eval type set, and the function pickled last, so each raises + # the error of its protobuf field or of its conversion in that order, and an int as + # `deterministic` is a bool. register_java sets the name, the class name, the return type and + # then `aggregate`, of no return type only; register_udtf reads the __name__ of the handler, + # sets the name and `deterministic`, converts the return type and sets the eval type before + # it pickles the handler. spark.udf.register and registerJavaFunction register through them. + import threading + + import pyspark.sql.connect.proto as pb2 + from pyspark.errors import PySparkPicklingError, PySparkValueError + from pyspark.sql.functions import udf + from pyspark.sql.types import IntegerType + + spark, service = mock_connect + client = spark.client + lock = threading.Lock() + + def unpicklable(x): + return lock + + class Good: + def eval(self): + yield (1,) + + class Bad: + def eval(self): + yield (lock,) + + def raised(call, kind, message): + with pytest.raises(kind) as error: + call() + assert str(error.value).startswith(message), str(error.value) + return error.value + + bad_name = "bad argument type for built-in operation" + not_an_int = "'str' object cannot be interpreted as an integer" + sent = len(service.execute_requests), len(service.analyze_requests) + raised(lambda: client.register_udf(unpicklable, "int", name=5), TypeError, bad_name) + raised(lambda: client.register_udf(unpicklable, None, name=5), TypeError, bad_name) + raised(lambda: client.register_udf(unpicklable, "int", deterministic="x"), TypeError, not_an_int) + raised(lambda: client.register_udf(unpicklable, 5, deterministic="x"), TypeError, not_an_int) + raised(lambda: client.register_udf(unpicklable, IntegerType(), eval_type="x"), TypeError, not_an_int) + raised( + lambda: client.register_udf(unpicklable, IntegerType(), eval_type=True), + TypeError, + "Field spark.connect.PythonUDF.eval_type: Expected an int, got a boolean.", + ) + raised( + lambda: client.register_udf(unpicklable, IntegerType(), eval_type=2**40), + ValueError, + "Value out of range: 1099511627776", + ) + raised( + lambda: client.register_udf(unpicklable, IntegerType(), eval_type=100.0), + TypeError, + "'float' object cannot be interpreted as an integer", + ) + raised(lambda: client.register_java(5, "x.Y", aggregate="x"), TypeError, bad_name) + raised(lambda: client.register_java("j", 5, return_type=5), TypeError, bad_name) + error = raised(lambda: client.register_java("j", "x.Y", return_type=5), PySparkValueError, "") + assert error.getCondition() == "UNSUPPORTED_OPERATION" + raised(lambda: client.register_java("j", "x.Y", aggregate="x"), TypeError, not_an_int) + raised(lambda: client.register_udtf(object(), "x int", 5), AttributeError, "'object' object") + raised(lambda: client.register_udtf(Bad, "x int", 5, deterministic="x"), TypeError, bad_name) + raised(lambda: client.register_udtf(Bad, "x int", "t", deterministic="x"), TypeError, not_an_int) + error = raised(lambda: client.register_udtf(Bad, 5, "t", eval_type=True), PySparkValueError, "") + assert error.getCondition() == "UNSUPPORTED_OPERATION" + raised( + lambda: client.register_udtf(Bad, "x int", "t", eval_type=True), + TypeError, + "Field spark.connect.PythonUDTF.eval_type: Expected an int, got a boolean.", + ) + raised( + lambda: client.register_udtf(Good, "x int", "t", eval_type=2**40), + ValueError, + "Value out of range: 1099511627776", + ) + error = raised( + lambda: client.register_udtf(Bad, "x int", "t", deterministic=1), PySparkPicklingError, "" + ) + assert error.getCondition() == "UDTF_SERIALIZATION_ERROR" + raised(lambda: spark.udf.register(5, udf(unpicklable)), TypeError, bad_name) + raised(lambda: spark.udf.register("\udc80", udf(unpicklable)), UnicodeEncodeError, "'utf-8'") + raised(lambda: spark.udf.registerJavaFunction(5, "x.Y"), TypeError, bad_name) + raised(lambda: spark.udf.registerJavaUDAF("j", 5), TypeError, bad_name) + not_none = "'NoneType' object cannot be interpreted as an integer" + raised(lambda: client.register_udf(unpicklable, 5, deterministic=None), TypeError, not_none) + raised(lambda: client.register_udtf(Bad, 5, "t", deterministic=None), TypeError, not_none) + raised(lambda: client.register_java("j", "x.Y", aggregate=None), TypeError, not_none) + assert (len(service.execute_requests), len(service.analyze_requests)) == sent + + def command(): + return pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]).plan.command + + client.register_udf(lambda x: x, IntegerType(), name="d1", deterministic=1) + assert command().register_function.deterministic is True + client.register_udf(lambda x: x, IntegerType(), name="d0", deterministic=0) + assert command().register_function.deterministic is False + client.register_java("j1", "x.Y", aggregate=1) + assert command().register_function.java_udf.aggregate is True + client.register_java("j2", "x.Y", return_type="int", aggregate="ignored") + java = command().register_function.java_udf + assert java.aggregate is False and java.output_type.unparsed.data_type_string == "int" + client.register_udtf(Good, "x int", "t1", deterministic=1) + assert command().register_table_function.deterministic is True + + def test_client_registers_a_udtf_in_the_order_of_pyspark(mock_connect): # As PySpark Connect's register_udtf: its PythonUDTF reads the name of the handler class when # it is made, and its udtf_plan sets the name in the string field of its command and then @@ -25608,6 +25740,345 @@ def name(cls): client.copy_from_local_to_fs(str(path), "/dest/a.py") +def test_the_attributes_of_the_clients_of_a_session_are_its_own_as_pyspark(mock_connect): + # As PySpark Connect's session, of one client: the clients of a session that SparkSession.client + # gives share its attributes, also once the program dropped them, so an attribute set on one, + # the lock and the list of the global user context extensions are those of the next. A list + # read before an extension is added is the one that it is added to, and the one that the + # requests carry; a list that the attribute is no longer set to is not. + import gc + import threading + + import pyspark.sql.connect.proto as pb2 + from pyspark.sql.connect.proto._rust import Any + + spark, service = mock_connect + + def sent(): + request = pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]) + return [extension.type_url for extension in request.user_context.extensions] + + def run(): + spark.range(1).collect() + return sent() + + client = spark.client + client.attribute = 1 + lock = client.global_user_context_extensions_lock + assert isinstance(lock, type(threading.Lock())) + held = client.global_user_context_extensions + del client + gc.collect() + client = spark.client + assert client.attribute == 1 and client.global_user_context_extensions_lock is lock + assert client.global_user_context_extensions is held + first = client.add_global_user_context_extension(Any(type_url="type.googleapis.com/e1")) + held.append(("global_2", Any(type_url="type.googleapis.com/e2"))) + assert [e.type_url for _, e in spark.client.global_user_context_extensions] == [ + "type.googleapis.com/e1", + "type.googleapis.com/e2", + ] + assert run() == ["type.googleapis.com/e1", "type.googleapis.com/e2"] + # Rebinding the attribute detaches the list that was held, as in PySpark. + client.remove_user_context_extension(first) + assert client.global_user_context_extensions is not held + held.append(("global_3", Any(type_url="type.googleapis.com/e3"))) + assert run() == ["type.googleapis.com/e2"] + # A change of an extension in place is sent, as PySpark serializes them for each request. + client.global_user_context_extensions[0][1].type_url = "type.googleapis.com/changed" + assert run() == ["type.googleapis.com/changed"] + client.clear_user_context_extensions() + assert run() == [] + del client.attribute + + +def test_the_thread_local_of_a_client_is_a_threading_local_of_live_tags_and_extensions_as_pyspark( + mock_connect, +): + # As PySpark Connect's thread_local, a threading.local: its tags and user_context_extensions + # exist once a thread uses them, are the set and the list that the requests of the thread + # carry, also as they are changed or set, and are the thread's own, as are its other + # attributes. The tags of SparkSession.addTag are those of the client, and getTags gives their + # set. A thread of inheritable_thread_target shares the copies of the attributes of the thread + # that decorated its function. Deleting the attributes leaves the thread with none. + import threading + + import pyspark.sql.connect.proto as pb2 + from pyspark.sql.connect.proto._rust import Any + from pyspark.util import inheritable_thread_target + + spark, service = mock_connect + client = spark.client + local = client.thread_local + + def run(): + spark.range(1).collect() + request = pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]) + return sorted(request.tags), [e.type_url for e in request.user_context.extensions] + + assert isinstance(local, threading.local) + assert not hasattr(local, "tags") and not hasattr(local, "user_context_extensions") + first = Any(type_url="type.googleapis.com/t1") + client.add_threadlocal_user_context_extension(first) + extensions = local.user_context_extensions + assert extensions[0][1] is first and local.user_context_extensions is extensions + extensions.append(("threadlocal_x", Any(type_url="type.googleapis.com/t2"))) + assert run() == ([], ["type.googleapis.com/t1", "type.googleapis.com/t2"]) + local.user_context_extensions = [("threadlocal_y", Any(type_url="type.googleapis.com/t3"))] + assert run() == ([], ["type.googleapis.com/t3"]) + local.user_context_extensions[0][1].type_url = "type.googleapis.com/t4" + assert run() == ([], ["type.googleapis.com/t4"]) + + spark.addTag("a") + tags = local.tags + assert tags is client.get_tags() is spark.getTags() and tags == {"a"} + assert repr(tags) == "{'a'}" + tags.add("b") + assert run() == (["a", "b"], ["type.googleapis.com/t4"]) + local.tags = {"z"} + assert run() == (["z"], ["type.googleapis.com/t4"]) and spark.getTags() == {"z"} + # A tag that the field of the tags of a request cannot take is refused, and the set is kept. + with pytest.raises(TypeError, match="bad argument type for built-in operation"): + local.tags.add(("not", "a", "str")) + assert local.tags == {"z"} and run()[0] == ["z"] + + seen = {} + + def other(): + seen["tags"] = hasattr(local, "tags") + seen["extensions"] = hasattr(local, "user_context_extensions") + local.other = 1 + seen["run"] = run() + + thread = threading.Thread(target=other) + thread.start() + thread.join() + assert seen == {"tags": False, "extensions": False, "run": ([], [])} + assert not hasattr(local, "other") + assert sorted(vars(local)) == ["tags", "user_context_extensions"] + + @inheritable_thread_target(spark) + def inherited(): + local.tags.add(threading.current_thread().name) + return sorted(spark.getTags()), run() + + results = {} + threads = [ + threading.Thread(target=lambda n=n: results.setdefault(n, inherited()), name=f"t{n}") + for n in range(2) + ] + for thread in threads: + thread.start() + thread.join() + # The threads share the copy of the set of tags, as PySpark's, and not the parent's set. + assert results[1][0] == ["t0", "t1", "z"] + assert results[1][1] == (["t0", "t1", "z"], ["type.googleapis.com/t4"]) + assert spark.getTags() == {"z"} + + del local.tags + del local.user_context_extensions + assert not hasattr(local, "tags") and run() == ([], []) + spark.clearTags() + assert spark.getTags() == set() and run() == ([], []) + + +def test_client_members_that_pysparks_tests_read_are_pysparks(mock_connect): + # As PySpark Connect's SparkConnectClient, whose tests and libraries read these members: + # _builder is the DefaultChannelBuilder of the connection; _session_id the session ID of the + # requests, which a set changes for them while the session's session_id stays; _user_id the + # user of the requests; _use_reattachable_execute whether the executions are reattachable, + # of enable_reattachable_execute and of a set; and _closed closes the session, and opens it + # again when it is set to False. + import uuid + + import pyspark.sql.connect.proto as pb2 + from pyspark.errors.exceptions.connect import SparkConnectException + from pyspark.sql.connect.client import DefaultChannelBuilder + + spark, service = mock_connect + client = spark.client + assert isinstance(client._builder, DefaultChannelBuilder) + assert client._builder.endpoint == service.remote[len("sc://"):] + assert client._builder.userAgent.startswith("_SPARK_CONNECT_PYTHON spark/") + assert client.host == client._builder.host + assert client._session_id == spark.session_id + assert client._user_id == os.environ.get("SPARK_USER", os.environ.get("USER")) + assert client._use_reattachable_execute is True + client._use_reattachable_execute = False + assert client._use_reattachable_execute is False + client.enable_reattachable_execute() + assert client._use_reattachable_execute is True + + original = spark.session_id + changed = str(uuid.uuid4()) + client._session_id = changed + spark.range(1).collect() + request = pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]) + assert request.session_id == changed and spark.session_id == original + assert client._session_id == changed + with pytest.raises(TypeError, match="bad argument type for built-in operation"): + client._session_id = 5 + client._session_id = original + + assert client._closed is False + client._closed = True + with pytest.raises(SparkConnectException) as error: + spark.range(1).collect() + assert error.value.getCondition() == "NO_ACTIVE_SESSION" + assert client._closed is True and client.is_closed and spark.is_stopped + client._closed = False + assert client._closed is False and not client.is_closed and not spark.is_stopped + spark.range(1).collect() + + +def test_client_get_operation_statuses_sends_pysparks_get_status_request(mock_connect): + # As PySpark Connect's SparkConnectClient._get_operation_statuses (client/core.py:2183-2237): + # a GetStatusRequest of the session ID, of the client type and of the user of the client's + # requests, of an operation_status that is set even of no operation ID, which asks of all of + # them, of the operation IDs, and of the extensions of each level; the GetStatusResponse is + # what it gives. + import pyspark.sql.connect.proto as pb2 + from pyspark.sql.connect.proto._rust import Any + + spark, service = mock_connect + client = spark.client + spark.range(1).collect() + execute = pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]) + every = client._get_operation_statuses() + assert type(every).__name__ == "GetStatusResponse" + assert every.session_id == spark.session_id and len(every.operation_statuses) == 0 + asked = client._get_operation_statuses( + ["op-1", "op-2"], + [Any(type_url="type.googleapis.com/operation")], + [Any(type_url="type.googleapis.com/request")], + ) + unknown = pb2.GetStatusResponse.OperationStatus.OPERATION_STATE_UNKNOWN + assert [(status.operation_id, status.state) for status in asked.operation_statuses] == [ + ("op-1", unknown), + ("op-2", unknown), + ] + first, second = (pb2.GetStatusRequest.FromString(r) for r in service.get_status_requests) + for request in (first, second): + assert request.session_id == spark.session_id + assert request.client_type == execute.client_type + assert request.user_context.user_id == execute.user_context.user_id == client._user_id + assert request.HasField("operation_status") + assert list(first.operation_status.operation_ids) == [] + assert len(first.operation_status.extensions) == 0 and len(first.extensions) == 0 + assert list(second.operation_status.operation_ids) == ["op-1", "op-2"] + assert [e.type_url for e in second.operation_status.extensions] == [ + "type.googleapis.com/operation" + ] + assert [e.type_url for e in second.extensions] == ["type.googleapis.com/request"] + + +def test_the_tags_and_extensions_of_a_thread_copy_and_pickle_as_pysparks_set_and_list( + mock_connect, +): + # As PySpark Connect's thread_local.tags and user_context_extensions, a plain set and a plain + # list: their copies and deep copies are a set and a list of the same members, and their + # pickles a plain set and a plain list. + import copy + import pickle + + from pyspark.sql.connect.proto._rust import Any + + spark, _ = mock_connect + client = spark.client + local = client.thread_local + client.add_tag("a") + client.add_threadlocal_user_context_extension(Any(type_url="type.googleapis.com/t")) + for value, kind in ((local.tags, set), (local.user_context_extensions, list)): + for copied in (copy.copy(value), copy.deepcopy(value)): + assert type(copied) is kind and copied == value + pickled = pickle.loads(pickle.dumps(local.tags)) + assert type(pickled) is set and pickled == {"a"} + local.user_context_extensions = [] + pickled = pickle.loads(pickle.dumps(local.user_context_extensions)) + assert type(pickled) is list and pickled == [] + + +def test_the_tags_and_extensions_of_a_thread_that_has_none_are_pysparks(mock_connect): + # As PySpark Connect's client in a thread whose thread_local has neither: the removal of a + # thread-local extension and the clear of the extensions leave the thread without a list of + # them, while the removal of a tag and get_tags give it an empty set of tags. + import threading + + spark, _ = mock_connect + client = spark.client + seen = {} + + def other(): + local = client.thread_local + client.remove_user_context_extension("threadlocal_missing") + client.clear_user_context_extensions() + seen["extensions"] = hasattr(local, "user_context_extensions") + client.remove_tag("missing") + seen["removed"] = set(local.tags) + del local.tags + seen["got"] = set(client.get_tags()) + + thread = threading.Thread(target=other) + thread.start() + thread.join() + assert seen == {"extensions": False, "removed": set(), "got": set()} + + +def test_the_other_attributes_of_a_thread_local_are_copied_and_deleted_as_pysparks(mock_connect): + # As PySpark Connect's thread_local, each attribute of which inheritable_thread_target copies + # with copy.deepcopy, when it decorates a function, to the threads that run the function, and + # an attribute of which a thread deletes is gone. + import threading + + from pyspark.util import inheritable_thread_target + + spark, _ = mock_connect + local = spark.client.thread_local + local.marker = ["parent"] + + @inheritable_thread_target(spark) + def child(): + return local.marker + + seen = [] + thread = threading.Thread(target=lambda: seen.append(child())) + thread.start() + thread.join() + assert seen == [["parent"]] and seen[0] is not local.marker + del local.marker + assert not hasattr(local, "marker") + + +def test_the_thread_local_of_a_client_outlives_its_session_as_pysparks(mock_connect): + # As PySpark Connect's thread_local, a threading.local, and its global list of extensions, + # which a caller may keep: once the session is gone, the tags and the extensions that a thread + # sets and deletes, and a change of the list, are those of plain ones. + import gc + import weakref + + from pyspark.sql.connect.proto._rust import Any + + spark, _ = mock_connect + other = spark.newSession() + local = other.client.thread_local + extensions = other.client.global_user_context_extensions + local.tags = {"a"} + gone = weakref.ref(other) + other.stop() + del other + gc.collect() + assert gone() is None + local.tags = {"b"} + assert type(local.tags) is set and local.tags == {"b"} + del local.tags + assert not hasattr(local, "tags") + local.user_context_extensions = [("threadlocal_a", Any(type_url="type.googleapis.com/a"))] + assert type(local.user_context_extensions) is list + del local.user_context_extensions + extensions.append(("global_a", Any(type_url="type.googleapis.com/a"))) + assert [extension_id for extension_id, _ in extensions] == ["global_a"] + + def test_client_tags_interrupts_and_sessions_are_those_of_its_session_as_pyspark(mock_connect): # As PySpark Connect's SparkConnectClient, whose tags, interrupts and sessions are those of # SparkSession.addTag, interruptOperation, newSession and cloneSession. @@ -30059,6 +30530,78 @@ def test_copy_from_local_to_fs_requires_an_absolute_destination(mock_connect, tm spark.copyFromLocalToFs(str(local), "relative/out.txt") +def test_copy_from_local_to_fs_raises_pysparks_errors_of_a_destination(mock_connect, tmp_path): + # As PySpark Connect's copyFromLocalToFs, which parses the destination with urlparse before + # its client's copy_from_local_to_fs: of an object that is not a str, the AttributeError of + # urlparse, or of bytes, the TypeError of the Path of the destination. The client's, which + # parses nothing, raises the TypeError of the name of the artifact, the concatenation of a + # prefix and the destination, for a Path. + import pathlib + import re + + spark, _ = mock_connect + local = tmp_path / "local.txt" + local.write_text("x") + destination = pathlib.Path("/dest/out.txt") + with pytest.raises(AttributeError, match="'PosixPath' object has no attribute 'decode'"): + spark.copyFromLocalToFs(str(local), destination) + with pytest.raises(AttributeError, match="'int' object has no attribute 'decode'"): + spark.copyFromLocalToFs(str(local), 5) + with pytest.raises(TypeError, match="argument should be a str"): + spark.copyFromLocalToFs(str(local), b"/dest/out.txt") + with pytest.raises( + TypeError, match=re.escape('can only concatenate str (not "PosixPath") to str') + ): + spark.client.copy_from_local_to_fs(str(local), destination) + with pytest.raises(TypeError, match="argument should be a str"): + spark.client.copy_from_local_to_fs(str(local), b"/dest/out.txt") + + +def test_client_executions_read_their_observations_as_pyspark(mock_connect): + # As PySpark Connect's to_table and to_table_as_iterator, which look the names of observed + # metrics up in `observations` when they arrive: None and an empty list execute a plan whose + # results have none, an empty list one whose results have some, and None and an int raise + # the TypeError of `in` when they arrive. The Observations of other names are not set. (The + # e2e test covers to_pandas, whose configurations the mock server does not have.) + import pyspark.sql.connect.proto as pb2 + from pyspark.sql import Observation + from pyspark.sql import functions as F + + spark, service = mock_connect + client = spark.client + observation = Observation("o") + df = spark.range(3).observe(observation, F.count(F.lit(1)).alias("c")) + batch, metrics = _count_batch_and_observed_metrics(service, df) + plan = pb2.Plan() + plan.root.CopyFrom(df._plan.plan(client)) + service.execute_payload = batch + assert client.to_table(plan, None)[0].num_rows == 1 + assert len(list(client.to_table_as_iterator(plan, None))) >= 1 + service.execute_payload = batch + metrics("o", varint_field(7, 3), "c") + assert client.to_table(plan, [])[0].num_rows == 1 + for observations, kind in ((None, "NoneType"), (5, "int")): + with pytest.raises(TypeError, match=f"argument of type '{kind}' is not iterable"): + client.to_table(plan, observations) + with pytest.raises(TypeError, match="argument of type 'NoneType' is not iterable"): + list(client.to_table_as_iterator(plan, None)) + assert observation.get == {} + client.to_table(plan, df._plan.observations) + assert observation.get == {"c": 3} + # The observed metrics of the accumulators of the Python workers are not looked up, as + # PySpark's client reads the updates of its UDF profiler of them instead; those of another + # accumulator than the profiler's are ignored. + from pyspark.accumulators import specialAccumulatorSer + + # ExecutePlanResponse.observed_metrics=6 -> ObservedMetrics name=1, values=2 (Expression + # .Literal binary=2). + update = message_field(2, message_field(2, specialAccumulatorSer.dumps((999, None)))) + service.execute_payload = batch + message_field( + 6, message_field(1, "__python_accumulator__") + update + ) + assert client.to_table(plan, None)[0].num_rows == 1 + assert len(list(client.to_table_as_iterator(plan, None))) >= 1 + + def test_requests_of_a_stopped_session_raise_no_active_session_as_pyspark(): # As PySpark's closed client: stop() releases the session and closes the client, so each # request of the stopped session, and of its DataFrames, raises NO_ACTIVE_SESSION. diff --git a/python/tests/test_ml_offline.py b/python/tests/test_ml_offline.py index 4cf09781..f905ccd9 100644 --- a/python/tests/test_ml_offline.py +++ b/python/tests/test_ml_offline.py @@ -2942,17 +2942,18 @@ def packed(graph_id): analysis = pb2.AnalyzePlanRequest.FromString(service.analyze_requests[-1]) assert list(analysis.user_context.extensions) == [packed("global"), packed("local")] - # Another thread has the global extensions only. + # Another thread has the global extensions only, and no list of its own until it adds one: + # PySpark's thread_local, a threading.local, has no user_context_extensions there. seen = [] def other(): - seen.append(client.thread_local.user_context_extensions) + seen.append(hasattr(client.thread_local, "user_context_extensions")) client.execute_command(command) thread = threading.Thread(target=other) thread.start() thread.join() - assert seen == [[]] + assert seen == [False] request = _last_request_but_deletes(service) assert list(request.user_context.extensions) == [packed("global")]