diff --git a/AGENTS.md b/AGENTS.md index 953120eb..2c2a1be0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -145,13 +145,14 @@ those modules (of protoc 3.20 and later, which build their classes with it) get classes, whose fields take ours as they are: their files are added to the runtime's pool at run time (`pbmsg::pool::register`; see `docs/compatibility.md`). `DataFrame(plan, session)`, `DataFrame._plan` and `Column._expr` are the other half of -that bridge: a `LogicalPlan` of Python sends its `plan(session)` relation as it is, of the shallow -placeholder that `DataFrame._plan.plan(session)` gives of the relation of a DataFrame, which the -core replaces wherever a plan, a command or an analysis built in Python holds a relation: every -input of every relation (`plan::relation_inputs_of`), the relation of each command that holds one, -the inputs of the plans of `pyspark.ml`, and the messages of the extensions of a plugin's relations -and commands (`plan::resolve_relation`, `resolve_command`, `resolve_analysis`), but not in an -expression's extension: "Plans of a DataFrame's relation" in `dev/parity/known-differences.md`. +that bridge: a `LogicalPlan` of Python sends its `plan(session)` relation as it is, of the +placeholder that `DataFrame._plan.plan(session)` gives, of one level, which holds the relation of +the DataFrame encoded, built at the call, and which the core replaces wherever a plan, a command or +an analysis built in Python holds a relation: every input of every relation +(`plan::relation_inputs_of`), the relation of each command that holds one, the inputs of the plans +of `pyspark.ml`, and the messages of the extensions of a plugin's relations and commands +(`plan::resolve_relation`, `resolve_command`, `resolve_analysis`), but not in an expression's +extension: "Plans of a DataFrame's relation" in `dev/parity/known-differences.md`. ## 3. Close API gaps (three layers) diff --git a/crates/pyspark-rs/src/dataframe.rs b/crates/pyspark-rs/src/dataframe.rs index e3e4fc0b..9bc48df9 100644 --- a/crates/pyspark-rs/src/dataframe.rs +++ b/crates/pyspark-rs/src/dataframe.rs @@ -967,11 +967,19 @@ impl PyDataFrame { } /// The bytes of the placeholder of the relation of this DataFrame, which the plan of `_plan` - /// sends in its place: shallow, whatever the depth of the DataFrame, and replaced by the - /// core with that relation when the plan reaches it. - fn _relation_placeholder<'py>(&self, py: Python<'py>) -> Bound<'py, pyo3::types::PyBytes> { + /// sends in its place: one extension that holds the relation encoded, which the core builds + /// now, as PySpark's `plan(session)` builds the relation, with its errors, and replaces the + /// placeholder with when the plan reaches it, also after this DataFrame is gone. + fn _relation_placeholder<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { use prost::Message; - pyo3::types::PyBytes::new(py, &self.dataframe.placeholder_relation().encode_to_vec()) + let placeholder = self + .dataframe + .standalone_placeholder_relation() + .to_pyerr()?; + Ok(pyo3::types::PyBytes::new(py, &placeholder.encode_to_vec())) } /// A DataFrame pickles, as PySpark Connect's, of its session, which gives back the active diff --git a/crates/spark-connect/Cargo.toml b/crates/spark-connect/Cargo.toml index 8881f4f5..ff1bd44a 100644 --- a/crates/spark-connect/Cargo.toml +++ b/crates/spark-connect/Cargo.toml @@ -42,6 +42,9 @@ base64 = "0.22" futures.workspace = true # The gRPC servers of the tests. tonic = { workspace = true, features = ["server", "router"] } +# The tests decode the relations of the placeholders of deep DataFrames, as the drop-in does, +# whose `pyspark-rs` lifts prost's limit of 100 nested messages: so do the tests. +prost = { workspace = true, features = ["no-recursion-limit"] } [features] # Run Rust UDFs on Spark via WebAssembly (the `wasm_udf` module). Off by diff --git a/crates/spark-connect/src/dataframe.rs b/crates/spark-connect/src/dataframe.rs index 680b6afe..f1307cfb 100644 --- a/crates/spark-connect/src/dataframe.rs +++ b/crates/spark-connect/src/dataframe.rs @@ -1771,14 +1771,24 @@ impl DataFrame { /// A relation that stands for the relation of this DataFrame, `to_relation`, in a relation /// built outside the core, such as a plan of `pyspark.ml`, which the core replaces by that - /// relation when the relation reaches it (`from_relation`, `execute_command_proto`). It - /// is shallow, so it is cheap to build and to decode whatever the depth of this DataFrame. - /// Not a stable API. + /// relation when the relation reaches it (`from_relation`, `execute_command_proto`), while + /// this DataFrame is alive. It is shallow, so it is cheap to build and to decode whatever the + /// depth of this DataFrame. Not a stable API. #[doc(hidden)] pub fn placeholder_relation(&self) -> proto::Relation { crate::plan::placeholder_relation(&self.plan, self.session.plan_ids_shared()) } + /// As [`DataFrame::placeholder_relation`], a placeholder that holds the relation of this + /// DataFrame, as it is built now, encoded in the value of its extension, with the errors of + /// the build: it stands for that relation after this DataFrame is gone, and its encoding + /// too, as the relation that PySpark's `plan(session)` gives does, and it is of one level + /// for a decoder of protobuf outside the core. Not a stable API. + #[doc(hidden)] + pub fn standalone_placeholder_relation(&self) -> Result { + crate::plan::standalone_placeholder_relation(&self.plan, self.session.plan_ids_shared()) + } + /// Get the schema of this DataFrame. /// /// The first successful lookup is cached and shared with exact clones. diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs index 46116fcf..cb83bab1 100644 --- a/crates/spark-connect/src/plan.rs +++ b/crates/spark-connect/src/plan.rs @@ -781,15 +781,23 @@ impl PlanIds { /// The `Any` type of a placeholder: the relation that stands for the relation of a DataFrame, in a /// relation that is built outside the core, such as a plan of `pyspark.ml`, which the Python -/// client builds over the placeholder of each DataFrame it takes as input. The value of a -/// placeholder is its marker. The core replaces it by the relation of its DataFrame when the -/// relation or the command that holds it reaches the core, so the nesting of a deep DataFrame is -/// never decoded: a protobuf decoder limits the depth of what it decodes. +/// client builds over the placeholder of each DataFrame it takes as input. The core builds the +/// relation of a DataFrame, and the Python client has no messages of it, so a placeholder is of +/// one level, an extension, whose value is the marker of its DataFrame, and, of the placeholder +/// of `DataFrame._plan.plan(session)`, the relation encoded too (see +/// [`standalone_placeholder_relation`]): the Python client never decodes the nesting of a deep +/// DataFrame, which a protobuf decoder of Python limits. The core replaces the placeholder by +/// the relation when the relation or the command that holds it reaches the core: by the relation +/// that it holds, so a standalone placeholder stands for the relation as it was built after its +/// DataFrame is gone too, as the relation of PySpark's `plan(session)` does, or else by the +/// relation of its DataFrame, which must be alive (see [`placeholder_relation`]). const PLACEHOLDER_TYPE_URL: &str = "type.googleapis.com/spark.connect.rust.DataFramePlaceholder"; /// The placeholders of all the sessions, by the addresses of their plan node and of their -/// session's plan IDs, each with its marker. Both are held weakly, as `PlanIds` holds its nodes, -/// so an entry goes with its DataFrame, and its address is not reused while the entry exists. +/// session's plan IDs, each with its marker, by which a placeholder of a marker alone finds its +/// DataFrame, and [`child_observations`] the Observations of the DataFrame of any placeholder. +/// Both are held weakly, as `PlanIds` holds its nodes, so an entry goes with its DataFrame, and +/// its address is not reused while the entry exists. #[derive(Default)] struct Placeholders { entries: HashMap<(usize, usize), Placeholder>, @@ -805,8 +813,8 @@ struct Placeholder { static PLACEHOLDERS: LazyLock> = LazyLock::new(|| Mutex::new(Placeholders::default())); -/// The placeholder of the relation of `node`, a DataFrame of the session of `ids`: the same -/// marker each time the DataFrame gives it. +/// The marker of the placeholder of the relation of `node`, a DataFrame of the session of `ids`: +/// the same marker each time the DataFrame gives it. fn placeholder_marker(node: &Arc, ids: &Arc) -> i64 { const MIN_PRUNE_AT: usize = 1024; let mut placeholders = PLACEHOLDERS.lock().unwrap_or_else(|e| e.into_inner()); @@ -844,19 +852,79 @@ fn placeholder_target(marker: i64) -> Option<(Arc, Arc)> { .and_then(|entry| Some((entry.node.upgrade()?, entry.ids.upgrade()?))) } -/// The placeholder of the relation of `node`, a DataFrame of the session of `ids`, as the -/// relation that the Python client sends in its place. +/// The placeholder of the relation of `node`, a DataFrame of the session of `ids`, of its marker +/// alone: the relation that a plan that the core builds over the DataFrame, and resolves while +/// the DataFrame is alive, holds in its place, as the binding's ML transforms of a DataFrame of +/// another session do. pub(crate) fn placeholder_relation(node: &Arc, ids: &Arc) -> proto::Relation { + placeholder_of_value(placeholder_marker(node, ids).to_le_bytes().to_vec()) +} + +/// The placeholder of the relation of `node`, a DataFrame of the session of `ids`, that holds the +/// relation, as `DataFrame._plan.plan(session)` gives it to the Python client: the extension's +/// value is the marker of the DataFrame, the levels of its relation and the encoding of the +/// relation, which is built now, as PySpark's `plan(session)` builds it, with its errors, and +/// pickles the functions of its UDFs then. So it stands for the relation after the DataFrame is +/// gone, its encoding too, while it holds no object of the DataFrame. +pub(crate) fn standalone_placeholder_relation( + node: &Arc, + ids: &Arc, +) -> Result { + use prost::Message; + let (relation, levels) = relation_with_ids_and_levels(node, ids)?; let marker = placeholder_marker(node, ids); + // The relation is encoded, and dropped, on a stack of its levels, as both recurse. + let stack = raw_relation_stack(levels); + let value = stacker::maybe_grow(stack, stack, move || { + let mut value = Vec::with_capacity(16 + relation.encoded_len()); + value.extend_from_slice(&marker.to_le_bytes()); + value.extend_from_slice(&(levels as u64).to_le_bytes()); + relation.encode(&mut value).map(|()| value) + }) + .map_err(|error| SparkError::value_msg(error.to_string()))?; + Ok(placeholder_of_value(value)) +} + +/// The placeholder relation of the extension value `value`. +fn placeholder_of_value(value: Vec) -> proto::Relation { proto::Relation { common: None, rel_type: Some(proto::relation::RelType::Extension(prost_types::Any { type_url: PLACEHOLDER_TYPE_URL.to_string(), - value: marker.to_le_bytes().to_vec(), + value, })), } } +/// The relation that the placeholder of the extension value `value` stands for, and its levels: +/// that which it holds, decoded, or else that of its DataFrame, which must be alive. The decoder +/// of the drop-in has no limit of depth (`pyspark-rs` lifts prost's), and the core's own limits +/// a relation of more than 100 levels, which only the Python client makes standalone +/// placeholders of. +fn placeholder_contents(value: &[u8]) -> Result<(proto::Relation, usize)> { + use prost::Message; + let marker = <[u8; 8]>::try_from(value.get(..8).unwrap_or_default()) + .map(i64::from_le_bytes) + .map_err(|_| SparkError::value_msg("a placeholder of no marker"))?; + if value.len() == 8 { + let (node, ids) = placeholder_target(marker).ok_or_else(|| { + SparkError::value_msg("a placeholder of a DataFrame that no longer exists") + })?; + return relation_with_ids_and_levels(&node, &ids); + } + let malformed = || SparkError::value_msg("a placeholder of no relation"); + let levels = value + .get(8..16) + .and_then(|levels| <[u8; 8]>::try_from(levels).ok()) + .and_then(|levels| usize::try_from(u64::from_le_bytes(levels)).ok()) + .ok_or_else(malformed)?; + let encoded = &value[16..]; + let stack = raw_relation_stack(levels); + let relation = stacker::maybe_grow(stack, stack, || proto::Relation::decode(encoded)) + .map_err(|_| malformed())?; + Ok((relation, levels)) +} + /// The Observations of the DataFrames whose placeholders `relation`, a relation built in Python, holds /// under it, as PySpark's `LogicalPlan.observations` collects them: those of the child of a /// relation (its input; the input of a transform or the dataset of a model summary of @@ -875,8 +943,10 @@ pub(crate) fn child_observations( }; let (left, right) = match &relation.rel_type { Some(RelType::Extension(any)) if any.type_url == PLACEHOLDER_TYPE_URL => { - let target = <[u8; 8]>::try_from(any.value.as_slice()) - .ok() + let target = any + .value + .get(..8) + .and_then(|marker| <[u8; 8]>::try_from(marker).ok()) .and_then(|marker| placeholder_target(i64::from_le_bytes(marker))); return target .map(|(node, _)| node.observations()) @@ -1033,16 +1103,7 @@ pub(crate) fn resolve_relation_levels(relation: &mut proto::Relation) -> Result< fn resolve_relation_at(relation: &mut proto::Relation, level: usize) -> Result { if let Some(proto::relation::RelType::Extension(any)) = &relation.rel_type { if any.type_url == PLACEHOLDER_TYPE_URL { - let marker = i64::from_le_bytes( - any.value - .as_slice() - .try_into() - .map_err(|_| SparkError::value_msg("a placeholder of no marker"))?, - ); - let (node, ids) = placeholder_target(marker).ok_or_else(|| { - SparkError::value_msg("a placeholder of a DataFrame that no longer exists") - })?; - let (resolved, levels) = relation_with_ids_and_levels(&node, &ids)?; + let (resolved, levels) = placeholder_contents(&any.value)?; *relation = resolved; // The relation of the DataFrame takes the place of the placeholder, at its level. return Ok(level - 1 + levels); @@ -1109,17 +1170,21 @@ fn is_placeholder(relation: &proto::Relation) -> bool { ) } -/// The length of the encoding of a placeholder, whose marker is of 8 bytes. -static PLACEHOLDER_LEN: LazyLock = LazyLock::new(|| { +/// The start of the encoding of a placeholder as `placeholder_relation` and +/// `standalone_placeholder_relation` give it, of no `common`: the key of the extension of a +/// relation, its first field. +static PLACEHOLDER_START: LazyLock> = LazyLock::new(|| { use prost::Message; - proto::Relation { + // A relation of an empty extension encodes the key of the extension and the length 0. + let mut start = proto::Relation { common: None, - rel_type: Some(proto::relation::RelType::Extension(prost_types::Any { - type_url: PLACEHOLDER_TYPE_URL.to_string(), - value: vec![0; 8], - })), + rel_type: Some(proto::relation::RelType::Extension( + prost_types::Any::default(), + )), } - .encoded_len() + .encode_to_vec(); + start.pop(); + start }); /// `encoded`, the encoding of a message of a type that the core need not know, with the @@ -1138,11 +1203,11 @@ fn resolve_encoded(encoded: &[u8]) -> Result>> { if !encoded.windows(url.len()).any(|window| window == url) { return Ok(None); } - if encoded.len() == *PLACEHOLDER_LEN { + if encoded.starts_with(&PLACEHOLDER_START) { if let Ok(mut relation) = proto::Relation::decode(encoded) { if is_placeholder(&relation) && relation.encode_to_vec() == encoded { let stack = raw_relation_stack(resolve_relation_levels(&mut relation)?); - return Ok(Some(stacker::maybe_grow(stack, stack, || { + return Ok(Some(stacker::maybe_grow(stack, stack, move || { relation.encode_to_vec() }))); } @@ -4396,7 +4461,7 @@ mod placeholder_tests { /// The placeholder of `node`, of the session of `ids`, as the relation that the plans of /// `pyspark.ml` send in its place. fn placeholder(node: &Arc, ids: &Arc) -> proto::Relation { - placeholder_relation(node, ids) + standalone_placeholder_relation(node, ids).unwrap() } /// The `ml_relation` of a transform of `input`, as `pyspark.ml.connect.proto` builds it. @@ -4419,7 +4484,7 @@ mod placeholder_tests { } #[test] - fn a_placeholder_is_shallow_and_stands_for_the_relation_of_its_node() { + fn a_placeholder_is_of_one_level_and_stands_for_the_relation_of_its_node() { // The plan is converted recursively, so a plan of 200 levels takes more than the 2 MiB // stack of a test thread in a debug build, whose frames are much larger than those of a // release build. @@ -4433,11 +4498,25 @@ mod placeholder_tests { }); } let relation = relation_with_ids(&node, &ids).unwrap(); - let shallow = placeholder(&node, &ids); - assert!(shallow.encoded_len() < 128, "{}", shallow.encoded_len()); - let mut resolved = transform_of(shallow.clone()); + let placeholder = placeholder(&node, &ids); + // An extension of the encoded relation: a decoder of Python reads one level of it. + let Some(proto::relation::RelType::Extension(any)) = &placeholder.rel_type else { + panic!("not an extension: {placeholder:?}"); + }; + assert!(any.value.ends_with(&relation.encode_to_vec())); + let mut resolved = transform_of(placeholder.clone()); resolve_relation(&mut resolved).unwrap(); - assert_eq!(resolved, transform_of(relation)); + assert_eq!(resolved, transform_of(relation.clone())); + // As PySpark's relation of `plan(session)`, it stands for the relation after its + // DataFrame is gone, and so do its bytes, decoded again; it keeps nothing alive. + let (weak_node, weak_ids) = (Arc::downgrade(&node), Arc::downgrade(&ids)); + std::mem::drop((node, ids)); + assert!(weak_node.upgrade().is_none() && weak_ids.upgrade().is_none()); + let decoded = proto::Relation::decode(placeholder.encode_to_vec().as_slice()).unwrap(); + for mut kept in [transform_of(placeholder), transform_of(decoded)] { + resolve_relation(&mut kept).unwrap(); + assert_eq!(kept, transform_of(relation.clone())); + } }; std::thread::Builder::new() .stack_size(64 * 1024 * 1024) @@ -4447,6 +4526,27 @@ mod placeholder_tests { .unwrap_or_else(|panic| std::panic::resume_unwind(panic)); } + #[test] + fn a_placeholder_of_a_marker_alone_stands_for_the_relation_of_its_node_while_it_lives() { + // As the core's own plans over a DataFrame hold it, which they resolve at once. + let ids = Arc::new(PlanIds::new()); + let node = Arc::new(range(0, 3, 1)); + let relation = relation_with_ids(&node, &ids).unwrap(); + let placeholder = placeholder_relation(&node, &ids); + assert!( + placeholder.encoded_len() < 128, + "{}", + placeholder.encoded_len() + ); + let mut resolved = transform_of(placeholder.clone()); + resolve_relation(&mut resolved).unwrap(); + assert_eq!(resolved, transform_of(relation)); + std::mem::drop(node); + let mut gone = transform_of(placeholder); + let error = resolve_relation(&mut gone).unwrap_err(); + assert!(error.message().contains("no longer exists"), "{error}"); + } + #[test] fn a_placeholder_is_the_same_for_the_same_node_and_session() { let ids = Arc::new(PlanIds::new()); @@ -4704,11 +4804,16 @@ mod placeholder_tests { resolve_relation(&mut text).unwrap(); assert_eq!(text, before); - // A placeholder of a dropped DataFrame in an extension is the error of one elsewhere. + // A placeholder of a dropped DataFrame in an extension stands for its relation as one + // elsewhere, and one of a marker alone is the error of one elsewhere. let gone = Arc::new(range(0, 4, 1)); - let mut plan = extension_of(&join(&placeholder(&gone, &ids))); + let gone_relation = relation_with_ids(&gone, &ids).unwrap(); + let mut standalone = extension_of(&join(&placeholder(&gone, &ids))); + let mut marked = extension_of(&join(&placeholder_relation(&gone, &ids))); std::mem::drop(gone); - let error = resolve_relation(&mut plan).unwrap_err(); + resolve_relation(&mut standalone).unwrap(); + assert_eq!(standalone, extension_of(&join(&gone_relation))); + let error = resolve_relation(&mut marked).unwrap_err(); assert!(error.to_string().contains("no longer exists"), "{error}"); } @@ -4976,16 +5081,17 @@ mod placeholder_tests { } #[test] - fn a_placeholder_of_a_dropped_node_is_an_error_and_keeps_nothing_alive() { + fn a_placeholder_of_a_dropped_node_stands_for_its_relation_and_keeps_nothing_alive() { let ids = Arc::new(PlanIds::new()); let node = Arc::new(range(0, 3, 1)); + let relation = relation_with_ids(&node, &ids).unwrap(); let alive = Arc::downgrade(&node); - let shallow = placeholder(&node, &ids); + let standalone = placeholder(&node, &ids); std::mem::drop(node); assert!(alive.upgrade().is_none()); - let mut relation = transform_of(shallow); - let error = resolve_relation(&mut relation).unwrap_err(); - assert!(error.to_string().contains("no longer exists"), "{error}"); + let mut resolved = transform_of(standalone); + resolve_relation(&mut resolved).unwrap(); + assert_eq!(resolved, transform_of(relation)); } #[test] diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index 443b2851..d599b096 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -202,11 +202,12 @@ difference is intentional or tracked for a fix. Columns keeps the origin of its call site at the pickle or the copy, where PySpark's has the call site of each request. The pickle itself does not warn, as PySpark's does not, but PySpark's unpickle of a DataFrame warns once, as it builds the plan to read the columns of the - new DataFrame, where the drop-in's does not. The `plan` of `DataFrame._plan` is a placeholder of - the relation (see "Plan logging of a deep DataFrame" below), so it does not warn, and a request - of a plan over it warns once, where PySpark's warns at both. The `UserWarning` is of - `pyspark.sql.connect.expressions` and of its line 1126, as PySpark's, in the file of the - drop-in's module, which ends before the line, so it shows no source line. + new DataFrame, where the drop-in's does not. The `plan` of `DataFrame._plan` builds the relation, + and warns, as PySpark's does; a `DataFrame(plan, session)` of a plan over it warns once, when it + is made, as the core keeps the relation that it built then, where PySpark's warns at each + request, which builds the plan. The `UserWarning` is of `pyspark.sql.connect.expressions` and of + its line 1126, as PySpark's, in the file of the drop-in's module, which ends before the line, so + it shows no source line. - **Public base and native MRO** (intentional; a PyO3 class cannot have a Python class as its base, so the public base can only be a virtual one). `pyspark.sql.DataFrame` is implemented as a @@ -425,23 +426,28 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan to remote host: Connection refused`. - **`SparkConnectClient.get_tags`** (intentional). A copy of the tags of the thread, where PySpark returns the set that its client keeps, which its later changes change too, as - `SparkSession.getTags` does. + `SparkSession.getTags` does, of which the drop-in returns a copy too: the tags are those of the + core session, which its requests read, and not a Python set. - **Plans of a DataFrame's relation** (intentional; DESIGN). `df._plan.plan(client)` is a - placeholder of the DataFrame's relation, which the core replaces by the relation while the - DataFrame is alive, wherever `plan::resolve_relation`, `resolve_command` and `resolve_analysis` - find it: as the root relation of a plan or of an analysis, as the input of every relation of - Spark Connect under it (`relation_inputs_of`: both sides of joins, set operations and zips, the - other inputs of `GroupMap` and `CoGroupMap`, the root and references of a `WithRelations`, so - the views of `SQL`), as the relation of each command that holds one (writes, views, streams, - `SqlCommand`, checkpoints, `MergeIntoTableCommand`, the flows of pipelines), as the dataset and - the fetched arguments of the ML commands, and anywhere in the message of the extension of a - relation or of a command (a plugin's, as Delta Lake's `merge`) that holds it as `plan(session)` - gives it. The core does not look into expressions, which hold no relation, so a placeholder in - an expression's extension is sent as it is, and so is one that the message of an extension - changes (`plan.common.plan_id = ...`); the server rejects them with - `CONNECT_INVALID_PLAN.NO_HANDLER_FOR_EXTENSION`. A plan of it that `client.to_table`, - `client.schema` or another method of the client sends after the DataFrame is freed fails with - "a placeholder of a DataFrame that no longer exists". + placeholder that holds the DataFrame's relation, which the core builds at the call, as + PySpark's `plan` does, with its pickles, warnings and errors: the core builds the relation, and + Python has no messages of it, so it is one message, an extension whose value holds the relation + encoded, which Python's protobuf does not parse, rather than PySpark's messages of each level. + The core replaces it by that relation, also after the DataFrame is gone, wherever + `plan::resolve_relation`, `resolve_command` and `resolve_analysis` find it: as the root relation + of a plan or of an analysis, as the input of every relation of Spark Connect under it + (`relation_inputs_of`: both sides of joins, set operations and zips, the other inputs of + `GroupMap` and `CoGroupMap`, the root and references of a `WithRelations`, so the views of `SQL`), + as the relation of each command that holds one (writes, views, streams, `SqlCommand`, checkpoints, + `MergeIntoTableCommand`, the flows of pipelines), as the dataset and the fetched arguments of the + ML commands, and anywhere in the message of the extension of a relation or of a command (a + plugin's, as Delta Lake's `merge`) that holds it as `plan(session)` gives it. The core does not + decode the expressions of such a plan, which hold no relation, so as not to decode every + expression of each plan for what only a plugin's expression extension could hold, and it takes a + placeholder in an extension's message by its encoding, which a change alters: so a placeholder in + an expression's extension is sent as it is, and so is one that the message of an extension changes + (`plan.common.plan_id = ...`), whose change the core does not apply to the relation; the server + rejects them with `CONNECT_INVALID_PLAN.NO_HANDLER_FOR_EXTENSION`. - **Default port** (intentional). `DefaultChannelBuilder.default_port()` is 15002. PySpark takes the port of a local JVM server in its testing mode; the drop-in has no JVM. - **Missing URL error** (intentional). The one builder of the drop-in serves @@ -451,13 +457,14 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan builder's `SESSION_NEED_CONN_STR_OR_BUILDER` family). Giving both a channel builder and a URL raises `SESSION_NEED_CONN_STR_OR_BUILDER`. A channel builder without a host is rejected at `channelBuilder(...)`, not when the session is created. -- **Plan logging of a deep DataFrame** (intentional; DESIGN). `DataFrame._plan` is a - placeholder over the relation of the Rust core, so `_plan.to_proto` is not provided. +- **Plan logging of a deep DataFrame** (intentional; DESIGN). `DataFrame._plan` is a plan of the + relation of the Rust core, whose `plan` is a placeholder that holds the relation encoded, so + `_plan.to_proto` is not provided. Python protobuf refuses to decode a relation nested beyond its limit (`too many levels - of nesting`), and the drop-in never decodes a deep plan. `SparkConnectClient._proto_to_string` - renders any message, including a deep one built in Python, as PySpark's does. Plan - logging of a deep DataFrame therefore does not reach the recursion handling that the - test `test_recursion_handling_for_plan_logging` exercises. + of nesting`), and the drop-in never decodes a deep plan in Python. + `SparkConnectClient._proto_to_string` renders any message, including a deep one built in + Python, as PySpark's does. Plan logging of a deep DataFrame therefore does not reach the + recursion handling that the test `test_recursion_handling_for_plan_logging` exercises. - **Client session ID** (not provided). `SparkConnectClient._session_id` is not exposed and is not settable; the session ID belongs to the Rust core. `_server_session_id` is exposed and settable, as PySpark's, and a changed value closes the session. @@ -574,12 +581,13 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan the modules of protoc 3.20 and later, so its message fields take the messages of `pyspark.sql.connect.proto` as they are, as PySpark's plugins take PySpark's, and the server receives the plans of Delta Lake 4.0's `delete`, `update` and `merge` as PySpark sends them: the - core replaces the shallow placeholder of the relation of a DataFrame, which + core replaces the placeholder that holds the relation of a DataFrame, which `DataFrame._plan.plan(session)` gives, in the message of an extension too. What stays different (see `docs/compatibility.md`): - the core replaces that placeholder where it replaces the plan of a DataFrame (see "Plans of a DataFrame's relation" above). One that a plugin changes in the message of an extension reaches - the server as a placeholder, which it refuses, where PySpark's plugins send the relation; + the server as a placeholder, which it refuses, where PySpark's plugins send the relation: the + core takes a placeholder there by its encoding, which the change alters; - its classes are the runtime's, not subclasses of `google.protobuf.message.Message`, while its `DESCRIPTOR`, its `_*` descriptors, its extensions and services stay `protobuf`'s, and `protobuf`'s `message_factory` and `symbol_database` give `protobuf`'s classes of its types, diff --git a/docs/compatibility.md b/docs/compatibility.md index 004a53da..a656efb8 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -1549,13 +1549,13 @@ files are added to the runtime's pool. So their message fields take the messages PySpark's: Delta Lake 4.0's `DeltaTable.delete`, `update` and `merge` build their plans with `CopyFrom` of the package's messages, of the bytes, text and JSON of PySpark's, with either implementation of `protobuf`. The relation of a DataFrame that such a message takes -(`df._plan.plan(session)`, as `merge` takes the plan of its source DataFrame) is a shallow -placeholder of the relation of the DataFrame of this client, as in the plans of `pyspark.ml`: the -core replaces it when the plan or the command reaches it, so the server receives the relation, as -PySpark's plugins send it: also anywhere in the message of the extension of a relation or of a -command, of a type that it does not know, as long as the message holds the placeholder as -`plan(session)` gives it. "SparkSession.client" below lists the places where the core replaces it: -under any relation and in any command that holds one, but not one that a plugin changes +(`df._plan.plan(session)`, as `merge` takes the plan of its source DataFrame) is a placeholder that +holds the relation of the DataFrame (see "SparkSession.client" below), as in the plans of +`pyspark.ml`: the core replaces it when the plan or the command reaches it, so the server receives +the relation, as PySpark's plugins send it: also anywhere in the message of the extension of a +relation or of a command, of a type that it does not know, as long as the message holds the +placeholder as `plan(session)` gives it. "SparkSession.client" below lists the places where the core +replaces it: under any relation and in any command that holds one, but not one that a plugin changes (`plan.common.plan_id = ...`) in the message of an extension, which is sent as it is and which the server refuses. Its classes are the runtime's, as those of the package (not `google.protobuf.message.Message`'s), @@ -1609,7 +1609,12 @@ a name bound to `protobuf`'s class before the package is imported (`from google.protobuf.unknown_fields import UnknownFieldSet`) refuses the package's messages. A DataFrame of a plan of `pyspark.ml` (`DataFrame(plan, session)`) takes the plan ID -of this client rather than the one of the plan, which its columns refer to. +of this client rather than the one of the plan, which its columns refer to: the core keeps the +relation of the plan as a node of its own plans, which it numbers from the counter of the plan +IDs of its sessions, and the plans of `pyspark.sql.connect.plan` number theirs from `1 << 48`, +past the core's, so that the two never give a relation the same ID. So the relation that it sends +and the columns of its `df["name"]` have the client's plan ID, and its `_plan` is the plan of that +relation, not `plan`, as PySpark's is. ## SparkSession.client @@ -1650,16 +1655,25 @@ parameters of the first `DefaultPolicy` of the list, its retries, backoffs, jitt that the server asks for, as PySpark's `Retrying` does, of the failures that `DefaultPolicy.can_retry` retries; a `RetryPolicy` retries none. It reads the parameters when the policies are set, so a change to a policy afterwards applies once the policies are set again. A -policy of its own `can_retry` or `to_state`, which the transport cannot call, and a backoff, jitter -or delay that is not a whole number of milliseconds raise `NOT_IMPLEMENTED`. The client of a new -session has the default policy, as PySpark's `newSession` and `clone` do. With +policy of its own `can_retry` or `to_state`, which the transport does not call, as it decides each +retry in Rust, without the GIL, of the parameters that it read of the policies, and a backoff, +jitter or delay that is not a whole number of milliseconds raise `NOT_IMPLEMENTED`. The client of a +new session has the default policy, as PySpark's `newSession` and `clone` do. With `disable_reattachable_execute`, an execution has no reattach option and no deadline, is never released, and is sent again when it fails in a way that its policy retries, as PySpark's. -The relation of a DataFrame's plan (`df._plan.plan(client)`) is a placeholder of the relation of -the DataFrame, which the core replaces by that relation when it sends a plan, a command or an -analysis that holds it, while the DataFrame is alive: keep the DataFrame while a plan of it is -sent. It replaces it wherever PySpark's plans and commands hold the relation of a DataFrame: +The relation of a DataFrame's plan (`df._plan.plan(client)`) is a placeholder that holds the +relation of the DataFrame, which the core builds at the call, as PySpark's `plan` builds the +relation then: the call pickles the functions of its UDFs, warns of a window of no partitions and +raises the errors of the build, and no request is sent. The core builds the relation, and Python +has no messages of it, so the placeholder is one message, an extension whose value holds the +relation encoded, which Python's protobuf does not parse, whatever the depth of the DataFrame, +where PySpark's `plan` gives the messages of each level of the relation: so its fields are not +those of the relation (`WhichOneof("rel_type")` is `"extension"`). The core replaces the +placeholder by the relation that it holds when it sends a plan, a command or an analysis that +holds it, also after the DataFrame is gone, and when the message is parsed again of its bytes, as +PySpark's relation is sent. It replaces it wherever PySpark's plans and commands hold the relation +of a DataFrame: - as the root relation of a plan, of an execution, of an analysis (`schema`, `explain_string`, `same_semantics`, `semantic_hash`) or of `DataFrame(plan, session)`, and as the relation of the @@ -1678,11 +1692,15 @@ sent. It replaces it wherever PySpark's plans and commands hold the relation of as Delta Lake's `MergeIntoTable` (see the Delta Lake paragraph above), where the message holds it as `plan(session)` gives it. -The core does not look into expressions, which hold no relation (a subquery refers to its plan by -its plan ID in a `WithRelations`), so a placeholder in the extension of an expression is sent as it -is, and so is one that the message of an extension changes (`plan.common.plan_id = ...`): the server -rejects them with `CONNECT_INVALID_PLAN.NO_HANDLER_FOR_EXTENSION`, where PySpark sends the relation. -Delta Lake's plans hold neither. +The core does not decode the expressions of a plan built in Python, which hold no relation of Spark +Connect (a subquery refers to its plan by its plan ID in a `WithRelations`), so as not to decode +every expression of each such plan for a relation that only the extension of an expression of a +plugin could hold; and it takes a placeholder in the message of an extension by its encoding, as +`plan(session)` gives it, which a change of the placeholder alters, and does not apply that change +to the relation that the placeholder holds. So a placeholder in the extension of an expression is +sent as it is, and so is one that the message of an extension changes (`plan.common.plan_id = ...`): +the server rejects them with `CONNECT_INVALID_PLAN.NO_HANDLER_FOR_EXTENSION`, where PySpark sends +the relation. Delta Lake's plans hold neither. The `observations` of a DataFrame's plan (`df._plan.observations`) are the Observations of the DataFrame by their names, as PySpark's `LogicalPlan.observations`: those of `observe` on it and on @@ -1710,12 +1728,13 @@ RPCs: they are the exception of their error class, such as `UnknownException` or the list attribute of PySpark's client, whose changes, and a list that it is set to, the requests of all the threads carry. -`get_tags` returns a copy of the tags of the thread, where PySpark returns the set that its client -keeps, which the later changes of the tags change too. The official tests of the client -(`sql/tests/connect/client/test_client*.py`) mock the gRPC stub and the reattachable iterator of -PySpark's client, which the drop-in does not have; those of its retries (`test_connect_retry.py`) -and of `pyspark.errors.exceptions.connect.convert_exception` (`test_connect_errors_conversion.py`) -pass. +`get_tags`, as `SparkSession.getTags`, returns a copy of the tags of the thread, where PySpark +returns the set that its client keeps, which the later changes of the tags change too: the tags +are those of the core session, which its requests read, and not a Python set. The official tests of +the client (`sql/tests/connect/client/test_client*.py`) mock the gRPC stub and the reattachable +iterator of PySpark's client, which the drop-in does not have; those of its retries +(`test_connect_retry.py`) and of `pyspark.errors.exceptions.connect.convert_exception` +(`test_connect_errors_conversion.py`) pass. ## Spark Declarative Pipelines CLI diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py index b14302e4..39208de8 100644 --- a/python/pyspark/sql/connect/plan.py +++ b/python/pyspark/sql/connect/plan.py @@ -130,10 +130,12 @@ class _DataFramePlan(LogicalPlan): """The plan of a DataFrame of the Rust core, as its ``DataFrame._plan``: its relation, as the core sends it, of the plan IDs of its session. - Its ``plan`` is a placeholder of that relation, which the core replaces by the relation when - the plan built over it reaches the core, as ``DataFrame(plan, session)`` or a command does. - So a plan of ``pyspark.ml`` over a deep DataFrame is never decoded in Python or in Rust, as - protobuf decoders limit the depth of what they decode.""" + Its ``plan`` is a placeholder that holds that relation, which the core builds at the call, as + PySpark's ``plan`` builds the relation, and replaces the placeholder with when the plan built + over it reaches the core, as ``DataFrame(plan, session)`` or a command does, also after the + DataFrame is gone. It is one message, an extension whose value holds the relation encoded, so + a plan of ``pyspark.ml`` over a deep DataFrame is never decoded in Python, as protobuf decoders + limit the depth of what they decode.""" def __init__(self, df: Any) -> None: super().__init__(None) diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py index 19dd1e08..bd1da3f2 100644 --- a/python/tests/test_dropin_e2e.py +++ b/python/tests/test_dropin_e2e.py @@ -330,6 +330,24 @@ def test_client_sends_the_relations_of_dataframes_under_the_plans_it_is_given_as assert properties["checkpoint_command_result"].count() == 5 +def test_the_relations_of_the_plans_of_dataframes_are_sent_after_the_dataframes_are_gone(session): + # As PySpark Connect's df._plan.plan(client), a relation of the DataFrame built at the call: + # the client sends it after the DataFrame is gone, and the bytes of a plan that holds it. + import gc + + import pyspark.sql.connect.proto as pb2 + + client = session.client + table = client.to_table(pb2.Plan(root=session.sql("SELECT 1 AS a")._plan.plan(client)), {})[0] + assert table.to_pydict() == {"a": [1]} + relation = session.range(3).filter("id > 0")._plan.plan(client) + gc.collect() + plan = pb2.Plan(root=relation) + for sent in (plan, plan, pb2.Plan.FromString(plan.SerializeToString())): + assert client.to_table(sent, {})[0].to_pydict() == {"id": [1, 2]} + assert client.schema(plan).simpleString() == "struct" + + def test_the_plans_of_dataframes_give_the_client_their_observations_as_pyspark(session): # As PySpark Connect's DataFrame._plan, whose observations are the Observations of the plan # by their names, which its DataFrame gives the executions of its client: an execution of diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index aa8a1143..627b5181 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -21950,9 +21950,11 @@ def test_arrow_input_explicit_schema_matching_width(schema, values): def test_ml_plans_send_a_deep_dataframe_without_parsing_it_as_pyspark(mock_connect): # As PySpark's pyspark.ml, which builds its plans over the relation of a DataFrame without # parsing it: a DataFrame of 120 chained withColumn calls is nested deeper than a protobuf - # decoder parses. A plan over it holds a shallow placeholder of that relation, which the core - # replaces with the relation when the plan reaches it, so the server receives the relation. + # decoder parses. A plan over it holds a placeholder of one level, the extension that holds + # the relation encoded, which the core replaces with the relation when the plan reaches it, + # so the server receives the relation. import gc + import weakref import pyspark.sql.connect.proto as pb2 from pyspark.sql import functions as F @@ -21976,8 +21978,11 @@ def plan(self, session): for i in range(120): deep = deep.withColumn(f"c{i}", F.lit(i)) relation = deep._relation_bytes() - # The placeholder of the relation is shallow, whatever the depth of the DataFrame. - assert len(deep._plan.plan(spark.client).SerializeToString()) < 128 + # The placeholder of the relation is an extension, whose value holds the relation, which + # protobuf does not parse, whatever the depth of the DataFrame. + placeholder = deep._plan.plan(spark.client) + assert placeholder.WhichOneof("rel_type") == "extension" + assert placeholder.extension.value.endswith(relation) # Two plans of transformers, one over the other, over the DataFrame. The request is read as # bytes, as the deep relation is not parsed by protobuf: ExecutePlanRequest.plan=3, # Plan.root=1, Relation.ml_relation=300, MlRelation.transform=1, Transform.input=3, @@ -21992,18 +21997,70 @@ def plan(self, session): assert parse_message(inner_transform[1][0]) == {1: [b"m1"]} assert inner_transform[3][0] == relation - # A placeholder of a DataFrame that is gone is an error, and keeps nothing alive. + # A placeholder of a DataFrame that is gone stands for its relation, as the relation of + # PySpark's plan(session) does, and keeps nothing alive. gone = spark.range(3) + gone_relation = gone._relation_bytes() placeholder = gone._plan.plan(spark.client) + freed = weakref.ref(gone) del gone gc.collect() + assert freed() is None class Gone(LogicalPlan): def plan(self, session): return placeholder - with pytest.raises(Exception, match="no longer exists"): - DataFrame(Gone(None), spark) + DataFrame(Transform(Gone(None), "m3"), spark).collect() + root = parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0] + transform = parse_message(parse_message(parse_message(root)[300][0])[1][0]) + assert transform[3][0] == gone_relation + + +def test_the_relations_of_the_plans_of_dataframes_are_built_when_they_are_given_as_pyspark( + mock_connect, +): + # As PySpark's df._plan.plan(session), which builds the relation of the DataFrame when it is + # called: the relation is sent, after the DataFrame is gone too, and of the bytes of a + # message that holds it; it is that of the call, of the pickles of its UDFs then, and the + # errors of its build are raised by the call, with no request sent. + import gc + import pickle + import threading + + import pyspark.sql.connect.proto as pb2 + from pyspark.sql import functions as F + + spark, service = mock_connect + client = spark.client + + def sent(): + return pb2.ExecutePlanRequest.FromString(service.execute_requests[-1]).plan + + list(client.to_table_as_iterator(pb2.Plan(root=spark.range(2)._plan.plan(client)), {})) + assert sent().root.range.end == 2 + kept = spark.range(3).filter("id > 0")._plan.plan(client) + gc.collect() + plan = pb2.Plan(root=kept) + for message in (plan, pb2.Plan.FromString(plan.SerializeToString())): + list(client.to_table_as_iterator(message, {})) + assert sent().root.filter.input.range.end == 3 + + state = ["at the call"] + df = spark.range(1).select(F.udf(lambda x: state[0], "string")("id")) + relation = df._plan.plan(client) + state[0] = "after the call" + list(client.to_table_as_iterator(pb2.Plan(root=relation), {})) + function = sent().root.project.expressions[0].common_inline_user_defined_function + assert b"at the call" in function.python_udf.command + assert b"after the call" not in function.python_udf.command + + lock = threading.Lock() + unpicklable = spark.range(1).select(F.udf(lambda x: lock is not None, "boolean")("id")) + requests = len(service.execute_requests) + with pytest.raises(pickle.PicklingError): + unpicklable._plan.plan(client) + assert len(service.execute_requests) == requests def test_plans_of_plugins_send_the_relations_of_their_dataframes_in_their_extensions(