From 51fedc3922bbcc7153453c10de190d929697206a Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 5 Nov 2026 19:01:30 +0900 Subject: [PATCH] [SPARK-60898][CONNECT] Keep plan IDs stable across re-executions to reuse the server plan cache As PySpark assigns a plan ID once per plan, give each plan node its ID the first time it is sent, from a registry of the session keyed by the node's allocation, and keep it, so that re-executing a DataFrame sends the same relation and a DataFrame built on another keeps that input's plan IDs. The server's per-session plan cache can then reuse the plans it already resolved. The registry holds weak references and prunes the nodes that were dropped as it grows. Co-authored-by: Isaac --- crates/spark-connect/src/dataframe.rs | 366 +++++------------- crates/spark-connect/src/merge.rs | 2 +- crates/spark-connect/src/ml.rs | 20 +- crates/spark-connect/src/pipelines.rs | 5 +- crates/spark-connect/src/plan.rs | 189 ++++++--- crates/spark-connect/src/readwriter.rs | 4 +- crates/spark-connect/src/session.rs | 22 +- crates/spark-connect/src/streaming.rs | 5 +- crates/spark-connect/tests/e2e_ops_breadth.rs | 8 +- docs/architecture.md | 12 + 10 files changed, 281 insertions(+), 352 deletions(-) diff --git a/crates/spark-connect/src/dataframe.rs b/crates/spark-connect/src/dataframe.rs index 3c5b0305..c72d870b 100644 --- a/crates/spark-connect/src/dataframe.rs +++ b/crates/spark-connect/src/dataframe.rs @@ -902,11 +902,15 @@ impl DataFrame { Ok(fields.into_iter().map(|field| field.name).collect()) } + /// This DataFrame's relation for a request. Its plan nodes keep the same plan IDs each + /// time, as in PySpark, so the server can reuse the plans it already resolved. + pub(crate) fn relation(&self) -> proto::Relation { + crate::plan::relation_with_ids(&self.plan, self.session.plan_ids()) + } + /// Build an ExecutePlanRequest from the logical plan. fn build_execute_request(&self) -> Result { - // Build proto from plan with plan_id assignment - let mut relation = self.plan.to_proto(); - assign_plan_ids(&mut relation, &self.session)?; + let relation = self.relation(); let mut request = proto::ExecutePlanRequest::default(); request.session_id = self.session.client().session_id().to_string(); @@ -922,8 +926,7 @@ impl DataFrame { /// Build an AnalyzePlanRequest from the logical plan. fn build_analyze_request(&self) -> Result { - let mut relation = self.plan.to_proto(); - assign_plan_ids(&mut relation, &self.session)?; + let relation = self.relation(); let mut plan = proto::Plan::default(); plan.op_type = Some(proto::plan::OpType::Root(relation)); @@ -990,14 +993,6 @@ impl DataFrame { } } - /// Build this DataFrame's plan as a proto `Relation` with plan ids assigned - /// (the shape the `Persist`/`Unpersist`/`GetStorageLevel` analyze ops take). - fn analyze_relation(&self) -> Result { - let mut relation = self.plan.to_proto(); - assign_plan_ids(&mut relation, &self.session)?; - Ok(relation) - } - fn analyze_request( &self, analyze: proto::analyze_plan_request::Analyze, @@ -1022,7 +1017,7 @@ impl DataFrame { /// Mirrors `pyspark.sql.DataFrame.persist(storageLevel)`. pub fn persist(&self, storage_level: proto::StorageLevel) -> Result { let persist = proto::analyze_plan_request::Persist { - relation: Some(self.analyze_relation()?), + relation: Some(self.relation()), storage_level: Some(storage_level), }; let request = self.analyze_request(proto::analyze_plan_request::Analyze::Persist(persist)); @@ -1033,7 +1028,7 @@ impl DataFrame { /// Remove this DataFrame from cache. Mirrors `DataFrame.unpersist(blocking)`. pub fn unpersist(&self, blocking: bool) -> Result { let unpersist = proto::analyze_plan_request::Unpersist { - relation: Some(self.analyze_relation()?), + relation: Some(self.relation()), blocking: Some(blocking), }; let request = @@ -1057,7 +1052,7 @@ impl DataFrame { /// materialize server-side and return a handle to the checkpointed data. fn checkpoint_impl(&self, local: bool, eager: bool) -> Result { let mut cmd = proto::CheckpointCommand::default(); - cmd.relation = Some(self.plan.to_proto()); + cmd.relation = Some(self.relation()); cmd.local = local; cmd.eager = eager; let responses = execute_command_collect( @@ -1106,8 +1101,7 @@ impl DataFrame { /// Build + execute a real `CreateDataFrameViewCommand` (was previously a /// silent no-op that passed the query relation through and never created a view). fn create_view(&self, name: &str, replace: bool, global: bool) -> Result<()> { - let mut input = self.plan.to_proto(); - assign_plan_ids(&mut input, &self.session)?; + let input = self.relation(); let mut cmd = proto::CreateDataFrameViewCommand::default(); cmd.input = Some(input); cmd.name = name.to_string(); @@ -1143,8 +1137,7 @@ impl DataFrame { )) } }; - let mut relation = self.plan.to_proto(); - assign_plan_ids(&mut relation, &self.session)?; + let relation = self.relation(); let mut plan = proto::Plan::default(); plan.op_type = Some(proto::plan::OpType::Root(relation)); let mut ex = proto::analyze_plan_request::Explain::default(); @@ -1226,8 +1219,7 @@ impl DataFrame { /// Get the input files for this DataFrame. Mirrors `pyspark.sql.DataFrame.inputFiles`. pub fn input_files(&self) -> Result> { - let mut relation = self.plan.to_proto(); - assign_plan_ids(&mut relation, &self.session)?; + let relation = self.relation(); let mut plan = proto::Plan::default(); plan.op_type = Some(proto::plan::OpType::Root(relation)); let mut inp = proto::analyze_plan_request::InputFiles::default(); @@ -1506,7 +1498,7 @@ impl DataFrame { /// Get the storage level of this DataFrame. Mirrors `DataFrame.storageLevel`. pub fn storage_level(&self) -> Result { let get = proto::analyze_plan_request::GetStorageLevel { - relation: Some(self.analyze_relation()?), + relation: Some(self.relation()), }; let request = self.analyze_request(proto::analyze_plan_request::Analyze::GetStorageLevel(get)); @@ -1542,8 +1534,7 @@ impl DataFrame { /// Compute the server-side semantic hash of this DataFrame's logical plan, /// mirroring `DataFrame.semanticHash()` (an AnalyzePlan request). pub fn semantic_hash(&self) -> Result { - let mut relation = self.plan.to_proto(); - assign_plan_ids(&mut relation, &self.session)?; + let relation = self.relation(); let mut plan = proto::Plan::default(); plan.op_type = Some(proto::plan::OpType::Root(relation)); let mut request = proto::AnalyzePlanRequest::default(); @@ -1564,10 +1555,8 @@ impl DataFrame { /// Whether two DataFrames have the same semantics, mirroring /// `DataFrame.sameSemantics(other)` (a server-side AnalyzePlan comparison). pub fn same_semantics(&self, other: &DataFrame) -> Result { - let mut self_rel = self.plan.to_proto(); - assign_plan_ids(&mut self_rel, &self.session)?; - let mut other_rel = other.plan.to_proto(); - assign_plan_ids(&mut other_rel, &other.session)?; + let self_rel = self.relation(); + let other_rel = other.relation(); let mut target_plan = proto::Plan::default(); target_plan.op_type = Some(proto::plan::OpType::Root(self_rel)); let mut other_plan = proto::Plan::default(); @@ -1858,12 +1847,10 @@ impl DataFrame { /// Shared by write operations (which embed a fully-formed input relation inside /// a `Command`) so they go through the same plan-id assignment as `collect()`. pub(crate) fn build_input_relation( - plan: &LogicalPlan, + plan: &Arc, session: &SparkSession, -) -> Result { - let mut relation = plan.to_proto(); - assign_plan_ids(&mut relation, session)?; - Ok(relation) +) -> proto::Relation { + crate::plan::relation_with_ids(plan, session.plan_ids()) } /// Execute a `Command` against the server and drain the response stream. @@ -1936,214 +1923,6 @@ fn capture_execution( } } -/// Assign unique plan_ids to all relations in a tree (post-order traversal). -pub(crate) fn assign_plan_ids( - relation: &mut proto::Relation, - session: &SparkSession, -) -> Result<()> { - if let Some(rel_type) = &mut relation.rel_type { - use proto::relation::RelType; - match rel_type { - RelType::Range(_) => {} - RelType::Sql(_) => {} - RelType::LocalRelation(_) => {} - RelType::CachedRemoteRelation(_) => {} - RelType::Project(proj) => { - if let Some(input) = &mut proj.input { - assign_plan_ids(input, session)?; - } - } - RelType::Filter(filter) => { - if let Some(input) = &mut filter.input { - assign_plan_ids(input, session)?; - } - } - RelType::Join(join) => { - if let Some(left) = &mut join.left { - assign_plan_ids(left, session)?; - } - if let Some(right) = &mut join.right { - assign_plan_ids(right, session)?; - } - } - RelType::SetOp(set_op) => { - if let Some(left) = &mut set_op.left_input { - assign_plan_ids(left, session)?; - } - if let Some(right) = &mut set_op.right_input { - assign_plan_ids(right, session)?; - } - } - RelType::Aggregate(agg) => { - if let Some(input) = &mut agg.input { - assign_plan_ids(input, session)?; - } - } - RelType::Sort(sort) => { - if let Some(input) = &mut sort.input { - assign_plan_ids(input, session)?; - } - } - RelType::Limit(limit) => { - if let Some(input) = &mut limit.input { - assign_plan_ids(input, session)?; - } - } - RelType::Offset(offset) => { - if let Some(input) = &mut offset.input { - assign_plan_ids(input, session)?; - } - } - RelType::Tail(tail) => { - if let Some(input) = &mut tail.input { - assign_plan_ids(input, session)?; - } - } - RelType::Deduplicate(dedup) => { - if let Some(input) = &mut dedup.input { - assign_plan_ids(input, session)?; - } - } - RelType::Repartition(repartition) => { - if let Some(input) = &mut repartition.input { - assign_plan_ids(input, session)?; - } - } - RelType::RepartitionByExpression(repart_expr) => { - if let Some(input) = &mut repart_expr.input { - assign_plan_ids(input, session)?; - } - } - RelType::WithColumns(with_cols) => { - if let Some(input) = &mut with_cols.input { - assign_plan_ids(input, session)?; - } - } - RelType::WithColumnsRenamed(with_renamed) => { - if let Some(input) = &mut with_renamed.input { - assign_plan_ids(input, session)?; - } - } - RelType::Drop(drop) => { - if let Some(input) = &mut drop.input { - assign_plan_ids(input, session)?; - } - } - RelType::ToDf(to_df) => { - if let Some(input) = &mut to_df.input { - assign_plan_ids(input, session)?; - } - } - RelType::ToSchema(to_schema) => { - if let Some(input) = &mut to_schema.input { - assign_plan_ids(input, session)?; - } - } - RelType::Hint(hint) => { - if let Some(input) = &mut hint.input { - assign_plan_ids(input, session)?; - } - } - RelType::Unpivot(unpivot) => { - if let Some(input) = &mut unpivot.input { - assign_plan_ids(input, session)?; - } - } - RelType::Sample(sample) => { - if let Some(input) = &mut sample.input { - assign_plan_ids(input, session)?; - } - } - RelType::FillNa(fill_na) => { - if let Some(input) = &mut fill_na.input { - assign_plan_ids(input, session)?; - } - } - RelType::DropNa(drop_na) => { - if let Some(input) = &mut drop_na.input { - assign_plan_ids(input, session)?; - } - } - RelType::Replace(replace) => { - if let Some(input) = &mut replace.input { - assign_plan_ids(input, session)?; - } - } - RelType::Describe(describe) => { - if let Some(input) = &mut describe.input { - assign_plan_ids(input, session)?; - } - } - RelType::Summary(summary) => { - if let Some(input) = &mut summary.input { - assign_plan_ids(input, session)?; - } - } - RelType::SubqueryAlias(sq_alias) => { - if let Some(input) = &mut sq_alias.input { - assign_plan_ids(input, session)?; - } - } - RelType::CachedLocalRelation(_cached) => { - // CachedLocalRelation doesn't have nested plans, it's just a reference with a hash - } - RelType::WithWatermark(watermark) => { - if let Some(input) = &mut watermark.input { - assign_plan_ids(input, session)?; - } - } - RelType::Crosstab(stat) => { - if let Some(input) = &mut stat.input { - assign_plan_ids(input, session)?; - } - } - RelType::FreqItems(stat) => { - if let Some(input) = &mut stat.input { - assign_plan_ids(input, session)?; - } - } - RelType::ApproxQuantile(stat) => { - if let Some(input) = &mut stat.input { - assign_plan_ids(input, session)?; - } - } - RelType::Corr(stat) => { - if let Some(input) = &mut stat.input { - assign_plan_ids(input, session)?; - } - } - RelType::Cov(stat) => { - if let Some(input) = &mut stat.input { - assign_plan_ids(input, session)?; - } - } - RelType::SampleBy(stat) => { - if let Some(input) = &mut stat.input { - assign_plan_ids(input, session)?; - } - } - RelType::CollectMetrics(metrics) => { - if let Some(input) = &mut metrics.input { - assign_plan_ids(input, session)?; - } - } - _ => { - // Handle any other relation types that we haven't explicitly handled - } - } - } - - // Assign plan_id to this relation - if relation.common.is_none() { - relation.common = Some(proto::RelationCommon::default()); - } - if let Some(common) = &mut relation.common { - common.plan_id = Some(session.next_plan_id()); - } - - Ok(()) -} - /// Decode an Arrow batch into rows. pub(crate) fn decode_arrow_batch( batch: &proto::execute_plan_response::ArrowBatch, @@ -3419,24 +3198,31 @@ mod plan_construction_tests { assert!(std::ptr::eq(source.plan(), cloned_table_arg.plan())); } - #[test] - fn shared_self_join_serializes_each_occurrence_with_distinct_ids() { - use crate::functions::col; - use crate::plan; + /// Plan IDs in pre-order, for the relation shapes these tests build. + fn plan_ids_of(relation: &proto::Relation) -> Vec { use proto::relation::RelType; - - fn ids(relation: &proto::Relation, out: &mut Vec) { + fn walk(relation: &proto::Relation, out: &mut Vec) { out.push(relation.common.as_ref().unwrap().plan_id.unwrap()); match relation.rel_type.as_ref().unwrap() { RelType::Join(join) => { - ids(join.left.as_ref().unwrap(), out); - ids(join.right.as_ref().unwrap(), out); + walk(join.left.as_ref().unwrap(), out); + walk(join.right.as_ref().unwrap(), out); } - RelType::Project(project) => ids(project.input.as_ref().unwrap(), out), + RelType::Project(project) => walk(project.input.as_ref().unwrap(), out), + RelType::Filter(filter) => walk(filter.input.as_ref().unwrap(), out), RelType::Range(_) => {} _ => panic!("unexpected relation"), } } + let mut out = Vec::new(); + walk(relation, &mut out); + out + } + + #[test] + fn shared_self_join_serializes_each_occurrence_with_the_same_ids() { + use crate::functions::col; + use crate::plan; let spark = session(); let projected = spark.range(10).unwrap().select(vec![col("id")]); @@ -3455,19 +3241,71 @@ mod plan_construction_tests { .to_proto(); assert_eq!(joined.plan.to_proto(), expected); - let first = build_input_relation(joined.plan(), &spark).unwrap(); - let second = build_input_relation(joined.plan(), &spark).unwrap(); - let mut all_ids = Vec::new(); - ids(&first, &mut all_ids); - ids(&second, &mut all_ids); - assert_eq!(all_ids.len(), 10); - all_ids.sort_unstable(); - all_ids.dedup(); - assert_eq!(all_ids.len(), 10); - // Assigning IDs mutates only the fresh protobuf, never the shared client plan. + // As in PySpark, a shared node has one plan ID wherever it occurs, and serializing + // the same plan again gives the same relation. + let first = build_input_relation(&joined.plan, &spark); + let second = build_input_relation(&joined.plan, &spark); + assert_eq!(first, second); + let ids = plan_ids_of(&first); + assert_eq!(ids.len(), 5); + assert_eq!(ids[1..3], ids[3..5]); + assert_ne!(ids[0], ids[1]); + assert_ne!(ids[1], ids[2]); + // Assigning IDs fills only the fresh protobuf, never the shared client plan. assert_eq!(joined.plan.to_proto(), expected); } + #[test] + fn derived_dataframes_keep_the_plan_ids_of_their_inputs() { + use crate::column::lit; + use crate::functions::col; + + let spark = session(); + let base = spark.range(3).unwrap().select(vec![col("id")]); + let base_relation = base.relation(); + let derived = base.filter(col("id").gt(lit(1))); + let derived_relation = derived.relation(); + let Some(proto::relation::RelType::Filter(filter)) = &derived_relation.rel_type else { + panic!("expected filter"); + }; + assert_eq!(filter.input.as_deref(), Some(&base_relation)); + assert_eq!(derived.relation(), derived_relation); + // Separately built plans, even equal ones, are distinct nodes with distinct IDs. + let other = spark.range(3).unwrap().select(vec![col("id")]); + assert_eq!(other.plan.to_proto(), base.plan.to_proto()); + let (mine, theirs) = (plan_ids_of(&base_relation), plan_ids_of(&other.relation())); + assert!(mine.iter().all(|id| !theirs.contains(id))); + } + + #[test] + fn plan_ids_of_dropped_nodes_are_pruned_and_never_reused() { + // A session of its own, so that other tests' plans do not change its entries. + let spark = session().new_session(); + let kept = spark.range(1).unwrap(); + let kept_ids = plan_ids_of(&kept.relation()); + let mut seen: std::collections::HashSet = kept_ids.iter().copied().collect(); + // Each plan is dropped at once, so pruning frees its allocation for a later plan to + // reuse. A new node must still never get the ID of a dropped one. + for end in 0..5000 { + for id in plan_ids_of(&spark.range(end).unwrap().relation()) { + assert!(seen.insert(id), "plan ID {id} was assigned twice"); + } + // With one live node, pruning keeps the map at most MIN_PRUNE_AT entries. + assert!(spark.plan_ids().len() <= 1024, "{}", spark.plan_ids().len()); + } + assert_eq!(plan_ids_of(&kept.relation()), kept_ids); + } + + #[test] + fn a_new_session_numbers_its_own_plans() { + let spark = session(); + let (one, two) = (spark.new_session(), spark.new_session()); + assert_eq!(plan_ids_of(&one.range(1).unwrap().relation()), vec![1]); + assert_eq!(plan_ids_of(&one.range(2).unwrap().relation()), vec![2]); + // The plans of one session do not advance the numbering of another. + assert_eq!(plan_ids_of(&two.range(1).unwrap().relation()), vec![1]); + } + #[test] fn with_metadata_plan() { let spark = session(); @@ -3535,7 +3373,7 @@ mod plan_construction_tests { let df2 = spark.range(5).unwrap(); let e = || col("id").expression().clone(); let ser = |d: &DataFrame| { - build_input_relation(d.plan(), &spark).expect("plan serializes to a relation"); + build_input_relation(&d.plan, &spark); }; ser(&df.select(vec![col("id")])); @@ -3601,7 +3439,7 @@ mod plan_construction_tests { use crate::streaming::Trigger; let spark = session(); let ser = |d: &DataFrame| { - build_input_relation(d.plan(), &spark).expect("stream plan serializes"); + build_input_relation(&d.plan, &spark); }; ser(&spark .read_stream() @@ -3706,7 +3544,7 @@ mod plan_construction_tests { let spark = session(); let ser = |d: &DataFrame| { - build_input_relation(d.plan(), &spark).expect("exotic plan serializes"); + build_input_relation(&d.plan, &spark); }; let df = spark.range(5).unwrap(); let df2 = spark.range(5).unwrap(); diff --git a/crates/spark-connect/src/merge.rs b/crates/spark-connect/src/merge.rs index fc57ad55..e03d914e 100644 --- a/crates/spark-connect/src/merge.rs +++ b/crates/spark-connect/src/merge.rs @@ -108,7 +108,7 @@ impl MergeIntoWriter { pub fn merge(self) -> Result<()> { let mut cmd = proto::MergeIntoTableCommand::default(); cmd.target_table_name = self.target_table; - cmd.source_table_plan = Some(build_input_relation(&self.source_plan, &self.session)?); + cmd.source_table_plan = Some(build_input_relation(&self.source_plan, &self.session)); cmd.merge_condition = Some(self.condition.to_proto()); cmd.match_actions = self.matched_actions; cmd.not_matched_actions = self.not_matched_actions; diff --git a/crates/spark-connect/src/ml.rs b/crates/spark-connect/src/ml.rs index 7ed20587..031ca3b2 100644 --- a/crates/spark-connect/src/ml.rs +++ b/crates/spark-connect/src/ml.rs @@ -257,12 +257,12 @@ pub trait Transformer: Send + Sync { } /// Build the MlRelation for this transformation. - fn build_ml_relation(&self, input_plan: &LogicalPlan) -> proto::MlRelation { + fn build_ml_relation(&self, input: &DataFrame) -> proto::MlRelation { let mut transform = proto::ml_relation::Transform::default(); transform.operator = Some(proto::ml_relation::transform::Operator::Transformer( self.operator().to_proto(), )); - transform.input = Some(Box::new(input_plan.to_proto())); + transform.input = Some(Box::new(input.relation())); transform.params = Some(self.params().to_proto()); let mut ml_relation = proto::MlRelation::default(); @@ -304,7 +304,7 @@ fn evaluate_via_command( params: &Params, df: &DataFrame, ) -> spark_connect_core::error::Result { - let dataset = crate::dataframe::build_input_relation(&df.plan, &df.session)?; + let dataset = df.relation(); let evaluate = proto::ml_command::Evaluate { evaluator: Some(operator.to_proto()), params: Some(params.to_proto()), @@ -448,7 +448,7 @@ impl Transformer for StandardScalerModel { fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result { // Build an MlRelation for transformation - let ml_relation = self.build_ml_relation(&df.plan); + let ml_relation = self.build_ml_relation(df); // Wrap it in a Relation let mut relation = proto::Relation::default(); @@ -539,7 +539,7 @@ impl Transformer for VectorAssembler { } fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result { - let ml_relation = self.build_ml_relation(&df.plan); + let ml_relation = self.build_ml_relation(df); let mut relation = proto::Relation::default(); relation.common = Some(proto::RelationCommon::default()); relation.rel_type = Some(proto::relation::RelType::MlRelation(Box::new(ml_relation))); @@ -658,7 +658,7 @@ impl Transformer for StringIndexerModel { } fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result { - let ml_relation = self.build_ml_relation(&df.plan); + let ml_relation = self.build_ml_relation(df); let mut relation = proto::Relation::default(); relation.common = Some(proto::RelationCommon::default()); relation.rel_type = Some(proto::relation::RelType::MlRelation(Box::new(ml_relation))); @@ -783,7 +783,7 @@ impl Transformer for MaxAbsScalerModel { } fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result { - let ml_relation = self.build_ml_relation(&df.plan); + let ml_relation = self.build_ml_relation(df); let mut relation = proto::Relation::default(); relation.common = Some(proto::RelationCommon::default()); relation.rel_type = Some(proto::relation::RelType::MlRelation(Box::new(ml_relation))); @@ -934,7 +934,7 @@ impl Transformer for LogisticRegressionModel { } fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result { - let ml_relation = self.build_ml_relation(&df.plan); + let ml_relation = self.build_ml_relation(df); let mut relation = proto::Relation::default(); relation.common = Some(proto::RelationCommon::default()); relation.rel_type = Some(proto::relation::RelType::MlRelation(Box::new(ml_relation))); @@ -1194,7 +1194,7 @@ impl Transformer for PipelineModel { } fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result { - let ml_relation = self.build_ml_relation(&df.plan); + let ml_relation = self.build_ml_relation(df); let mut relation = proto::Relation::default(); relation.common = Some(proto::RelationCommon::default()); relation.rel_type = Some(proto::relation::RelType::MlRelation(Box::new(ml_relation))); @@ -1383,7 +1383,7 @@ impl Transformer for CrossValidatorModel { &mut self.params } fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result { - let ml_relation = self.build_ml_relation(&df.plan); + let ml_relation = self.build_ml_relation(df); let mut relation = proto::Relation::default(); relation.common = Some(proto::RelationCommon::default()); relation.rel_type = Some(proto::relation::RelType::MlRelation(Box::new(ml_relation))); diff --git a/crates/spark-connect/src/pipelines.rs b/crates/spark-connect/src/pipelines.rs index 7c4d4923..a30d8e7c 100644 --- a/crates/spark-connect/src/pipelines.rs +++ b/crates/spark-connect/src/pipelines.rs @@ -13,7 +13,7 @@ use spark_connect_core::error::{Result, SparkError}; use spark_connect_core::runtime::block_on; use spark_connect_proto as proto; -use crate::dataframe::{assign_plan_ids, execute_command_collect, DataFrame}; +use crate::dataframe::{execute_command_collect, DataFrame}; use crate::session::SparkSession; use crate::types::DataType; @@ -149,8 +149,7 @@ pub fn define_flow( sql_conf: HashMap, source_code_location: Option, ) -> Result<()> { - let mut relation = relation_df.plan.to_proto(); - assign_plan_ids(&mut relation, &relation_df.session)?; + let relation = relation_df.relation(); let details = define_flow::Details::RelationFlowDetails(define_flow::WriteRelationFlowDetails { relation: Some(relation), diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs index 7167eb6e..49cc45eb 100644 --- a/crates/spark-connect/src/plan.rs +++ b/crates/spark-connect/src/plan.rs @@ -5,7 +5,7 @@ use spark_connect_proto as proto; use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{Arc, Mutex, Weak}; use crate::column::Column; use crate::expression::{Expression, ExpressionNode}; @@ -25,8 +25,8 @@ pub struct TransformWithStateInfo { /// /// Each variant represents a different relation type that can be executed. /// Child plans are immutable shared nodes, so adding a transformation does not clone -/// its ancestors. Call `to_proto()` to materialize a fresh protobuf tree; sharing -/// does not deduplicate relation occurrences or their execution-time plan IDs. +/// its ancestors. Call `to_proto()` to materialize a fresh protobuf tree without plan +/// IDs; requests give every shared node its stable plan ID (see `PlanIds`). #[derive(Debug, Clone)] pub enum LogicalPlan { /// A range of integers: `range(start, end, step)`. @@ -407,9 +407,86 @@ pub enum SetOpType { Except, } +/// Plan IDs that stay the same each time a plan node is serialized, as PySpark assigns +/// them once per plan. The server caches resolved plans keyed by the whole relation, so +/// re-executing a DataFrame, or building on one it already resolved, reuses that work. +/// +/// A node is identified by its `Arc` allocation. Each entry holds a `Weak` to the node, +/// which keeps the allocation, and so the key, from being reused while the entry exists. +/// Entries of dropped nodes are pruned as the map grows. +#[derive(Debug)] +pub(crate) struct PlanIds { + nodes: Mutex, +} + +/// The state of `PlanIds`, all guarded by its one lock. +#[derive(Debug)] +struct PlanIdNodes { + next: i64, + ids: HashMap, i64)>, + prune_at: usize, +} + +impl PlanIds { + const MIN_PRUNE_AT: usize = 1024; + + pub(crate) fn new() -> Self { + PlanIds { + nodes: Mutex::new(PlanIdNodes { + next: 1, + ids: HashMap::new(), + prune_at: Self::MIN_PRUNE_AT, + }), + } + } + + /// The plan ID of `node`, assigned the first time it is serialized. + pub(crate) fn id_of(&self, node: &Arc) -> i64 { + let key = Arc::as_ptr(node) as usize; + let mut nodes = self.nodes.lock().unwrap_or_else(|e| e.into_inner()); + // The entry's `Weak` keeps its allocation from being freed, so a live node at this + // address is the node the entry was made for. + if let Some((_, id)) = nodes.ids.get(&key) { + return *id; + } + if nodes.ids.len() >= nodes.prune_at { + nodes.ids.retain(|_, (node, _)| node.strong_count() > 0); + nodes.prune_at = (nodes.ids.len() * 2).max(Self::MIN_PRUNE_AT); + } + let id = nodes.next; + nodes.next += 1; + nodes.ids.insert(key, (Arc::downgrade(node), id)); + id + } + + #[cfg(test)] + pub(crate) fn len(&self) -> usize { + self.nodes.lock().unwrap().ids.len() + } +} + +/// Serialize a shared plan node for a request: the node and every node below it carry +/// their stable plan IDs. +pub(crate) fn relation_with_ids(node: &Arc, ids: &PlanIds) -> proto::Relation { + relation_of(node, Some(ids)) +} + +/// Serialize a child node, with its stable plan ID when `ids` is given. +fn relation_of(node: &Arc, ids: Option<&PlanIds>) -> proto::Relation { + let mut relation = node.to_proto_with(ids); + if let Some(ids) = ids { + relation.common.get_or_insert_with(Default::default).plan_id = Some(ids.id_of(node)); + } + relation +} + impl LogicalPlan { - /// Convert this logical plan to a `spark.connect.Relation` protobuf. + /// Convert this logical plan to a `spark.connect.Relation` protobuf, without plan IDs. pub fn to_proto(&self) -> proto::Relation { + self.to_proto_with(None) + } + + fn to_proto_with(&self, plan_ids: Option<&PlanIds>) -> proto::Relation { let mut relation = proto::Relation::default(); relation.common = Some(proto::RelationCommon::default()); @@ -448,7 +525,7 @@ impl LogicalPlan { LogicalPlan::Project { input, columns } => { let mut project = proto::Project::default(); - project.input = Some(Box::new(input.to_proto())); + project.input = Some(Box::new(relation_of(input, plan_ids))); for col in columns { project.expressions.push(col.to_proto()); } @@ -457,7 +534,7 @@ impl LogicalPlan { LogicalPlan::Filter { input, condition } => { let mut filter = proto::Filter::default(); - filter.input = Some(Box::new(input.to_proto())); + filter.input = Some(Box::new(relation_of(input, plan_ids))); filter.condition = Some(condition.to_proto()); relation.rel_type = Some(proto::relation::RelType::Filter(Box::new(filter))); } @@ -472,7 +549,7 @@ impl LogicalPlan { grouping_sets, } => { let mut agg = proto::Aggregate::default(); - agg.input = Some(Box::new(input.to_proto())); + agg.input = Some(Box::new(relation_of(input, plan_ids))); agg.group_type = match group_type { AggregateGroupType::GroupBy => proto::aggregate::GroupType::Groupby as i32, AggregateGroupType::Rollup => proto::aggregate::GroupType::Rollup as i32, @@ -523,8 +600,8 @@ impl LogicalPlan { using_columns, } => { let mut join = proto::Join::default(); - join.left = Some(Box::new(left.to_proto())); - join.right = Some(Box::new(right.to_proto())); + join.left = Some(Box::new(relation_of(left, plan_ids))); + join.right = Some(Box::new(relation_of(right, plan_ids))); join.join_type = match join_type { JoinType::Inner => proto::join::JoinType::Inner as i32, JoinType::LeftOuter => proto::join::JoinType::LeftOuter as i32, @@ -548,8 +625,8 @@ impl LogicalPlan { on, } => { let mut lj = proto::LateralJoin::default(); - lj.left = Some(Box::new(left.to_proto())); - lj.right = Some(Box::new(right.to_proto())); + lj.left = Some(Box::new(relation_of(left, plan_ids))); + lj.right = Some(Box::new(relation_of(right, plan_ids))); lj.join_type = match join_type { JoinType::Inner => proto::join::JoinType::Inner as i32, JoinType::LeftOuter => proto::join::JoinType::LeftOuter as i32, @@ -574,8 +651,8 @@ impl LogicalPlan { allow_missing_columns, } => { let mut set_op = proto::SetOperation::default(); - set_op.left_input = Some(Box::new(left.to_proto())); - set_op.right_input = Some(Box::new(right.to_proto())); + set_op.left_input = Some(Box::new(relation_of(left, plan_ids))); + set_op.right_input = Some(Box::new(relation_of(right, plan_ids))); set_op.set_op_type = match set_op_type { SetOpType::Union => proto::set_operation::SetOpType::Union as i32, SetOpType::Intersect => proto::set_operation::SetOpType::Intersect as i32, @@ -589,21 +666,21 @@ impl LogicalPlan { LogicalPlan::Limit { input, limit } => { let mut lim = proto::Limit::default(); - lim.input = Some(Box::new(input.to_proto())); + lim.input = Some(Box::new(relation_of(input, plan_ids))); lim.limit = *limit; relation.rel_type = Some(proto::relation::RelType::Limit(Box::new(lim))); } LogicalPlan::Offset { input, offset } => { let mut off = proto::Offset::default(); - off.input = Some(Box::new(input.to_proto())); + off.input = Some(Box::new(relation_of(input, plan_ids))); off.offset = *offset; relation.rel_type = Some(proto::relation::RelType::Offset(Box::new(off))); } LogicalPlan::Tail { input, limit } => { let mut tail = proto::Tail::default(); - tail.input = Some(Box::new(input.to_proto())); + tail.input = Some(Box::new(relation_of(input, plan_ids))); tail.limit = *limit; relation.rel_type = Some(proto::relation::RelType::Tail(Box::new(tail))); } @@ -615,7 +692,7 @@ impl LogicalPlan { within_watermark, } => { let mut dedup = proto::Deduplicate::default(); - dedup.input = Some(Box::new(input.to_proto())); + dedup.input = Some(Box::new(relation_of(input, plan_ids))); dedup.all_columns_as_keys = Some(*all_columns_as_keys); dedup.column_names.extend(column_names.clone()); dedup.within_watermark = Some(*within_watermark); @@ -631,7 +708,7 @@ impl LogicalPlan { // numbers, so a future proto reorder can't silently change meaning. use proto::expression::sort_order::{NullOrdering as PbNulls, SortDirection}; let mut sort = proto::Sort::default(); - sort.input = Some(Box::new(input.to_proto())); + sort.input = Some(Box::new(relation_of(input, plan_ids))); for expr in order { // Extract SortOrder from Expression if it's SortOrder type if let ExpressionNode::SortOrder(so) = expr.node() { @@ -680,7 +757,7 @@ impl LogicalPlan { seed, } => { let mut sample = proto::Sample::default(); - sample.input = Some(Box::new(input.to_proto())); + sample.input = Some(Box::new(relation_of(input, plan_ids))); sample.lower_bound = *lower_bound; sample.upper_bound = *upper_bound; sample.with_replacement = Some(*with_replacement); @@ -696,7 +773,7 @@ impl LogicalPlan { shuffle, } => { let mut repart = proto::Repartition::default(); - repart.input = Some(Box::new(input.to_proto())); + repart.input = Some(Box::new(relation_of(input, plan_ids))); repart.num_partitions = *num_partitions; repart.shuffle = Some(*shuffle); relation.rel_type = Some(proto::relation::RelType::Repartition(Box::new(repart))); @@ -708,7 +785,7 @@ impl LogicalPlan { expressions, } => { let mut repart = proto::RepartitionByExpression::default(); - repart.input = Some(Box::new(input.to_proto())); + repart.input = Some(Box::new(relation_of(input, plan_ids))); // num_partitions is optional in the proto: <= 0 means "unset" so the // server uses the default (the `df.repartition(*cols)` column-only form, // where no partition count is given). A real count is always positive. @@ -731,7 +808,7 @@ impl LogicalPlan { columns, } => { let mut wc = proto::WithColumns::default(); - wc.input = Some(Box::new(input.to_proto())); + wc.input = Some(Box::new(relation_of(input, plan_ids))); for (name, col) in column_names.iter().zip(columns.iter()) { // Create an Alias expression for each column let mut alias = proto::expression::Alias::default(); @@ -748,7 +825,7 @@ impl LogicalPlan { metadata_json, } => { let mut wc = proto::WithColumns::default(); - wc.input = Some(Box::new(input.to_proto())); + wc.input = Some(Box::new(relation_of(input, plan_ids))); let col_expr = crate::expression::Expression::column(column_name.clone()); let mut alias = proto::expression::Alias::default(); alias.expr = Some(Box::new(col_expr.to_proto())); @@ -760,7 +837,7 @@ impl LogicalPlan { LogicalPlan::WithColumnsRenamed { input, renames } => { let mut wcr = proto::WithColumnsRenamed::default(); - wcr.input = Some(Box::new(input.to_proto())); + wcr.input = Some(Box::new(relation_of(input, plan_ids))); for (old_name, new_name) in renames.iter() { let mut rename = proto::with_columns_renamed::Rename::default(); rename.col_name = old_name.clone(); @@ -773,7 +850,7 @@ impl LogicalPlan { LogicalPlan::Drop { input, columns } => { let mut drop = proto::Drop::default(); - drop.input = Some(Box::new(input.to_proto())); + drop.input = Some(Box::new(relation_of(input, plan_ids))); drop.column_names.extend(columns.clone()); relation.rel_type = Some(proto::relation::RelType::Drop(Box::new(drop))); } @@ -783,14 +860,14 @@ impl LogicalPlan { column_names, } => { let mut to_df = proto::ToDf::default(); - to_df.input = Some(Box::new(input.to_proto())); + to_df.input = Some(Box::new(relation_of(input, plan_ids))); to_df.column_names.extend(column_names.clone()); relation.rel_type = Some(proto::relation::RelType::ToDf(Box::new(to_df))); } LogicalPlan::ToSchema { input, schema } => { let mut to_schema = proto::ToSchema::default(); - to_schema.input = Some(Box::new(input.to_proto())); + to_schema.input = Some(Box::new(relation_of(input, plan_ids))); to_schema.schema = Some(schema.to_proto()); relation.rel_type = Some(proto::relation::RelType::ToSchema(Box::new(to_schema))); } @@ -801,7 +878,7 @@ impl LogicalPlan { parameters, } => { let mut hint = proto::Hint::default(); - hint.input = Some(Box::new(input.to_proto())); + hint.input = Some(Box::new(relation_of(input, plan_ids))); hint.name = name.clone(); // Hint parameters are literal Expressions (e.g. `REPARTITION 10`). // An integer-looking parameter becomes an Integer (int32) literal - @@ -827,7 +904,7 @@ impl LogicalPlan { value_column_name, } => { let mut unpivot = proto::Unpivot::default(); - unpivot.input = Some(Box::new(input.to_proto())); + unpivot.input = Some(Box::new(relation_of(input, plan_ids))); for col in ids { unpivot.ids.push(col.to_proto()); } @@ -849,7 +926,7 @@ impl LogicalPlan { columns, } => { let mut na_fill = proto::NaFill::default(); - na_fill.input = Some(Box::new(input.to_proto())); + na_fill.input = Some(Box::new(relation_of(input, plan_ids))); na_fill.cols.extend(columns.clone()); na_fill.values.push(value_to_proto_literal(fill_value)); relation.rel_type = Some(proto::relation::RelType::FillNa(Box::new(na_fill))); @@ -863,7 +940,7 @@ impl LogicalPlan { // Per-column fill: cols[i] is filled with values[i] (Spark aligns them // positionally when both are non-empty). let mut na_fill = proto::NaFill::default(); - na_fill.input = Some(Box::new(input.to_proto())); + na_fill.input = Some(Box::new(relation_of(input, plan_ids))); na_fill.cols.extend(cols.clone()); na_fill .values @@ -878,7 +955,7 @@ impl LogicalPlan { columns, } => { let mut na_drop = proto::NaDrop::default(); - na_drop.input = Some(Box::new(input.to_proto())); + na_drop.input = Some(Box::new(relation_of(input, plan_ids))); na_drop.cols.extend(columns.clone()); // Mirror pyspark: an explicit `thresh` (min_non_null) wins; otherwise // `how="all"` -> min_non_nulls=1 and `how="any"` -> unset (server default). @@ -898,7 +975,7 @@ impl LogicalPlan { columns, } => { let mut na_replace = proto::NaReplace::default(); - na_replace.input = Some(Box::new(input.to_proto())); + na_replace.input = Some(Box::new(relation_of(input, plan_ids))); for (old_val, new_val) in replacements.iter() { let mut replace = proto::na_replace::Replacement::default(); // A numeric-looking value becomes a Double literal; otherwise a @@ -914,14 +991,14 @@ impl LogicalPlan { LogicalPlan::Describe { input, columns } => { let mut describe = proto::StatDescribe::default(); - describe.input = Some(Box::new(input.to_proto())); + describe.input = Some(Box::new(relation_of(input, plan_ids))); describe.cols.extend(columns.clone()); relation.rel_type = Some(proto::relation::RelType::Describe(Box::new(describe))); } LogicalPlan::Summary { input, percentiles } => { let mut summary = proto::StatSummary::default(); - summary.input = Some(Box::new(input.to_proto())); + summary.input = Some(Box::new(relation_of(input, plan_ids))); summary.statistics.extend(percentiles.clone()); relation.rel_type = Some(proto::relation::RelType::Summary(Box::new(summary))); } @@ -929,7 +1006,7 @@ impl LogicalPlan { LogicalPlan::ColRegex { input, col_name } => { // ColRegex is implemented as a Project with UnresolvedRegex expressions let mut project = proto::Project::default(); - project.input = Some(Box::new(input.to_proto())); + project.input = Some(Box::new(relation_of(input, plan_ids))); let mut expr = proto::Expression::default(); expr.expr_type = Some(proto::expression::ExprType::UnresolvedRegex( proto::expression::UnresolvedRegex { @@ -943,7 +1020,7 @@ impl LogicalPlan { LogicalPlan::SubqueryAlias { input, alias } => { let mut sq_alias = proto::SubqueryAlias::default(); - sq_alias.input = Some(Box::new(input.to_proto())); + sq_alias.input = Some(Box::new(relation_of(input, plan_ids))); sq_alias.alias = alias.clone(); relation.rel_type = Some(proto::relation::RelType::SubqueryAlias(Box::new(sq_alias))); @@ -1042,7 +1119,7 @@ impl LogicalPlan { delay_threshold, } => { let mut watermark = proto::WithWatermark::default(); - watermark.input = Some(Box::new(input.to_proto())); + watermark.input = Some(Box::new(relation_of(input, plan_ids))); watermark.event_time = time_column.clone(); watermark.delay_threshold = delay_threshold.clone(); relation.rel_type = @@ -1055,7 +1132,7 @@ impl LogicalPlan { partition_exprs, } => { let mut repart = proto::RepartitionByExpression::default(); - repart.input = Some(Box::new(input.to_proto())); + repart.input = Some(Box::new(relation_of(input, plan_ids))); if let Some(n) = num_partitions { repart.num_partitions = Some(*n); } @@ -1069,7 +1146,7 @@ impl LogicalPlan { LogicalPlan::StatCrosstab { input, col1, col2 } => { let mut stat = proto::StatCrosstab::default(); - stat.input = Some(Box::new(input.to_proto())); + stat.input = Some(Box::new(relation_of(input, plan_ids))); stat.col1 = col1.clone(); stat.col2 = col2.clone(); relation.rel_type = Some(proto::relation::RelType::Crosstab(Box::new(stat))); @@ -1081,7 +1158,7 @@ impl LogicalPlan { support, } => { let mut stat = proto::StatFreqItems::default(); - stat.input = Some(Box::new(input.to_proto())); + stat.input = Some(Box::new(relation_of(input, plan_ids))); stat.cols.extend(columns.clone()); stat.support = Some(*support); relation.rel_type = Some(proto::relation::RelType::FreqItems(Box::new(stat))); @@ -1094,7 +1171,7 @@ impl LogicalPlan { relative_error, } => { let mut stat = proto::StatApproxQuantile::default(); - stat.input = Some(Box::new(input.to_proto())); + stat.input = Some(Box::new(relation_of(input, plan_ids))); stat.cols.extend(columns.clone()); stat.probabilities.extend(probabilities.clone()); stat.relative_error = *relative_error; @@ -1103,7 +1180,7 @@ impl LogicalPlan { LogicalPlan::StatCorr { input, col1, col2 } => { let mut stat = proto::StatCorr::default(); - stat.input = Some(Box::new(input.to_proto())); + stat.input = Some(Box::new(relation_of(input, plan_ids))); stat.col1 = col1.clone(); stat.col2 = col2.clone(); relation.rel_type = Some(proto::relation::RelType::Corr(Box::new(stat))); @@ -1111,7 +1188,7 @@ impl LogicalPlan { LogicalPlan::StatCov { input, col1, col2 } => { let mut stat = proto::StatCov::default(); - stat.input = Some(Box::new(input.to_proto())); + stat.input = Some(Box::new(relation_of(input, plan_ids))); stat.col1 = col1.clone(); stat.col2 = col2.clone(); relation.rel_type = Some(proto::relation::RelType::Cov(Box::new(stat))); @@ -1124,7 +1201,7 @@ impl LogicalPlan { seed, } => { let mut stat = proto::StatSampleBy::default(); - stat.input = Some(Box::new(input.to_proto())); + stat.input = Some(Box::new(relation_of(input, plan_ids))); // col field expects an Expression; create one from the column name let mut col_expr = proto::Expression::default(); col_expr.expr_type = Some(proto::expression::ExprType::UnresolvedAttribute( @@ -1153,7 +1230,7 @@ impl LogicalPlan { LogicalPlan::Observe { input, name, exprs } => { let mut collect_metrics = proto::CollectMetrics::default(); - collect_metrics.input = Some(Box::new(input.to_proto())); + collect_metrics.input = Some(Box::new(relation_of(input, plan_ids))); collect_metrics.name = name.clone(); for expr in exprs { collect_metrics.metrics.push(expr.to_proto()); @@ -1175,8 +1252,8 @@ impl LogicalPlan { LogicalPlan::Zip { left, right } => { let mut zip = proto::Zip::default(); - zip.left = Some(Box::new(left.to_proto())); - zip.right = Some(Box::new(right.to_proto())); + zip.left = Some(Box::new(relation_of(left, plan_ids))); + zip.right = Some(Box::new(relation_of(right, plan_ids))); relation.rel_type = Some(proto::relation::RelType::Zip(Box::new(zip))); } @@ -1192,7 +1269,7 @@ impl LogicalPlan { index_columns, } => { let mut transpose = proto::Transpose::default(); - transpose.input = Some(Box::new(input.to_proto())); + transpose.input = Some(Box::new(relation_of(input, plan_ids))); transpose.index_columns = index_columns.iter().map(|e| e.to_proto()).collect(); relation.rel_type = Some(proto::relation::RelType::Transpose(Box::new(transpose))); return relation; @@ -1203,7 +1280,7 @@ impl LogicalPlan { is_barrier, } => { let mut mp = proto::MapPartitions::default(); - mp.input = Some(Box::new(input.to_proto())); + mp.input = Some(Box::new(relation_of(input, plan_ids))); mp.func = Some(func.to_proto()); mp.is_barrier = Some(*is_barrier); relation.rel_type = Some(proto::relation::RelType::MapPartitions(Box::new(mp))); @@ -1222,7 +1299,7 @@ impl LogicalPlan { transform_with_state_info, } => { let mut gm = proto::GroupMap::default(); - gm.input = Some(Box::new(input.to_proto())); + gm.input = Some(Box::new(relation_of(input, plan_ids))); for e in grouping_expressions { gm.grouping_expressions.push(e.to_proto()); } @@ -1231,7 +1308,7 @@ impl LogicalPlan { gm.sorting_expressions.push(e.to_proto()); } if let Some(ii) = initial_input { - gm.initial_input = Some(Box::new(ii.to_proto())); + gm.initial_input = Some(Box::new(relation_of(ii, plan_ids))); } for e in initial_grouping_expressions { gm.initial_grouping_expressions.push(e.to_proto()); @@ -1258,11 +1335,11 @@ impl LogicalPlan { func, } => { let mut cg = proto::CoGroupMap::default(); - cg.input = Some(Box::new(input.to_proto())); + cg.input = Some(Box::new(relation_of(input, plan_ids))); for e in input_grouping_expressions { cg.input_grouping_expressions.push(e.to_proto()); } - cg.other = Some(Box::new(other.to_proto())); + cg.other = Some(Box::new(relation_of(other, plan_ids))); for e in other_grouping_expressions { cg.other_grouping_expressions.push(e.to_proto()); } @@ -1279,8 +1356,8 @@ impl LogicalPlan { direction, } => { let mut nbj = proto::NearestByJoin::default(); - nbj.left = Some(Box::new(left.to_proto())); - nbj.right = Some(Box::new(right.to_proto())); + nbj.left = Some(Box::new(relation_of(left, plan_ids))); + nbj.right = Some(Box::new(relation_of(right, plan_ids))); nbj.ranking_expression = Some(ranking_expression.to_proto()); nbj.num_results = *num_results; nbj.join_type = join_type.clone(); diff --git a/crates/spark-connect/src/readwriter.rs b/crates/spark-connect/src/readwriter.rs index f103bb1a..ad08c94d 100644 --- a/crates/spark-connect/src/readwriter.rs +++ b/crates/spark-connect/src/readwriter.rs @@ -443,7 +443,7 @@ impl DataFrameWriter { save_type: Option, ) -> Result { let mut op = proto::WriteOperation::default(); - op.input = Some(build_input_relation(&self.input_plan, &self.session)?); + op.input = Some(build_input_relation(&self.input_plan, &self.session)); op.source = self.format.clone(); op.mode = self.mode.to_proto(); op.sort_column_names = self.sort_cols.clone(); @@ -612,7 +612,7 @@ impl DataFrameWriterV2 { overwrite_condition: Option, ) -> Result { let mut op = proto::WriteOperationV2::default(); - op.input = Some(build_input_relation(&self.input_plan, &self.session)?); + op.input = Some(build_input_relation(&self.input_plan, &self.session)); op.table_name = self.table_name.clone(); op.provider = self.provider.clone(); op.partitioning_columns = self.partition_cols.iter().map(|c| c.to_proto()).collect(); diff --git a/crates/spark-connect/src/session.rs b/crates/spark-connect/src/session.rs index 7a3cdc6a..5927b348 100644 --- a/crates/spark-connect/src/session.rs +++ b/crates/spark-connect/src/session.rs @@ -2,7 +2,7 @@ //! //! Provides the entry point for DataFrame operations and SQL queries. -use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use spark_connect_core::channel::ChannelBuilder; @@ -12,7 +12,7 @@ use spark_connect_core::runtime::block_on; use spark_connect_proto as proto; use crate::dataframe::DataFrame; -use crate::plan::LogicalPlan; +use crate::plan::{LogicalPlan, PlanIds}; use crate::profiler::ProfilerCollector; use crate::row::{Row, Value}; use crate::types::DataType; @@ -44,8 +44,8 @@ pub struct ExecutionInfo { pub struct SparkSession { /// Shared gRPC client client: Arc, - /// Plan ID counter for unique IDs in nested plans - plan_id_counter: Arc, + /// Stable plan IDs for the plan nodes this session serializes. + plan_ids: Arc, /// Operation tags attached to every ExecutePlan request from this session. /// Mirrors `pyspark.sql.connect.session.SparkSession`'s addTag/removeTag/getTags. tags: Arc>>, @@ -69,7 +69,7 @@ impl SparkSession { fn new(client: SparkConnectClient) -> Self { SparkSession { client: Arc::new(client), - plan_id_counter: Arc::new(AtomicI64::new(1)), + plan_ids: Arc::new(PlanIds::new()), tags: Arc::new(Mutex::new(Vec::new())), last_execution: Arc::new(Mutex::new(None)), progress_handlers: Arc::new(Mutex::new(Vec::new())), @@ -197,11 +197,11 @@ impl SparkSession { /// Start a brand-new session over the same connection. Mirrors /// `SparkSession.newSession` — a fresh server-side session (new session id), - /// with its own tags and plan-id counter. + /// with its own tags and plan IDs. pub fn new_session(&self) -> SparkSession { SparkSession { client: Arc::new(self.client.with_new_session_id()), - plan_id_counter: Arc::new(AtomicI64::new(1)), + plan_ids: Arc::new(PlanIds::new()), tags: Arc::new(Mutex::new(Vec::new())), last_execution: Arc::new(Mutex::new(None)), progress_handlers: Arc::new(Mutex::new(Vec::new())), @@ -217,9 +217,9 @@ impl SparkSession { self.new_session() } - /// Get the next plan ID (atomic post-increment). - pub(crate) fn next_plan_id(&self) -> i64 { - self.plan_id_counter.fetch_add(1, Ordering::SeqCst) + /// The stable plan IDs of the plan nodes this session serializes. + pub(crate) fn plan_ids(&self) -> &PlanIds { + &self.plan_ids } /// Get the underlying client. @@ -598,7 +598,7 @@ impl Clone for SparkSession { fn clone(&self) -> Self { SparkSession { client: Arc::clone(&self.client), - plan_id_counter: Arc::clone(&self.plan_id_counter), + plan_ids: Arc::clone(&self.plan_ids), tags: Arc::clone(&self.tags), last_execution: Arc::clone(&self.last_execution), progress_handlers: Arc::clone(&self.progress_handlers), diff --git a/crates/spark-connect/src/streaming.rs b/crates/spark-connect/src/streaming.rs index 9a033233..18b54949 100644 --- a/crates/spark-connect/src/streaming.rs +++ b/crates/spark-connect/src/streaming.rs @@ -375,7 +375,10 @@ impl DataStreamWriter { /// Internal method to build and execute the write stream command. fn _start_internal(self) -> Result { let mut write_op = proto::WriteStreamOperationStart::default(); - write_op.input = Some(self.plan.to_proto()); + write_op.input = Some(crate::plan::relation_with_ids( + &self.plan, + self.session.plan_ids(), + )); if let Some(fmt) = &self.format { write_op.format = fmt.clone(); diff --git a/crates/spark-connect/tests/e2e_ops_breadth.rs b/crates/spark-connect/tests/e2e_ops_breadth.rs index 58db3506..d3da8c01 100644 --- a/crates/spark-connect/tests/e2e_ops_breadth.rs +++ b/crates/spark-connect/tests/e2e_ops_breadth.rs @@ -1,6 +1,6 @@ //! Server-gated breadth coverage: build DataFrames with (nearly) every operation and -//! execute them, so the per-operation execution methods and the recursive -//! `assign_plan_ids` match over `RelType` variants in `dataframe.rs` actually run. +//! execute them, so the per-operation execution methods and the serialization of +//! every plan node, with its plan ID, actually run. //! Correctness is asserted elsewhere; here the point is to exercise the reachable //! execution surface, so most calls are resilient (`let _ = ...`) and a few safe ones //! are asserted. Run with SPARK_REMOTE set. @@ -35,8 +35,8 @@ fn expr(c: spark_connect::column::Column) -> spark_connect::expression::Expressi } /// Relational transforms: project, filter, joins, set ops, aggregates, sort, slicing, -/// dedup, repartition, sample, reshape. Each is collected so `assign_plan_ids` walks -/// the corresponding `RelType` arm. +/// dedup, repartition, sample, reshape. Each is collected so the corresponding plan +/// node is serialized with its plan ID. #[test] fn transforms_execute() { if !should_run() { diff --git a/docs/architecture.md b/docs/architecture.md index 32d6168d..8405256e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -110,6 +110,18 @@ last response it saw, so long results survive transient disconnects. Like the reference client, it releases each response it has read, and the whole execution once the result is complete, so the server does not keep them. +Each plan node gets its plan ID once, the first time it is sent, and keeps it, as +in the reference client. Re-executing a DataFrame therefore sends the same +relation, and a DataFrame built on another keeps that input's plan IDs, so the +server's per-session plan cache (`spark.connect.session.planCache.*`) can reuse +the plans it already resolved instead of resolving them again. As with the +reference client, a re-executed DataFrame can then use its earlier resolution. +A DataFrame from `spark.read.parquet(path)`, for example, keeps the file listing +it resolved: once the files under `path` are overwritten, running it again fails +with `FAILED_READ_FILE.FILE_NOT_EXIST`, while a new `spark.read.parquet(path)` +reads the new files. Setting `spark.connect.session.planCache.enabled` to `false` +turns the cache off, on the server or for a session with `spark.conf.set`. + ### Python Arrow table handoff `toArrow()` collects decoded batches and uses arrow-rs's `IntoPyArrow` for one