Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
366 changes: 102 additions & 264 deletions crates/spark-connect/src/dataframe.rs

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion crates/spark-connect/src/merge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ impl MergeIntoWriter {
pub fn merge(self) -> Result<()> {
let mut cmd = proto::MergeIntoTableCommand::default();
cmd.target_table_name = self.target_table;
cmd.source_table_plan = Some(build_input_relation(&self.source_plan, &self.session)?);
cmd.source_table_plan = Some(build_input_relation(&self.source_plan, &self.session));
cmd.merge_condition = Some(self.condition.to_proto());
cmd.match_actions = self.matched_actions;
cmd.not_matched_actions = self.not_matched_actions;
Expand Down
20 changes: 10 additions & 10 deletions crates/spark-connect/src/ml.rs
Original file line number Diff line number Diff line change
Expand Up @@ -257,12 +257,12 @@ pub trait Transformer: Send + Sync {
}

/// Build the MlRelation for this transformation.
fn build_ml_relation(&self, input_plan: &LogicalPlan) -> proto::MlRelation {
fn build_ml_relation(&self, input: &DataFrame) -> proto::MlRelation {
let mut transform = proto::ml_relation::Transform::default();
transform.operator = Some(proto::ml_relation::transform::Operator::Transformer(
self.operator().to_proto(),
));
transform.input = Some(Box::new(input_plan.to_proto()));
transform.input = Some(Box::new(input.relation()));
transform.params = Some(self.params().to_proto());

let mut ml_relation = proto::MlRelation::default();
Expand Down Expand Up @@ -304,7 +304,7 @@ fn evaluate_via_command(
params: &Params,
df: &DataFrame,
) -> spark_connect_core::error::Result<f64> {
let dataset = crate::dataframe::build_input_relation(&df.plan, &df.session)?;
let dataset = df.relation();
let evaluate = proto::ml_command::Evaluate {
evaluator: Some(operator.to_proto()),
params: Some(params.to_proto()),
Expand Down Expand Up @@ -448,7 +448,7 @@ impl Transformer for StandardScalerModel {

fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result<DataFrame> {
// Build an MlRelation for transformation
let ml_relation = self.build_ml_relation(&df.plan);
let ml_relation = self.build_ml_relation(df);

// Wrap it in a Relation
let mut relation = proto::Relation::default();
Expand Down Expand Up @@ -539,7 +539,7 @@ impl Transformer for VectorAssembler {
}

fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result<DataFrame> {
let ml_relation = self.build_ml_relation(&df.plan);
let ml_relation = self.build_ml_relation(df);
let mut relation = proto::Relation::default();
relation.common = Some(proto::RelationCommon::default());
relation.rel_type = Some(proto::relation::RelType::MlRelation(Box::new(ml_relation)));
Expand Down Expand Up @@ -658,7 +658,7 @@ impl Transformer for StringIndexerModel {
}

fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result<DataFrame> {
let ml_relation = self.build_ml_relation(&df.plan);
let ml_relation = self.build_ml_relation(df);
let mut relation = proto::Relation::default();
relation.common = Some(proto::RelationCommon::default());
relation.rel_type = Some(proto::relation::RelType::MlRelation(Box::new(ml_relation)));
Expand Down Expand Up @@ -783,7 +783,7 @@ impl Transformer for MaxAbsScalerModel {
}

fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result<DataFrame> {
let ml_relation = self.build_ml_relation(&df.plan);
let ml_relation = self.build_ml_relation(df);
let mut relation = proto::Relation::default();
relation.common = Some(proto::RelationCommon::default());
relation.rel_type = Some(proto::relation::RelType::MlRelation(Box::new(ml_relation)));
Expand Down Expand Up @@ -934,7 +934,7 @@ impl Transformer for LogisticRegressionModel {
}

fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result<DataFrame> {
let ml_relation = self.build_ml_relation(&df.plan);
let ml_relation = self.build_ml_relation(df);
let mut relation = proto::Relation::default();
relation.common = Some(proto::RelationCommon::default());
relation.rel_type = Some(proto::relation::RelType::MlRelation(Box::new(ml_relation)));
Expand Down Expand Up @@ -1194,7 +1194,7 @@ impl Transformer for PipelineModel {
}

fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result<DataFrame> {
let ml_relation = self.build_ml_relation(&df.plan);
let ml_relation = self.build_ml_relation(df);
let mut relation = proto::Relation::default();
relation.common = Some(proto::RelationCommon::default());
relation.rel_type = Some(proto::relation::RelType::MlRelation(Box::new(ml_relation)));
Expand Down Expand Up @@ -1383,7 +1383,7 @@ impl Transformer for CrossValidatorModel {
&mut self.params
}
fn transform_impl(&mut self, df: &DataFrame) -> spark_connect_core::error::Result<DataFrame> {
let ml_relation = self.build_ml_relation(&df.plan);
let ml_relation = self.build_ml_relation(df);
let mut relation = proto::Relation::default();
relation.common = Some(proto::RelationCommon::default());
relation.rel_type = Some(proto::relation::RelType::MlRelation(Box::new(ml_relation)));
Expand Down
5 changes: 2 additions & 3 deletions crates/spark-connect/src/pipelines.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use spark_connect_core::error::{Result, SparkError};
use spark_connect_core::runtime::block_on;
use spark_connect_proto as proto;

use crate::dataframe::{assign_plan_ids, execute_command_collect, DataFrame};
use crate::dataframe::{execute_command_collect, DataFrame};
use crate::session::SparkSession;
use crate::types::DataType;

Expand Down Expand Up @@ -149,8 +149,7 @@ pub fn define_flow(
sql_conf: HashMap<String, String>,
source_code_location: Option<proto::SourceCodeLocation>,
) -> Result<()> {
let mut relation = relation_df.plan.to_proto();
assign_plan_ids(&mut relation, &relation_df.session)?;
let relation = relation_df.relation();
let details =
define_flow::Details::RelationFlowDetails(define_flow::WriteRelationFlowDetails {
relation: Some(relation),
Expand Down
Loading
Loading