Repository navigation
fix: move native shuffle over Spark's RowToColumnar to the JVM columnar shuffle #7704
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
56ca6b2
89964fb
f674109
bf78701
f9a7a65
2e47eaa
75ec828
d731a15
961c4d8
eb80d82
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -25,7 +25,7 @@ import org.apache.spark.sql.SparkSession | |
| import org.apache.spark.sql.catalyst.rules.Rule | ||
| import org.apache.spark.sql.catalyst.util.sideBySide | ||
| import org.apache.spark.sql.comet.{CometCollectLimitExec, CometColumnarToRowExec, CometIcebergWriteExec, CometMapInBatchExec, CometNativeWriteExec, CometPlan, CometRangeExec, CometSparkToColumnarExec} | ||
| import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, CometShuffleExchangeExec} | ||
| import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, CometNativeShuffle, CometShuffleExchangeExec} | ||
| import org.apache.spark.sql.comet.shims.{MapInBatchInfo, ShimCometMapInBatch} | ||
| import org.apache.spark.sql.comet.util.Utils.containsVariantType | ||
| import org.apache.spark.sql.execution.{ColumnarToRowExec, RowToColumnarExec, SparkPlan} | ||
|
|
@@ -176,6 +176,15 @@ case class EliminateRedundantTransitions(session: SparkSession) | |
| CometColumnarShuffle, | ||
| _) => | ||
| s.withNewChildren(Seq(child)) | ||
|
|
||
| // Spark also adds `RowToColumnar` even when comet chose native shuffle. | ||
| // Native shuffle reads only Comet vectors, not Spark's vectors. | ||
| // Switch to row and drop the transition. | ||
| case s @ CometShuffleExchangeExec(_, RowToColumnarExec(child), _, _, CometNativeShuffle, _) | ||
| if CometShuffleExchangeExec.columnarShuffleSupports(s, child) => | ||
|
Comment on lines
+183
to
+184
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. When |
||
| val columnar = s.copy(child = child, shuffleType = CometColumnarShuffle) | ||
| columnar.copyTagsFrom(s) | ||
| columnar | ||
|
Comment on lines
+185
to
+187
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The rewrite turns |
||
| } | ||
|
|
||
| eliminatedPlan match { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -575,6 +575,15 @@ object CometShuffleExchangeExec | |
| nativeShuffleFailureReasons(s).isEmpty && | ||
| !hashesDifferentlyFromSpark(s) | ||
|
|
||
| /** | ||
| * Whether the JVM columnar shuffle can take `s` with `rowChild`, a row-based plan, as its | ||
| * child. Pure: does not tag the node | ||
| */ | ||
| def columnarShuffleSupports(s: CometShuffleExchangeExec, rowChild: SparkPlan): Boolean = | ||
| !isCometCelebornShuffleManagerEnabled(s.conf) && | ||
| columnarShuffleFailureReasons( | ||
| ShuffleExchangeExec(s.outputPartitioning, rowChild, s.shuffleOrigin)).isEmpty | ||
|
Comment on lines
+582
to
+585
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can the |
||
|
|
||
| /** | ||
| * Whether the shuffle hashes into more than one partition a key that native shuffle, reading | ||
| * converted rows, may put in a different partition from Spark's partitioner. Native shuffle | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,9 +19,13 @@ | |
|
|
||
| package org.apache.comet.rules | ||
|
|
||
| import org.apache.spark.SparkException | ||
| import org.apache.spark.sql.CometTestBase | ||
| import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning | ||
| import org.apache.spark.sql.comet.{CometColumnarToRowExec, CometPlan} | ||
| import org.apache.spark.sql.execution.{ColumnarToRowExec, SparkPlan, UnionExec} | ||
| import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, CometNativeShuffle, CometShuffleExchangeExec} | ||
| import org.apache.spark.sql.execution.{ColumnarToRowExec, RowToColumnarExec, SparkPlan, UnionExec} | ||
| import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec | ||
|
|
||
| import org.apache.comet.CometConf | ||
|
|
||
|
|
@@ -64,4 +68,31 @@ class EliminateRedundantTransitionsSuite extends CometTestBase { | |
| s"Spark-only branch should be left untouched:\n${result.treeString}") | ||
| } | ||
| } | ||
|
|
||
| // A native shuffle reads Arrow-backed Comet vectors. If its child turns row-based after Comet | ||
| // chose native shuffle, Spark add its own `RowToColumnar` under it, whose vectors are unable | ||
| // to be read. | ||
| test("native shuffle over Spark's RowToColumnar moves to the JVM columnar shuffle") { | ||
| val rowChild = sparkPlanFor("SELECT id, CAST(id AS STRING) AS s FROM range(10)") | ||
| assert(!rowChild.supportsColumnar) | ||
| val partitioning = HashPartitioning(rowChild.output.take(1), 4) | ||
| val shuffle = CometShuffleExchangeExec( | ||
| partitioning, | ||
| RowToColumnarExec(rowChild), | ||
| ShuffleExchangeExec(partitioning, rowChild), | ||
| shuffleType = CometNativeShuffle) | ||
|
|
||
| val result = EliminateRedundantTransitions(spark).apply(shuffle) | ||
|
|
||
| result match { | ||
| case CometShuffleExchangeExec(_, child, _, _, CometColumnarShuffle, _) => | ||
| assert(child == rowChild, s"RowToColumnar should be removed:\n${result.treeString}") | ||
| case _ => | ||
| fail(s"Expected the JVM columnar shuffle:\n${result.treeString}") | ||
| } | ||
|
Comment on lines
+85
to
+92
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This test only takes the branch where |
||
| assert(result.executeColumnar().map(_.numRows()).sum() == 10) | ||
|
|
||
| val e = intercept[SparkException](shuffle.executeColumnar().count()) | ||
| assert(e.getMessage.contains("expects CometVector type"), e.getMessage) | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This adds a second way for a native exchange to end up as
CometColumnarExchange, and a second producer of the SparkRowToColumnarExecthat native shuffle cannot read. The input-contract paragraph indocs/source/contributor-guide/native_shuffle.md(just before "When Native Shuffle is Used") only describes theRevertNativeForTransitionHeavyStagescase, and the "When JVM Shuffle is Used" list injvm_shuffle.mddoes not have this one. Could you add it to both pages in this PR? Also, "Switch to the JVM columnar shuffle, which reads the child's rows" would be clearer here than "Switch to row".