From 0fcf57e312dee049a7ed99c62c5b22bf108a5378 Mon Sep 17 00:00:00 2001 From: Dustin Smith Date: Sun, 15 Nov 2026 00:13:23 +0700 Subject: [PATCH] test: check that a bucketed native scan keeps its partitioning The bucketed DPP test from #6268 checks for one sort-merge join and no shuffle, but Spark plans those shuffles before Comet replaces the scan, so it still passed if the bucketed scan stopped reporting its bucket layout. Assert the scan's HashPartitioning on the bucket column directly. --- .../org/apache/comet/exec/CometExecSuite.scala | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 7b30248b833..36bde25b94e 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -32,9 +32,10 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogStatistics, CatalogTable} -import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression, ExpressionInfo, Hex, Literal} +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, DynamicPruningExpression, Expression, ExpressionInfo, Hex, Literal} import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateMode, BloomFilterAggregate, Final} import org.apache.spark.sql.catalyst.plans.logical.Expand +import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning import org.apache.spark.sql.comet._ import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, CometNativeShuffle, CometShuffleExchangeExec} import org.apache.spark.sql.connector.catalog.InMemoryTableCatalog @@ -1422,6 +1423,16 @@ class CometExecSuite extends CometTestBase { assert( dppScans.head.bucketedScan, s"Expected a bucketed scan:\n${cometPlan.treeString}") + // Spark plans the join's shuffles before Comet replaces the scan, so the plan shape + // above holds even if the scan stops reporting its bucket layout. Check it directly. + val partitioning = dppScans.head.outputPartitioning + val isBucketLayout = partitioning match { + case HashPartitioning(Seq(key: AttributeReference), 4) => key.name == "k" + case _ => false + } + assert( + isBucketLayout, + s"Expected HashPartitioning(k, 4) from the bucketed scan, got $partitioning") assertAqeDppShape(cometPlan, expectedCometSubqueryBroadcasts = Some(1)) // The dimension filter keeps store 3 only, so DPP reads 1 of the 10 partitions. assert(