From 48ee06f3fd28fa742fd53ec6575b8d748b10fb70 Mon Sep 17 00:00:00 2001 From: Steve Vaughan Jr Date: Mon, 16 Nov 2026 17:48:21 -0500 Subject: [PATCH 1/3] fix: keep a DPP build side's storage-partitioned join free of exchanges AQE plans a DPP subquery's converted build side again, and EnsureRequirements then puts an exchange under each side of a Comet join that Spark planned as a storage-partitioned join, because the join declares the original operator's required distribution since #7231. AQE has no logical link for those exchanges and asserts. Spark only accepts a key-grouped join as co-partitioned when the parent is its own SortMergeJoinExec or ShuffledHashJoinExec, and in Spark 4.1 when every leaf is also a KeyGroupedPartitionedScan. Reporting the scan's partitioning therefore cannot help a join. Declare no required distribution on CometSortMergeJoinExec and CometHashJoinExec when both sides are storage-partitioned. An aggregate above such a join does need the scan's partitioning, or the final aggregate gets an exchange inside an already serialized native block. So CometIcebergNativeScanExec now reports whatever the BatchScanExec it replaces reports, which generalizes #7424 from SinglePartition. Add a test for the join and one for an aggregate above it. Closes #7441. --- .../comet/CometIcebergNativeScanExec.scala | 25 ++-- .../apache/spark/sql/comet/operators.scala | 44 +++++- .../comet/CometIcebergNativeSuite.scala | 127 ++++++++++++++++++ 3 files changed, 183 insertions(+), 13 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergNativeScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergNativeScanExec.scala index e3466c71d27..1344652d9c5 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergNativeScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergNativeScanExec.scala @@ -26,7 +26,7 @@ import org.apache.spark.rdd.RDD import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, SortOrder} import org.apache.spark.sql.catalyst.plans.QueryPlan -import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, SinglePartition, UnknownPartitioning} +import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning} import org.apache.spark.sql.execution.{SparkPlan, SQLExecution} import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} @@ -136,16 +136,23 @@ case class CometIcebergNativeScanExec( // tasks here would run a placeholder subquery. At execution, CometNativeExec sizes the native // RDD from perPartitionData instead. // - // Spark 3.4's BatchScanExec reports SinglePartition for one input partition, which it counts - // before any DPP filtering, and Spark plans no exchange above it on that basis. The scan reports - // the same. Otherwise EnsureRequirements, which AQE runs again over a DPP subquery's converted - // plan, inserts an exchange inside a native block that is already serialized (#7422). - // serializePartitions keeps that one partition when DPP prunes it, as Spark does. + // Spark 3.4's BatchScanExec reports SinglePartition for one input partition, and every supported + // version reports a key-grouped partitioning (KeyGroupedPartitioning, called KeyedPartitioning + // from Spark 4.2) for a storage-partitioned scan. Spark takes both from the partitions it plans, + // before any DPP filtering, and plans the operators above the scan on that basis, with no + // exchange for a join or aggregate that the partitioning already satisfies. The scan reports the + // same, so reading it resolves no DPP subquery. Otherwise EnsureRequirements, which AQE runs + // again over a DPP subquery's converted plan, inserts exchanges inside a native block that is + // already serialized (#7422, #7441). serializePartitions keeps one partition for each of Spark's + // partitions when DPP prunes some, as Spark does. override lazy val outputPartitioning: Partitioning = - if (originalPlan != null && originalPlan.outputPartitioning == SinglePartition) { - SinglePartition - } else { + if (originalPlan == null) { UnknownPartitioning(0) + } else { + originalPlan.outputPartitioning match { + case _: UnknownPartitioning => UnknownPartitioning(0) + case reported => reported + } } override lazy val outputOrdering: Seq[SortOrder] = Nil diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala index f0f7e900aa8..bb80de57fd0 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala @@ -45,7 +45,7 @@ import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AQEShuffl import org.apache.spark.sql.execution.aggregate.{BaseAggregateExec, HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec} import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase -import org.apache.spark.sql.execution.exchange.ReusedExchangeExec +import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeLike} import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec, CartesianProductExec, HashJoin, ShuffledHashJoinExec, SortMergeJoinExec} import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.internal.SQLConf @@ -741,17 +741,53 @@ abstract class CometExec extends CometPlan { * Declares the distribution of the Spark operator that a Comet operator replaces. Without it * AQE's OptimizeSkewInRebalancePartitions can split a skewed partition under an operator that * needs its input clustered, because ValidateRequirements has nothing to reject, and one key then - * yields several rows. Mix it into operators that replace a clustering aggregate, window or join. + * yields several rows. Mix it into operators that replace a clustering aggregate or window. A + * join mixes in [[CometDelegatesJoinChildDistribution]] instead. */ trait CometDelegatesRequiredChildDistribution extends CometExec { override def requiredChildDistribution: Seq[Distribution] = originalPlan.requiredChildDistribution } +/** + * [[CometDelegatesRequiredChildDistribution]] for a join. A storage-partitioned join, whose sides + * are key-grouped scans that Spark joins with no exchange, declares no requirement. No shuffle + * sits below it for AQE to split. EnsureRequirements only accepts such a join when it is Spark's + * own join operator, and in Spark 4.1 only over Spark's own scans. So when AQE plans a DPP + * subquery's converted plan again, it would put an exchange under each side that it has no + * logical link for (#7441). The sides are the join's current children, which are Comet operators, + * because conversion works bottom-up and AQE can later swap in query stages. + */ +trait CometDelegatesJoinChildDistribution extends CometDelegatesRequiredChildDistribution { + override def requiredChildDistribution: Seq[Distribution] = + if (children.forall(CometExec.isStoragePartitioned)) { + children.map(_ => UnspecifiedDistribution) + } else { + super.requiredChildDistribution + } +} + object CometExec { final class InvalidSparkFallbackException(message: String) extends IllegalArgumentException(message) + /** + * Whether `plan` reads only key-grouped scans, with no shuffle above them. This is how Spark + * plans each side of a storage-partitioned join. A scan counts only when the scan it replaces + * or wraps was planned key-grouped, so a table read without grouping does not. + */ + private[comet] def isStoragePartitioned(plan: SparkPlan): Boolean = + !plan.exists(_.isInstanceOf[ShuffleExchangeLike]) && + plan.collectLeaves().forall { + // Spark's own scans. Comet's non-native V2 scan, CometBatchScanExec, is a leaf under a + // CometScanWrapper that this does not unwrap, so a join over it keeps its requirement. + case scan: DataSourceV2ScanExecBase => scan.keyGroupedPartitioning.isDefined + // originalPlan is null on a canonicalized copy + case scan: CometIcebergNativeScanExec => + scan.originalPlan != null && scan.originalPlan.keyGroupedPartitioning.isDefined + case _ => false + } + // An unique id for each CometExecIterator, used to identify the native query execution. private val curId = new java.util.concurrent.atomic.AtomicLong() @@ -3434,7 +3470,7 @@ case class CometHashJoinExec( override val right: SparkPlan, override val serializedPlanOpt: SerializedPlan) extends CometBinaryExec - with CometDelegatesRequiredChildDistribution { + with CometDelegatesJoinChildDistribution { override def outputPartitioning: Partitioning = joinType match { case _: InnerLike => @@ -3824,7 +3860,7 @@ case class CometSortMergeJoinExec( override val right: SparkPlan, override val serializedPlanOpt: SerializedPlan) extends CometBinaryExec - with CometDelegatesRequiredChildDistribution { + with CometDelegatesJoinChildDistribution { override def outputPartitioning: Partitioning = joinType match { case _: InnerLike => diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala index 3f2bd95b6be..aebc526a5df 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala @@ -7033,6 +7033,133 @@ class CometIcebergNativeSuite } } + // A storage-partitioned join is a sort-merge join that Spark plans without a shuffle, because + // both scans report a KeyGroupedPartitioning. On the build side of a DPP broadcast, AQE plans + // the converted join again, and EnsureRequirements has to find the join's required distribution + // already met rather than put an exchange under each side, which AQE has no logical link for + // (#7441). + test("AQE DPP - storage-partitioned join on the DPP build side (#7441)") { + assume(icebergAvailable, "Iceberg not available") + withTempIcebergDir { warehouseDir => + withSQLConf( + "spark.sql.catalog.aqe_cat" -> "org.apache.iceberg.spark.SparkCatalog", + "spark.sql.catalog.aqe_cat.type" -> "hadoop", + "spark.sql.catalog.aqe_cat.warehouse" -> warehouseDir.getAbsolutePath, + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", + SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true", + SQLConf.V2_BUCKETING_ENABLED.key -> "true", + SQLConf.V2_BUCKETING_PUSH_PART_VALUES_ENABLED.key -> "true", + "spark.sql.iceberg.planning.preserve-data-grouping" -> "true", + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true") { + + spark.sql(""" + CREATE TABLE aqe_cat.db.spj_dpp_fact (amount INT, store_id INT) + USING iceberg PARTITIONED BY (store_id) + """) + spark.sql(""" + CREATE TABLE aqe_cat.db.spj_dpp_dim_a (id INT, country STRING) + USING iceberg PARTITIONED BY (id) + """) + spark.sql(""" + CREATE TABLE aqe_cat.db.spj_dpp_dim_b (id INT, name STRING) + USING iceberg PARTITIONED BY (id) + """) + spark.sql( + "INSERT INTO aqe_cat.db.spj_dpp_fact SELECT CAST(id AS INT), CAST(id % 10 AS INT) " + + "FROM range(0, 100)") + spark.sql( + "INSERT INTO aqe_cat.db.spj_dpp_dim_a SELECT CAST(id AS INT), " + + "IF(id < 3, 'X', 'Y') FROM range(0, 10)") + spark.sql( + "INSERT INTO aqe_cat.db.spj_dpp_dim_b SELECT CAST(id AS INT), concat('n', id) " + + "FROM range(0, 10)") + + // The merge join of the two dimensions is the build side of the broadcast that prunes the + // fact scan. It needs no shuffle because both of its scans are key-grouped on id. + val query = + """SELECT /*+ BROADCAST(d) */ f.store_id, f.amount, d.name + |FROM aqe_cat.db.spj_dpp_fact f JOIN ( + | SELECT /*+ MERGE(a, b) */ a.id, b.name + | FROM aqe_cat.db.spj_dpp_dim_a a JOIN aqe_cat.db.spj_dpp_dim_b b ON a.id = b.id + | WHERE a.country = 'X' + |) d ON f.store_id = d.id""".stripMargin + val (_, cometPlan) = checkSparkAnswer(query) + assertIcebergNativeScanPresent(cometPlan) + assertNoLeftoverCSAB(cometPlan) + assert( + collectWithSubqueries(cometPlan) { case e: ShuffleExchangeLike => e }.isEmpty, + s"Expected no exchange in the plan or in the DPP subquery's plan:\n$cometPlan") + + spark.sql("DROP TABLE aqe_cat.db.spj_dpp_fact") + spark.sql("DROP TABLE aqe_cat.db.spj_dpp_dim_a") + spark.sql("DROP TABLE aqe_cat.db.spj_dpp_dim_b") + } + } + } + + // The aggregate above a storage-partitioned join on a DPP build side needs its input clustered + // as well. The join's partitioning comes from its scans, so they have to report the key-grouped + // partitioning that Spark planned the aggregate on, or EnsureRequirements puts an exchange + // between the two when AQE plans the converted subquery again (#7441). + test("AQE DPP - aggregate over a storage-partitioned join on the DPP build side (#7441)") { + assume(icebergAvailable, "Iceberg not available") + withTempIcebergDir { warehouseDir => + withSQLConf( + "spark.sql.catalog.aqe_cat" -> "org.apache.iceberg.spark.SparkCatalog", + "spark.sql.catalog.aqe_cat.type" -> "hadoop", + "spark.sql.catalog.aqe_cat.warehouse" -> warehouseDir.getAbsolutePath, + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", + SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true", + SQLConf.V2_BUCKETING_ENABLED.key -> "true", + SQLConf.V2_BUCKETING_PUSH_PART_VALUES_ENABLED.key -> "true", + "spark.sql.iceberg.planning.preserve-data-grouping" -> "true", + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true") { + + spark.sql(""" + CREATE TABLE aqe_cat.db.spj_dpp_fact (amount INT, store_id INT) + USING iceberg PARTITIONED BY (store_id) + """) + spark.sql(""" + CREATE TABLE aqe_cat.db.spj_dpp_dim_a (id INT, country STRING) + USING iceberg PARTITIONED BY (id) + """) + spark.sql(""" + CREATE TABLE aqe_cat.db.spj_dpp_dim_b (id INT, name STRING) + USING iceberg PARTITIONED BY (id) + """) + spark.sql( + "INSERT INTO aqe_cat.db.spj_dpp_fact SELECT CAST(id AS INT), CAST(id % 10 AS INT) " + + "FROM range(0, 100)") + spark.sql( + "INSERT INTO aqe_cat.db.spj_dpp_dim_a SELECT CAST(id AS INT), " + + "IF(id < 3, 'X', 'Y') FROM range(0, 10)") + spark.sql( + "INSERT INTO aqe_cat.db.spj_dpp_dim_b SELECT CAST(id AS INT), concat('n', id) " + + "FROM range(0, 10)") + + val query = + """SELECT /*+ BROADCAST(d) */ f.store_id, f.amount, d.cnt + |FROM aqe_cat.db.spj_dpp_fact f JOIN ( + | SELECT /*+ MERGE(a, b) */ a.id, count(*) AS cnt + | FROM aqe_cat.db.spj_dpp_dim_a a JOIN aqe_cat.db.spj_dpp_dim_b b ON a.id = b.id + | WHERE a.country = 'X' + | GROUP BY a.id + |) d ON f.store_id = d.id""".stripMargin + val (_, cometPlan) = checkSparkAnswer(query) + assertIcebergNativeScanPresent(cometPlan) + assertNoLeftoverCSAB(cometPlan) + + spark.sql("DROP TABLE aqe_cat.db.spj_dpp_fact") + spark.sql("DROP TABLE aqe_cat.db.spj_dpp_dim_a") + spark.sql("DROP TABLE aqe_cat.db.spj_dpp_dim_b") + } + } + } + // TPC-DS q64 shape (#6264): a broadcast of (fact SMJ other) per UNION ALL branch, with the // dimension join above it. The two broadcasts differ only in the DPP filter of the fact scan // inside a child shuffle stage, so they must not be reused for each other. From 7835efea9a10014cff97aa15ef39fcc4e201288e Mon Sep 17 00:00:00 2001 From: Steve Vaughan Jr Date: Tue, 17 Nov 2026 18:37:18 -0500 Subject: [PATCH 2/3] test: run the #7441 tests for both join hints and fix a comment Generate the join and the aggregate test for the MERGE and SHUFFLE_HASH hints, so CometSortMergeJoinExec and CometHashJoinExec are both covered. Each case asserts the join operator, that the fact scan carries a DPP filter, and that neither the plan nor the DPP subquery's plan has an exchange. The tables are created once per case and dropped in a finally, so a failing case no longer makes the next one fail on CREATE TABLE. Correct the comment in isStoragePartitioned. A declined Iceberg scan stays a bare CometBatchScanExec, which extends DataSourceV2ScanExecBase and is matched by that case. --- .../apache/spark/sql/comet/operators.scala | 3 +- .../comet/CometIcebergNativeSuite.scala | 172 +++++++----------- 2 files changed, 71 insertions(+), 104 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala index bb80de57fd0..5aa88f5a923 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala @@ -779,8 +779,7 @@ object CometExec { private[comet] def isStoragePartitioned(plan: SparkPlan): Boolean = !plan.exists(_.isInstanceOf[ShuffleExchangeLike]) && plan.collectLeaves().forall { - // Spark's own scans. Comet's non-native V2 scan, CometBatchScanExec, is a leaf under a - // CometScanWrapper that this does not unwrap, so a join over it keeps its requirement. + // Spark's own scans, and CometBatchScanExec, which extends DataSourceV2ScanExecBase. case scan: DataSourceV2ScanExecBase => scan.keyGroupedPartitioning.isDefined // originalPlan is null on a canonicalized copy case scan: CometIcebergNativeScanExec => diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala index aebc526a5df..053a5e10cfe 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala @@ -7033,13 +7033,11 @@ class CometIcebergNativeSuite } } - // A storage-partitioned join is a sort-merge join that Spark plans without a shuffle, because - // both scans report a KeyGroupedPartitioning. On the build side of a DPP broadcast, AQE plans - // the converted join again, and EnsureRequirements has to find the join's required distribution - // already met rather than put an exchange under each side, which AQE has no logical link for - // (#7441). - test("AQE DPP - storage-partitioned join on the DPP build side (#7441)") { - assume(icebergAvailable, "Iceberg not available") + // The fact table is partitioned on the column the DPP join uses and the two dimension tables on + // theirs, so Spark reads every scan key-grouped and joins the dimensions without a shuffle. The + // drops sit in a finally because a table left behind by a failing test makes the next test fail + // on CREATE TABLE, which hides that test's own result. + private def withStoragePartitionedDppTables(f: => Unit): Unit = { withTempIcebergDir { warehouseDir => withSQLConf( "spark.sql.catalog.aqe_cat" -> "org.apache.iceberg.spark.SparkCatalog", @@ -7053,109 +7051,79 @@ class CometIcebergNativeSuite CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true") { - - spark.sql(""" - CREATE TABLE aqe_cat.db.spj_dpp_fact (amount INT, store_id INT) - USING iceberg PARTITIONED BY (store_id) - """) - spark.sql(""" - CREATE TABLE aqe_cat.db.spj_dpp_dim_a (id INT, country STRING) - USING iceberg PARTITIONED BY (id) - """) - spark.sql(""" - CREATE TABLE aqe_cat.db.spj_dpp_dim_b (id INT, name STRING) - USING iceberg PARTITIONED BY (id) - """) - spark.sql( - "INSERT INTO aqe_cat.db.spj_dpp_fact SELECT CAST(id AS INT), CAST(id % 10 AS INT) " + - "FROM range(0, 100)") - spark.sql( - "INSERT INTO aqe_cat.db.spj_dpp_dim_a SELECT CAST(id AS INT), " + - "IF(id < 3, 'X', 'Y') FROM range(0, 10)") - spark.sql( - "INSERT INTO aqe_cat.db.spj_dpp_dim_b SELECT CAST(id AS INT), concat('n', id) " + - "FROM range(0, 10)") - - // The merge join of the two dimensions is the build side of the broadcast that prunes the - // fact scan. It needs no shuffle because both of its scans are key-grouped on id. - val query = - """SELECT /*+ BROADCAST(d) */ f.store_id, f.amount, d.name - |FROM aqe_cat.db.spj_dpp_fact f JOIN ( - | SELECT /*+ MERGE(a, b) */ a.id, b.name - | FROM aqe_cat.db.spj_dpp_dim_a a JOIN aqe_cat.db.spj_dpp_dim_b b ON a.id = b.id - | WHERE a.country = 'X' - |) d ON f.store_id = d.id""".stripMargin - val (_, cometPlan) = checkSparkAnswer(query) - assertIcebergNativeScanPresent(cometPlan) - assertNoLeftoverCSAB(cometPlan) - assert( - collectWithSubqueries(cometPlan) { case e: ShuffleExchangeLike => e }.isEmpty, - s"Expected no exchange in the plan or in the DPP subquery's plan:\n$cometPlan") - - spark.sql("DROP TABLE aqe_cat.db.spj_dpp_fact") - spark.sql("DROP TABLE aqe_cat.db.spj_dpp_dim_a") - spark.sql("DROP TABLE aqe_cat.db.spj_dpp_dim_b") + try { + spark.sql(""" + CREATE TABLE aqe_cat.db.spj_dpp_fact (amount INT, store_id INT) + USING iceberg PARTITIONED BY (store_id) + """) + spark.sql(""" + CREATE TABLE aqe_cat.db.spj_dpp_dim_a (id INT, country STRING) + USING iceberg PARTITIONED BY (id) + """) + spark.sql(""" + CREATE TABLE aqe_cat.db.spj_dpp_dim_b (id INT, name STRING) + USING iceberg PARTITIONED BY (id) + """) + spark.sql( + "INSERT INTO aqe_cat.db.spj_dpp_fact SELECT CAST(id AS INT), CAST(id % 10 AS INT) " + + "FROM range(0, 100)") + spark.sql( + "INSERT INTO aqe_cat.db.spj_dpp_dim_a SELECT CAST(id AS INT), " + + "IF(id < 3, 'X', 'Y') FROM range(0, 10)") + spark.sql( + "INSERT INTO aqe_cat.db.spj_dpp_dim_b SELECT CAST(id AS INT), concat('n', id) " + + "FROM range(0, 10)") + f + } finally { + Seq("spj_dpp_fact", "spj_dpp_dim_a", "spj_dpp_dim_b").foreach { table => + spark.sql(s"DROP TABLE IF EXISTS aqe_cat.db.$table") + } + } } } } - // The aggregate above a storage-partitioned join on a DPP build side needs its input clustered - // as well. The join's partitioning comes from its scans, so they have to report the key-grouped - // partitioning that Spark planned the aggregate on, or EnsureRequirements puts an exchange - // between the two when AQE plans the converted subquery again (#7441). - test("AQE DPP - aggregate over a storage-partitioned join on the DPP build side (#7441)") { - assume(icebergAvailable, "Iceberg not available") - withTempIcebergDir { warehouseDir => - withSQLConf( - "spark.sql.catalog.aqe_cat" -> "org.apache.iceberg.spark.SparkCatalog", - "spark.sql.catalog.aqe_cat.type" -> "hadoop", - "spark.sql.catalog.aqe_cat.warehouse" -> warehouseDir.getAbsolutePath, - SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", - SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true", - SQLConf.V2_BUCKETING_ENABLED.key -> "true", - SQLConf.V2_BUCKETING_PUSH_PART_VALUES_ENABLED.key -> "true", - "spark.sql.iceberg.planning.preserve-data-grouping" -> "true", - CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true") { - - spark.sql(""" - CREATE TABLE aqe_cat.db.spj_dpp_fact (amount INT, store_id INT) - USING iceberg PARTITIONED BY (store_id) - """) - spark.sql(""" - CREATE TABLE aqe_cat.db.spj_dpp_dim_a (id INT, country STRING) - USING iceberg PARTITIONED BY (id) - """) - spark.sql(""" - CREATE TABLE aqe_cat.db.spj_dpp_dim_b (id INT, name STRING) - USING iceberg PARTITIONED BY (id) - """) - spark.sql( - "INSERT INTO aqe_cat.db.spj_dpp_fact SELECT CAST(id AS INT), CAST(id % 10 AS INT) " + - "FROM range(0, 100)") - spark.sql( - "INSERT INTO aqe_cat.db.spj_dpp_dim_a SELECT CAST(id AS INT), " + - "IF(id < 3, 'X', 'Y') FROM range(0, 10)") - spark.sql( - "INSERT INTO aqe_cat.db.spj_dpp_dim_b SELECT CAST(id AS INT), concat('n', id) " + - "FROM range(0, 10)") - + // The build side of the broadcast that prunes the fact scan joins the two dimensions, which + // needs no shuffle because both scans are key-grouped on id, and so does an aggregate on id + // above that join. AQE plans the converted subquery again, and EnsureRequirements has to find + // each requirement already met rather than put an exchange in it, because AQE has no logical + // link for one (#7441). The hint picks the join operator, so both Comet joins are covered. + for { + (shape, column, projection, grouping) <- Seq( + ("storage-partitioned join", "name", "a.id, b.name", ""), + ( + "aggregate over a storage-partitioned join", + "cnt", + "a.id, count(*) AS cnt", + "GROUP BY a.id")) + (hint, joinClass) <- Seq[(String, Class[_ <: SparkPlan])]( + "MERGE" -> classOf[CometSortMergeJoinExec], + "SHUFFLE_HASH" -> classOf[CometHashJoinExec]) + } { + test(s"AQE DPP - $shape on the DPP build side, $hint (#7441)") { + assume(icebergAvailable, "Iceberg not available") + withStoragePartitionedDppTables { val query = - """SELECT /*+ BROADCAST(d) */ f.store_id, f.amount, d.cnt - |FROM aqe_cat.db.spj_dpp_fact f JOIN ( - | SELECT /*+ MERGE(a, b) */ a.id, count(*) AS cnt - | FROM aqe_cat.db.spj_dpp_dim_a a JOIN aqe_cat.db.spj_dpp_dim_b b ON a.id = b.id - | WHERE a.country = 'X' - | GROUP BY a.id - |) d ON f.store_id = d.id""".stripMargin + s"""SELECT /*+ BROADCAST(d) */ f.store_id, f.amount, d.$column + |FROM aqe_cat.db.spj_dpp_fact f JOIN ( + | SELECT /*+ $hint(a, b) */ $projection + | FROM aqe_cat.db.spj_dpp_dim_a a JOIN aqe_cat.db.spj_dpp_dim_b b ON a.id = b.id + | WHERE a.country = 'X' + | $grouping + |) d ON f.store_id = d.id""".stripMargin val (_, cometPlan) = checkSparkAnswer(query) assertIcebergNativeScanPresent(cometPlan) assertNoLeftoverCSAB(cometPlan) - - spark.sql("DROP TABLE aqe_cat.db.spj_dpp_fact") - spark.sql("DROP TABLE aqe_cat.db.spj_dpp_dim_a") - spark.sql("DROP TABLE aqe_cat.db.spj_dpp_dim_b") + assert( + collectIcebergDPPSubqueries(cometPlan).nonEmpty, + s"Expected the fact scan to carry a DPP filter:\n$cometPlan") + assert( + collectWithSubqueries(cometPlan) { case p if joinClass.isInstance(p) => p }.nonEmpty, + s"Expected a ${joinClass.getSimpleName} in the plan or in the DPP subquery's plan:\n" + + cometPlan) + assert( + collectWithSubqueries(cometPlan) { case e: ShuffleExchangeLike => e }.isEmpty, + s"Expected no exchange in the plan or in the DPP subquery's plan:\n$cometPlan") } } } From 6dab540e2c448d1ee0a1f0e7830bf14a21ec2dc1 Mon Sep 17 00:00:00 2001 From: Steve Vaughan Jr Date: Fri, 20 Nov 2026 09:10:02 -0500 Subject: [PATCH 3/3] refactor: report the replaced scan's partitioning as it is BatchScanExec reports UnknownPartitioning(0) when it has no partitioning, on every supported Spark version, so mapping an unknown partitioning to UnknownPartitioning(0) changed nothing. Return the replaced scan's outputPartitioning directly, as CometNativeScanExec does for a bucketed scan. --- .../apache/spark/sql/comet/CometIcebergNativeScanExec.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergNativeScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergNativeScanExec.scala index 67afc79f2ca..70a91cc7325 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergNativeScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergNativeScanExec.scala @@ -147,10 +147,7 @@ case class CometIcebergNativeScanExec( if (originalPlan == null) { UnknownPartitioning(0) } else { - originalPlan.outputPartitioning match { - case _: UnknownPartitioning => UnknownPartitioning(0) - case reported => reported - } + originalPlan.outputPartitioning } override lazy val outputOrdering: Seq[SortOrder] = Nil