From c73475513722b81651fb1c9f58b652058d761c34 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 5 Nov 2026 10:46:35 -0700 Subject: [PATCH 1/4] fix: resolve the three-argument ManifestFiles.read so native Iceberg writes work on 1.12 Iceberg 1.12 removed the public ManifestFiles.read(ManifestFile, FileIO) overload, so the executor-side reflection probe failed and every native Iceberg write fell back to iceberg-java. The three-argument overload (manifest, io, specsById) is public on every supported version (1.5.2 through 1.12), so resolve that and pass the table's PartitionSpec for the stamped spec id. DetectionDelegatingFileIO moves to Java: HadoopConfigurable.getConf is abstract on some supported versions and a default method on 1.12, and Scala has no spelling that compiles against both. --- .../comet/iceberg/IcebergReflection.scala | 37 ++++++--- .../sql/comet/CometIcebergWriteExec.scala | 2 +- .../comet/DetectionDelegatingFileIO.java | 80 +++++++++++++++++++ .../CometIcebergWriteDetectionSuite.scala | 30 +------ 4 files changed, 110 insertions(+), 39 deletions(-) create mode 100644 spark/src/test/java/org/apache/comet/DetectionDelegatingFileIO.java diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index 58278b488aa..aeb0d8492e0 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -1865,8 +1865,7 @@ object IcebergReflection extends Logging { genericManifestFile.getDeclaredConstructor(inputFileClass, classOf[Int]) } val fileIOClass = loadClass(ClassNames.FILE_IO) - val manifestReadMethod = loadClass(ClassNames.MANIFEST_FILES) - .getMethod("read", loadClass(ClassNames.MANIFEST_FILE), fileIOClass) + val manifestReadMethod = manifestReadMethodOf(fileIOClass) manifestReadMethod.getReturnType.getMethod("iterator") manifestReadMethod.getReturnType.getMethod("close") val contentFileClass = loadClass(ClassNames.CONTENT_FILE) @@ -1949,8 +1948,14 @@ object IcebergReflection extends Logging { * * `specId` is stamped onto the synthesised `ManifestFile` because the manifest reader uses it * to pick the correct partition spec when materialising partition data on each `DataFile`. - */ - def decodeManifestToDataFiles(bytes: Array[Byte], specId: Int): java.util.List[AnyRef] = { + * `spec` is the table's `PartitionSpec` for that id, handed to the reader through the + * `specsById` argument so it is not re-parsed from the manifest's metadata (Iceberg deprecated + * that fallback, and 1.12 removed the `read(ManifestFile, FileIO)` overload that implied it). + */ + def decodeManifestToDataFiles( + bytes: Array[Byte], + specId: Int, + spec: AnyRef): java.util.List[AnyRef] = { // Iceberg's `ManifestReader.open` infers format from the file extension; the in-memory path // must end in `.avro` for the v2 data-manifest path to be picked. The "memory:" scheme just // namespaces the location so it can't collide with on-disk paths in the same FileIO. @@ -1958,9 +1963,23 @@ object IcebergReflection extends Logging { val fileIO = newInMemoryFileIO(location, bytes) val inputFile = newInMemoryInputFile(location, bytes) val manifestFile = newDataManifestFile(inputFile, specId) - readDataFilesFromManifest(manifestFile, fileIO) + val specsById = java.util.Collections.singletonMap(Integer.valueOf(specId), spec) + readDataFilesFromManifest(manifestFile, fileIO, specsById) } + /** + * `ManifestFiles.read(ManifestFile, FileIO, Map)`, the only overload + * that is public on every supported Iceberg release (1.5.2 through 1.12; 1.12 dropped the + * two-argument one). + */ + private def manifestReadMethodOf(fileIOClass: Class[_]): java.lang.reflect.Method = + loadClass(ClassNames.MANIFEST_FILES) + .getMethod( + "read", + loadClass(ClassNames.MANIFEST_FILE), + fileIOClass, + classOf[java.util.Map[_, _]]) + private def newInMemoryFileIO(location: String, bytes: Array[Byte]): AnyRef = { val cls = loadClass(ClassNames.INMEMORY_FILE_IO) val instance = cls.getDeclaredConstructor().newInstance().asInstanceOf[AnyRef] @@ -2029,13 +2048,11 @@ object IcebergReflection extends Logging { private def readDataFilesFromManifest( manifestFile: AnyRef, - fileIO: AnyRef): java.util.List[AnyRef] = { - val manifestFileClass = loadClass(ClassNames.MANIFEST_FILE) + fileIO: AnyRef, + specsById: java.util.Map[Integer, AnyRef]): java.util.List[AnyRef] = { val fileIOClass = loadClass(ClassNames.FILE_IO) val contentFileClass = loadClass(ClassNames.CONTENT_FILE) - val readMethod = loadClass(ClassNames.MANIFEST_FILES) - .getMethod("read", manifestFileClass, fileIOClass) - val reader = readMethod.invoke(null, manifestFile, fileIO) + val reader = manifestReadMethodOf(fileIOClass).invoke(null, manifestFile, fileIO, specsById) try { val iterator = reader.getClass .getMethod("iterator") diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala index 98ba9babe77..9f794b61ab3 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergWriteExec.scala @@ -201,7 +201,7 @@ case class CometIcebergWriteExec( val proj = UnsafeProjection.create(schemaTypes) val decoded = if (manifestBytes.isEmpty) new java.util.ArrayList[AnyRef]() - else IcebergReflection.decodeManifestToDataFiles(manifestBytes, specId) + else IcebergReflection.decodeManifestToDataFiles(manifestBytes, specId, outputSpec) val dataFiles = IcebergReflection.rebuildDataFilesWithJavaMetrics( decoded, tableIO, diff --git a/spark/src/test/java/org/apache/comet/DetectionDelegatingFileIO.java b/spark/src/test/java/org/apache/comet/DetectionDelegatingFileIO.java new file mode 100644 index 00000000000..8d503f587d1 --- /dev/null +++ b/spark/src/test/java/org/apache/comet/DetectionDelegatingFileIO.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet; + +import java.util.Map; +import java.util.function.Function; + +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.hadoop.HadoopConfigurable; +import org.apache.iceberg.hadoop.HadoopFileIO; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.util.SerializableSupplier; + +/** + * A FileIO that works normally (delegating to HadoopFileIO) but whose class is not on Comet's + * recognized-FileIO allowlist. Composition rather than inheritance is the point: a HadoopFileIO + * SUBCLASS passes the hierarchy check by design, while this class must be declined. Instantiated + * reflectively by Iceberg's `CatalogUtil.loadFileIO`, hence public with a no-arg constructor. + * + *

This is Java because {@code getConf} is abstract on some supported Iceberg versions (1.8.1) + * and a default method on others (1.12). Scala has no spelling that compiles against both, since it + * requires {@code override} for a concrete member and rejects it for an abstract one. + */ +public class DetectionDelegatingFileIO implements FileIO, HadoopConfigurable { + private final HadoopFileIO delegate = new HadoopFileIO(); + + @Override + public InputFile newInputFile(String path) { + return delegate.newInputFile(path); + } + + @Override + public OutputFile newOutputFile(String path) { + return delegate.newOutputFile(path); + } + + @Override + public void deleteFile(String path) { + delegate.deleteFile(path); + } + + @Override + public void initialize(Map properties) { + delegate.initialize(properties); + } + + @Override + public void setConf(Configuration conf) { + delegate.setConf(conf); + } + + public Configuration getConf() { + return delegate.getConf(); + } + + @Override + public void serializeConfWith( + Function> confSerializer) { + delegate.serializeConfWith(confSerializer); + } +} diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index f2fcc3010fe..491e8dbf924 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -29,9 +29,8 @@ import org.apache.iceberg.{PartitionSpec, StructLike, TableMetadata, TableOperat import org.apache.iceberg.aws.s3.S3FileIO import org.apache.iceberg.catalog.TableIdentifier import org.apache.iceberg.encryption.EncryptionManager -import org.apache.iceberg.hadoop.{HadoopCatalog, HadoopConfigurable, HadoopFileIO} -import org.apache.iceberg.io.{FileIO, InputFile, LocationProvider, OutputFile, ResolvingFileIO} -import org.apache.iceberg.util.SerializableSupplier +import org.apache.iceberg.hadoop.{HadoopCatalog, HadoopFileIO} +import org.apache.iceberg.io.{FileIO, LocationProvider, ResolvingFileIO} import org.apache.spark.SparkConf import org.apache.spark.rdd.RDD import org.apache.spark.sql.CometTestBase @@ -1858,31 +1857,6 @@ case class TransitionProbeLeaf(columnar: Boolean) extends LeafExecNode { throw new UnsupportedOperationException("planning-only node") } -/** - * A FileIO that works normally (delegating to HadoopFileIO) but whose class is not on Comet's - * recognized-FileIO allowlist. Composition rather than inheritance is the point: a HadoopFileIO - * SUBCLASS passes the hierarchy check by design, while this class must be declined. Instantiated - * reflectively by Iceberg's `CatalogUtil.loadFileIO`, hence top-level with a no-arg constructor. - */ -class DetectionDelegatingFileIO extends FileIO with HadoopConfigurable { - private val delegate = new HadoopFileIO() - - override def newInputFile(path: String): InputFile = delegate.newInputFile(path) - override def newOutputFile(path: String): OutputFile = delegate.newOutputFile(path) - override def deleteFile(path: String): Unit = delegate.deleteFile(path) - override def initialize(properties: java.util.Map[String, String]): Unit = - delegate.initialize(properties) - override def setConf(conf: Configuration): Unit = delegate.setConf(conf) - // No `override` modifier: `Configurable.getConf` exists on some supported Iceberg versions - // (e.g. 1.8.1) and not others, and a plain def satisfies both shapes. - def getConf: Configuration = delegate.getConf - override def serializeConfWith( - confSerializer: java.util.function.Function[ - Configuration, - SerializableSupplier[Configuration]]): Unit = - delegate.serializeConfWith(confSerializer) -} - /** * A catalog whose TableOperations supplies a custom LocationProvider directly, without setting * `write.location-provider.impl`. This is the path the native write gate must inspect explicitly. From e66103c4256c5bbe8cede4174f55ca10b5ebbec0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 5 Nov 2026 10:46:36 -0700 Subject: [PATCH 2/4] ci: run Iceberg's Spark suites on Iceberg 1.12.0 and Spark 4.2.0 Add dev/diffs/iceberg/1.12.0.diff, ported from the 1.11.0 diff and regenerated against the apache-iceberg-1.12.0 tag, and an iceberg_1_12 job in the nightly tier and behind the run-iceberg-tests label. Spark 4.2 support is still experimental, so it does not join the merge queue. --- .github/workflows/README.md | 4 +- .github/workflows/ci.yml | 18 + dev/ci/check-ci-config.py | 4 +- dev/ci/compute-changes.py | 31 +- dev/diffs/iceberg/1.12.0.diff | 968 ++++++++++++++++++++++++++++++++++ 5 files changed, 1021 insertions(+), 4 deletions(-) create mode 100644 dev/diffs/iceberg/1.12.0.diff diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 6addfd923b9..41596097f78 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -29,7 +29,7 @@ ruleset in `.asf.yaml`. That splits CI into three tiers: - **Nightly tier** (`nightly`): the regression sweep of everything else, once a day against `main` as it stands. The Comet test suites against the other three Spark profiles, Spark SQL on Spark 3.5, 4.0 and 4.2, Iceberg - 1.9/1.10, and the Delta contrib suites on Spark 3.5, 4.0 and 4.1. See + 1.9/1.10/1.12, and the Delta contrib suites on Spark 3.5, 4.0 and 4.1. See [Nightly tier](#nightly-tier) below for how a failure surfaces. Every queue-only and nightly job has a `run-*` label that opts a pull request @@ -119,6 +119,7 @@ tiers partition the list and that the `pr` tier is exactly the default profile. spark_4_2 run-spark-4.2-tests iceberg_1_9 run-iceberg-tests iceberg_1_10 run-iceberg-tests + iceberg_1_12 run-iceberg-tests delta_3_5 run-delta-tests delta_4_0 run-delta-tests delta_4_1 run-delta-tests @@ -169,6 +170,7 @@ tiers partition the list and that the `pr` tier is exactly the default profile. | `iceberg_1_11` | PR or merge group, paths matched | Iceberg sources | | `iceberg_1_9` | nightly, **or** PR with `run-iceberg-tests` | Iceberg sources | | `iceberg_1_10` | nightly, **or** PR with `run-iceberg-tests` | Iceberg sources | +| `iceberg_1_12` | nightly, **or** PR with `run-iceberg-tests` | Iceberg sources | | `required_checks` | always, after every job above except `docs` | none (always runs) | | `nightly_report` | schedule only, after `required_checks`, when it is not green | none | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f644c8f7fa4..8acb453b971 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -299,6 +299,7 @@ jobs: iceberg_1_9: ${{ steps.compute.outputs.iceberg_1_9 }} iceberg_1_10: ${{ steps.compute.outputs.iceberg_1_10 }} iceberg_1_11: ${{ steps.compute.outputs.iceberg_1_11 }} + iceberg_1_12: ${{ steps.compute.outputs.iceberg_1_12 }} delta_3_5: ${{ steps.compute.outputs.delta_3_5 }} delta_4_0: ${{ steps.compute.outputs.delta_4_0 }} delta_4_1: ${{ steps.compute.outputs.delta_4_1 }} @@ -619,6 +620,22 @@ jobs: spark-full: '4.1.3' java: 17 + iceberg_1_12: + name: Iceberg Spark SQL Tests (Iceberg 1.12) + needs: changes + # Nightly by default; PRs need the `run-iceberg-tests` label. Iceberg 1.12 + # is the first release with a Spark 4.2 module, and Spark 4.2 support is + # still experimental, so this sits in the nightly tier like the Spark 4.2 + # SQL suite does. + if: needs.changes.outputs.iceberg_1_12 == 'true' + uses: ./.github/workflows/iceberg_spark_test_reusable.yml + with: + iceberg-short: '1.12' + iceberg-full: '1.12.0' + spark-short: '4.2' + spark-full: '4.2.0' + java: 17 + # --------------------------------------------------------------------------- # required_checks: the single context listed in `required_status_checks` for # `main` in `.asf.yaml`, and therefore the only thing the merge queue waits @@ -679,6 +696,7 @@ jobs: - iceberg_1_9 - iceberg_1_10 - iceberg_1_11 + - iceberg_1_12 runs-on: ubuntu-slim steps: - name: Summarize upstream results diff --git a/dev/ci/check-ci-config.py b/dev/ci/check-ci-config.py index dfdbb406427..d7916ba7003 100644 --- a/dev/ci/check-ci-config.py +++ b/dev/ci/check-ci-config.py @@ -128,6 +128,7 @@ "iceberg_1_9", "iceberg_1_10", "iceberg_1_11", + "iceberg_1_12", } # The two contrib/UDF gates also run ./mvnw, but consume no shared artifact. @@ -230,6 +231,7 @@ "spark_4_2", "iceberg_1_9", "iceberg_1_10", + "iceberg_1_12", "delta_3_5", "delta_4_0", "delta_4_1", @@ -237,7 +239,7 @@ } # One label opts a pull request into every nightly Iceberg version. Iceberg # 1.11 is in the PR tier, so it needs no label. -ICEBERG_OPT_IN = {"iceberg_1_9", "iceberg_1_10"} +ICEBERG_OPT_IN = {"iceberg_1_9", "iceberg_1_10", "iceberg_1_12"} # Likewise one label for every Delta profile. DELTA_OPT_IN = {"delta_3_5", "delta_4_0", "delta_4_1"} ALL_JOBS = QUEUE_TIER | NIGHTLY_TIER | {"docs"} diff --git a/dev/ci/compute-changes.py b/dev/ci/compute-changes.py index 42d4b859a37..1d291618343 100644 --- a/dev/ci/compute-changes.py +++ b/dev/ci/compute-changes.py @@ -385,6 +385,31 @@ ".mvn/**", "mvnw", ], + "iceberg_1_12": [ + "native/**/src/**", + "native/**/Cargo.toml", + "native/Cargo.lock", + "common/src/main/**", + "common/pom.xml", + "spark/src/main/**", + "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", + "spark/pom.xml", + "dev/diffs/iceberg/**", + "pom.xml", + "rust-toolchain.toml", + ".github/workflows/ci.yml", + ".github/workflows/iceberg_spark_test_reusable.yml", + ".github/actions/setup-builder/**", + ".github/actions/setup-iceberg-builder/**", + "dev/ci/iceberg-test-shards.gradle", + "dev/ci/check-iceberg-shards.py", + "dev/ci/test-iceberg-shards.py", + ".github/actions/upload-artifact-retry/**", + ".github/actions/download-artifact-retry/**", + ".github/actions/maven-bootstrap/**", + ".mvn/**", + "mvnw", + ], "delta_3_5": [ "contrib/delta-spark/**", "!**.md", @@ -547,11 +572,13 @@ "spark_4_2": ["nightly", "label:run-spark-4.2-tests"], # Same shape for Iceberg: 1.11 is the only Spark 4.1 coverage, so it is # the one Iceberg version every pull request and the queue run, for the - # same reason as Spark 4.1 above. The two older versions run nightly, - # and one label brings both forward onto a pull request. + # same reason as Spark 4.1 above. The two older versions run nightly, as + # does 1.12, which is the Spark 4.2 coverage and so shares that suite's + # tier. One label brings all three forward onto a pull request. "iceberg_1_9": ["nightly", "label:run-iceberg-tests"], "iceberg_1_10": ["nightly", "label:run-iceberg-tests"], "iceberg_1_11": ["pr", "queue"], + "iceberg_1_12": ["nightly", "label:run-iceberg-tests"], # The Delta contrib suites all run nightly. The PR tier's strict warnings # job compiles the contrib against Spark 3.5, so a change that breaks its # build on Spark 3.5 fails on its own pull request. One label opts a pull diff --git a/dev/diffs/iceberg/1.12.0.diff b/dev/diffs/iceberg/1.12.0.diff new file mode 100644 index 00000000000..66a6a8f8067 --- /dev/null +++ b/dev/diffs/iceberg/1.12.0.diff @@ -0,0 +1,968 @@ +diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml +index 2004e3a164..bf30b0766b 100644 +--- a/gradle/libs.versions.toml ++++ b/gradle/libs.versions.toml +@@ -40,6 +40,7 @@ bouncycastle = "1.86" + bson-ver = "4.11.5" + caffeine = "3.2.4" + calcite = "1.42.0" ++comet = "1.3.0-SNAPSHOT" + datasketches = "6.2.0" + delta-standalone = "3.3.3" + delta-spark = "3.3.3" +diff --git a/spark/v4.2/build.gradle b/spark/v4.2/build.gradle +index b800d87f75..94c3fb2d59 100644 +--- a/spark/v4.2/build.gradle ++++ b/spark/v4.2/build.gradle +@@ -109,6 +109,7 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") { + testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) + testImplementation libs.awaitility + testImplementation(testFixtures(project(':iceberg-parquet'))) ++ testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" + } + + test { +@@ -174,6 +175,7 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer + testImplementation libs.parquet.hadoop + testImplementation libs.awaitility + testImplementation(testFixtures(project(':iceberg-parquet'))) ++ testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" + + // Required because we remove antlr plugin dependencies from the compile configuration, see note above + runtimeOnly libs.antlr.runtime413 +@@ -256,6 +258,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio + integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') + integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') ++ integrationImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" + + // runtime dependencies for running Hive Catalog based integration test + integrationRuntimeOnly project(':iceberg-hive-metastore') +diff --git a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +index f766fbb79a..56efa0fadb 100644 +--- a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java ++++ b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java +@@ -71,6 +71,17 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { + .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") + .config( + SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .enableHiveSupport() + .getOrCreate(); +diff --git a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java +index 36f28acd99..7da7c3c3ef 100644 +--- a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java ++++ b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java +@@ -400,7 +400,8 @@ public abstract class SparkRowLevelOperationsTestBase extends ExtensionsTestBase + + protected void assertAllBatchScansVectorized(SparkPlan plan) { + List batchScans = SparkPlanUtil.collectBatchScans(plan); +- assertThat(batchScans).hasSizeGreaterThan(0).allMatch(SparkPlan::supportsColumnar); ++ // When Comet is enabled, its native scan replaces BatchScanExec nodes entirely ++ assertThat(batchScans).allMatch(SparkPlan::supportsColumnar); + } + + protected void createTableWithDeleteGranularity( +diff --git a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestPartitionedWritesToWapBranch.java b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestPartitionedWritesToWapBranch.java +index af065451ab..68e6dcadaf 100644 +--- a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestPartitionedWritesToWapBranch.java ++++ b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestPartitionedWritesToWapBranch.java +@@ -69,6 +69,17 @@ public class TestPartitionedWritesToWapBranch extends PartitionedWritesTestBase + .config("spark.sql.shuffle.partitions", "4") + .config("spark.sql.hive.metastorePartitionPruningFallbackOnException", "true") + .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .enableHiveSupport() + .getOrCreate(); +diff --git a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +index 934220e5d3..56e375e430 100644 +--- a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java ++++ b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java +@@ -40,6 +40,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; + import org.apache.spark.sql.catalyst.expressions.Expression; + import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; + import org.apache.spark.sql.execution.CommandResultExec; ++import org.apache.spark.sql.execution.SparkPlan; + import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; + import org.junit.jupiter.api.AfterEach; + import org.junit.jupiter.api.BeforeEach; +@@ -330,10 +331,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe + + private List executeAndCollectFunctionCalls(String query, Object... args) { + CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); +- V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); ++ // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not ++ // a V2TableWriteExec; collect over the whole command plan in that case. ++ SparkPlan plan = command.commandPhysicalPlan(); ++ SparkPlan queryPlan = ++ plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; + return SparkPlanUtil.collectExprs( +- write.query(), +- expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); ++ queryPlan, expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); + } + + private List findIrrelevantFileLocations(String cond) { +diff --git a/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +index 6cf3b2fa2d..ad3cf9bc48 100644 +--- a/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java ++++ b/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java +@@ -180,6 +180,17 @@ public class DeleteOrphanFilesBenchmark { + .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) + .config("spark.sql.catalog.spark_catalog.type", "hadoop") + .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .master("local"); + spark = builder.getOrCreate(); +diff --git a/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergCompactionBenchmark.java b/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergCompactionBenchmark.java +index 1da85e0c59..5fe77c8cd7 100644 +--- a/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergCompactionBenchmark.java ++++ b/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergCompactionBenchmark.java +@@ -132,6 +132,17 @@ public abstract class IcebergCompactionBenchmark { + .config( + "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") + .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .master(sparkMaster()); + extraCatalogProperties() +diff --git a/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +index 13a575f343..29687af297 100644 +--- a/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java ++++ b/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +@@ -240,6 +240,17 @@ public class DVReaderBenchmark { + .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) + .config("spark.sql.catalog.spark_catalog.type", "hadoop") + .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .master("local[*]") + .getOrCreate(); + } +diff --git a/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +index 3793101724..4d3b1ea3a1 100644 +--- a/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java ++++ b/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +@@ -224,6 +224,17 @@ public class DVWriterBenchmark { + .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) + .config("spark.sql.catalog.spark_catalog.type", "hadoop") + .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .master("local[*]") + .getOrCreate(); + } +diff --git a/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +index debe37866f..c203878223 100644 +--- a/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java ++++ b/spark/v4.2/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java +@@ -95,7 +95,20 @@ public abstract class IcebergSourceBenchmark { + } + + protected void setupSpark(boolean enableDictionaryEncoding) { +- SparkSession.Builder builder = SparkSession.builder().config(TestBase.DISABLE_UI); ++ SparkSession.Builder builder = ++ SparkSession.builder() ++ .config(TestBase.DISABLE_UI) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g"); + if (!enableDictionaryEncoding) { + builder + .config("parquet.dictionary.page.size", "1") +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +index d1c724425c..ea2d079301 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java +@@ -90,6 +90,17 @@ public abstract class SparkDistributedDataScanTestBase + .master("local[2]") + .config("spark.serializer", serializer) + .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + } +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +index a21c6a08ec..9f1e8774c5 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java +@@ -73,6 +73,17 @@ public class TestSparkDistributedDataScanDeletes + .master("local[2]") + .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + } +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +index 5edf482822..51c50904cd 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java +@@ -62,6 +62,17 @@ public class TestSparkDistributedDataScanFilterFiles + .master("local[2]") + .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + } +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +index e6f3c75475..44a3e15688 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java +@@ -63,6 +63,17 @@ public class TestSparkDistributedDataScanReporting + .master("local[2]") + .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + } +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +index 507d7b313b..cea692da3f 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestBase.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestBase.java +@@ -86,6 +86,17 @@ public abstract class TestBase extends SparkTestHelperBase { + .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") + .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) + .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config("spark.ui.enabled", "false") + .config(DISABLE_UI) + .enableHiveSupport() +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java +index 3bd902fafc..a3dc6ed069 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java +@@ -133,6 +133,7 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; + import org.apache.spark.sql.internal.SQLConf; + import org.apache.spark.sql.types.DataType; + import org.apache.spark.sql.types.DataTypes; ++import org.junit.jupiter.api.AfterEach; + import org.junit.jupiter.api.BeforeAll; + import org.junit.jupiter.api.BeforeEach; + import org.junit.jupiter.api.TestTemplate; +@@ -144,6 +145,9 @@ import org.mockito.Mockito; + @ExtendWith(ParameterizedTestExtension.class) + public class TestRewriteDataFilesAction extends TestBase { + ++ private static final String COMET_NATIVE_ICEBERG_WRITE_ENABLED = ++ "spark.comet.write.iceberg.enabled"; ++ + @TempDir private File tableDir; + private static final int SCALE = 400; + private static final int LARGE_SCALE = 400000; +@@ -177,6 +181,15 @@ public class TestRewriteDataFilesAction extends TestBase { + @BeforeEach + public void setupTableLocation() { + this.tableLocation = tableDir.toURI().toString(); ++ // These tests calibrate bin-packing thresholds against parquet-mr file sizes. Comet's native ++ // writer produces different-sized Parquet files, so keep v3 setup writes on parquet-mr while ++ // the Comet suites cover native v3 writes and row-lineage fallback directly. ++ spark.conf().set(COMET_NATIVE_ICEBERG_WRITE_ENABLED, Boolean.toString(formatVersion != 3)); ++ } ++ ++ @AfterEach ++ public void restoreCometNativeIcebergWrite() { ++ spark.conf().set(COMET_NATIVE_ICEBERG_WRITE_ENABLED, "true"); + } + + private RewriteDataFilesSparkAction basicRewrite(Table table) { +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTestBase.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTestBase.java +index 8ce60f6275..75112b4354 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTestBase.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTestBase.java +@@ -48,7 +48,6 @@ import org.apache.iceberg.types.Types.LongType; + import org.apache.iceberg.types.Types.MapType; + import org.apache.iceberg.types.Types.StructType; + import org.apache.iceberg.util.DateTimeUtil; +-import org.assertj.core.api.Condition; + import org.junit.jupiter.api.Test; + import org.junit.jupiter.api.io.TempDir; + import org.junit.jupiter.params.ParameterizedTest; +@@ -385,12 +384,6 @@ public abstract class AvroDataTestBase { + .build()); + + assertThatThrownBy(() -> writeAndValidate(writeSchema, expectedSchema)) +- .has( +- new Condition<>( +- t -> +- IllegalArgumentException.class.isInstance(t) +- || IllegalArgumentException.class.isInstance(t.getCause()), +- "Expecting a throwable or cause that is an instance of IllegalArgumentException")) + .hasMessageContaining("Missing required field: missing_str"); + } + +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java +index b61ecfa2f4..78198044f7 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java +@@ -66,6 +66,17 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect + SparkSession.builder() + .master("local[2]") + .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + } +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +index da9cd63921..79d996db28 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java +@@ -62,6 +62,17 @@ public abstract class ScanTestBase extends AvroDataTestBase { + ScanTestBase.spark = + SparkSession.builder() + .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .master("local[2]") + .config(TestBase.DISABLE_UI) + .getOrCreate(); +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +index d381822483..4debfa07bd 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +@@ -144,7 +144,21 @@ public class TestCompressionSettings extends CatalogTestBase { + + @BeforeAll + public static void startSpark() { +- TestCompressionSettings.spark = SparkSession.builder().master("local[2]").getOrCreate(); ++ TestCompressionSettings.spark = ++ SparkSession.builder() ++ .master("local[2]") ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") ++ .getOrCreate(); + } + + @BeforeEach +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +index e67ec5fd62..f47ae6b21b 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java +@@ -79,7 +79,21 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { + + @BeforeAll + public static void startSpark() { +- TestDataSourceOptions.spark = SparkSession.builder().master("local[2]").getOrCreate(); ++ TestDataSourceOptions.spark = ++ SparkSession.builder() ++ .master("local[2]") ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") ++ .getOrCreate(); + } + + @AfterAll +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +index 57b833a16c..96cd119f5b 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +@@ -122,6 +122,17 @@ public class TestFilteredScan { + SparkSession.builder() + .master("local[2]") + .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + } +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +index d0103ff46e..78a3711d03 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java +@@ -99,6 +99,17 @@ public class TestForwardCompatibility { + SparkSession.builder() + .master("local[2]") + .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + } +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +index 19931f0ebd..4d4581eb6b 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java +@@ -52,6 +52,17 @@ public class TestIcebergSpark { + SparkSession.builder() + .master("local[2]") + .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + } +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +index 8098db81f9..7c0cc6046f 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +@@ -120,6 +120,17 @@ public class TestPartitionPruning { + SparkSession.builder() + .master("local[2]") + .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +index 9b5b22a73f..bf3b6d4e0f 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +@@ -113,6 +113,17 @@ public class TestPartitionValues { + SparkSession.builder() + .master("local[2]") + .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + } +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +index 3004e8fa5c..422126ed93 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java +@@ -98,6 +98,17 @@ public class TestSnapshotSelection { + SparkSession.builder() + .master("local[2]") + .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + } +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +index 8b55cc0fe6..df4b961293 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java +@@ -127,6 +127,17 @@ public class TestSparkDataFile { + SparkSession.builder() + .master("local[2]") + .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +index aacdabfe60..bde9a4f8d0 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +@@ -104,6 +104,17 @@ public class TestSparkDataWrite { + SparkSession.builder() + .master("local[2]") + .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + } +@@ -149,7 +160,7 @@ public class TestSparkDataWrite { + Dataset result = spark.read().format("iceberg").load(targetLocation); + + List actual = +- result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); ++ result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); + assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); + for (ManifestFile manifest : + SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) { +@@ -219,7 +230,7 @@ public class TestSparkDataWrite { + Dataset result = spark.read().format("iceberg").load(targetLocation); + + List actual = +- result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); ++ result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); + assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); + } + +@@ -264,7 +275,7 @@ public class TestSparkDataWrite { + Dataset result = spark.read().format("iceberg").load(targetLocation); + + List actual = +- result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); ++ result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); + assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); + } + +@@ -316,7 +327,7 @@ public class TestSparkDataWrite { + Dataset result = spark.read().format("iceberg").load(targetLocation); + + List actual = +- result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); ++ result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); + assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); + } + +@@ -358,7 +369,7 @@ public class TestSparkDataWrite { + Dataset result = spark.read().format("iceberg").load(targetLocation); + + List actual = +- result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); ++ result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); + assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); + } + +@@ -397,7 +408,7 @@ public class TestSparkDataWrite { + Dataset result = spark.read().format("iceberg").load(targetLocation); + + List actual = +- result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); ++ result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); + assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); + + List files = Lists.newArrayList(); +@@ -461,7 +472,7 @@ public class TestSparkDataWrite { + Dataset result = spark.read().format("iceberg").load(targetLocation); + + List actual = +- result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); ++ result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); + assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); + } + +@@ -814,7 +825,7 @@ public class TestSparkDataWrite { + // Since write and commit succeeded, the rows should be readable + Dataset result = spark.read().format("iceberg").load(targetLocation); + List actual = +- result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); ++ result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); + assertThat(actual) + .hasSize(records.size() + records2.size()) + .containsExactlyInAnyOrder( +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +index de6a5e5902..fe4212148a 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +@@ -89,6 +89,17 @@ public class TestSparkReadProjection extends TestReadProjection { + SparkSession.builder() + .master("local[2]") + .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + ImmutableMap config = +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +index 0d61930571..12a464909f 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +@@ -138,6 +138,17 @@ public class TestSparkReaderDeletes extends DeleteReadTests { + .config("spark.ui.liveUpdate.period", 0) + .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") + .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .enableHiveSupport() + .getOrCreate(); +@@ -210,7 +221,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { + } + + protected boolean countDeletes() { +- return true; ++ // TODO: Enable once iceberg-rust exposes delete count metrics to Comet ++ return false; + } + + @Override +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +index cb2f866fab..fcba981b20 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java +@@ -183,6 +183,17 @@ public class TestSparkReaderWithBloomFilter { + SparkSession.builder() + .master("local[2]") + .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .enableHiveSupport() + .getOrCreate(); +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +index d9b8815ed8..cf97f01940 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java +@@ -75,6 +75,17 @@ public class TestStructuredStreaming { + .master("local[2]") + .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) + .config("spark.sql.shuffle.partitions", 4) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + } +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +index 79781a8fc3..3674800ab8 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java +@@ -75,7 +75,21 @@ public class TestTimestampWithoutZone extends TestBase { + + @BeforeAll + public static void startSpark() { +- TestTimestampWithoutZone.spark = SparkSession.builder().master("local[2]").getOrCreate(); ++ TestTimestampWithoutZone.spark = ++ SparkSession.builder() ++ .master("local[2]") ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") ++ .getOrCreate(); + } + + @AfterAll +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +index ab2479d610..2c65d0298c 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java +@@ -86,6 +86,17 @@ public class TestWriteMetricsConfig { + SparkSession.builder() + .master("local[2]") + .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .getOrCreate(); + TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +index 1669301d2d..dcd60ba357 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java +@@ -63,6 +63,17 @@ public class TestAggregatePushDown extends CatalogTestBase { + SparkSession.builder() + .master("local[2]") + .config("spark.sql.iceberg.aggregate_pushdown", "true") ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .config(TestBase.DISABLE_UI) + .enableHiveSupport() + .getOrCreate(); +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java +index 55d5199c96..7870c861ea 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java +@@ -936,9 +936,11 @@ public class TestFilterPushDown extends TestBaseWithCatalog { + String planAsString = sparkPlan.toString().replaceAll("#(\\d+L?)", ""); + + if (sparkFilter != null) { ++ // Comet accelerates the post-scan filter as CometFilter; predicates it does not support ++ // (e.g. on variant columns) fall back to a Spark Filter. + assertThat(planAsString) + .as("Post scan filter should match") +- .containsAnyOf("Filter (" + sparkFilter + ")", "Filter " + sparkFilter); ++ .containsAnyOf("CometFilter", "Filter (" + sparkFilter + ")", "Filter " + sparkFilter); + } else { + assertThat(planAsString).as("Should be no post scan filter").doesNotContain("Filter ("); + } +diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java +index c7617274c0..fb055fb524 100644 +--- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java ++++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/variant/TestVariantShredding.java +@@ -100,6 +100,17 @@ public class TestVariantShredding extends CatalogTestBase { + .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) + .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") + .config(DISABLE_UI) ++ .config("spark.plugins", "org.apache.spark.CometPlugin") ++ .config( ++ "spark.shuffle.manager", ++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") ++ .config("spark.comet.explainFallback.enabled", "true") ++ .config("spark.comet.scan.icebergNative.enabled", "true") ++ .config("spark.comet.write.iceberg.splitOperator.enabled", "true") ++ .config("spark.comet.write.iceberg.enabled", "true") ++ .config("spark.comet.exec.localTableScan.enabled", "true") ++ .config("spark.memory.offHeap.enabled", "true") ++ .config("spark.memory.offHeap.size", "10g") + .enableHiveSupport() + .getOrCreate(); + From 439104f9f4b16848885af2c83795a270deed997b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 5 Nov 2026 10:46:36 -0700 Subject: [PATCH 3/4] docs: document Iceberg 1.12 coverage and why the Spark 4.2 profile stays on the 4.0 runtime Iceberg 1.12.0 has a Spark 4.2 module in source, but its release script only publishes the 3.5, 4.0 and 4.1 runtimes, so iceberg-spark-runtime-4.2 does not exist on Maven Central. --- docs/source/contributor-guide/ci.md | 34 +++++++++---------- .../contributor-guide/iceberg-spark-tests.md | 15 ++++---- .../contributor-guide/iceberg-writes.md | 9 +++-- docs/source/user-guide/latest/iceberg.md | 2 +- spark/pom.xml | 4 ++- 5 files changed, 35 insertions(+), 29 deletions(-) diff --git a/docs/source/contributor-guide/ci.md b/docs/source/contributor-guide/ci.md index ca9b6d28dd6..662a4a6835c 100644 --- a/docs/source/contributor-guide/ci.md +++ b/docs/source/contributor-guide/ci.md @@ -44,27 +44,27 @@ flowchart LR PRTIER["PR tier
Linux build, lint, Rust tests
TPC-H / TPC-DS
Comet suites, Spark 4.1
Spark SQL, Spark 4.1 catalyst and sql_core
Iceberg 1.11"] QUEUE["Queue tier, on top of the PR tier
Spark SQL, Spark 4.1 sql_hive
macOS build and Comet suites
Benchmark check, Delta gate
PyArrow UDF, Spark 4.0 / 4.1 / 4.2"] - NIGHTLY["Nightly tier
Comet suites, Spark 3.5 / 4.0 / 4.2
Spark SQL, Spark 3.5 / 4.0 / 4.2
Iceberg 1.9 / 1.10
Delta contrib, Spark 3.5 / 4.0 / 4.1"] + NIGHTLY["Nightly tier
Comet suites, Spark 3.5 / 4.0 / 4.2
Spark SQL, Spark 3.5 / 4.0 / 4.2
Iceberg 1.9 / 1.10 / 1.12
Delta contrib, Spark 3.5 / 4.0 / 4.1"] CACHE["Cache-refresh-only mode
the three cache-writing jobs, plus Lint"] ``` Suite by suite: -| Suite | Pull request | Merge queue | Nightly | -| ------------------------------------------------- | ------------ | ----------- | ------- | -| Linux build, lint, Rust tests, TPC-H/TPC-DS | yes | yes | no | -| Comet test suites, Spark 4.1 | yes | yes | no | -| Spark SQL tests, Spark 4.1, catalyst and sql_core | yes | yes | no | -| Spark SQL tests, Spark 4.1, sql_hive | with label | yes | no | -| Iceberg Spark SQL tests, Iceberg 1.11 | yes | yes | no | -| macOS build and Comet test suites | with label | yes | no | -| Benchmark compile and lint check | with label | yes | no | -| Delta contrib build gate | with label | yes | no | -| PyArrow UDF tests, Spark 4.0 / 4.1 / 4.2 | with label | yes | no | -| Comet test suites, Spark 3.5 / 4.0 / 4.2 | with label | no | yes | -| Spark SQL tests, Spark 3.5 / 4.0 / 4.2 | with label | no | yes | -| Iceberg Spark SQL tests, Iceberg 1.9 / 1.10 | with label | no | yes | -| Delta contrib tests, Spark 3.5 / 4.0 / 4.1 | with label | no | yes | +| Suite | Pull request | Merge queue | Nightly | +| -------------------------------------------------- | ------------ | ----------- | ------- | +| Linux build, lint, Rust tests, TPC-H/TPC-DS | yes | yes | no | +| Comet test suites, Spark 4.1 | yes | yes | no | +| Spark SQL tests, Spark 4.1, catalyst and sql_core | yes | yes | no | +| Spark SQL tests, Spark 4.1, sql_hive | with label | yes | no | +| Iceberg Spark SQL tests, Iceberg 1.11 | yes | yes | no | +| macOS build and Comet test suites | with label | yes | no | +| Benchmark compile and lint check | with label | yes | no | +| Delta contrib build gate | with label | yes | no | +| PyArrow UDF tests, Spark 4.0 / 4.1 / 4.2 | with label | yes | no | +| Comet test suites, Spark 3.5 / 4.0 / 4.2 | with label | no | yes | +| Spark SQL tests, Spark 3.5 / 4.0 / 4.2 | with label | no | yes | +| Iceberg Spark SQL tests, Iceberg 1.9 / 1.10 / 1.12 | with label | no | yes | +| Delta contrib tests, Spark 3.5 / 4.0 / 4.1 | with label | no | yes | The **PR tier** is the fast feedback loop while a change is being iterated on. The **queue tier** is the authoritative gate: everything the PR tier runs plus the suites that must pass before @@ -303,7 +303,7 @@ re-armed automatically. `ci.yml` also runs on a schedule, at 06:00 UTC every day, against what landed on `main` since the last successful scheduled run. That run executes only the nightly tier: the Comet test suites against the Spark profiles other than 4.1, the Spark SQL suites for Spark 3.5, 4.0 and 4.2, the Iceberg suites for -1.9 and 1.10, and the Delta contrib suites for Spark 3.5, 4.0 and 4.1, each only when the day's +1.9, 1.10 and 1.12, and the Delta contrib suites for Spark 3.5, 4.0 and 4.1, each only when the day's changes touched files it covers. The queue already ran everything else against the same tree, so nothing in the queue tier is repeated. A day with no merges, or with only documentation changes, runs nothing. diff --git a/docs/source/contributor-guide/iceberg-spark-tests.md b/docs/source/contributor-guide/iceberg-spark-tests.md index d36b75bd8a6..3b8a307e47b 100644 --- a/docs/source/contributor-guide/iceberg-spark-tests.md +++ b/docs/source/contributor-guide/iceberg-spark-tests.md @@ -123,17 +123,18 @@ git add -A git diff apache-iceberg-1.9.1 > ../datafusion-comet/dev/diffs/iceberg/1.9.1.diff ``` -Repeat for each Iceberg version (1.9.1, 1.10.0, 1.11.0). The file contents differ between versions, so each +Repeat for each Iceberg version (1.9.1, 1.10.0, 1.11.0, 1.12.0). The file contents differ between versions, so each diff must be generated against its own tag. ## Running Tests in CI -The `iceberg_1_9`, `iceberg_1_10` and `iceberg_1_11` jobs in `ci.yml` apply these diffs and +The `iceberg_1_9`, `iceberg_1_10`, `iceberg_1_11` and `iceberg_1_12` jobs in `ci.yml` apply these diffs and run the three Gradle targets above against each Iceberg version, all with Java 17. Iceberg 1.9.1 and 1.10.0 run -against Spark 3.5.9; Iceberg 1.11.0 runs against Spark 4.1.3. -Iceberg 1.11.0 runs on every pull request whose paths match and in the merge queue; 1.9.1 and -1.10.0 run once a night against `main`. Those two run earlier on a pull request labeled -`run-iceberg-tests`. Each job +against Spark 3.5.9; Iceberg 1.11.0 runs against Spark 4.1.3; Iceberg 1.12.0, the first release with a Spark 4.2 +module, runs against Spark 4.2.0. +Iceberg 1.11.0 runs on every pull request whose paths match and in the merge queue; 1.9.1, 1.10.0 and +1.12.0 run once a night against `main`, 1.12.0 because Spark 4.2 support is still experimental. Those three run +earlier on a pull request labeled `run-iceberg-tests`. Each job calls `iceberg_spark_test_reusable.yml`, which holds the build and test job logic. See [.github/workflows/README.md](https://github.com/apache/datafusion-comet/blob/main/.github/workflows/README.md) for how the pull-request, merge-queue and nightly tiers differ. @@ -164,7 +165,7 @@ Apply the `run-iceberg-tests` label to a pull request whenever it touches the Ic path, reflection code (`org.apache.comet.iceberg.IcebergReflection`), or other logic whose behavior can differ across Iceberg versions. The Comet test suites in the Linux build do not exercise Iceberg's own Spark tests, but the Iceberg 1.11 job does and every pull request gets its verdict. Without the -label, the first verdict on the older Iceberg versions is the nightly run's, after the change has +label, the first verdict on the other Iceberg versions is the nightly run's, after the change has landed. ### Which writer ran each Iceberg write diff --git a/docs/source/contributor-guide/iceberg-writes.md b/docs/source/contributor-guide/iceberg-writes.md index 0fad3c35370..5f5389836c4 100644 --- a/docs/source/contributor-guide/iceberg-writes.md +++ b/docs/source/contributor-guide/iceberg-writes.md @@ -455,9 +455,12 @@ the writer: run the write suites and the Iceberg Spark tests. The pin policy is | `CometIcebergEndToEndBenchmark` | An Iceberg read, hash repartition and Iceberg write against stock Spark, over primitive, struct, array, map and nested columns. It checks each arm's plan and the rows it wrote before timing it. | The Comet suites run against the Iceberg version each Spark profile pins in `spark/pom.xml`: 1.8.1 -for Spark 3.5, 1.10.0 for 4.0, and 1.11.0 for 4.1. No Iceberg runtime is published -for Spark 4.2, so the 4.2 profile builds against the 4.0 runtime and the Iceberg suites skip there -(`icebergAvailable` is false on Spark 4.2). Only the default profile runs on every pull request. +for Spark 3.5, 1.10.0 for 4.0, and 1.11.0 for 4.1. Iceberg 1.12.0 has a +Spark 4.2 module in its source tree, but its release does not publish the runtime +(`iceberg-spark-runtime-4.2_2.13`), so the 4.2 profile builds against the 4.0 runtime and the Comet +Iceberg suites skip there (`icebergAvailable` is false on Spark 4.2). Iceberg's own Spark 4.2 suites +do run, from source, in the `iceberg_1_12` CI job. Only the default profile runs on every pull +request. Behaviour that differs between Iceberg versions, such as reflection targets, metrics conventions and partition path spelling, needs a test that runs on each. diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 4c603f79b86..11f02e4d372 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -36,7 +36,7 @@ then serialized to Comet's native execution engine (see [PR #2528](https://github.com/apache/datafusion-comet/pull/2528)). The example below uses Spark's package downloader to retrieve Comet $COMET_VERSION and Iceberg -1.8.1, but Comet has been tested with Iceberg 1.5, 1.8, 1.9, 1.10, and 1.11. The native Iceberg +1.8.1, but Comet has been tested with Iceberg 1.5, 1.8, 1.9, 1.10, 1.11, and 1.12. The native Iceberg reader is enabled by default. To disable it, set `spark.comet.scan.icebergNative.enabled=false`. The example uses the Spark 3.5 / Scala 2.12 build of Comet; substitute the Comet artifact diff --git a/spark/pom.xml b/spark/pom.xml index e2c51bf3c5f..026c2897908 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -506,7 +506,9 @@ under the License. ${amazon-awssdk-v2.version} provided - + org.apache.iceberg iceberg-spark-runtime-4.0_${scala.binary.version} From 6e991c51810b05254346174ecba936b83ed259ea Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 15 Nov 2026 13:08:22 -0700 Subject: [PATCH 4/4] fix: address review on Iceberg 1.12 support - Fail the Iceberg shard coverage and extensions jobs when no write ran on the native writer (summarize-iceberg-writes.py --require-native), so a release that makes the gate decline every write no longer passes silently. - Resolve ManifestFiles.read through the cached getMethod helper, and drop 1.5.2 from its supported-version range. - Move DetectionDelegatingFileIO back to Scala: override def getConf compiles against Iceberg 1.8.1 through 1.12.0. - Say which suites test which Iceberg versions in the user guide. - Define the Iceberg path filter once in compute-changes.py. Co-Authored-By: Claude Opus 5.5 --- .../workflows/iceberg_spark_test_reusable.yml | 7 +- dev/ci/compute-changes.py | 84 ++----------------- dev/ci/summarize-iceberg-writes.py | 32 ++++++- dev/ci/test-summarize-iceberg-writes.py | 28 +++++++ .../contributor-guide/iceberg-spark-tests.md | 5 +- docs/source/user-guide/latest/iceberg.md | 4 +- .../comet/iceberg/IcebergReflection.scala | 14 ++-- .../comet/DetectionDelegatingFileIO.java | 80 ------------------ .../CometIcebergWriteDetectionSuite.scala | 31 ++++++- 9 files changed, 113 insertions(+), 172 deletions(-) delete mode 100644 spark/src/test/java/org/apache/comet/DetectionDelegatingFileIO.java diff --git a/.github/workflows/iceberg_spark_test_reusable.yml b/.github/workflows/iceberg_spark_test_reusable.yml index faa7ae610e5..5def2f4228b 100644 --- a/.github/workflows/iceberg_spark_test_reusable.yml +++ b/.github/workflows/iceberg_spark_test_reusable.yml @@ -200,8 +200,11 @@ jobs: --task :iceberg-spark:iceberg-spark-${{ inputs.spark-short }}_${{ inputs.scala }}:test - name: Summarize Iceberg writes across shards if: ${{ !cancelled() }} + # --require-native fails the job when no write in any shard ran on the + # native writer. Iceberg's tests pass on either writer, so without it a + # release that makes the gate decline every write goes unnoticed. run: | - python3 dev/ci/summarize-iceberg-writes.py --title "iceberg-spark, all shards" \ + python3 dev/ci/summarize-iceberg-writes.py --require-native --title "iceberg-spark, all shards" \ iceberg-shard-reports iceberg-spark-extensions: @@ -242,7 +245,7 @@ jobs: - name: Summarize Iceberg writes if: ${{ !cancelled() }} run: | - python3 dev/ci/summarize-iceberg-writes.py --title "iceberg-spark-extensions" \ + python3 dev/ci/summarize-iceberg-writes.py --require-native --title "iceberg-spark-extensions" \ apache-iceberg/build/comet-iceberg-writes - name: Upload Iceberg write report if: ${{ !cancelled() }} diff --git a/dev/ci/compute-changes.py b/dev/ci/compute-changes.py index 1d291618343..dd19ee71449 100644 --- a/dev/ci/compute-changes.py +++ b/dev/ci/compute-changes.py @@ -335,81 +335,12 @@ ".mvn/**", "mvnw", ], - "iceberg_1_10": [ - "native/**/src/**", - "native/**/Cargo.toml", - "native/Cargo.lock", - "common/src/main/**", - "common/pom.xml", - "spark/src/main/**", - "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", - "spark/pom.xml", - "dev/diffs/iceberg/**", - "pom.xml", - "rust-toolchain.toml", - ".github/workflows/ci.yml", - ".github/workflows/iceberg_spark_test_reusable.yml", - ".github/actions/setup-builder/**", - ".github/actions/setup-iceberg-builder/**", - "dev/ci/iceberg-test-shards.gradle", - "dev/ci/check-iceberg-shards.py", - "dev/ci/test-iceberg-shards.py", - ".github/actions/upload-artifact-retry/**", - ".github/actions/download-artifact-retry/**", - ".github/actions/maven-bootstrap/**", - ".mvn/**", - "mvnw", - ], - "iceberg_1_11": [ - "native/**/src/**", - "native/**/Cargo.toml", - "native/Cargo.lock", - "common/src/main/**", - "common/pom.xml", - "spark/src/main/**", - "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", - "spark/pom.xml", - "dev/diffs/iceberg/**", - "pom.xml", - "rust-toolchain.toml", - ".github/workflows/ci.yml", - ".github/workflows/iceberg_spark_test_reusable.yml", - ".github/actions/setup-builder/**", - ".github/actions/setup-iceberg-builder/**", - "dev/ci/iceberg-test-shards.gradle", - "dev/ci/check-iceberg-shards.py", - "dev/ci/test-iceberg-shards.py", - ".github/actions/upload-artifact-retry/**", - ".github/actions/download-artifact-retry/**", - ".github/actions/maven-bootstrap/**", - ".mvn/**", - "mvnw", - ], - "iceberg_1_12": [ - "native/**/src/**", - "native/**/Cargo.toml", - "native/Cargo.lock", - "common/src/main/**", - "common/pom.xml", - "spark/src/main/**", - "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", - "spark/pom.xml", - "dev/diffs/iceberg/**", - "pom.xml", - "rust-toolchain.toml", - ".github/workflows/ci.yml", - ".github/workflows/iceberg_spark_test_reusable.yml", - ".github/actions/setup-builder/**", - ".github/actions/setup-iceberg-builder/**", - "dev/ci/iceberg-test-shards.gradle", - "dev/ci/check-iceberg-shards.py", - "dev/ci/test-iceberg-shards.py", - ".github/actions/upload-artifact-retry/**", - ".github/actions/download-artifact-retry/**", - ".github/actions/maven-bootstrap/**", - ".mvn/**", - "mvnw", - ], + # The Iceberg versions share one path filter: each call builds the same + # Comet sources against a different Iceberg release. Populated below, + # after the dict, so the lists cannot drift. + "iceberg_1_10": [], + "iceberg_1_11": [], + "iceberg_1_12": [], "delta_3_5": [ "contrib/delta-spark/**", "!**.md", @@ -442,6 +373,9 @@ FILTERS["build_linux_all_profiles"] = FILTERS["build_linux"] FILTERS["delta_4_0"] = FILTERS["delta_3_5"] FILTERS["delta_4_1"] = FILTERS["delta_3_5"] +FILTERS["iceberg_1_10"] = FILTERS["iceberg_1_9"] +FILTERS["iceberg_1_11"] = FILTERS["iceberg_1_9"] +FILTERS["iceberg_1_12"] = FILTERS["iceberg_1_9"] # Which events may run each job, independent of the path filters above. # diff --git a/dev/ci/summarize-iceberg-writes.py b/dev/ci/summarize-iceberg-writes.py index c94638b4e12..380b1d4ff52 100644 --- a/dev/ci/summarize-iceberg-writes.py +++ b/dev/ci/summarize-iceberg-writes.py @@ -30,8 +30,13 @@ jobs does not count a shard twice. The latest attempt is the newest such directory, whether or not it holds any report files, so a rerun that recorded no writes is reported as missing rather than replaced by an earlier attempt. -The summary is also appended to $GITHUB_STEP_SUMMARY when that is set. It -never fails the job. +The summary is also appended to $GITHUB_STEP_SUMMARY when that is set. + +By default it never fails the job. With --require-native it exits non-zero, +after printing the summary, when no write ran on Comet's native writer. That +guards against the native writer silently declining every write, as happened +when an Iceberg release removed a method the eligibility probe resolves: the +upstream tests still pass, because they do not assert which writer ran. """ import argparse @@ -40,6 +45,7 @@ import os from pathlib import Path import re +import sys SHARD_ATTEMPT = re.compile(r"-shard-(\d+)-attempt-(\d+)$") @@ -153,19 +159,37 @@ def summarize(title, writes, missing=()): return "\n".join(lines) + "\n" +def native_writes(writes): + return sum(1 for w in writes if w["writer"] == "native") + + def main(): parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--title", required=True, help="heading for the summary") + parser.add_argument( + "--require-native", + action="store_true", + help="exit non-zero when no write ran on Comet's native writer", + ) parser.add_argument("roots", nargs="+", help="directories holding the report files") args = parser.parse_args() - summary = summarize(args.title, *load(args.roots)) + writes, missing = load(args.roots) + summary = summarize(args.title, writes, missing) print(summary) step_summary = os.environ.get("GITHUB_STEP_SUMMARY") if step_summary: with open(step_summary, "a", encoding="utf-8") as out: out.write(summary + "\n") + if args.require_native and native_writes(writes) == 0: + print( + f"error: none of the {len(writes)} recorded Iceberg writes ran on Comet's native " + "writer, so the native writer is declining every write", + file=sys.stderr, + ) + return 1 + return 0 if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/dev/ci/test-summarize-iceberg-writes.py b/dev/ci/test-summarize-iceberg-writes.py index cce4ada6630..210c352d292 100644 --- a/dev/ci/test-summarize-iceberg-writes.py +++ b/dev/ci/test-summarize-iceberg-writes.py @@ -19,9 +19,12 @@ """Fast regression tests for the Iceberg write report summary.""" +import contextlib import importlib.util +import io import json from pathlib import Path +import sys import tempfile import unittest @@ -96,6 +99,31 @@ def test_files_outside_shard_artifacts_always_count(self): (self.root / "iceberg-writes-local.jsonl").write_text(records("native", "jvm")) self.assertEqual(self.load(), (["jvm", "native"], [])) + def run_main(self, *args): + argv = sys.argv + sys.argv = ["summarize-iceberg-writes.py", "--title", "fixture", *args, str(self.root)] + try: + with contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(io.StringIO()): + return SUMMARIZE.main() + finally: + sys.argv = argv + + def test_require_native_passes_with_a_native_write(self): + self.attempt(1, 1, "jvm", "native") + self.attempt(2, 1, "jvm") + self.assertEqual(self.run_main("--require-native"), 0) + + def test_require_native_fails_when_every_write_falls_back(self): + self.attempt(1, 1, "jvm", "spark") + self.attempt(2, 1, "jvm") + self.assertEqual(self.run_main("--require-native"), 1) + self.assertEqual(self.run_main(), 0) + + def test_require_native_fails_when_no_writes_were_recorded(self): + self.attempt(1, 1) + self.assertEqual(self.run_main("--require-native"), 1) + def test_no_writes_at_all(self): summary = SUMMARIZE.summarize("fixture", *SUMMARIZE.load([self.root / "missing"])) self.assertIn("No Iceberg writes were recorded", summary) diff --git a/docs/source/contributor-guide/iceberg-spark-tests.md b/docs/source/contributor-guide/iceberg-spark-tests.md index 3b8a307e47b..74ad3a61629 100644 --- a/docs/source/contributor-guide/iceberg-spark-tests.md +++ b/docs/source/contributor-guide/iceberg-spark-tests.md @@ -189,5 +189,8 @@ shows the count and share of each writer, the most common fallback reasons, and operators. Each shard and the extensions job gets its own table. The shard coverage job adds one for all shards together, counting only the latest attempt of each shard. A shard whose latest attempt recorded no writes is named above the table rather than counted from an earlier attempt. The raw -records are uploaded with the job's other reports. The summary never fails a job. +records are uploaded with the job's other reports. The shard coverage job and the extensions job fail +when none of their writes ran on the native writer, because Iceberg's tests pass on either writer and +would not otherwise show the native writer declining every write. A shard's own summary never fails +its job. `dev/local-ci.sh iceberg` prints the same summary after each shard and after the extensions target. diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 11f02e4d372..7016915038f 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -36,7 +36,9 @@ then serialized to Comet's native execution engine (see [PR #2528](https://github.com/apache/datafusion-comet/pull/2528)). The example below uses Spark's package downloader to retrieve Comet $COMET_VERSION and Iceberg -1.8.1, but Comet has been tested with Iceberg 1.5, 1.8, 1.9, 1.10, 1.11, and 1.12. The native Iceberg +1.8.1. Comet's own test suites run against Iceberg 1.8, 1.10 and 1.11, and Iceberg's Spark test +suites run with Comet against Iceberg 1.9, 1.10, 1.11 and 1.12. Iceberg 1.12 is tested only through +Iceberg's suites, with Spark 4.2, whose support in Comet is experimental. The native Iceberg reader is enabled by default. To disable it, set `spark.comet.scan.icebergNative.enabled=false`. The example uses the Spark 3.5 / Scala 2.12 build of Comet; substitute the Comet artifact diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index aeb0d8492e0..2dd91f3e112 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -1969,16 +1969,16 @@ object IcebergReflection extends Logging { /** * `ManifestFiles.read(ManifestFile, FileIO, Map)`, the only overload - * that is public on every supported Iceberg release (1.5.2 through 1.12; 1.12 dropped the + * that is public on every supported Iceberg release (1.8.1 through 1.12; 1.12 dropped the * two-argument one). */ private def manifestReadMethodOf(fileIOClass: Class[_]): java.lang.reflect.Method = - loadClass(ClassNames.MANIFEST_FILES) - .getMethod( - "read", - loadClass(ClassNames.MANIFEST_FILE), - fileIOClass, - classOf[java.util.Map[_, _]]) + getMethod( + loadClass(ClassNames.MANIFEST_FILES), + "read", + loadClass(ClassNames.MANIFEST_FILE), + fileIOClass, + classOf[java.util.Map[_, _]]) private def newInMemoryFileIO(location: String, bytes: Array[Byte]): AnyRef = { val cls = loadClass(ClassNames.INMEMORY_FILE_IO) diff --git a/spark/src/test/java/org/apache/comet/DetectionDelegatingFileIO.java b/spark/src/test/java/org/apache/comet/DetectionDelegatingFileIO.java deleted file mode 100644 index 8d503f587d1..00000000000 --- a/spark/src/test/java/org/apache/comet/DetectionDelegatingFileIO.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.comet; - -import java.util.Map; -import java.util.function.Function; - -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.hadoop.HadoopConfigurable; -import org.apache.iceberg.hadoop.HadoopFileIO; -import org.apache.iceberg.io.FileIO; -import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.io.OutputFile; -import org.apache.iceberg.util.SerializableSupplier; - -/** - * A FileIO that works normally (delegating to HadoopFileIO) but whose class is not on Comet's - * recognized-FileIO allowlist. Composition rather than inheritance is the point: a HadoopFileIO - * SUBCLASS passes the hierarchy check by design, while this class must be declined. Instantiated - * reflectively by Iceberg's `CatalogUtil.loadFileIO`, hence public with a no-arg constructor. - * - *

This is Java because {@code getConf} is abstract on some supported Iceberg versions (1.8.1) - * and a default method on others (1.12). Scala has no spelling that compiles against both, since it - * requires {@code override} for a concrete member and rejects it for an abstract one. - */ -public class DetectionDelegatingFileIO implements FileIO, HadoopConfigurable { - private final HadoopFileIO delegate = new HadoopFileIO(); - - @Override - public InputFile newInputFile(String path) { - return delegate.newInputFile(path); - } - - @Override - public OutputFile newOutputFile(String path) { - return delegate.newOutputFile(path); - } - - @Override - public void deleteFile(String path) { - delegate.deleteFile(path); - } - - @Override - public void initialize(Map properties) { - delegate.initialize(properties); - } - - @Override - public void setConf(Configuration conf) { - delegate.setConf(conf); - } - - public Configuration getConf() { - return delegate.getConf(); - } - - @Override - public void serializeConfWith( - Function> confSerializer) { - delegate.serializeConfWith(confSerializer); - } -} diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index 491e8dbf924..c67d66915ff 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -29,8 +29,9 @@ import org.apache.iceberg.{PartitionSpec, StructLike, TableMetadata, TableOperat import org.apache.iceberg.aws.s3.S3FileIO import org.apache.iceberg.catalog.TableIdentifier import org.apache.iceberg.encryption.EncryptionManager -import org.apache.iceberg.hadoop.{HadoopCatalog, HadoopFileIO} -import org.apache.iceberg.io.{FileIO, LocationProvider, ResolvingFileIO} +import org.apache.iceberg.hadoop.{HadoopCatalog, HadoopConfigurable, HadoopFileIO} +import org.apache.iceberg.io.{FileIO, InputFile, LocationProvider, OutputFile, ResolvingFileIO} +import org.apache.iceberg.util.SerializableSupplier import org.apache.spark.SparkConf import org.apache.spark.rdd.RDD import org.apache.spark.sql.CometTestBase @@ -1857,6 +1858,32 @@ case class TransitionProbeLeaf(columnar: Boolean) extends LeafExecNode { throw new UnsupportedOperationException("planning-only node") } +/** + * A FileIO that works normally (delegating to HadoopFileIO) but whose class is not on Comet's + * recognized-FileIO allowlist. Composition rather than inheritance is the point: a HadoopFileIO + * SUBCLASS passes the hierarchy check by design, while this class must be declined. Instantiated + * reflectively by Iceberg's `CatalogUtil.loadFileIO`, hence top-level with a no-arg constructor. + */ +class DetectionDelegatingFileIO extends FileIO with HadoopConfigurable { + private val delegate = new HadoopFileIO() + + override def newInputFile(path: String): InputFile = delegate.newInputFile(path) + override def newOutputFile(path: String): OutputFile = delegate.newOutputFile(path) + override def deleteFile(path: String): Unit = delegate.deleteFile(path) + override def initialize(properties: java.util.Map[String, String]): Unit = + delegate.initialize(properties) + override def setConf(conf: Configuration): Unit = delegate.setConf(conf) + // `getConf` is abstract through Hadoop's `Configurable` up to Iceberg 1.11 and a default method + // of `HadoopConfigurable` in 1.12. Scala requires `override` for the default method and accepts + // it for the abstract one, so this spelling compiles against both. + override def getConf: Configuration = delegate.getConf + override def serializeConfWith( + confSerializer: java.util.function.Function[ + Configuration, + SerializableSupplier[Configuration]]): Unit = + delegate.serializeConfWith(confSerializer) +} + /** * A catalog whose TableOperations supplies a custom LocationProvider directly, without setting * `write.location-provider.impl`. This is the path the native write gate must inspect explicitly.