diff --git a/.ai/skills/review-comet-pr/SKILL.md b/.ai/skills/review-comet-pr/SKILL.md index 2c6bb6f0db3..7571db5cf58 100644 --- a/.ai/skills/review-comet-pr/SKILL.md +++ b/.ai/skills/review-comet-pr/SKILL.md @@ -244,8 +244,10 @@ config, not a testing one, to turn it off before merge (#6466). - Do the tests compare against Spark? A comparison with Comet's own accumulator, or with a second implementation such as iceberg-rust instead of the Iceberg Java that Spark runs, can agree on the wrong answer (#6423, #6426). -- New suites must be registered in both `.github/workflows/pr_build_linux.yml` and - `pr_build_macos.yml`. +- Each new concrete suite needs exactly one test-group tag (`@ScansTestGroup`, `@ShuffleTestGroup`, + `@ExecTestGroup` or `@ExpressionsTestGroup`), and a listing under that group in both + `.github/workflows/pr_build_linux.yml` and `pr_build_macos.yml`. `dev/ci/check-suites.py` + enforces both; the group should match the area the suite covers. For a change on a default path, look for tests with the inputs that broke earlier changes: diff --git a/dev/ci/check-suites.py b/dev/ci/check-suites.py index 68bf972c5cf..97d6a51c460 100644 --- a/dev/ci/check-suites.py +++ b/dev/ci/check-suites.py @@ -14,90 +14,172 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""Check that every Comet test suite belongs to exactly one CI test group. + +CI runs the Comet test suites in groups, one job per group. Each concrete suite +carries the ScalaTest tag of its group (see +spark/src/test/java/org/apache/comet/tags/*TestGroup.java), and the CI workflows +list it in the group of the same name. This fails on a suite with no group tag, +which would run in no job, on one with two, and on a suite whose workflow +listing does not match its tag. Suites deliberately left out of CI are listed +in IGNORE_LIST and must carry no group tag, so that no group picks them up. +""" import re import sys from pathlib import Path -def file_to_class_name(path: Path) -> str | None: - parts = path.parts - if "org" not in parts or "apache" not in parts: - return None - org_index = parts.index("org") - package_parts = parts[org_index:] - class_name = ".".join(package_parts) - class_name = class_name.replace(".scala", "") - return class_name +REPO_ROOT = Path(__file__).resolve().parents[2] -if __name__ == "__main__": +# One ScalaTest tag annotation per group, such as ExecTestGroup for the `exec` group. +TAGS_DIR = REPO_ROOT / "spark/src/test/java/org/apache/comet/tags" +GROUP_TAGS = sorted(path.stem for path in TAGS_DIR.glob("*TestGroup.java")) - # ignore traits, abstract classes, and intentionally skipped test suites - ignore_list = [ - "org.apache.comet.parquet.ParquetReadSuite", # abstract - "org.apache.comet.parquet.ParquetReadFromS3Suite", # manual test suite - "org.apache.comet.parquet.ParquetWriteWithMagicCommitterSuite", # manual test suite - "org.apache.comet.csv.CsvReadFromS3Suite", # manual test suite - "org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite", # manual test suite (loads libhdfs, see #5023) - "org.apache.comet.IcebergReadFromS3Suite", # manual test suite - "org.apache.comet.CometIcebergSchedulerFailureSuite", # manual, multiple hosts + SIGKILL - "org.apache.comet.cloud.s3.CometS3CredentialBridgeSuite", # manual test suite - "org.apache.comet.cloud.s3.HadoopS3ACredentialProviderAdapterBridgeSuite", # manual test suite - "org.apache.comet.shuffle.CelebornReflectionCompatibilitySuite", # dedicated version matrix - "org.apache.spark.sql.comet.CometPlanStabilitySuite", # abstract - "org.apache.spark.sql.comet.ParquetDatetimeRebaseSuite", # abstract - "org.apache.comet.exec.CometColumnarShuffleSuite" # abstract - ] - - for workflow_filename in [".github/workflows/pr_build_linux.yml", ".github/workflows/pr_build_macos.yml"]: - workflow = open(workflow_filename, encoding="utf-8").read() - - root = Path(".") - for path in root.rglob("*Suite.scala"): - # contrib suites run via their own module-level workflows - # (e.g. delta_contrib_test.yml), not the main PR build matrix - if path.parts[0] == "contrib": - continue - class_name = file_to_class_name(path) - if class_name: - if "Shim" in class_name: - continue - if class_name in ignore_list: - continue - if class_name not in workflow: - print(f"Suite not found in workflow {workflow_filename}: {class_name}") - sys.exit(-1) - print(f"Found {class_name} in {workflow_filename}") - - # Forward check: every suite named in a workflow must be declared in a source file. - # Filename-based discovery is not enough here, because several suites are declared - # inside a file named after a different class (e.g. CometShuffleSuite is declared in - # CometColumnarShuffleSuite.scala), and version-specific suites such as - # CometStringDecodeSuite live only in the spark-3.x sourceset. - declared = set() - suite_declaration = re.compile( - r"^\s*(?:(?:final|sealed|abstract|private|case)\s+)*(?:class|trait|object)\s+(\w*Suite)\b" - ) - for path in Path(".").rglob("*.scala"): - if "target" in path.parts: +# Suites that CI does not run, each with the reason. +IGNORE_LIST = { + "org.apache.comet.parquet.ParquetReadFromS3Suite": "manual test suite", + "org.apache.comet.parquet.ParquetWriteWithMagicCommitterSuite": "manual test suite", + "org.apache.comet.csv.CsvReadFromS3Suite": "manual test suite", + "org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite": "manual test suite (loads libhdfs, see #5023)", + "org.apache.comet.IcebergReadFromS3Suite": "manual test suite", + "org.apache.comet.CometIcebergSchedulerFailureSuite": "manual, multiple hosts + SIGKILL", + "org.apache.comet.cloud.s3.CometS3CredentialBridgeSuite": "manual test suite", + "org.apache.comet.cloud.s3.HadoopS3ACredentialProviderAdapterBridgeSuite": "manual test suite", + "org.apache.comet.shuffle.CelebornReflectionCompatibilitySuite": "dedicated version matrix", +} + +CLASS_DECL = re.compile( + r"^(?P(?:(?:final|sealed|abstract|private(?:\[\w+\])?|case)\s+)*)" + r"(?Pclass|trait|object)\s+(?P\w+)" +) +ANNOTATION = re.compile(r"^@(\w+)") + + +def test_sources(): + """Every Scala test source outside contrib, in all source sets (scala, spark-3.4, spark-4.x, ...). + + The contrib modules run their suites from their own workflows, such as delta_contrib_test.yml. + """ + for path in sorted(REPO_ROOT.rglob("*.scala")): + parts = path.relative_to(REPO_ROOT).parts + if parts[0] != "contrib" and "target" not in parts and "test" in parts: + yield path + + +def declarations(text): + """Yield (name, concrete, annotations, line) for each top-level type in a source file. + + `concrete` is true for a class that is neither abstract nor a trait or object. + """ + lines = text.splitlines() + for number, line in enumerate(lines): + match = CLASS_DECL.match(line) + if not match: continue - class_name = file_to_class_name(path) - if class_name: - declared.add(class_name.rsplit(".", 1)[-1]) - for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): - match = suite_declaration.match(line) - if match: - declared.add(match.group(1)) - - listed_suite = re.compile(r"\borg\.apache\.[A-Za-z0-9_.]*Suite\b") - undeclared = [] - for workflow_filename in [".github/workflows/pr_build_linux.yml", ".github/workflows/pr_build_macos.yml"]: - workflow = open(workflow_filename, encoding="utf-8").read() - for name in sorted(set(listed_suite.findall(workflow))): - if name.rsplit(".", 1)[-1] not in declared: - undeclared.append((workflow_filename, name)) - - if undeclared: - for workflow_filename, name in undeclared: - print(f"Workflow lists an undeclared suite {name} in {workflow_filename}") - sys.exit(-1) - print("All workflow-listed suites are declared") + annotations = [] + above = number - 1 + while above >= 0 and (annotation := ANNOTATION.match(lines[above])): + annotations.append(annotation.group(1)) + above -= 1 + concrete = match.group("kind") == "class" and "abstract" not in match.group("mods") + yield match.group("name"), concrete, annotations, number + 1 + + +# The linux-test and macOS test matrices list each group's suites by name. +WORKFLOWS = [".github/workflows/pr_build_linux.yml", ".github/workflows/pr_build_macos.yml"] +MATRIX_GROUP = re.compile(r'^\s+- name: "(\w+)"\n\s+value: \|\n((?:\s+[\w.]+\n)+)', re.M) + + +def workflow_groups(workflow): + """Map each suite a workflow's test matrix lists to the group it is listed in.""" + text = (REPO_ROOT / workflow).read_text(encoding="utf-8") + return { + suite: group + for group, suites in MATRIX_GROUP.findall(text) + for suite in suites.split() + } + + +def group_name(tag): + """ExecTestGroup -> exec""" + return tag[: -len("TestGroup")].lower() + + +def main(): + failures = [] + members = {tag: set() for tag in GROUP_TAGS} + # Where each tag was found for each suite. A suite can be declared in several source sets + # (for one Spark version each), and every copy has to carry the same tag. + declared = {} + seen_ignored = set() + for path in test_sources(): + text = path.read_text(encoding="utf-8", errors="ignore") + package = re.search(r"^package\s+([\w.]+)", text, re.M) + prefix = package.group(1) + "." if package else "" + for name, concrete, annotations, line in declarations(text): + fqn = prefix + name + where = f"{path.relative_to(REPO_ROOT)}:{line}" + tags = [a for a in annotations if a in members] + if not (concrete and re.search(r"(Suite|ITCase)$", name)): + if tags: + failures.append( + f"{where}: {fqn} is not a concrete suite, so its @{tags[0]} does not reach " + "the suites that extend it. Tag each concrete suite instead." + ) + continue + if fqn in IGNORE_LIST: + seen_ignored.add(fqn) + if tags: + failures.append( + f"{where}: {fqn} is in IGNORE_LIST ({IGNORE_LIST[fqn]}) but carries @{tags[0]}" + ) + continue + if len(tags) != 1: + found = ", ".join("@" + t for t in tags) if tags else "none" + failures.append( + f"{where}: {fqn} needs exactly one group tag ({', '.join('@' + t for t in GROUP_TAGS)}); " + f"found {found}" + ) + continue + members[tags[0]].add(fqn) + declared.setdefault(fqn, {}).setdefault(tags[0], []).append(where) + + for fqn, by_tag in sorted(declared.items()): + if len(by_tag) > 1: + detail = "; ".join( + f"@{tag} at {', '.join(wheres)}" for tag, wheres in sorted(by_tag.items()) + ) + failures.append(f"{fqn} is declared with different group tags: {detail}") + + for fqn in sorted(set(IGNORE_LIST) - seen_ignored): + failures.append(f"IGNORE_LIST names {fqn}, which no longer exists") + + tagged = { + fqn: group_name(tag) + for tag, suites in members.items() + for fqn in suites + if len(declared[fqn]) == 1 + } + # A suite whose copies disagree is already reported, so it is left out of the comparison with + # the workflows, but it is still a tagged suite. + all_tagged = {fqn for suites in members.values() for fqn in suites} + for workflow in WORKFLOWS: + listed = workflow_groups(workflow) + for fqn, group in sorted(tagged.items()): + if fqn not in listed: + failures.append(f"{workflow} does not list {fqn}, which its tag puts in {group}") + elif listed[fqn] != group: + failures.append(f"{workflow} lists {fqn} in {listed[fqn]}, but its tag puts it in {group}") + for fqn in sorted(set(listed) - all_tagged): + failures.append(f"{workflow} lists {fqn}, which is not a tagged concrete suite") + + if failures: + for failure in failures: + print(failure) + sys.exit(1) + summary = ", ".join(f"{tag} {len(suites)}" for tag, suites in members.items()) + print(f"Every suite has one group tag: {summary}") + + +if __name__ == "__main__": + main() diff --git a/dev/ci/run-comet-tests.sh b/dev/ci/run-comet-tests.sh new file mode 100755 index 00000000000..50249fa04c2 --- /dev/null +++ b/dev/ci/run-comet-tests.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Run one group of the Comet JVM test suites. Each concrete suite carries one +# group tag, such as org.apache.comet.tags.ExecTestGroup, and CI runs each group as +# its own job. +# Arguments after the group go to Maven: +# +# dev/ci/run-comet-tests.sh exec -Pspark-4.0 +# +# The group tag does not select integration suites (@IntegrationTestSuite, such as +# ParquetEncryptionITCase). The ScalaTest plugin excludes them, and they run under +# maven-failsafe-plugin, which ignores the tag. So the default `test` goal never runs +# them, and `--goal install` runs all of them whichever group is named. +# +# Build the native library first: `make core`, or `make release` with -Prelease. +# Written for bash 3.2. + +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# Each group is a tag annotation here, such as ExecTestGroup.java for `exec`. +TAGS_DIR="$REPO/spark/src/test/java/org/apache/comet/tags" + +usage() { + groups="" + for tag in "$TAGS_DIR"/*TestGroup.java; do + tag="$(basename "$tag" TestGroup.java)" + groups="$groups $(printf '%s' "$tag" | tr '[:upper:]' '[:lower:]')" + done + cat >&2 <] [maven args...] + + groups $groups + --goal the Maven goal to run (default: test). install also runs the + spark-integration checks, which are not selected by the group: + every *ITCase suite runs, whichever group is named. +USAGE + exit 2 +} + +GOAL="test" +while [ $# -gt 0 ]; do + case "$1" in + --goal) + [ $# -ge 2 ] || usage + GOAL="$2" + shift 2 + ;; + -h | --help) usage ;; + *) break ;; + esac +done +[ $# -ge 1 ] || usage + +# exec -> ExecTestGroup +TAG="$(printf '%s' "${1:0:1}" | tr '[:lower:]' '[:upper:]')${1:1}TestGroup" +if [ ! -f "$TAGS_DIR/$TAG.java" ]; then + echo "Unknown test group: $1" >&2 + usage +fi +shift + +cd "$REPO" + +export SPARK_HOME="${SPARK_HOME:-$REPO}" +export SPARK_LOCAL_HOSTNAME="${SPARK_LOCAL_HOSTNAME:-localhost}" +export SPARK_LOCAL_IP="${SPARK_LOCAL_IP:-127.0.0.1}" +export MAVEN_OPTS="${MAVEN_OPTS:--Xmx4G -Xms2G -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log}" + +set -x +exec ./mvnw -B "$GOAL" "-DtagsToInclude=org.apache.comet.tags.$TAG" "$@" diff --git a/docs/source/contributor-guide/ci.md b/docs/source/contributor-guide/ci.md index 8604280ced6..284a154f5bf 100644 --- a/docs/source/contributor-guide/ci.md +++ b/docs/source/contributor-guide/ci.md @@ -545,7 +545,7 @@ python3 dev/ci/check-ci-config.py actionlint --shellcheck=off ``` -A new test suite has to be registered in the workflow files by hand; see +A new test suite needs a group tag and a listing in the workflow files; see [Register New Test Suites in CI](development.md#6-register-new-test-suites-in-ci). A new job in `ci.yml` needs an entry in both `FILTERS` and `POLICY` in `dev/ci/compute-changes.py`, a case in `dev/ci/check-ci-config.py`, and a line in `required_checks.needs`. The check will tell you which diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index 4eb6224c57a..dc9aba0d9b9 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -710,19 +710,28 @@ make test-jvm ### 6. Register New Test Suites in CI -Comet's CI does not automatically discover test suites. Instead, test suites are explicitly listed -in the GitHub Actions workflow files so they can be grouped by category and run as separate parallel -jobs. This reduces overall CI time. - -If you add a new Comet Scala Test suite, you must add it to the `suite` matrix in **both** workflow files: +CI runs Comet's Scala test suites in four groups, each as its own parallel job. A new suite joins a +group in two places: its class carries the group's ScalaTest tag, and its fully qualified class name +is listed under the group in the `suite` matrix of **both** workflow files: - `.github/workflows/pr_build_linux.yml` - `.github/workflows/pr_build_macos.yml` -Each file contains a `suite` matrix with named groups such as `fuzz`, `shuffle`, `parquet`, `csv`, -`exec`, `expressions`, and `sql`. Add your new suite's fully qualified class name to the -appropriate group. For example, if you add a new expression test suite, add it to the -`expressions` group: +| Tag | Group | Covers | +| ----------------------- | ------------- | -------------------------------------------------------------- | +| `@ScansTestGroup` | `scans` | Parquet, CSV and Iceberg reads and writes, and the fuzz suites | +| `@ShuffleTestGroup` | `shuffle` | Native and columnar shuffle, and the Celeborn integration | +| `@ExecTestGroup` | `exec` | Operators, plan rules, plugins, memory, and TPC queries | +| `@ExpressionsTestGroup` | `expressions` | Expression evaluation, casts, codegen, and the SQL file tests | + +For example, a new expression suite goes in the `expressions` group: + +```scala +import org.apache.comet.tags.ExpressionsTestGroup + +@ExpressionsTestGroup +class YourNewExpressionSuite extends CometTestBase { +``` ```yaml - name: "expressions" @@ -732,27 +741,42 @@ appropriate group. For example, if you add a new expression test suite, add it t org.apache.comet.YourNewExpressionSuite # <-- add here ``` -Choose the group that best matches the area your test covers: +Tag each concrete suite class. A tag on an abstract base class or a trait does not reach the suites +that extend it. A suite that has a copy in more than one Spark version source set, such as +`spark-3.5` and `spark-4.0`, needs the tag in each copy, and the copies must carry the same tag. + +The `preflight` job in `.github/workflows/ci.yml` runs `dev/ci/check-suites.py` on every pull +request. It fails when a concrete suite has no group tag or more than one, when the copies of a +suite in different source sets carry different tags, or when either workflow file does not list it +under the group its tag names. + +#### Test Groups + +`dev/ci/run-comet-tests.sh` runs one group locally: the suites that carry its tag, which +`dev/ci/check-suites.py` keeps the same as the group's CI job. + +```sh +make +dev/ci/run-comet-tests.sh expressions # the default Spark profile +dev/ci/run-comet-tests.sh exec -Pspark-4.0 # arguments after the group go to Maven +``` + +The script runs `mvnw test` by default. Pass `--goal install` to also run the `spark-integration` +checks, and `-Prelease` to use the library that `make release` builds. -| Group | Covers | -| ------------- | ---------------------------------------------------------- | -| `fuzz` | Fuzz testing and data generation | -| `shuffle` | Shuffle operators and related exchange behavior | -| `parquet` | Parquet read/write and native reader tests | -| `csv` | CSV native read tests | -| `exec` | Execution operators, joins, aggregates, plan rules, TPC-\* | -| `expressions` | Expression evaluation, casts, and Comet SQL Tests | -| `sql` | SQL-level behavior tests | +The group tag does not select an integration suite, one marked `@IntegrationTestSuite` such as +`ParquetEncryptionITCase`. The ScalaTest plugin excludes those, and Maven runs them under +`maven-failsafe-plugin` by their `*ITCase` name, which ignores the tag. The default `test` goal +therefore never runs them, and `--goal install` runs every one of them whichever group you name. -**Important:** The suite lists in both workflow files must stay in sync. The `preflight` job in -`.github/workflows/ci.yml` runs `dev/ci/check-suites.py` on every pull request. It scans for all -`*Suite.scala` files in the repository and verifies that each one appears in both workflow files. -If any suite is missing, this check will fail and block the PR. +CI still selects suites by the lists in the workflow files, not by tag, and runs `install`. An +integration suite is excluded from that selection too, so it runs in every group's job whichever +group the list names it under. A small number of suites are deliberately **not** run in CI, because they need infrastructure CI -does not have or because running them there is not worth the cost. These are listed in the -`ignore_list` in `dev/ci/check-suites.py`, and each one documents in its own scaladoc why it is -excluded and how to run it. Run a manual suite with: +does not have or because running them there is not worth the cost. These are listed in +`IGNORE_LIST` in `dev/ci/check-suites.py` and carry no group tag. Each one documents in its own +scaladoc why it is excluded and how to run it. Run a manual suite with: ```sh ./mvnw test -Dtest=none -Dsuites="org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite" diff --git a/spark/src/test/java/org/apache/comet/tags/ExecTestGroup.java b/spark/src/test/java/org/apache/comet/tags/ExecTestGroup.java new file mode 100644 index 00000000000..16f946ff088 --- /dev/null +++ b/spark/src/test/java/org/apache/comet/tags/ExecTestGroup.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.scalatest.TagAnnotation; + +/** Puts a suite in the `exec` CI test group. See "Test Groups" in the development guide. */ +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface ExecTestGroup {} diff --git a/spark/src/test/java/org/apache/comet/tags/ExpressionsTestGroup.java b/spark/src/test/java/org/apache/comet/tags/ExpressionsTestGroup.java new file mode 100644 index 00000000000..b9fac52b36c --- /dev/null +++ b/spark/src/test/java/org/apache/comet/tags/ExpressionsTestGroup.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.scalatest.TagAnnotation; + +/** Puts a suite in the `expressions` CI test group. See "Test Groups" in the development guide. */ +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface ExpressionsTestGroup {} diff --git a/spark/src/test/java/org/apache/comet/tags/ScansTestGroup.java b/spark/src/test/java/org/apache/comet/tags/ScansTestGroup.java new file mode 100644 index 00000000000..efe21288d24 --- /dev/null +++ b/spark/src/test/java/org/apache/comet/tags/ScansTestGroup.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.scalatest.TagAnnotation; + +/** Puts a suite in the `scans` CI test group. See "Test Groups" in the development guide. */ +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface ScansTestGroup {} diff --git a/spark/src/test/java/org/apache/comet/tags/ShuffleTestGroup.java b/spark/src/test/java/org/apache/comet/tags/ShuffleTestGroup.java new file mode 100644 index 00000000000..310f531b0a9 --- /dev/null +++ b/spark/src/test/java/org/apache/comet/tags/ShuffleTestGroup.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.tags; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.scalatest.TagAnnotation; + +/** Puts a suite in the `shuffle` CI test group. See "Test Groups" in the development guide. */ +@TagAnnotation +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface ShuffleTestGroup {} diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala index a3f81b92df5..03acde69611 100644 --- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala @@ -37,6 +37,7 @@ import org.apache.spark.sql.types.{ArrayType, DataType, DoubleType, FloatType, I import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.serde.{ArrayElementEqualitySupport, ArraySetSupport, CometArrayDistinct, CometArrayExcept, CometArrayJoin, CometArrayRemove, CometArrayReverse, CometArraysOverlap, CometArrayUnion, CometFlatten, CometSequence, Compatible, ExprOuterClass, Incompatible, SparkFixes} +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} /** Counts calls to a test UDF. Comet tests run in local mode, so tasks see this object. */ @@ -44,6 +45,7 @@ object SizeChildCounter { val calls = new AtomicLong(0) } +@ExpressionsTestGroup class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("array set signed-zero support levels") { diff --git a/spark/src/test/scala/org/apache/comet/CometBitwiseExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometBitwiseExpressionSuite.scala index 36540bc4997..d51a9b1e03a 100644 --- a/spark/src/test/scala/org/apache/comet/CometBitwiseExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometBitwiseExpressionSuite.scala @@ -25,8 +25,10 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} +@ExpressionsTestGroup class CometBitwiseExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("bitwise expressions") { diff --git a/spark/src/test/scala/org/apache/comet/CometBroadcastBuildSideSuite.scala b/spark/src/test/scala/org/apache/comet/CometBroadcastBuildSideSuite.scala index b4bf8afd1ff..e2b905196d4 100644 --- a/spark/src/test/scala/org/apache/comet/CometBroadcastBuildSideSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometBroadcastBuildSideSuite.scala @@ -32,10 +32,13 @@ import org.apache.spark.sql.execution.exchange.ReusedExchangeExec import org.apache.spark.sql.execution.joins.BroadcastNestedLoopJoinExec import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.tags.ScansTestGroup + // Tests for the broadcast build-side bridge, spark.comet.convert.broadcastBuildSide.enabled (issue // #6008). A build-side scan that Comet cannot read natively, such as ORC or an Iceberg table // the native reader declines, is converted to Arrow so that the broadcast join runs natively. Each // query is compared with Spark, and the tests assert where the bridge must and must not appear. +@ScansTestGroup class CometBroadcastBuildSideSuite extends CometTestBase with CometIcebergTestBase { // Most tests use an ORC build side. Text, Parquet, JSON and CSV scans are converted only by their diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenFuzzSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenFuzzSuite.scala index 2f1c605cb01..10d4eb00de0 100644 --- a/spark/src/test/scala/org/apache/comet/CometCodegenFuzzSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCodegenFuzzSuite.scala @@ -37,6 +37,7 @@ import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.codegen.CometBatchKernelCodegen +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, ParquetGenerator, SchemaGenOptions} import org.apache.comet.vector.CometVector @@ -47,6 +48,7 @@ import org.apache.comet.vector.CometVector * (not [[CometFuzzTestBase]]) because the base's per-shuffle-mode runs are irrelevant for * projection-only queries. */ +@ExpressionsTestGroup class CometCodegenFuzzSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenHOFSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenHOFSuite.scala index 49fadd16e0e..fd29d7b3b1f 100644 --- a/spark/src/test/scala/org/apache/comet/CometCodegenHOFSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCodegenHOFSuite.scala @@ -23,6 +23,8 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ExpressionsTestGroup + /** * Higher-order function regression coverage for the codegen dispatcher. * @@ -41,6 +43,7 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper * the per-task instance, not the companion, so concurrent partitions cannot race on a shared * `NamedLambdaVariable.value`. The two-collects test below regresses this. */ +@ExpressionsTestGroup class CometCodegenHOFSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenSourceSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenSourceSuite.scala index 54b59dec9dd..230bf253ada 100644 --- a/spark/src/test/scala/org/apache/comet/CometCodegenSourceSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCodegenSourceSuite.scala @@ -31,6 +31,7 @@ import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.codegen.CometBatchKernelCodegen import org.apache.comet.codegen.CometBatchKernelCodegen.{ArrayColumnSpec, ArrowColumnSpec, MapColumnSpec, ScalarColumnSpec, StructColumnSpec, StructFieldSpec} +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.udf.codegen.CometScalaUDFCodegen // Resolve Arrow vector classes through the codegen object so tests see the same `Class` objects @@ -53,6 +54,7 @@ import org.apache.comet.udf.codegen.CometScalaUDFCodegen * These are the smallest durable tests that the claimed optimizations actually reach the * generated Java, and they document the shapes future contributors should preserve. */ +@ExpressionsTestGroup class CometCodegenSourceSuite extends AnyFunSuite { private val varCharVectorClass = diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala index f0551a46125..21b6ad39e60 100644 --- a/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala @@ -45,6 +45,7 @@ import org.apache.comet.codegen.{CometBatchKernelCodegen, DispatchOccurrence} import org.apache.comet.codegen.CometBatchKernelCodegen.ArrowColumnSpec import org.apache.comet.serde.{CometInvokeTargets, CometScalaUDF, QueryPlanSerde} import org.apache.comet.serde.ExprOuterClass.Expr.ExprStructCase +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.udf.CometUdfBridge import org.apache.comet.udf.codegen.CometScalaUDFCodegen import org.apache.comet.vector.CometVector @@ -58,6 +59,7 @@ import org.apache.comet.vector.CometVector * rather than `checkSparkAnswerAndOperator` because ScalaUDF has no Comet-native path. Under * fallback the project runs on the JVM Spark path. */ +@ExpressionsTestGroup class CometCodegenSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometConfSuite.scala b/spark/src/test/scala/org/apache/comet/CometConfSuite.scala index 5c269589310..a44da43d5e4 100644 --- a/spark/src/test/scala/org/apache/comet/CometConfSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometConfSuite.scala @@ -23,6 +23,9 @@ import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.tags.ExecTestGroup + +@ExecTestGroup class CometConfSuite extends AnyFunSuite { test("primary key wins over alternative when both are set") { diff --git a/spark/src/test/scala/org/apache/comet/CometCoverageStatsSuite.scala b/spark/src/test/scala/org/apache/comet/CometCoverageStatsSuite.scala index b3a9b020775..2de20f93a98 100644 --- a/spark/src/test/scala/org/apache/comet/CometCoverageStatsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCoverageStatsSuite.scala @@ -26,10 +26,13 @@ import org.apache.spark.sql.execution.{ReusedSubqueryExec, SparkPlan, SubqueryEx import org.apache.spark.sql.execution.exchange.ReusedExchangeExec import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.tags.ExpressionsTestGroup + /** * Tests for how [[CometCoverageStats]] categorizes plan nodes into accelerated operators, * un-accelerated Spark operators, and transitions. */ +@ExpressionsTestGroup class CometCoverageStatsSuite extends CometTestBase { private def somePlan: SparkPlan = diff --git a/spark/src/test/scala/org/apache/comet/CometCsvExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometCsvExpressionSuite.scala index 6e50e7cb9ae..217f0833e88 100644 --- a/spark/src/test/scala/org/apache/comet/CometCsvExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCsvExpressionSuite.scala @@ -29,8 +29,10 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StringType +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} +@ExpressionsTestGroup class CometCsvExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("to_csv - default options") { diff --git a/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala b/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala index 072009d7ab6..3a7569ea31f 100644 --- a/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala @@ -26,11 +26,14 @@ import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.DataTypes +import org.apache.comet.tags.ExpressionsTestGroup + /** * Tests for string-to-timestamp (and string-to-date) cast correctness, ported from Spark's * DateTimeUtilsSuite. That suite lives in sql/catalyst and does not extend CometTestBase, so it * never runs in Comet CI. */ +@ExpressionsTestGroup class CometDateTimeUtilsSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 4abd96100b4..5eef0540d4a 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -36,8 +36,10 @@ import org.apache.spark.sql.types._ import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus, isSpark42Plus} import org.apache.comet.serde.{CometAdd, CometDivide, CometIntegralDivide, CometMultiply, CometRemainder, CometSubtract, Compatible, Unsupported} +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} +@ExpressionsTestGroup class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/CometFallbackInvarianceSuite.scala b/spark/src/test/scala/org/apache/comet/CometFallbackInvarianceSuite.scala index eb56881db81..a047911ab34 100644 --- a/spark/src/test/scala/org/apache/comet/CometFallbackInvarianceSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFallbackInvarianceSuite.scala @@ -26,6 +26,8 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.tags.ExpressionsTestGroup + /** * Checks that forcing an expression from Comet's native path back to Spark does not change the * outcome of a query. @@ -50,6 +52,7 @@ import org.apache.spark.sql.internal.SQLConf * everything and the gate above would be inoperative. Row order is canonicalised in the * comparator instead. */ +@ExpressionsTestGroup class CometFallbackInvarianceSuite extends CometFuzzTestBase { /** diff --git a/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala b/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala index 442cb7c41a7..eb04f30ca51 100644 --- a/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFloatSemanticsSuite.scala @@ -23,6 +23,8 @@ import org.scalatest.exceptions.TestFailedException import org.apache.spark.sql.CometTestBase +import org.apache.comet.tags.ExpressionsTestGroup + /** * Sweeps Spark's `-0.0` and NaN semantics across operator contexts and the expressions that * accept `FLOAT` or `DOUBLE` input (#6385). @@ -36,6 +38,7 @@ import org.apache.spark.sql.CometTestBase * answer from Spark. The case fails once it matches, so that a fix, or a DataFusion upgrade that * changes float behavior, removes its entry from the list instead of leaving it stale. */ +@ExpressionsTestGroup class CometFloatSemanticsSuite extends CometTestBase { import CometFloatSemanticsSuite._ diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzAggregateSuite.scala index 3674887f80f..e532a6d12e9 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzAggregateSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzAggregateSuite.scala @@ -20,7 +20,9 @@ package org.apache.comet import org.apache.comet.DataTypeSupport.isComplexType +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometFuzzAggregateSuite extends CometFuzzTestBase { test("count distinct - simple columns") { diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala index ef1f0d40253..474277c3ade 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzIcebergSuite.scala @@ -28,7 +28,9 @@ import org.apache.spark.sql.types._ import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.IcebergScanPlanData.distinctResiduals +import org.apache.comet.tags.ScansTestGroup +@ScansTestGroup class CometFuzzIcebergSuite extends CometFuzzIcebergBase { test("select *") { diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzMathSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzMathSuite.scala index 50a4425f973..b849f1288aa 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzMathSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzMathSuite.scala @@ -23,6 +23,9 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DecimalType, IntegerType, LongType} +import org.apache.comet.tags.ExpressionsTestGroup + +@ExpressionsTestGroup class CometFuzzMathSuite extends CometFuzzTestBase { // The integer-math fuzz tests intentionally cover overflowing inputs; under Spark 4's diff --git a/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala index f8d8f307934..508c4b737c0 100644 --- a/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala @@ -28,9 +28,11 @@ import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType import org.apache.spark.sql.types._ import org.apache.comet.DataTypeSupport.isComplexType +import org.apache.comet.tags.ScansTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator} import org.apache.comet.testing.FuzzDataGenerator.{doubleNaNLiteral, floatNaNLiteral} +@ScansTestGroup class CometFuzzTestSuite extends CometFuzzTestBase { test("select *") { diff --git a/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala index 7e823fb347c..57303d7e069 100644 --- a/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometHashExpressionSuite.scala @@ -24,6 +24,7 @@ import scala.util.Random import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, ParquetGenerator, SchemaGenOptions} import org.apache.comet.udf.codegen.CometScalaUDFCodegen @@ -33,6 +34,7 @@ import org.apache.comet.udf.codegen.CometScalaUDFCodegen * These tests verify that Comet's native implementations produce identical results to Spark for * all supported data types. */ +@ExpressionsTestGroup class CometHashExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { private def checkNativeHashes(query: String): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergHdfsSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergHdfsSuite.scala index 8addcd2e9a7..eb64c8deeb7 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergHdfsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergHdfsSuite.scala @@ -45,6 +45,7 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus import org.apache.comet.iceberg.IcebergReflection +import org.apache.comet.tags.ScansTestGroup /** * End-to-end coverage of the native Iceberg scan against an `hdfs://` warehouse, backed by @@ -85,6 +86,7 @@ import org.apache.comet.iceberg.IcebergReflection * the fix once an Iceberg runtime exists. Until then this suite skips there and starts * nothing. */ +@ScansTestGroup class CometIcebergHdfsSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala index 46740ddb20e..5d59ef40287 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala @@ -54,6 +54,7 @@ import org.apache.comet.cloud.s3.MinioLocationScopedCredentialProvider import org.apache.comet.iceberg.{IcebergReflection, RESTCatalogHelper} import org.apache.comet.rules.CometScanRule import org.apache.comet.serde.{OperatorOuterClass, QueryPlanSerde} +import org.apache.comet.tags.ScansTestGroup import org.apache.comet.testing.{FuzzDataGenerator, SchemaGenOptions} /** @@ -61,6 +62,7 @@ import org.apache.comet.testing.{FuzzDataGenerator, SchemaGenOptions} * * Note: Requires Iceberg dependencies to be added to pom.xml */ +@ScansTestGroup class CometIcebergNativeSuite extends CometTestBase with RESTCatalogHelper diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergResidualPushdownSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergResidualPushdownSuite.scala index cdeaaa9cd40..a0d3d0cbc08 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergResidualPushdownSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergResidualPushdownSuite.scala @@ -24,6 +24,8 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.comet.CometIcebergNativeScanExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ScansTestGroup + /** * Residual pushdown for predicates that only reach the Iceberg scan filter when Iceberg's SQL * extensions are installed. @@ -33,6 +35,7 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper * so Iceberg can push the comparison, which puts an `UnboundTransform` term into each file's * residual. CometIcebergNativeSuite runs without the extensions, so it never sees one. */ +@ScansTestGroup class CometIcebergResidualPushdownSuite extends CometTestBase with CometIcebergTestBase diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergRewriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergRewriteActionSuite.scala index 94bba27171d..1385062542d 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergRewriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergRewriteActionSuite.scala @@ -33,6 +33,7 @@ import org.apache.spark.sql.execution.SparkPlanInfo import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart import org.apache.comet.iceberg.IcebergReflection +import org.apache.comet.tags.ScansTestGroup /** * Verifies that Comet engages on the per-group reads issued by Iceberg's @@ -40,6 +41,7 @@ import org.apache.comet.iceberg.IcebergReflection * via ScanTaskSetManager and then reads them back through SparkStagedScan, which Comet's * CometScanRule recognises alongside SparkBatchQueryScan. */ +@ScansTestGroup class CometIcebergRewriteActionSuite extends CometTestBase with CometIcebergTestBase { private val catalog = "test_cat" diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionExtensionsSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionExtensionsSuite.scala index e71acf4e5c9..214fd8bb533 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionExtensionsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionExtensionsSuite.scala @@ -24,6 +24,8 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression import org.apache.spark.sql.catalyst.plans.logical.Filter +import org.apache.comet.tags.ScansTestGroup + /** * Iceberg's system functions once Iceberg's SQL extensions have rewritten them. * @@ -33,6 +35,7 @@ import org.apache.spark.sql.catalyst.plans.logical.Filter * it with the same native kernels as the `StaticInvoke` that CometIcebergSystemFunctionSuite * covers. `spark.sql.extensions` is static, so this path needs a suite of its own. */ +@ScansTestGroup class CometIcebergSystemFunctionExtensionsSuite extends CometTestBase with CometIcebergTestBase { override protected def sparkConf: SparkConf = diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala index 613f726767d..47d4c1ed35f 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala @@ -43,6 +43,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.serde.{CometApplyFunctionExpression, CometExpressionSerde, CometIcebergBucket, CometIcebergSystemFunctions, CometIcebergTruncate, CometStaticInvoke, Compatible, SupportLevel, Unsupported} +import org.apache.comet.tags.ScansTestGroup /** * Native support for Iceberg's system functions (`bucket`, `truncate`, `years`, `months`, `days`, @@ -55,6 +56,7 @@ import org.apache.comet.serde.{CometApplyFunctionExpression, CometExpressionSerd * out-of-order partitions); in a filter or projection it would be a silently wrong answer, which * is why the coverage is per type rather than a few hand-picked rows. */ +@ScansTestGroup class CometIcebergSystemFunctionSuite extends CometTestBase with CometIcebergTestBase diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala index 32fd168e5ab..82f7e540fe0 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala @@ -62,6 +62,7 @@ import org.apache.comet.CometSparkSessionExtensions.{isSpark41Plus, isSpark42Plu import org.apache.comet.iceberg.{IcebergReflection, IcebergWriteReportListener, PositionDeltaWrite} import org.apache.comet.serde.Unsupported import org.apache.comet.serde.operator.CometIcebergNativeWrite +import org.apache.comet.tags.ScansTestGroup private case class WriteSnapshot(snapshotDelta: Long, plans: Seq[SparkPlan]) @@ -71,6 +72,7 @@ private case class ReportedWrite( reasons: Seq[String], failed: Boolean) +@ScansTestGroup class CometIcebergWriteActionSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index 9bbacfb0df0..2625514360d 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -48,7 +48,9 @@ import org.apache.comet.rules.EliminateRedundantTransitions import org.apache.comet.serde.{Compatible, SupportLevel, Unsupported} import org.apache.comet.serde.OperatorOuterClass.Operator import org.apache.comet.serde.operator.CometIcebergNativeWrite +import org.apache.comet.tags.ScansTestGroup +@ScansTestGroup class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTestBase { override protected def sparkConf: SparkConf = { diff --git a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala index 2356a985ef7..11fa06e5929 100644 --- a/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometJsonExpressionSuite.scala @@ -31,8 +31,10 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions._ import org.apache.comet.serde.CometStructsToJson +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} +@ExpressionsTestGroup class CometJsonExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit diff --git a/spark/src/test/scala/org/apache/comet/CometJsonJvmSuite.scala b/spark/src/test/scala/org/apache/comet/CometJsonJvmSuite.scala index 4a4df2e9b1a..d37532b0f95 100644 --- a/spark/src/test/scala/org/apache/comet/CometJsonJvmSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometJsonJvmSuite.scala @@ -23,6 +23,9 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ExpressionsTestGroup + +@ExpressionsTestGroup class CometJsonJvmSuite extends CometTestBase with AdaptiveSparkPlanHelper { // No per-expression `allowIncompatible` is set, so all JSON expressions run through the diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index 6b8202d240c..cba9aa8f85d 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -29,8 +29,10 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.BinaryType import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} +@ExpressionsTestGroup class CometMapExpressionSuite extends CometTestBase { test("read map[int, int] from parquet") { diff --git a/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala index a8c145d1036..c692e7ff00c 100644 --- a/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMathExpressionSuite.scala @@ -26,8 +26,10 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataTypes, StructField, StructType} +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} +@ExpressionsTestGroup class CometMathExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("abs") { diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 894209204ac..7a018b535b4 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -40,6 +40,7 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.rules.CometScanTypeChecker import org.apache.comet.serde.{Compatible, Incompatible, QueryPlanSerde, SparkFixes, Unsupported} +import org.apache.comet.tags.ExpressionsTestGroup /** * Spark-parity coverage for Comet's **native** `Cast` implementation, and for the @@ -62,6 +63,7 @@ import org.apache.comet.serde.{Compatible, Incompatible, QueryPlanSerde, SparkFi * through the dispatcher instead. Adding a native cast implementation therefore means moving a * pair out of the `Unsupported` assertions and into the parity matrix below. */ +@ExpressionsTestGroup class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper diff --git a/spark/src/test/scala/org/apache/comet/CometNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeSuite.scala index 08e0a81ccae..01c6bbca5ea 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeSuite.scala @@ -30,6 +30,9 @@ import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.{LongType, StructField, StructType} import org.apache.spark.sql.vectorized.ColumnarBatch +import org.apache.comet.tags.ExecTestGroup + +@ExecTestGroup class CometNativeSuite extends CometTestBase { test("test handling NPE thrown by JVM") { val rdd = spark.range(0, 1).rdd.map { value => diff --git a/spark/src/test/scala/org/apache/comet/CometNativeUdfSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeUdfSuite.scala index 765476df231..dee24fbe65f 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeUdfSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeUdfSuite.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.comet.CometProjectExec import org.apache.spark.sql.functions.{col, expr, struct} import org.apache.spark.sql.types._ +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.udf.{CometNativeUDF, CometNativeUdfAbiException, CometNativeUdfLoadException, CometNativeUdfNotAllowedException} /** @@ -46,6 +47,7 @@ import org.apache.comet.udf.{CometNativeUDF, CometNativeUdfAbiException, CometNa * ./mvnw test -Dsuites="org.apache.comet.CometNativeUdfSuite" -Dtest=none * }}} */ +@ExpressionsTestGroup class CometNativeUdfSuite extends CometTestBase { private lazy val libPath: String = { diff --git a/spark/src/test/scala/org/apache/comet/CometPublicApiSuite.scala b/spark/src/test/scala/org/apache/comet/CometPublicApiSuite.scala index f6e15cfa8e6..42a0bdb2379 100644 --- a/spark/src/test/scala/org/apache/comet/CometPublicApiSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometPublicApiSuite.scala @@ -27,6 +27,7 @@ import scala.jdk.CollectionConverters._ import org.scalatest.funsuite.AnyFunSuite import org.apache.comet.annotation.Public +import org.apache.comet.tags.ExecTestGroup /** * Guards the enumerated public API described in `docs/source/about/versioning_policy.md`. @@ -36,6 +37,7 @@ import org.apache.comet.annotation.Public * accident, or the policy is genuinely changing and both the doc and this list need updating in * the same PR. */ +@ExecTestGroup class CometPublicApiSuite extends AnyFunSuite { private val expected: Set[String] = Set( diff --git a/spark/src/test/scala/org/apache/comet/CometRegExpJvmSuite.scala b/spark/src/test/scala/org/apache/comet/CometRegExpJvmSuite.scala index 46caadc31f8..f99a54f67a5 100644 --- a/spark/src/test/scala/org/apache/comet/CometRegExpJvmSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometRegExpJvmSuite.scala @@ -23,9 +23,11 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.functions.col import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus} +import org.apache.comet.tags.ExpressionsTestGroup // Regex expressions other than in-subset `rlike` run through the codegen dispatcher by default // (Spark's own code, enabled by default) rather than the native rust path. +@ExpressionsTestGroup class CometRegExpJvmSuite extends CometTestBase { // A Java-only pattern used to check the opt-in behavior when Rust rejects it. diff --git a/spark/src/test/scala/org/apache/comet/CometRegexParitySuite.scala b/spark/src/test/scala/org/apache/comet/CometRegexParitySuite.scala index 60e0369ee63..cb50ba13b7d 100644 --- a/spark/src/test/scala/org/apache/comet/CometRegexParitySuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometRegexParitySuite.scala @@ -26,12 +26,14 @@ import org.apache.spark.sql.types.{StringType, StructField, StructType} import org.apache.comet.expressions.{CometRegex, RegexFlavor} import org.apache.comet.serde.Compatible +import org.apache.comet.tags.ExpressionsTestGroup /** * Differential corpus: every pattern the plan-time analyzer admits is evaluated on Spark (Java * `Pattern`) and on Comet's native Rust path, and the results must match. Routing is asserted via * extended explain so a dispatcher fallback cannot silently satisfy the comparison. */ +@ExpressionsTestGroup class CometRegexParitySuite extends CometTestBase with AdaptiveSparkPlanHelper { // One pattern per whitelist production, plus the lexer-boundary cases. diff --git a/spark/src/test/scala/org/apache/comet/CometScalaUDFClassLoaderSuite.scala b/spark/src/test/scala/org/apache/comet/CometScalaUDFClassLoaderSuite.scala index 17fd0fea095..f75d9a748e0 100644 --- a/spark/src/test/scala/org/apache/comet/CometScalaUDFClassLoaderSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometScalaUDFClassLoaderSuite.scala @@ -29,6 +29,8 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.{CometTestBase, Encoders} import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.comet.tags.ExpressionsTestGroup + /** * Regression coverage for UDF closures whose capturing class lives in a user jar, which used to * fail with: @@ -50,6 +52,7 @@ import org.apache.spark.sql.catalyst.expressions.Expression * `MutableURLClassLoader`, which becomes the task thread's context ClassLoader while staying * invisible to the ClassLoader that loaded Comet. */ +@ExpressionsTestGroup class CometScalaUDFClassLoaderSuite extends CometTestBase { import CometScalaUDFClassLoaderSuite._ diff --git a/spark/src/test/scala/org/apache/comet/CometSetOpWithGroupBySuite.scala b/spark/src/test/scala/org/apache/comet/CometSetOpWithGroupBySuite.scala index c6595ba42e1..af35dd63b73 100644 --- a/spark/src/test/scala/org/apache/comet/CometSetOpWithGroupBySuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSetOpWithGroupBySuite.scala @@ -24,6 +24,8 @@ import org.apache.spark.sql.comet.CometUnionExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.tags.ExecTestGroup + /** * Regression test for issue #4122: on Spark 4.1 (SPARK-52921), EXCEPT ALL / INTERSECT ALL whose * sides are themselves GROUP BY aggregates are lowered to a plan where the union inherits a hash @@ -31,6 +33,7 @@ import org.apache.spark.sql.internal.SQLConf * If Comet's columnar Union concatenates partitions it breaks that partitioning invariant and the * resulting sums/counts collapse two sides into the wrong partitions. */ +@ExecTestGroup class CometSetOpWithGroupBySuite extends CometTestBase with AdaptiveSparkPlanHelper { test("issue #4122: EXCEPT ALL with GROUP BY under both sides") { diff --git a/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala b/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala index 92bac5dba86..5fba5c6eecb 100644 --- a/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSparkSessionExtensionsSuite.scala @@ -30,7 +30,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.LongType import org.apache.comet.serde.OperatorOuterClass +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometSparkSessionExtensionsSuite extends CometTestBase { import CometSparkSessionExtensions._ diff --git a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala index ba6e804d664..84ff0242dcb 100644 --- a/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala @@ -31,6 +31,9 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.ArrayType +import org.apache.comet.tags.ExpressionsTestGroup + +@ExpressionsTestGroup class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper { /** Check if the current Spark version meets a minimum version requirement. */ diff --git a/spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala index f921c7bda79..4dbf78f6a40 100644 --- a/spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometStringExpressionSuite.scala @@ -28,9 +28,11 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DataType, DataTypes, StructField, StructType} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} import org.apache.comet.udf.codegen.CometScalaUDFCodegen +@ExpressionsTestGroup class CometStringExpressionSuite extends CometTestBase with CometCodegenAssertions { // scalastyle:off private val edgeCases = Seq( diff --git a/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala index 72cbee24ac8..4f1c134eaea 100644 --- a/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala @@ -36,8 +36,10 @@ import org.apache.spark.sql.types.{DataTypes, StructField, StructType} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.{CometDateFormat, CometTimeZone, CometTruncDate, CometTruncTimestamp} +import org.apache.comet.tags.ExpressionsTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} +@ExpressionsTestGroup class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { /** Timezones used to verify that TimestampNTZ operations are timezone-independent. */ diff --git a/spark/src/test/scala/org/apache/comet/CometUuidExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometUuidExpressionSuite.scala index 0a4b164c1a9..74c91e3858c 100644 --- a/spark/src/test/scala/org/apache/comet/CometUuidExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometUuidExpressionSuite.scala @@ -24,6 +24,8 @@ import org.apache.spark.sql.catalyst.expressions.Uuid import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions.col +import org.apache.comet.tags.ExpressionsTestGroup + /** * Bit-for-bit comparisons of seeded `uuid` against Spark. * @@ -37,6 +39,7 @@ import org.apache.spark.sql.functions.col * form is not, so building the expression directly gives all profiles a real cross-engine * assertion and removes that dependence. */ +@ExpressionsTestGroup class CometUuidExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("seeded uuid matches Spark bit for bit") { diff --git a/spark/src/test/scala/org/apache/comet/CometVariantProjectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometVariantProjectionSuite.scala index 94fc0f88af8..a8c53cf708a 100644 --- a/spark/src/test/scala/org/apache/comet/CometVariantProjectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometVariantProjectionSuite.scala @@ -34,7 +34,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{IntegerType, StructField, StructType} import org.apache.comet.serde.operator.CometNativeScan +import org.apache.comet.tags.ScansTestGroup +@ScansTestGroup class CometVariantProjectionSuite extends CometTestBase { override protected def sparkConf: SparkConf = super.sparkConf .set(SQLConf.USE_V1_SOURCE_LIST.key, "parquet") diff --git a/spark/src/test/scala/org/apache/comet/CometVariantTypeSuite.scala b/spark/src/test/scala/org/apache/comet/CometVariantTypeSuite.scala index 222dba677ef..a23e0e07ca8 100644 --- a/spark/src/test/scala/org/apache/comet/CometVariantTypeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometVariantTypeSuite.scala @@ -33,7 +33,9 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, StructField, StructTyp import org.apache.comet.rules.CometScanTypeChecker import org.apache.comet.serde.{CometAttributeReference, QueryPlanSerde, Unsupported} +import org.apache.comet.tags.ExpressionsTestGroup +@ExpressionsTestGroup class CometVariantTypeSuite extends AnyFunSuite { private val storageType = StructType( Seq( diff --git a/spark/src/test/scala/org/apache/comet/DataGeneratorSuite.scala b/spark/src/test/scala/org/apache/comet/DataGeneratorSuite.scala index d1a321099c6..c04e865fda8 100644 --- a/spark/src/test/scala/org/apache/comet/DataGeneratorSuite.scala +++ b/spark/src/test/scala/org/apache/comet/DataGeneratorSuite.scala @@ -24,8 +24,10 @@ import scala.util.Random import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, MapType, StructField, StructType} +import org.apache.comet.tags.ScansTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, SchemaGenOptions} +@ScansTestGroup class DataGeneratorSuite extends CometTestBase { test("allowNull produces nulls in every type that honours it") { diff --git a/spark/src/test/scala/org/apache/comet/DirectColumnarToRowConverterSuite.scala b/spark/src/test/scala/org/apache/comet/DirectColumnarToRowConverterSuite.scala index 887ef5cb646..1dea95c6f83 100644 --- a/spark/src/test/scala/org/apache/comet/DirectColumnarToRowConverterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/DirectColumnarToRowConverterSuite.scala @@ -35,12 +35,14 @@ import org.apache.spark.sql.types._ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.unsafe.types.UTF8String +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.{CometDictionary, CometDictionaryVector, CometPlainVector} /** * Verifies that [[DirectColumnarToRowConverter]] produces rows byte-identical to * `UnsafeProjection`, which matters because `UnsafeRow` equality and hashing are byte-wise. */ +@ExecTestGroup class DirectColumnarToRowConverterSuite extends AnyFunSuite { private def assertMatchesUnsafeProjection( diff --git a/spark/src/test/scala/org/apache/comet/GenerateDocsSuite.scala b/spark/src/test/scala/org/apache/comet/GenerateDocsSuite.scala index 3d00d764900..0c999d831d1 100644 --- a/spark/src/test/scala/org/apache/comet/GenerateDocsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/GenerateDocsSuite.scala @@ -22,7 +22,9 @@ package org.apache.comet import org.scalatest.funsuite.AnyFunSuite import org.apache.comet.serde.{CometMakeTimestamp, CometRLike, CometToUnixTimestamp, NativeOptInAvailable} +import org.apache.comet.tags.ExpressionsTestGroup +@ExpressionsTestGroup class GenerateDocsSuite extends AnyFunSuite { private val configKey = "spark.comet.expression.RLike.allowIncompatible" diff --git a/spark/src/test/scala/org/apache/comet/QueryContextInternerSuite.scala b/spark/src/test/scala/org/apache/comet/QueryContextInternerSuite.scala index fd33f249947..10e11d331e6 100644 --- a/spark/src/test/scala/org/apache/comet/QueryContextInternerSuite.scala +++ b/spark/src/test/scala/org/apache/comet/QueryContextInternerSuite.scala @@ -25,7 +25,9 @@ import org.apache.spark.sql.{CometTestBase, DataFrame} import org.apache.spark.sql.comet.CometNativeExec import org.apache.comet.serde.{ExprOuterClass, OperatorOuterClass} +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class QueryContextInternerSuite extends CometTestBase { private val sqlText = diff --git a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala index e44978c1d56..d8dd766194d 100644 --- a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala @@ -26,7 +26,9 @@ import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.{SparkException, SparkThrowable} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ExpressionsTestGroup +@ExpressionsTestGroup class SparkErrorConverterSuite extends AnyFunSuite { test("ArithmeticOverflow preserves the suggested function") { diff --git a/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala b/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala index 61d2877ee0c..39a3a35dea8 100644 --- a/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala +++ b/spark/src/test/scala/org/apache/comet/SqlFileTestParserSuite.scala @@ -25,11 +25,14 @@ import java.nio.file.Files import org.scalatest.funsuite.AnyFunSuite +import org.apache.comet.tags.ExpressionsTestGroup + /** * Unit tests for [[SqlFileTestParser]]. Pure text parsing, so no Spark session is needed. The * end-to-end behaviour of each query mode is covered by `CometSqlFileTestSuite` running the * fixtures themselves. */ +@ExpressionsTestGroup class SqlFileTestParserSuite extends AnyFunSuite { private def parseQueries(lines: String*): Seq[SqlQuery] = diff --git a/spark/src/test/scala/org/apache/comet/codegen/CometSpecializedGettersDispatchSuite.scala b/spark/src/test/scala/org/apache/comet/codegen/CometSpecializedGettersDispatchSuite.scala index 7ad93fd9fc1..ca65055b132 100644 --- a/spark/src/test/scala/org/apache/comet/codegen/CometSpecializedGettersDispatchSuite.scala +++ b/spark/src/test/scala/org/apache/comet/codegen/CometSpecializedGettersDispatchSuite.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.comet.util.Utils import org.apache.spark.sql.types.{DataType, DateType, LongType} import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus +import org.apache.comet.tags.ExpressionsTestGroup /** * Unit coverage for the generic `SpecializedGetters.get(ordinal, dataType)` dispatch the kernel @@ -35,6 +36,7 @@ import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus * has to match what `CometBatchKernelCodegen.isSupportedDataType` admits at plan time. A type * accepted there but missing here throws at execute time, when there is no longer a fallback. */ +@ExpressionsTestGroup class CometSpecializedGettersDispatchSuite extends AnyFunSuite { /** Minimal `CometInternalRow` over one fixed long, to drive the dispatch through `get`. */ diff --git a/spark/src/test/scala/org/apache/comet/csv/CometCsvNativeReadSuite.scala b/spark/src/test/scala/org/apache/comet/csv/CometCsvNativeReadSuite.scala index 7cdfcb99187..3f062f55242 100644 --- a/spark/src/test/scala/org/apache/comet/csv/CometCsvNativeReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/csv/CometCsvNativeReadSuite.scala @@ -29,7 +29,9 @@ import org.apache.spark.sql.types.{IntegerType, StringType, StructType, Timestam import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus +import org.apache.comet.tags.ScansTestGroup +@ScansTestGroup class CometCsvNativeReadSuite extends CometTestBase { private val TEST_CSV_PATH_NO_HEADER = "src/test/resources/test-data/csv-test-1.csv" private val TEST_CSV_PATH_HAS_HEADER = "src/test/resources/test-data/csv-test-2.csv" diff --git a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala index da5ed7e6ce6..2ad2c614445 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala @@ -47,11 +47,13 @@ import org.apache.comet.CometConf.COMET_EXEC_STRICT_FLOATING_POINT import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus, isSpark42Plus} import org.apache.comet.rules.CometExecRule import org.apache.comet.serde.{CometCorr, CometRegrR2, ExprOuterClass, SparkFixes} +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, ParquetGenerator, SchemaGenOptions} /** * Test suite dedicated to Comet native aggregate operator */ +@ExecTestGroup class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala index 48202acea0d..5a5dbe3f34f 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala @@ -40,6 +40,7 @@ import org.apache.spark.sql.types._ import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ShuffleTestGroup abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper { protected val adaptiveExecutionEnabled: Boolean @@ -1065,6 +1066,7 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar } } +@ShuffleTestGroup class CometShuffleSuite extends CometColumnarShuffleSuite { protected val adaptiveExecutionEnabled: Boolean = true @@ -1116,6 +1118,7 @@ class CometShuffleSuite extends CometColumnarShuffleSuite { } } +@ShuffleTestGroup class DisableAQECometShuffleSuite extends CometColumnarShuffleSuite { protected val adaptiveExecutionEnabled: Boolean = false @@ -1162,6 +1165,7 @@ class DisableAQECometShuffleSuite extends CometColumnarShuffleSuite { } } +@ShuffleTestGroup class CometShuffleEncryptionSuite extends CometTestBase { import testImplicits._ @@ -1197,6 +1201,7 @@ class CometShuffleEncryptionSuite extends CometTestBase { } } +@ShuffleTestGroup class CometShuffleManagerSuite extends CometTestBase { test("should not bypass merge sort if executor cores are too high") { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometDirectColumnarToRowSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometDirectColumnarToRowSuite.scala index 3a4ee1f00a5..9982c5b0b19 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometDirectColumnarToRowSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometDirectColumnarToRowSuite.scala @@ -24,12 +24,14 @@ import org.apache.spark.sql.comet.CometColumnarToRowExec import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf +import org.apache.comet.tags.ShuffleTestGroup /** * End-to-end tests for `spark.comet.exec.columnarToRow.direct.enabled`, which routes * `CometColumnarToRowExec`'s non-codegen conversion through `DirectColumnarToRowConverter`. * Whole-stage codegen is disabled so the operator's `doExecute` conversion path runs. */ +@ShuffleTestGroup class CometDirectColumnarToRowSuite extends CometTestBase { private def withDirectConverter(minBatchSize: Int = 0)(f: => Unit): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecCompatibilitySuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecCompatibilitySuite.scala index 18eec3f32f1..8d00cf07b2a 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecCompatibilitySuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecCompatibilitySuite.scala @@ -35,8 +35,10 @@ import org.apache.spark.util.sketch.BloomFilter import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus +import org.apache.comet.tags.ExecTestGroup /** Tests compatibility-sensitive execution paths shared by the supported Spark versions. */ +@ExecTestGroup class CometExecCompatibilitySuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala index 8e78885e6ce..3d117c7b251 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala @@ -56,8 +56,10 @@ import org.apache.comet.{CometConf, CometExecIterator, CometNativeException, Ext import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus, isSpark42Plus} import org.apache.comet.rules.CometCoalesceShufflePartitions import org.apache.comet.serde.Config.ConfigMap +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} +@ExecTestGroup class CometExecSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala index e7038db6eb9..b0026319885 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometGenerateExecSuite.scala @@ -27,7 +27,9 @@ import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometGenerateExecSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheKryoSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheKryoSuite.scala index 4455fad88ef..cd86c258682 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheKryoSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheKryoSuite.scala @@ -32,6 +32,7 @@ import org.apache.spark.unsafe.types.UTF8String import com.esotericsoftware.kryo.Kryo import org.apache.comet.{CometConf, CometKryoRegistrator} +import org.apache.comet.tags.ExecTestGroup /** * Covers Comet's cached batch format under `spark.kryo.registrationRequired=true`. @@ -47,6 +48,7 @@ import org.apache.comet.{CometConf, CometKryoRegistrator} * This needs its own suite because `spark.serializer` and `spark.kryo.registrator` are read when * `SparkEnv` builds the serializer, so they cannot be changed per test. */ +@ExecTestGroup class CometInMemoryCacheKryoSuite extends CometTestBase { override protected def beforeAll(): Unit = { @@ -270,6 +272,7 @@ abstract class CometInMemoryCacheKryoRegistrationSuite(expectedBatch: String) } /** Registers Spark's cached batch and nothing of Comet's, so the plugin keeps Spark's format. */ +@ExecTestGroup class CometInMemoryCacheKryoUnregisteredSuite extends CometInMemoryCacheKryoRegistrationSuite(classOf[DefaultCachedBatch].getName) { override protected def registrations: Seq[(String, String)] = @@ -281,6 +284,7 @@ class CometInMemoryCacheKryoUnregisteredSuite * [[CometKryoRegistrator]], and not Spark's cached batch, so before Spark 4.1 only Comet's format * can be stored, and the plugin has to install it. */ +@ExecTestGroup class CometInMemoryCacheKryoClassesToRegisterSuite extends CometInMemoryCacheKryoRegistrationSuite( ArrowCachedBatchSerializer.cachedBatchClass.getName) { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCachePruningSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCachePruningSuite.scala index 0f9464b89b7..dcde6912b77 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCachePruningSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCachePruningSuite.scala @@ -31,7 +31,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.CometConf +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometInMemoryCachePruningSuite extends CometTestBase { override protected def beforeAll(): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala index 29a788577ec..04703009704 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala @@ -58,10 +58,12 @@ import org.apache.spark.storage.StorageLevel import org.apache.comet.{CometArrowAllocator, CometConf, CometKryoRegistrator, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.rules.{CometCacheBuildRule, CometCacheColumnarRule} +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.{CometPlainVector, CometVector} import net.jpountz.lz4.LZ4FrameInputStream +@ExecTestGroup class CometInMemoryCacheSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index e57d2eee864..5d2d222cf62 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -53,7 +53,9 @@ import org.apache.comet.{CometConf, CometExecIterator, CometExplainInfo, Extende import org.apache.comet.CometSparkSessionExtensions.hasFallbackReason import org.apache.comet.rules.RewriteJoin import org.apache.comet.serde.OperatorOuterClass +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometJoinSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala index 7e35d22002d..0741cb3c677 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala @@ -38,7 +38,9 @@ import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, CometNativeException, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus +import org.apache.comet.tags.ScansTestGroup +@ScansTestGroup class CometNativeReaderSuite extends CometTestBase with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala index f3ced4e3012..565a0eaa22b 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala @@ -59,7 +59,9 @@ import org.apache.comet.{CometConf, CometExecIterator, CometExplainInfo, CometSh import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, withFallbackReason} import org.apache.comet.serde.{OperatorOuterClass, PartitioningOuterClass} import org.apache.comet.shuffle.ShufflePartitionPusher +import org.apache.comet.tags.ShuffleTestGroup +@ShuffleTestGroup class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometRangeExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometRangeExecSuite.scala index 15d13ced8e4..982df0bf786 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometRangeExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometRangeExecSuite.scala @@ -29,7 +29,9 @@ import org.apache.spark.sql.functions.{col, count, sum} import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometRangeExecSuite extends CometTestBase { // CometTestBase enables the Spark-to-Arrow conversions, including the one for Range. Use their diff --git a/spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala index e626410aae9..6cdb841606f 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala @@ -33,11 +33,13 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, import org.apache.spark.unsafe.types.CalendarInterval import org.apache.comet.{CometConf, ExtendedExplainInfo} +import org.apache.comet.tags.ShuffleTestGroup // Top-level, so the encoder needs no outer pointer. case class ShuffleInputRec(a: Int, b: String) /** Tests for [[CometConf.COMET_CONVERT_FROM_SHUFFLE_INPUT_ENABLED]]. */ +@ShuffleTestGroup class CometShuffleInputConversionSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala index 4bd78cf7aee..2593dbd18e4 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometTopKSuite.scala @@ -35,7 +35,9 @@ import org.apache.spark.sql.types.{ByteType, IntegerType, LongType, ShortType} import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometTopKSuite extends CometTestBase { private def localTopK(plan: SparkPlan): CometLocalTopKExec = { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometTypedDatasetSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometTypedDatasetSuite.scala index f2222b37f5d..94972c81a53 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometTypedDatasetSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometTypedDatasetSuite.scala @@ -30,6 +30,7 @@ import org.apache.spark.sql.functions.{broadcast, col, size, sum} import org.apache.spark.sql.internal.SQLConf import org.apache.comet.{CometConf, ExtendedExplainInfo} +import org.apache.comet.tags.ExecTestGroup // Top-level, so the encoders need no outer pointer, which is the ordinary user shape. case class TypedDsRec(a: Int, b: String) @@ -50,6 +51,7 @@ object TypedDsCounter { } /** Tests for [[CometConf.COMET_CONVERT_FROM_TYPED_DATASET_ENABLED]]. */ +@ExecTestGroup class CometTypedDatasetSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala index 62c87654b96..c6688112665 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometWindowExecSuite.scala @@ -37,7 +37,9 @@ import org.apache.spark.sql.types.DecimalType import org.apache.comet.{CometConf, CometExplainInfo, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometWindowExecSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/expressions/CometRegexSuite.scala b/spark/src/test/scala/org/apache/comet/expressions/CometRegexSuite.scala index 4ad8eb63b18..cc1b8393730 100644 --- a/spark/src/test/scala/org/apache/comet/expressions/CometRegexSuite.scala +++ b/spark/src/test/scala/org/apache/comet/expressions/CometRegexSuite.scala @@ -22,7 +22,9 @@ package org.apache.comet.expressions import org.scalatest.funsuite.AnyFunSuite import org.apache.comet.serde.{Compatible, Incompatible} +import org.apache.comet.tags.ExpressionsTestGroup +@ExpressionsTestGroup class CometRegexSuite extends AnyFunSuite { private def assertCompatible(pattern: String): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCaseWhenSuite.scala b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCaseWhenSuite.scala index 8e825462c4e..9d9b0639920 100644 --- a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCaseWhenSuite.scala +++ b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCaseWhenSuite.scala @@ -22,6 +22,9 @@ package org.apache.comet.expressions.conditional import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ExpressionsTestGroup + +@ExpressionsTestGroup class CometCaseWhenSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("case_when") { diff --git a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCoalesceSuite.scala b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCoalesceSuite.scala index b4e4ceb6d9c..1e137772d71 100644 --- a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCoalesceSuite.scala +++ b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometCoalesceSuite.scala @@ -24,6 +24,9 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.SQLConf +import org.apache.comet.tags.ExpressionsTestGroup + +@ExpressionsTestGroup class CometCoalesceSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("coalesce should return correct datatype") { diff --git a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometIfSuite.scala b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometIfSuite.scala index 788667aad2e..aca4d7e2788 100644 --- a/spark/src/test/scala/org/apache/comet/expressions/conditional/CometIfSuite.scala +++ b/spark/src/test/scala/org/apache/comet/expressions/conditional/CometIfSuite.scala @@ -22,6 +22,9 @@ package org.apache.comet.expressions.conditional import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.tags.ExpressionsTestGroup + +@ExpressionsTestGroup class CometIfSuite extends CometTestBase with AdaptiveSparkPlanHelper { test("if expression") { diff --git a/spark/src/test/scala/org/apache/comet/iceberg/IcebergReflectionSuite.scala b/spark/src/test/scala/org/apache/comet/iceberg/IcebergReflectionSuite.scala index 02ffb737a7d..09e4d4b54f1 100644 --- a/spark/src/test/scala/org/apache/comet/iceberg/IcebergReflectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/iceberg/IcebergReflectionSuite.scala @@ -34,6 +34,9 @@ import org.apache.iceberg.TableMetadata import org.apache.iceberg.io.FileIO import org.apache.iceberg.types.Types +import org.apache.comet.tags.ScansTestGroup + +@ScansTestGroup class IcebergReflectionSuite extends AnyFunSuite { /** Mimics HiveTableOperations/GlueTableOperations which inherit current(). */ diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala index eeca4d0fb99..94959cff7af 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthParitySuite.scala @@ -44,6 +44,7 @@ import org.apache.comet.objectstore.AbfsAuthParityCases._ import org.apache.comet.objectstore.AbfsAuthResolver._ import org.apache.comet.objectstore.AbfsReflection.Handles import org.apache.comet.tags.AzureTest +import org.apache.comet.tags.ExecTestGroup /** * Pins the resolver's behaviour per hadoop-azure version and cross-checks it against the real @@ -68,6 +69,7 @@ import org.apache.comet.tags.AzureTest * assert while recording, so a recorded file never contradicts the hadoop-azure source. The file * is written only when every case ran, so a filtered run cannot leave a partial file behind. */ +@ExecTestGroup class AbfsAuthParitySuite extends AnyFunSuite with Matchers with BeforeAndAfterAll { import AbfsAuthParitySuite._ diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala index 60ca6aece03..63abfee9761 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsAuthResolverSuite.scala @@ -44,12 +44,14 @@ import org.apache.logging.log4j.core.config.Property import org.apache.comet.objectstore.AbfsAuthResolver._ import org.apache.comet.objectstore.AbfsReflection.Handles +import org.apache.comet.tags.ExecTestGroup /** * Runs the resolver against the hadoop-azure on the test classpath (3.3.4 on the spark-3.x * profiles, 3.4.1, 3.4.2 and 3.5.0 on spark-4.0, 4.1 and 4.2). Cases that depend on a feature one * line lacks are gated on the capability probes rather than on a version number. */ +@ExecTestGroup class AbfsAuthResolverSuite extends AnyFunSuite with Matchers { private val account = "myacct.dfs.core.windows.net" diff --git a/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala index 6751d5aaa77..1775f5ca5cb 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/AbfsScanFallbackSuite.scala @@ -31,12 +31,14 @@ import org.apache.spark.sql.types.{IntegerType, StructType} import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.hadoop.fs.FakeAbfsFileSystem +import org.apache.comet.tags.ExecTestGroup /** * Plans scans over `abfss://` paths served by [[FakeAbfsFileSystem]], with authentication * resolved offline by the hadoop-azure on the test classpath. A scan Comet declines runs on * Spark's own scan against the local files. */ +@ExecTestGroup class AbfsScanFallbackSuite extends CometTestBase { private val multipleAuthoritiesReason = "more than one container or account" diff --git a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala index 34d885aa319..72dc9738899 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala @@ -28,7 +28,9 @@ import org.apache.hadoop.conf.Configuration import org.apache.comet.CometConf.{COMET_LIBHDFS_SCHEMES_KEY, COMET_S3_COMPLIANT_SCHEMES_KEY} import org.apache.comet.objectstore.AbfsAuthResolver.{Keys, Markers, Providers} +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class NativeConfigSuite extends AnyFunSuite with Matchers { /** diff --git a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterRetrySuite.scala b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterRetrySuite.scala index 40cd10e6ebd..5fe13903aad 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterRetrySuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterRetrySuite.scala @@ -26,12 +26,14 @@ import scala.jdk.CollectionConverters._ import org.apache.spark.SparkConf import org.apache.comet.hadoop.fs.RecordingFileSystem +import org.apache.comet.tags.ScansTestGroup /** * Native Parquet writes whose tasks Spark retries. A suite of its own because a task gets more * than one attempt only under a `local[N,M]` master, and the failure tests elsewhere expect a * failed task to fail its job. */ +@ScansTestGroup class CometParquetWriterRetrySuite extends CometParquetWriterTestBase { override protected def sparkConf: SparkConf = diff --git a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala index 5f4efe8ce57..1396d341cce 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala @@ -58,8 +58,10 @@ import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.hadoop.fs.RecordingFileSystem import org.apache.comet.rules.RevertNativeForTransitionHeavyStages import org.apache.comet.serde.operator.NativeWriteUtils +import org.apache.comet.tags.ScansTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator, SchemaGenOptions} +@ScansTestGroup class CometParquetWriterSuite extends CometParquetWriterTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index 10772988d3a..17c7d9aa799 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -54,6 +54,7 @@ import com.google.common.primitives.UnsignedLong import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus} +import org.apache.comet.tags.ScansTestGroup import org.apache.comet.vector.CometVector abstract class ParquetReadSuite extends CometTestBase { @@ -3111,6 +3112,7 @@ abstract class ParquetReadSuite extends CometTestBase { } } +@ScansTestGroup class ParquetReadV1Suite extends ParquetReadSuite with AdaptiveSparkPlanHelper { override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala index 4305ff5d872..926aa13609a 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetTimestampLtzAsNtzSuite.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ScansTestGroup /** * Tests for reading Parquet TimestampLTZ columns as TimestampNTZ. @@ -37,6 +38,7 @@ import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus * * See https://github.com/apache/datafusion-comet/issues/4219 */ +@ScansTestGroup class ParquetTimestampLtzAsNtzSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala index 0ad4c50a2c1..8a106264b99 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala @@ -54,6 +54,7 @@ import org.apache.comet.{CometConf, CometCoverageStats, CometExplainInfo, CometS import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark42Plus, withFallbackReason} import org.apache.comet.serde.{CometAggregateExpressionSerde, Compatible, ExprOuterClass, OperatorOuterClass, QueryPlanSerde, Unsupported} import org.apache.comet.shims.ShimCometWindowGroupLimit +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} /** @@ -61,6 +62,7 @@ import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} * transform Spark operators to Comet operators, fallback mechanisms, configuration handling, and * edge cases. */ +@ExecTestGroup class CometExecRuleSuite extends CometTestBase { // The observers are active only during the DPP lifecycle regression below. AQE can prepare diff --git a/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala index a650ebcf2f5..dff23d3c2e2 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala @@ -43,12 +43,14 @@ import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.CometConf.{COMET_LIBHDFS_SCHEMES_KEY, COMET_S3_COMPLIANT_SCHEMES_KEY} import org.apache.comet.hadoop.fs.{FakeAbfsFileSystem, FakeHdfsAuthorityFileSystem} import org.apache.comet.objectstore.NativeConfig +import org.apache.comet.tags.ExecTestGroup /** * Native scans over files in more than one object store, without a cloud store: `hdfs://nn1` and * `hdfs://nn2` are two native stores backed by the local disk. Native execution cannot read them, * so claimed scans are checked on their plans, and declined scans are run by Spark. */ +@ExecTestGroup class CometMultiStoreScanSuite extends CometTestBase with AdaptiveSparkPlanHelper { private var rootDir: File = _ diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala index 4e41a9b147a..9c6febb9015 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala @@ -44,6 +44,7 @@ import org.apache.spark.sql.execution.datasources.HadoopFsRelation import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.comet.ContribServices +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.util.ClassLoaders /** @@ -62,6 +63,7 @@ import org.apache.comet.util.ClassLoaders * Sibling to `CometScanWithPlanDataSuite`, which covers the other half of the contrib SPI (plan * data). Everything here runs on the default build. */ +@ExecTestGroup class CometScanContribSuite extends AnyFunSuite { /** diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala index 52c6159d11e..96b0dfb1fb7 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala @@ -30,11 +30,13 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, DataTypeSupport} +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} /** * Test suite specifically for CometScanRule transformation logic. */ +@ExecTestGroup class CometScanRuleSuite extends CometTestBase { /** Helper method to apply CometExecRule and return the transformed plan */ diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala index 8536206d1b5..22a1ef21f98 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala @@ -44,6 +44,7 @@ import org.apache.comet.{CometConf, CometExplainInfo, CometIcebergTestBase, Exte import org.apache.comet.hadoop.fs.{FakeHdfsAuthorityFileSystem, FakeHDFSFileSystem, FakeHdfsSchemeFileSystem, FakeWasbSchemeFileSystem} import org.apache.comet.iceberg.IcebergStorageSchemes import org.apache.comet.objectstore.NativeConfig +import org.apache.comet.tags.ExecTestGroup /** * Comet's native readers go through object_store, which understands a fixed set of URL schemes. A @@ -55,6 +56,7 @@ import org.apache.comet.objectstore.NativeConfig * native probe is cached per probe URL so an authorityless URL can't poison the authority-bearing * form of the same scheme. */ +@ExecTestGroup class CometScanSchemeFallbackSuite extends CometTestBase with CometIcebergTestBase { private var fakeRootDir: File = _ diff --git a/spark/src/test/scala/org/apache/comet/rules/EliminateRedundantTransitionsSuite.scala b/spark/src/test/scala/org/apache/comet/rules/EliminateRedundantTransitionsSuite.scala index b6625d4fd02..d349f6e1644 100644 --- a/spark/src/test/scala/org/apache/comet/rules/EliminateRedundantTransitionsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/EliminateRedundantTransitionsSuite.scala @@ -24,7 +24,9 @@ import org.apache.spark.sql.comet.{CometColumnarToRowExec, CometPlan} import org.apache.spark.sql.execution.{ColumnarToRowExec, SparkPlan, UnionExec} import org.apache.comet.CometConf +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class EliminateRedundantTransitionsSuite extends CometTestBase { private def sparkPlanFor(sql: String): SparkPlan = { diff --git a/spark/src/test/scala/org/apache/comet/rules/RevertNativeForTransitionHeavyStagesSuite.scala b/spark/src/test/scala/org/apache/comet/rules/RevertNativeForTransitionHeavyStagesSuite.scala index 573751dae65..f910d5bfbe7 100644 --- a/spark/src/test/scala/org/apache/comet/rules/RevertNativeForTransitionHeavyStagesSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/RevertNativeForTransitionHeavyStagesSuite.scala @@ -43,6 +43,7 @@ import org.apache.spark.sql.util.QueryExecutionListener import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.tags.ExecTestGroup private case class AliasingFallbackCometExec( override val originalPlan: SparkPlan, @@ -67,6 +68,7 @@ private case class BothFormatsExec(child: SparkPlan) extends UnaryExecNode { copy(child = newChild) } +@ExecTestGroup class RevertNativeForTransitionHeavyStagesSuite extends CometTestBase { // Spark 4.0+ builds a cache through columnar output only when the cache serializer accepts diff --git a/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala b/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala index 44c8691ed47..37e426173ef 100644 --- a/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala @@ -32,6 +32,7 @@ import org.apache.spark.sql.types._ import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus} import org.apache.comet.serde.literals.CometLiteral import org.apache.comet.shims.CometTypeShim +import org.apache.comet.tags.ExpressionsTestGroup /** * Pins `CometLiteral.listLiteralElementSupported` against the arms of @@ -40,6 +41,7 @@ import org.apache.comet.shims.CometTypeShim * raises a `MatchError` in the middle of planning, which Spark surfaces as `[INTERNAL_ERROR] The * Spark SQL phase planning failed with an internal error`. */ +@ExpressionsTestGroup class CometLiteralSuite extends CometTestBase with CometTypeShim { test("a null literal with duplicate struct field names is unsupported") { diff --git a/spark/src/test/scala/org/apache/comet/serde/CometScalarFunctionSuite.scala b/spark/src/test/scala/org/apache/comet/serde/CometScalarFunctionSuite.scala index 8cf88ec5593..44b723d241f 100644 --- a/spark/src/test/scala/org/apache/comet/serde/CometScalarFunctionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/CometScalarFunctionSuite.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.expressions.{Abs, Cos, Expression, Literal, import org.apache.spark.sql.types.{DataType, DoubleType, IntegerType} import org.apache.comet.{CometExplainInfo, CometSparkSessionExtensions} +import org.apache.comet.tags.ExpressionsTestGroup /** * Synthetic expression whose constructor declares `evalMode`, used to prove class-level detection @@ -85,6 +86,7 @@ case class TestEvalContextExpression(child: Expression, evalContext: Any) copy(child = newChildren.head) } +@ExpressionsTestGroup class CometScalarFunctionSuite extends CometTestBase { private def fallbackReasons(expr: Expression): Set[String] = { diff --git a/spark/src/test/scala/org/apache/comet/serde/QueryPlanSerdeSuite.scala b/spark/src/test/scala/org/apache/comet/serde/QueryPlanSerdeSuite.scala index d7e595d4a83..b6d6ffb8baf 100644 --- a/spark/src/test/scala/org/apache/comet/serde/QueryPlanSerdeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/QueryPlanSerdeSuite.scala @@ -33,7 +33,9 @@ import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus} import org.apache.comet.serde.QueryPlanSerde.{canShortCircuitNulls, canSkipEvaluation, supportedDataType, withSharedSubexpressions} +import org.apache.comet.tags.ExpressionsTestGroup +@ExpressionsTestGroup class QueryPlanSerdeSuite extends AnyFunSuite { test("make_date IN evaluates the probe but can skip list expressions") { diff --git a/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala index 300d19081f1..e71b5547bdc 100644 --- a/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/SerdeRegistrationSuite.scala @@ -21,6 +21,9 @@ package org.apache.comet.serde import org.scalatest.funsuite.AnyFunSuite +import org.apache.comet.tags.ExpressionsTestGroup + +@ExpressionsTestGroup class SerdeRegistrationSuite extends AnyFunSuite { // A version shim only adds serdes for classes the shared map cannot name on every Spark diff --git a/spark/src/test/scala/org/apache/comet/serde/SparkFixesSuite.scala b/spark/src/test/scala/org/apache/comet/serde/SparkFixesSuite.scala index 40a6b12ee31..f28472f03a1 100644 --- a/spark/src/test/scala/org/apache/comet/serde/SparkFixesSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/SparkFixesSuite.scala @@ -21,10 +21,13 @@ package org.apache.comet.serde import org.scalatest.funsuite.AnyFunSuite +import org.apache.comet.tags.ExpressionsTestGroup + /** * Pins the version table of every fix in `SparkFixes`, so that the boundaries are checked on all * Spark versions in one run and not only on the one a test suite runs against. */ +@ExpressionsTestGroup class SparkFixesSuite extends AnyFunSuite { private def checkVersions(name: String, fixed: String => Boolean)( diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala index 978b1fb89df..3649d4e7ba1 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala @@ -36,8 +36,10 @@ import org.apache.spark.sql.catalyst.expressions.AttributeReference import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType} import org.apache.comet.iceberg.IcebergReflection +import org.apache.comet.tags.ExecTestGroup /** Unit tests for Iceberg native-scan serde helpers that do not require a Spark session. */ +@ExecTestGroup class CometIcebergNativeScanSuite extends AnyFunSuite with Matchers { test("complex type null residuals are not serialized") { diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala index fba592852f6..fe8b46da782 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala @@ -30,7 +30,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{IntegerType, LongType, MetadataBuilder, StringType, StructField, StructType} import org.apache.comet.parquet.CometParquetUtils +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometNativeScanSuite extends AnyFunSuite { import CometNativeScan.{constantMetadataFieldPrefix, uniqueConstantMetadataFields} diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala index f18e33f76b6..c670633cb9f 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/IcebergWriteProtoTranslationSuite.scala @@ -22,11 +22,13 @@ package org.apache.comet.serde.operator import org.scalatest.funsuite.AnyFunSuite import org.apache.comet.serde.OperatorOuterClass.{CompressionCodec, IcebergWriterMode} +import org.apache.comet.tags.ScansTestGroup /** * Unit tests for the protobuf translation. Iceberg must be on the classpath because the property * keys / numeric defaults are looked up reflectively from `org.apache.iceberg.TableProperties`. */ +@ScansTestGroup class IcebergWriteProtoTranslationSuite extends AnyFunSuite { import IcebergWriteProtoTranslation._ diff --git a/spark/src/test/scala/org/apache/comet/shuffle/CelebornShufflePartitionPusherSuite.scala b/spark/src/test/scala/org/apache/comet/shuffle/CelebornShufflePartitionPusherSuite.scala index d2da70a4e3b..5dda97e324f 100644 --- a/spark/src/test/scala/org/apache/comet/shuffle/CelebornShufflePartitionPusherSuite.scala +++ b/spark/src/test/scala/org/apache/comet/shuffle/CelebornShufflePartitionPusherSuite.scala @@ -32,6 +32,9 @@ import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.SparkConf +import org.apache.comet.tags.ShuffleTestGroup + +@ShuffleTestGroup class CelebornShufflePartitionPusherSuite extends AnyFunSuite { private val encodedAttemptId = (4 << 16) | 7 diff --git a/spark/src/test/scala/org/apache/comet/udf/CometUdfBridgeSuite.scala b/spark/src/test/scala/org/apache/comet/udf/CometUdfBridgeSuite.scala index 71ec1f856e5..88aec7e5fd1 100644 --- a/spark/src/test/scala/org/apache/comet/udf/CometUdfBridgeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/udf/CometUdfBridgeSuite.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.types.IntegerType import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.comet.{CometArrowAllocator, CometArrowImportAllocator} +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.NativeUtil /** @@ -81,6 +82,7 @@ class IdentityUdf extends CometUDF { override def evaluate(inputs: Array[ValueVector], numRows: Int): ValueVector = inputs.head } +@ExecTestGroup class CometUdfBridgeSuite extends AnyFunSuite { /** Exports a single-column batch and invokes the bridge against it. */ diff --git a/spark/src/test/scala/org/apache/comet/vector/CometVectorUtilsSuite.scala b/spark/src/test/scala/org/apache/comet/vector/CometVectorUtilsSuite.scala index 8e61b3e0bcd..55521f60347 100644 --- a/spark/src/test/scala/org/apache/comet/vector/CometVectorUtilsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/vector/CometVectorUtilsSuite.scala @@ -33,7 +33,10 @@ import org.apache.arrow.vector.dictionary.Dictionary import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider import org.apache.arrow.vector.types.pojo.{ArrowType, DictionaryEncoding, FieldType} +import org.apache.comet.tags.ExecTestGroup + /** Check failure cleanup without transferring ownership of the helper's source vectors. */ +@ExecTestGroup class CometVectorUtilsSuite extends AnyFunSuite with Matchers { for (missingDictionary <- Seq(true, false)) { diff --git a/spark/src/test/scala/org/apache/comet/vector/NativeUtilSuite.scala b/spark/src/test/scala/org/apache/comet/vector/NativeUtilSuite.scala index 50eb2959cf1..822d005263d 100644 --- a/spark/src/test/scala/org/apache/comet/vector/NativeUtilSuite.scala +++ b/spark/src/test/scala/org/apache/comet/vector/NativeUtilSuite.scala @@ -42,7 +42,9 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.comet.{CometArrowAllocator, CometArrowImportAllocator, CometConf, CometRuntimeException} import org.apache.comet.serde.{OperatorOuterClass, QueryPlanSerde} +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class NativeUtilSuite extends CometTestBase { private def withIsolatedStructAllocator( diff --git a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala index 1caaec887fa..8ab788bc66b 100644 --- a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala @@ -47,6 +47,7 @@ import org.apache.spark.util.JsonProtocol import org.apache.comet.{CometArrowAllocator, CometConf, CometExecIterator, CometExecutorMemoryUsage, CometNativeException, CometShuffleBlockIterator, Native} import org.apache.comet.serde.Config.ConfigMap import org.apache.comet.serde.OperatorOuterClass +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.CometPlainVector /** @@ -54,6 +55,7 @@ import org.apache.comet.vector.CometPlainVector * one release of the native execution context and of its task-shared memory pool reference, even * when a step of the lifecycle fails partway through. See issue #5212 (positions 2, 3 and 8). */ +@ExecTestGroup class CometExecIteratorLifecycleSuite extends CometTestBase { private def withTaskContext[T](taskAttemptId: Long)(f: => T): T = { diff --git a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala index dc228b98916..64b5816ec65 100644 --- a/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometPluginsSuite.scala @@ -35,7 +35,9 @@ import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} import org.apache.spark.util.{JsonProtocol, ManualClock, Utils} import org.apache.comet.{COMET_VERSION, CometConf, CometExecIterator, CometExecutorMemoryUsage, CometSparkSessionExtensions} +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometPluginsSuite extends CometTestBase { override protected def sparkConf: SparkConf = { val conf = new SparkConf() @@ -176,6 +178,7 @@ class CometPluginsSuite extends CometTestBase { } } +@ExecTestGroup class CometPluginsDefaultSuite extends CometTestBase { override protected def sparkConf: SparkConf = { val conf = new SparkConf() @@ -194,6 +197,7 @@ class CometPluginsDefaultSuite extends CometTestBase { } } +@ExecTestGroup class CometPluginsMemoryOverheadWarningSuite extends CometTestBase { private val warning = @@ -277,6 +281,7 @@ class CometPluginsMemoryOverheadWarningSuite extends CometTestBase { } } +@ExecTestGroup class CometPluginsDeprecatedConfigWarningSuite extends CometTestBase { private def warningsFor(conf: SparkConf): Seq[String] = { @@ -303,6 +308,7 @@ class CometPluginsDeprecatedConfigWarningSuite extends CometTestBase { } } +@ExecTestGroup class CometPluginsUnifiedModeSuite extends CometTestBase { override protected def sparkConf: SparkConf = { val conf = new SparkConf() @@ -376,6 +382,7 @@ class CometDriverPluginShuffleManagerSuite extends SparkFunSuite { } } +@ExecTestGroup class CometPluginsExtensionOnlySuite extends CometTestBase { // No plugin and no off-heap memory. CometTestBase registers CometSparkSessionExtensions // directly, as an application can with spark.sql.extensions. @@ -437,6 +444,7 @@ class CometPluginsExtensionOnlySuite extends CometTestBase { } } +@ExecTestGroup class CometPluginsSparkShuffleManagerSuite extends CometTestBase { // The application runs Spark's own shuffle manager. override protected def sparkConf: SparkConf = { @@ -478,6 +486,7 @@ class CometPluginsSparkShuffleManagerSuite extends CometTestBase { } } +@ExecTestGroup class CometPluginsEventLogSuite extends SparkFunSuite { import CometExecIterator.{memoryUsageEvent, JvmArrowMemory} diff --git a/spark/src/test/scala/org/apache/spark/CometRuntimeShutdownSuite.scala b/spark/src/test/scala/org/apache/spark/CometRuntimeShutdownSuite.scala index 69425c7eae4..383b906957b 100644 --- a/spark/src/test/scala/org/apache/spark/CometRuntimeShutdownSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometRuntimeShutdownSuite.scala @@ -33,6 +33,8 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.functions.{expr, sum} import org.apache.spark.util.Utils +import org.apache.comet.tags.ExecTestGroup + /** * Entry point run in a child JVM by [[CometRuntimeShutdownSuite]]. Executes a native Comet query * and then returns from `main` without calling `SparkContext.stop()`. @@ -78,6 +80,7 @@ object CometRuntimeShutdownMain { * Runs Comet in a child JVM because the failure mode under test, the JVM being unable to exit, * cannot be observed from inside the affected JVM. */ +@ExecTestGroup class CometRuntimeShutdownSuite extends AnyFunSuite { test("JVM exits when main returns without SparkContext.stop()") { diff --git a/spark/src/test/scala/org/apache/spark/CometTaskMemoryManagerSuite.scala b/spark/src/test/scala/org/apache/spark/CometTaskMemoryManagerSuite.scala index 82c5731da7a..38ce8c7f922 100644 --- a/spark/src/test/scala/org/apache/spark/CometTaskMemoryManagerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometTaskMemoryManagerSuite.scala @@ -26,6 +26,9 @@ import org.apache.logging.log4j.Level import org.apache.logging.log4j.core.LogEvent import org.apache.spark.memory.{MemoryConsumer, TaskMemoryManager, TestMemoryManager} +import org.apache.comet.tags.ExecTestGroup + +@ExecTestGroup class CometTaskMemoryManagerSuite extends TaskMemoryTestUtils { test("native memory usage is visible to Spark's memory consumer") { diff --git a/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnboundedShuffleMemoryAllocatorSuite.scala b/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnboundedShuffleMemoryAllocatorSuite.scala index 7b2be4fbdff..748dd91774a 100644 --- a/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnboundedShuffleMemoryAllocatorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnboundedShuffleMemoryAllocatorSuite.scala @@ -27,6 +27,9 @@ import org.apache.spark.SparkConf import org.apache.spark.memory.{SparkOutOfMemoryError, TaskMemoryManager, TestMemoryManager} import org.apache.spark.unsafe.memory.MemoryBlock +import org.apache.comet.tags.ShuffleTestGroup + +@ShuffleTestGroup class CometUnboundedShuffleMemoryAllocatorSuite extends AnyFunSuite { private val pageSize = 4096L diff --git a/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocatorSuite.scala b/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocatorSuite.scala index ecf2e47be21..34ba0cc03cf 100644 --- a/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocatorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocatorSuite.scala @@ -24,6 +24,9 @@ import org.apache.logging.log4j.core.LogEvent import org.apache.spark.{CometTaskMemoryManager, TaskContext, TaskContextImpl, TaskMemoryTestUtils} import org.apache.spark.memory.SparkOutOfMemoryError +import org.apache.comet.tags.ShuffleTestGroup + +@ShuffleTestGroup class CometUnifiedShuffleMemoryAllocatorSuite extends TaskMemoryTestUtils { private val PageSize = 20L diff --git a/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala b/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala index bf691530ccd..aa6baa40ced 100644 --- a/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/shuffle/sort/SpillSorterSuite.scala @@ -44,6 +44,7 @@ import org.apache.spark.unsafe.Platform import org.apache.spark.unsafe.UnsafeAlignedOffset import org.apache.comet.Native +import org.apache.comet.tags.ShuffleTestGroup import org.apache.comet.vector.NativeUtil /** @@ -52,6 +53,7 @@ import org.apache.comet.vector.NativeUtil * These tests verify SpillSorter behavior using Spark's test memory management infrastructure, * without needing a full SparkContext. */ +@ShuffleTestGroup class SpillSorterSuite extends AnyFunSuite with BeforeAndAfterEach { private val INITIAL_SIZE = 1024 diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala index bf80f12e9bb..328bf8634f3 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala @@ -24,7 +24,9 @@ import org.apache.spark.internal.config.{MEMORY_OFFHEAP_ENABLED, MEMORY_OFFHEAP_ import org.apache.spark.sql.comet.shims.ShimCometTPCDSQuerySuite import org.apache.comet.CometConf +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometTPCDSQuerySuite extends { val tpcdsAllQueries: Seq[String] = Seq( diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala index 0e2fd64683e..a05308696ec 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala @@ -31,11 +31,13 @@ import org.apache.spark.sql.test.TestSparkSession import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ExecTestGroup /** * Because we need to modify some methods of Spark `TPCDSQueryTestSuite` but they are private, we * copy Spark `TPCDSQueryTestSuite`. */ +@ExecTestGroup class CometTPCDSQueryTestSuite extends QueryTest with TPCDSBase with CometSQLQueryTestHelper { val tpcdsExtendedQueries: Seq[String] = Seq("q72") diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala index 0f942927091..faa4b95205b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCHQuerySuite.scala @@ -33,6 +33,7 @@ import org.apache.spark.sql.test.TestSparkSession import org.apache.comet.CometConf import org.apache.comet.shims.ShimCometTPCHQuerySuite +import org.apache.comet.tags.ExecTestGroup /** * End-to-end tests to check TPCH query results. @@ -49,6 +50,7 @@ import org.apache.comet.shims.ShimCometTPCHQuerySuite * ./mvnw -Dsuites=org.apache.spark.sql.CometTPCHQuerySuite test * }}} */ +@ExecTestGroup class CometTPCHQuerySuite extends QueryTest with TPCBase with ShimCometTPCHQuerySuite { private val tpchDataPath = sys.env.get("SPARK_TPCH_DATA") diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala index 15b7c706490..5a3f5e67afb 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala @@ -42,8 +42,10 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.util.io.ChunkedByteBuffer import org.apache.comet.{CometArrowAllocator, CometKryoRegistrator} +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.CometVector +@ExecTestGroup class CometBroadcastValueSuite extends CometTestBase { private val Timeout = 30.seconds diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometCollectMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometCollectMetricsSuite.scala index 0e50d55f40e..e1f66f3d236 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometCollectMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometCollectMetricsSuite.scala @@ -36,7 +36,9 @@ import org.apache.spark.sql.util.QueryExecutionListener import org.apache.comet.CometConf import org.apache.comet.rules.CometExecRule +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometCollectMetricsSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometDecimalPromotionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometDecimalPromotionSuite.scala index 68f88e18255..e3623574c6e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometDecimalPromotionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometDecimalPromotionSuite.scala @@ -27,7 +27,9 @@ import org.apache.spark.sql.types.{DecimalType, IntegerType} import org.apache.comet.{CometConf, CometExplainInfo} import org.apache.comet.serde.{CometDivide, ExprOuterClass, QueryPlanSerde, Unsupported} +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometDecimalPromotionSuite extends CometTestBase { private case class Operation( diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala index f672ebc082f..75896b13476 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala @@ -32,6 +32,7 @@ import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec import org.apache.spark.sql.internal.SQLConf import org.apache.comet.{CometConf, CometExplainInfo} +import org.apache.comet.tags.ExecTestGroup /** * Attempts an end-to-end reproduction of issue #3949. @@ -61,6 +62,7 @@ import org.apache.comet.{CometConf, CometExplainInfo} * shuffle whose child subtree (descending through `QueryStageExec.plan`) still contains a DPP * scan -- i.e. an inconsistency that the bug would produce. */ +@ExecTestGroup class CometDppFallbackRepro3949Suite extends CometTestBase { // ---------------------------------------------------------------------- diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometNativeWriteSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometNativeWriteSuite.scala index 098b1c16bc5..ce8f188c705 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometNativeWriteSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometNativeWriteSuite.scala @@ -49,8 +49,10 @@ import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.serde.{Incompatible, Unsupported} import org.apache.comet.serde.OperatorOuterClass.Operator import org.apache.comet.serde.operator.{schema2Proto, CometDataWritingCommand, NativeWriteUtils} +import org.apache.comet.tags.ScansTestGroup /** Exercises the retained Spark 3.x writer, including both terminal execution entry points. */ +@ScansTestGroup class CometNativeWriteSuite extends CometTestBase { import NativeWriteCommitProtocol._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala index 52a5e1fe953..b104f40098d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala @@ -32,8 +32,10 @@ import org.apache.xbean.asm9.tree.{ClassNode, FieldInsnNode, MethodInsnNode} import com.google.common.reflect.ClassPath import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.util.ClassLoaders +@ExecTestGroup class CometPlanEqualitySuite extends AnyFunSuite { // Native serialization state and the source plan are deliberately not the identity of a diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala index 903f123360d..884fc316c05 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanStabilitySuite.scala @@ -36,6 +36,7 @@ import org.apache.spark.sql.test.TestSparkSession import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus, isSpark42Plus} +import org.apache.comet.tags.ExecTestGroup /** * Similar to [[org.apache.spark.sql.PlanStabilitySuite]], checks that TPC-DS Comet plans don't @@ -281,6 +282,7 @@ private object CometPlanStabilitySuite { } } +@ExecTestGroup class CometTPCDSV1_4_PlanStabilitySuite extends CometPlanStabilitySuite { private val planNames: Seq[String] = CometPlanStabilitySuite.planNameChain("v1_4") override val goldenFilePath: String = @@ -295,6 +297,7 @@ class CometTPCDSV1_4_PlanStabilitySuite extends CometPlanStabilitySuite { } } +@ExecTestGroup class CometTPCDSV2_7_PlanStabilitySuite extends CometPlanStabilitySuite { private val planNames: Seq[String] = CometPlanStabilitySuite.planNameChain("v2_7") override val goldenFilePath: String = diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala index c3633121621..77f257c7f92 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometScanWithPlanDataSuite.scala @@ -36,6 +36,7 @@ import org.apache.spark.sql.execution.{BinaryExecNode, SparkPlan} import org.apache.comet.CometRuntimeException import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.tags.ExecTestGroup /** * Unit coverage for the core SPI that lets out-of-tree contrib leaf scans (e.g. the Delta @@ -50,6 +51,7 @@ import org.apache.comet.serde.OperatorOuterClass.Operator * Deliberately does not exercise the per-op injector registry mechanics; that surface is owned by * `PlanDataInjectorSuite`. */ +@ExecTestGroup class CometScanWithPlanDataSuite extends AnyFunSuite { /** diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala index d6da464804b..ce8a491d901 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala @@ -25,7 +25,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometSparkSessionExtensions import org.apache.comet.rules.CometRule +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometSessionExtensionsSuite extends CometTestBase { test("captured configs do not override a SQL execution after a setting is unset") { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometShuffleFallbackStickinessSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometShuffleFallbackStickinessSuite.scala index 23f5ad5d413..cdd3ad84ad6 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometShuffleFallbackStickinessSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometShuffleFallbackStickinessSuite.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{hasFallbackReason, withFallbackReason} +import org.apache.comet.tags.ExecTestGroup /** * Pins the sticky-fallback invariant for Comet shuffle decisions: `shuffleSupported` must return @@ -46,6 +47,7 @@ import org.apache.comet.CometSparkSessionExtensions.{hasFallbackReason, withFall * The coordinator tags the node with `withFallbackReasons` only on total fallback and * short-circuits via `hasFallbackReason` on subsequent passes. */ +@ExecTestGroup class CometShuffleFallbackStickinessSuite extends CometTestBase { test("shuffleSupported returns None when the shuffle already carries explain info") { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala index 936033313f2..b8fac4aedc5 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala @@ -47,7 +47,9 @@ import org.apache.spark.unsafe.Platform import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, isSpark41Plus} import org.apache.comet.serde.{Metric, OperatorOuterClass} +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper { override protected def sparkConf: SparkConf = { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometUnsafeProjectionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometUnsafeProjectionSuite.scala index c592d5b0a35..b738848ee81 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometUnsafeProjectionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometUnsafeProjectionSuite.scala @@ -30,6 +30,9 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} import org.apache.spark.sql.types._ +import org.apache.comet.tags.ExecTestGroup + +@ExecTestGroup class CometUnsafeProjectionSuite extends CometTestBase with AdaptiveSparkPlanHelper { /** A schema with every writer GenerateUnsafeProjection nests: structs, arrays and maps. */ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/DecimalPrecisionSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/DecimalPrecisionSuite.scala index be4c7db5de8..682d242ff6e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/DecimalPrecisionSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/DecimalPrecisionSuite.scala @@ -23,6 +23,9 @@ import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.expressions.{Add, Alias, AttributeReference, CaseWhen, Cast, CheckOverflow, Coalesce, Divide, Expression, GreaterThan, IsNotNull, Literal, Multiply, Remainder, Subtract} import org.apache.spark.sql.types.{DecimalType, IntegerType, LongType, StringType} +import org.apache.comet.tags.ExecTestGroup + +@ExecTestGroup class DecimalPrecisionSuite extends SparkFunSuite { private val decA = AttributeReference("a", DecimalType(10, 2))() diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala b/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala index 1423981b1c4..20c0cfb1563 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/ParquetEncryptionITCase.scala @@ -37,12 +37,14 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SQLTestUtils import org.apache.comet.{CometConf, IntegrationTestSuite} +import org.apache.comet.tags.ScansTestGroup /** * A integration test suite that tests parquet modular encryption usage. */ @RunWith(classOf[JUnitRunner]) @IntegrationTestSuite +@ScansTestGroup class ParquetEncryptionITCase extends CometTestBase with SQLTestUtils { private val encoder = Base64.getEncoder private val footerKey = diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorShuffleLifecycleSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorShuffleLifecycleSuite.scala index ae7db078cb6..9f1ba00febb 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorShuffleLifecycleSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorShuffleLifecycleSuite.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.functions.col import org.apache.spark.util.Utils import org.apache.comet.CometConf +import org.apache.comet.tags.ExecTestGroup /** * End-to-end lifecycle of the executor-side plan data held by [[PlanDataInjector]]. A map-only @@ -37,6 +38,7 @@ import org.apache.comet.CometConf * The recreated-context test stops the suite's SparkContext, so it runs last and nothing else may * follow it. */ +@ExecTestGroup class PlanDataInjectorShuffleLifecycleSuite extends CometTestBase { private def withNativeShuffle[T](session: SparkSession)(f: => T): T = { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorSuite.scala index a6416ccc73b..58df60a3eef 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/PlanDataInjectorSuite.scala @@ -28,7 +28,9 @@ import org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager import org.apache.comet.serde.OperatorOuterClass import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class PlanDataInjectorSuite extends AnyFunSuite { /** Builds an un-injected IcebergScan operator: hasCommon, zero file_scan_tasks. */ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchRowIteratorSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchRowIteratorSuite.scala index 108694a7b5a..c305e3a8494 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchRowIteratorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchRowIteratorSuite.scala @@ -32,8 +32,10 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{IntegerType, LongType, StringType} import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.CometPlainVector +@ExecTestGroup class CachedBatchRowIteratorSuite extends AnyFunSuite { Seq("CODEGEN_ONLY", "NO_CODEGEN").foreach { mode => def withMode(settings: (String, String)*)(f: => Unit): Unit = { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala index 34bb32d908f..81a15d0a8db 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowStreamSuite.scala @@ -44,6 +44,7 @@ import org.apache.spark.sql.types.{ArrayType, BooleanType, ByteType, CalendarInt import org.apache.spark.sql.vectorized.{ColumnarArray, ColumnarBatch, ColumnarMap, ColumnVector} import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.{CometPlainVector, CometVector, NativeUtil} /** @@ -52,6 +53,7 @@ import org.apache.comet.vector.{CometPlainVector, CometVector, NativeUtil} * `CometMathExpressionSuite`, but that test only catches *one* function-level type drift. This * suite covers the boundary contract independently of any specific function. */ +@ExecTestGroup class CometArrowStreamSuite extends AnyFunSuite with Matchers { private def expectedSchema(types: (String, ArrowType)*): Schema = { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowWriterSuite.scala index 400d9aad797..e37fde2235a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowWriterSuite.scala @@ -43,12 +43,14 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.unsafe.types.UTF8String import org.apache.comet.CometSparkSessionExtensions.isSpark41Plus +import org.apache.comet.tags.ExecTestGroup /** * The columnar paths of [[ArrowWriter]] copy Spark's vectors in bulk where they can. Whatever * they take, they must produce what the row path writes for the same rows, which reads every * value through Spark's getters one at a time. */ +@ExecTestGroup class CometArrowWriterSuite extends AnyFunSuite with Matchers { private val numRows = 300 diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometStringWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometStringWriterSuite.scala index 3f23851d76f..21b70924c18 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometStringWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometStringWriterSuite.scala @@ -39,6 +39,9 @@ import org.apache.spark.sql.comet.util.Utils import org.apache.spark.sql.types.{ArrayType, StringType, StructField, StructType} import org.apache.spark.unsafe.types.UTF8String +import org.apache.comet.tags.ExecTestGroup + +@ExecTestGroup class CometStringWriterSuite extends AnyFunSuite with Matchers { private val sourceOffset = 7 private val schema = StructType(Seq(StructField("text", StringType))) diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala index 361ed654c52..92f8bbb166d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala @@ -35,6 +35,8 @@ import org.apache.spark.SparkException import com.github.luben.zstd.Zstd +import org.apache.comet.tags.ExecTestGroup + /** * Checks that the zstd codec never writes past the buffer it allocates for its output. * @@ -44,6 +46,7 @@ import com.github.luben.zstd.Zstd * The defect is explained on [[CometZstdCompressionCodec]]. The inputs are the ones that provoke * it: bytes that look structured enough for zstd to try entropy coding but do not compress. */ +@ExecTestGroup class CometZstdCompressionCodecSuite extends AnyFunSuite with Matchers { // zstd-jni unpacks its native library into java.io.tmpdir, which the test JVM points at diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReaderSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReaderSuite.scala index 6eda56cdb76..d86cc470e7b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReaderSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReaderSuite.scala @@ -24,6 +24,9 @@ import java.io.InputStream import org.scalatest.funsuite.AnyFunSuite +import org.apache.comet.tags.ShuffleTestGroup + +@ShuffleTestGroup class CometBlockStoreShuffleReaderSuite extends AnyFunSuite { test("closing a partially consumed raw stream does not fetch remaining blocks") { class TrackingStream extends ByteArrayInputStream(Array[Byte](1, 2, 3)) { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala index 7603c22d784..56f5534758a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornConcurrentMaterializationSuite.scala @@ -34,7 +34,9 @@ import org.apache.spark.sql.comet.shims.ShimCometShuffleMaterialization import org.apache.spark.util.ThreadUtils import org.apache.comet.CometConf +import org.apache.comet.tags.ShuffleTestGroup +@ShuffleTestGroup class CometCelebornConcurrentMaterializationSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala index a2f8f1e41e5..36e2dbfb7cd 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornLocalFetchFailureSuite.scala @@ -30,7 +30,9 @@ import org.apache.spark.shuffle.{FetchFailedException, ShuffleHandle, ShuffleRea import org.apache.spark.sql.CometTestBase import org.apache.comet.CometConf +import org.apache.comet.tags.ShuffleTestGroup +@ShuffleTestGroup class CometCelebornLocalFetchFailureSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornNativeShuffleWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornNativeShuffleWriterSuite.scala index 048b4c09a6e..a0e3271965d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornNativeShuffleWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornNativeShuffleWriterSuite.scala @@ -30,8 +30,10 @@ import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.{CometConf, CometShuffleSizeLimitException} import org.apache.comet.shuffle.{CelebornShufflePartitionPusher, CelebornShufflePusherFactory, RecordingCelebornPushClient} +import org.apache.comet.tags.ShuffleTestGroup /** Exercises real Spark map tasks, native RSS planning, and the Celeborn map lifecycle. */ +@ShuffleTestGroup class CometCelebornNativeShuffleWriterSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala index de3381febfe..afce8b7f025 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleFallbackSuite.scala @@ -38,10 +38,12 @@ import org.apache.spark.sql.CometTestBase import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.shuffle.{CelebornShufflePusherFactory, RecordingCelebornPushClient, ResolvedCelebornShufflePusher} +import org.apache.comet.tags.ShuffleTestGroup /** * Real Spark stage recovery and local Comet I/O; only the optional Celeborn client is replaced. */ +@ShuffleTestGroup class CometCelebornShuffleFallbackSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManagerSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManagerSuite.scala index 9c193a958ba..a9f48f67119 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManagerSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleManagerSuite.scala @@ -31,7 +31,9 @@ import org.apache.spark.shuffle.{BaseShuffleHandle, ShuffleBlockResolver, Shuffl import org.apache.spark.sql.comet.{NativeScanPlanDataInjector, PlanDataInjector} import org.apache.comet.serde.OperatorOuterClass +import org.apache.comet.tags.ShuffleTestGroup +@ShuffleTestGroup class CometCelebornShuffleManagerSuite extends AnyFunSuite { // Match the optional CelebornConf API, including its enum-valued policy and Long threshold. diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShufflePlanningSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShufflePlanningSuite.scala index 625013aa9e9..7e212ccd1c0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShufflePlanningSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShufflePlanningSuite.scala @@ -42,12 +42,14 @@ import org.apache.comet.{CometConf, CometExplainInfo} import org.apache.comet.CometSparkSessionExtensions.{isCometShuffleEnabled, isCometShuffleManagerEnabled} import org.apache.comet.rules.{CometExecRule, RevertNativeForTransitionHeavyStages} import org.apache.comet.serde.{Compatible, OperatorOuterClass} +import org.apache.comet.tags.ShuffleTestGroup /** * Plans against the actual composite manager without requiring the optional Celeborn client. Only * Spark-fallback queries execute: the local test backend must never receive a native dependency. * Native plan assertions below are not transport or live-cluster integration tests. */ +@ShuffleTestGroup class CometCelebornShufflePlanningSuite extends CometTestBase { override protected val shuffleManager: String = diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReaderSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReaderSuite.scala index d315bddfe02..5f83fc23b53 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReaderSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleReaderSuite.scala @@ -53,7 +53,9 @@ import com.google.flatbuffers.Table import org.apache.comet.{CometConf, CometExecIterator, CometShuffleBlockIterator, Native} import org.apache.comet.serde.{OperatorOuterClass, QueryPlanSerde} +import org.apache.comet.tags.ShuffleTestGroup +@ShuffleTestGroup class CometCelebornShuffleReaderSuite extends CometTestBase { import NativeShuffleTestFrames._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleStatisticsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleStatisticsSuite.scala index f4be2680366..a941533e782 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleStatisticsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometCelebornShuffleStatisticsSuite.scala @@ -27,7 +27,9 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.metric.SQLShuffleWriteMetricsReporter.{SHUFFLE_BYTES_WRITTEN, SHUFFLE_RECORDS_WRITTEN} import org.apache.comet.CometConf +import org.apache.comet.tags.ShuffleTestGroup +@ShuffleTestGroup class CometCelebornShuffleStatisticsSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala index f18e50658c7..15a6fc1c595 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala @@ -43,6 +43,9 @@ import org.apache.spark.sql.types.{BinaryType, IntegerType, MetadataBuilder, Str import org.apache.spark.unsafe.UnsafeAlignedOffset import org.apache.spark.util.Utils +import org.apache.comet.tags.ShuffleTestGroup + +@ShuffleTestGroup class CometDiskBlockWriterSuite extends AnyFunSuite { private val schema = StructType(Seq(StructField("a", BinaryType))) diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala index bfab3727cab..80198308b8f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativePositionalRoundRobinSuite.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.functions.{col, lit, rand, udf} import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf +import org.apache.comet.tags.ShuffleTestGroup /** * Positional round robin places rows by their ordinal within the map task rather than by hashing @@ -39,6 +40,7 @@ import org.apache.comet.CometConf * `CometShuffleExchangeExec.usesPositionalRoundRobin` directly rather than inferring the decision * from output. */ +@ShuffleTestGroup class CometNativePositionalRoundRobinSuite extends CometTestBase with AdaptiveSparkPlanHelper { private val numPartitions = 8 diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDDSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDDSuite.scala index 756da7badf2..8222f603a0d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDDSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleInputRDDSuite.scala @@ -28,6 +28,7 @@ import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.tags.ShuffleTestGroup /** * Ensure the serialized shuffle-map-stage task binary does not grow with the number of @@ -43,6 +44,7 @@ import org.apache.comet.serde.OperatorOuterClass.Operator * Lives in the `execution.shuffle` package so it can construct the `private[shuffle]` * [[CometNativeShuffleInputRDD]] and the `private[comet]` [[NativeExecContext]] directly. */ +@ShuffleTestGroup class CometNativeShuffleInputRDDSuite extends CometTestBase { test("native shuffle input preserves its parents' determinism") { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala index 8e72e37384a..6ed184b6487 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala @@ -24,11 +24,13 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.CometConf +import org.apache.comet.tags.ShuffleTestGroup /** * Checks the plan [[CometNativeShuffleWriter]] hands to native code. Lives in the * `execution.shuffle` package so it can call the `private[shuffle]` `buildUnifiedPlan`. */ +@ShuffleTestGroup class CometNativeShuffleWriterSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIteratorSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIteratorSuite.scala index 865d1a34cb8..26531a97649 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIteratorSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIteratorSuite.scala @@ -28,8 +28,10 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.metric.SQLMetrics import org.apache.comet.Native +import org.apache.comet.tags.ShuffleTestGroup import org.apache.comet.vector.NativeUtil +@ShuffleTestGroup class NativeBatchDecoderIteratorSuite extends CometTestBase { import NativeShuffleTestFrames.{intFrame, TrackingInputStream} diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala index 86a4fe7a899..c7b1e7d4d06 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala @@ -39,8 +39,10 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.util.io.ChunkedByteBuffer import org.apache.comet.CometArrowAllocator +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.CometVector +@ExecTestGroup class UtilsSuite extends CometTestBase { test("serializeBatches round trips schema metadata larger than the output chunk estimate") { diff --git a/spark/src/test/spark-3.5/org/apache/comet/CometWidthBucketSuite.scala b/spark/src/test/spark-3.5/org/apache/comet/CometWidthBucketSuite.scala index c2a0034a01b..34b6258acae 100644 --- a/spark/src/test/spark-3.5/org/apache/comet/CometWidthBucketSuite.scala +++ b/spark/src/test/spark-3.5/org/apache/comet/CometWidthBucketSuite.scala @@ -22,6 +22,9 @@ package org.apache.comet import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} +import org.apache.comet.tags.ExpressionsTestGroup + +@ExpressionsTestGroup class CometWidthBucketSuite extends CometTestBase { private def countSparkProjectExec(plan: SparkPlan): Int = diff --git a/spark/src/test/spark-3.5/org/apache/comet/exec/CometMergeRowsSuite.scala b/spark/src/test/spark-3.5/org/apache/comet/exec/CometMergeRowsSuite.scala index a621b5a9f52..986eb8c9462 100644 --- a/spark/src/test/spark-3.5/org/apache/comet/exec/CometMergeRowsSuite.scala +++ b/spark/src/test/spark-3.5/org/apache/comet/exec/CometMergeRowsSuite.scala @@ -19,5 +19,8 @@ package org.apache.comet.exec +import org.apache.comet.tags.ExecTestGroup + /** Spark 3.5.x entry point for the shared native MergeRows contract suite. */ +@ExecTestGroup class CometMergeRowsSuite extends CometMergeRowsNativeSuiteBase diff --git a/spark/src/test/spark-3.5/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala b/spark/src/test/spark-3.5/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala index 6f645747394..6e5b976b42c 100644 --- a/spark/src/test/spark-3.5/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala +++ b/spark/src/test/spark-3.5/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala @@ -22,6 +22,9 @@ package org.apache.spark.sql import org.apache.spark.sql.catalyst.expressions.Empty2Null import org.apache.spark.sql.functions.col +import org.apache.comet.tags.ExecTestGroup + +@ExecTestGroup class CometSparkInternalFunctionsSuite extends CometTestBase { test("empty2null is offloaded to Comet") { diff --git a/spark/src/test/spark-3.5/org/apache/spark/sql/CometToPrettyStringSuite.scala b/spark/src/test/spark-3.5/org/apache/spark/sql/CometToPrettyStringSuite.scala index beb37211029..28c7094d236 100644 --- a/spark/src/test/spark-3.5/org/apache/spark/sql/CometToPrettyStringSuite.scala +++ b/spark/src/test/spark-3.5/org/apache/spark/sql/CometToPrettyStringSuite.scala @@ -30,7 +30,9 @@ import org.apache.comet.CometConf import org.apache.comet.CometFuzzTestBase import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.serde.Compatible +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometToPrettyStringSuite extends CometFuzzTestBase { test("ToPrettyString") { diff --git a/spark/src/test/spark-3.x/org/apache/comet/CometStringDecodeSuite.scala b/spark/src/test/spark-3.x/org/apache/comet/CometStringDecodeSuite.scala index 40c238e6566..d4d70d8f0c9 100644 --- a/spark/src/test/spark-3.x/org/apache/comet/CometStringDecodeSuite.scala +++ b/spark/src/test/spark-3.x/org/apache/comet/CometStringDecodeSuite.scala @@ -23,7 +23,9 @@ import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.tags.ExpressionsTestGroup +@ExpressionsTestGroup class CometStringDecodeSuite extends CometTestBase { private def countSparkProjectExec(plan: SparkPlan): Int = diff --git a/spark/src/test/spark-4.0/org/apache/comet/exec/CometMergeRowsSuite.scala b/spark/src/test/spark-4.0/org/apache/comet/exec/CometMergeRowsSuite.scala index f0da046a99d..2aafd71e107 100644 --- a/spark/src/test/spark-4.0/org/apache/comet/exec/CometMergeRowsSuite.scala +++ b/spark/src/test/spark-4.0/org/apache/comet/exec/CometMergeRowsSuite.scala @@ -19,5 +19,8 @@ package org.apache.comet.exec +import org.apache.comet.tags.ExecTestGroup + /** Spark 4.0 entry point for the shared native MergeRows contract suite. */ +@ExecTestGroup class CometMergeRowsSuite extends CometMergeRowsNativeSuiteBase diff --git a/spark/src/test/spark-4.0/org/apache/spark/sql/CometCollationSuite.scala b/spark/src/test/spark-4.0/org/apache/spark/sql/CometCollationSuite.scala index 8502eb44c0b..b69990c4333 100644 --- a/spark/src/test/spark-4.0/org/apache/spark/sql/CometCollationSuite.scala +++ b/spark/src/test/spark-4.0/org/apache/spark/sql/CometCollationSuite.scala @@ -30,7 +30,9 @@ import org.apache.spark.sql.types.StringType import org.apache.comet.{CometConf, CometExplainInfo, ExtendedExplainInfo} import org.apache.comet.serde.OperatorOuterClass +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometCollationSuite extends CometTestBase { // Queries that group, sort, or shuffle on a non-default collated string must fall back to diff --git a/spark/src/test/spark-4.1+/org/apache/comet/exec/CometMergeRowsSuite.scala b/spark/src/test/spark-4.1+/org/apache/comet/exec/CometMergeRowsSuite.scala index 49d2f2d4c10..d57585a9a67 100644 --- a/spark/src/test/spark-4.1+/org/apache/comet/exec/CometMergeRowsSuite.scala +++ b/spark/src/test/spark-4.1+/org/apache/comet/exec/CometMergeRowsSuite.scala @@ -35,11 +35,13 @@ import org.apache.spark.sql.util.QueryExecutionListener import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus import org.apache.comet.shims.{MergeRowsMetricsShim, ShimCometMergeRows} +import org.apache.comet.tags.ExecTestGroup /** * Spark 4.1+ stock V2 writers build MergeSummary by locating the concrete Spark MergeRowsExec. * Native MergeRows is therefore kept on Spark when the enclosing writer remains on Spark. */ +@ExecTestGroup class CometMergeRowsSuite extends CometTestBase with AdaptiveSparkPlanHelper { private val catalog = "generic_rowlevel" diff --git a/spark/src/test/spark-4.1+/org/apache/spark/sql/CometCollationSuite.scala b/spark/src/test/spark-4.1+/org/apache/spark/sql/CometCollationSuite.scala index 9ebd3002905..203577ffabf 100644 --- a/spark/src/test/spark-4.1+/org/apache/spark/sql/CometCollationSuite.scala +++ b/spark/src/test/spark-4.1+/org/apache/spark/sql/CometCollationSuite.scala @@ -20,7 +20,9 @@ package org.apache.spark.sql import org.apache.comet.{CometConf, ExtendedExplainInfo} +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometCollationSuite extends CometTestBase { // Queries that group, sort, or shuffle on a non-default collated string must fall back to diff --git a/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala b/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala index 1e742873a5a..412c9abbe15 100644 --- a/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala +++ b/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometArrowPythonUdfSuite.scala @@ -36,7 +36,9 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, import org.apache.comet.{CometConf, CometExplainInfo, NativeBase, SparkErrorConverter} import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus import org.apache.comet.exceptions.CometQueryExecutionException +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometArrowPythonUdfSuite extends CometTestBase { private def python: String = sys.env.getOrElse("PYSPARK_PYTHON", "python3") diff --git a/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometDecimalArithmeticViewSuite.scala b/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometDecimalArithmeticViewSuite.scala index 3e68ffb80b0..a869807a22c 100644 --- a/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometDecimalArithmeticViewSuite.scala +++ b/spark/src/test/spark-4.1+/org/apache/spark/sql/comet/CometDecimalArithmeticViewSuite.scala @@ -25,7 +25,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.DecimalType import org.apache.comet.serde.{CometScalarFunction, ExprOuterClass, QueryPlanSerde} +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometDecimalArithmeticViewSuite extends CometTestBase { // Spark 4.1+ (SPARK-53968) stores `spark.sql.decimalOperations.allowPrecisionLoss` per diff --git a/spark/src/test/spark-4.1/org/apache/comet/CometIcebergEncryptionSuite.scala b/spark/src/test/spark-4.1/org/apache/comet/CometIcebergEncryptionSuite.scala index 1b221bf454d..662b9e04bd5 100644 --- a/spark/src/test/spark-4.1/org/apache/comet/CometIcebergEncryptionSuite.scala +++ b/spark/src/test/spark-4.1/org/apache/comet/CometIcebergEncryptionSuite.scala @@ -32,12 +32,14 @@ import org.apache.spark.sql.comet.CometIcebergNativeScanExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.comet.iceberg.CometTestKMS +import org.apache.comet.tags.ScansTestGroup /** * Encryption-specific Iceberg tests. Lives in the spark-4.1 source set because Iceberg table * encryption is a V3 feature and the encryption APIs (KeyManagementClient, StandardEncryption * manager) are only public from Iceberg 1.11, which Comet pairs with Spark 4.1. */ +@ScansTestGroup class CometIcebergEncryptionSuite extends CometTestBase with CometIcebergTestBase diff --git a/spark/src/test/spark-4.2/org/apache/comet/exec/CometInMemoryCacheLastAttemptMetricSuite.scala b/spark/src/test/spark-4.2/org/apache/comet/exec/CometInMemoryCacheLastAttemptMetricSuite.scala index 3f30e44aa06..1b751de08b1 100644 --- a/spark/src/test/spark-4.2/org/apache/comet/exec/CometInMemoryCacheLastAttemptMetricSuite.scala +++ b/spark/src/test/spark-4.2/org/apache/comet/exec/CometInMemoryCacheLastAttemptMetricSuite.scala @@ -27,8 +27,10 @@ import org.apache.spark.sql.execution.columnar.CometInMemoryRelationHelper import org.apache.spark.sql.execution.metric.SQLLastAttemptMetrics import org.apache.comet.CometConf +import org.apache.comet.tags.ExecTestGroup /** Spark 4.2's last-attempt metrics over a relation cached in Comet's format. */ +@ExecTestGroup class CometInMemoryCacheLastAttemptMetricSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/spark-4.x/org/apache/comet/CometCastCollatedStringSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/CometCastCollatedStringSuite.scala index f547836a7b7..0b140b3e47d 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/CometCastCollatedStringSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/CometCastCollatedStringSuite.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, IntegerType, import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.serde.{Compatible, Unsupported} +import org.apache.comet.tags.ExpressionsTestGroup /** * https://github.com/apache/datafusion-comet/issues/4489 @@ -62,6 +63,7 @@ import org.apache.comet.serde.{Compatible, Unsupported} * This lives under `spark-4.x`, shared by every 4.x profile, rather than `spark-4.1+`, because * collation is a Spark 4.0 feature and `StringType(collationName)` already resolves there. */ +@ExpressionsTestGroup class CometCastCollatedStringSuite extends CometTestBase { private val lcase = StringType("UTF8_LCASE") diff --git a/spark/src/test/spark-4.x/org/apache/comet/CometWidthBucketSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/CometWidthBucketSuite.scala index c2a0034a01b..34b6258acae 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/CometWidthBucketSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/CometWidthBucketSuite.scala @@ -22,6 +22,9 @@ package org.apache.comet import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.{ProjectExec, SparkPlan} +import org.apache.comet.tags.ExpressionsTestGroup + +@ExpressionsTestGroup class CometWidthBucketSuite extends CometTestBase { private def countSparkProjectExec(plan: SparkPlan): Int = diff --git a/spark/src/test/spark-4.x/org/apache/comet/exec/CometEmptyRelationExecSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/exec/CometEmptyRelationExecSuite.scala index 76b378e4181..359ef10a8a8 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/exec/CometEmptyRelationExecSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/exec/CometEmptyRelationExecSuite.scala @@ -28,7 +28,9 @@ import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometConf +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometEmptyRelationExecSuite extends CometTestBase { // CometTestBase enables the Spark-to-Arrow conversions; use their production defaults here. diff --git a/spark/src/test/spark-4.x/org/apache/comet/exec/CometShuffle4_0Suite.scala b/spark/src/test/spark-4.x/org/apache/comet/exec/CometShuffle4_0Suite.scala index 8629d1a78c1..46c268cb5ba 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/exec/CometShuffle4_0Suite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/exec/CometShuffle4_0Suite.scala @@ -28,6 +28,9 @@ import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{FloatType, LongType, StringType, TimestampType} +import org.apache.comet.tags.ShuffleTestGroup + +@ShuffleTestGroup class CometShuffle4_0Suite extends CometColumnarShuffleSuite { protected val adaptiveExecutionEnabled: Boolean = true diff --git a/spark/src/test/spark-4.x/org/apache/comet/parquet/CometEmptyRelationParquetWriterSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/parquet/CometEmptyRelationParquetWriterSuite.scala index c739e420315..c61ceda7649 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/parquet/CometEmptyRelationParquetWriterSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/parquet/CometEmptyRelationParquetWriterSuite.scala @@ -27,7 +27,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.comet.CometConf +import org.apache.comet.tags.ScansTestGroup +@ScansTestGroup class CometEmptyRelationParquetWriterSuite extends CometParquetWriterTestBase { test("EmptyRelationExec keeps AQE empty parquet output readable") { diff --git a/spark/src/test/spark-4.x/org/apache/comet/rules/CometEmptyRelationExecRuleSuite.scala b/spark/src/test/spark-4.x/org/apache/comet/rules/CometEmptyRelationExecRuleSuite.scala index 77811d9e4f4..e30e6770a51 100644 --- a/spark/src/test/spark-4.x/org/apache/comet/rules/CometEmptyRelationExecRuleSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/comet/rules/CometEmptyRelationExecRuleSuite.scala @@ -34,7 +34,9 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, BooleanType, DecimalType, IntegerType, StructType} import org.apache.comet.{CometConf, CometCoverageStats, CometExplainInfo, ExtendedExplainInfo} +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometEmptyRelationExecRuleSuite extends CometTestBase { import testImplicits._ diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/CometSortCollationSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/CometSortCollationSuite.scala index 6916ae72b3c..d521db80f06 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/CometSortCollationSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/CometSortCollationSuite.scala @@ -28,7 +28,9 @@ import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, CometExplainInfo} import org.apache.comet.serde.QueryPlanSerde +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometSortCollationSuite extends CometTestBase { private val sortCollationReason = "Sort does not support non-default string collation" diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala index b8b107f2e8d..5c522e3b5ca 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/CometSparkInternalFunctionsSuite.scala @@ -23,6 +23,9 @@ import org.apache.spark.sql.catalyst.expressions.Empty2Null import org.apache.spark.sql.classic.ExpressionUtils import org.apache.spark.sql.functions._ +import org.apache.comet.tags.ExecTestGroup + +@ExecTestGroup class CometSparkInternalFunctionsSuite extends CometTestBase { test("empty2null is offloaded to Comet") { diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/CometToPrettyStringSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/CometToPrettyStringSuite.scala index 80063d08e54..f86cc680458 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/CometToPrettyStringSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/CometToPrettyStringSuite.scala @@ -33,7 +33,9 @@ import org.apache.comet.CometConf import org.apache.comet.CometFuzzTestBase import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.serde.Compatible +import org.apache.comet.tags.ExecTestGroup +@ExecTestGroup class CometToPrettyStringSuite extends CometFuzzTestBase { test("ToPrettyString") { diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/CometVariantShreddingSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/CometVariantShreddingSuite.scala index c5359c3512c..6ed56ca72bd 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/CometVariantShreddingSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/CometVariantShreddingSuite.scala @@ -27,8 +27,10 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.TestSparkSession import org.apache.comet.{CometConf, CometSparkSessionExtensions} +import org.apache.comet.tags.ScansTestGroup /** Run Spark's unchanged reconstruction assertions in Comet's regular Spark 4 CI jobs. */ +@ScansTestGroup class CometVariantShreddingSuite extends VariantShreddingSuite { override protected def sparkConf: SparkConf = super.sparkConf .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "false") diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/comet/CometMapInBatchSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/CometMapInBatchSuite.scala index 2a3a0f9920e..c467f6c3144 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/comet/CometMapInBatchSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/CometMapInBatchSuite.scala @@ -32,6 +32,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.{CometConf, ExtendedExplainInfo} import org.apache.comet.rules.EliminateRedundantTransitions +import org.apache.comet.tags.ExecTestGroup /** Minimal CometPlan leaf used to anchor the rule's transform without triggering execution. */ private case class StubCometLeaf(override val output: Seq[Attribute]) @@ -53,6 +54,7 @@ private case class StubCometLeaf(override val output: Seq[Attribute]) * `PythonFunction` / `PythonAccumulatorV2` / `PythonBroadcast` classes when fabricating a stub * `PythonUDF` for `MapInArrowExec` to wrap. */ +@ExecTestGroup class CometMapInBatchSuite extends CometTestBase { private def stubPythonUDF: PythonUDF = { diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerSuite.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerSuite.scala index cda8fb3834a..8cb569fce41 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerSuite.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/execution/python/CometArrowPythonRunnerSuite.scala @@ -52,8 +52,10 @@ import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.util.DirectByteBufferOutputStream import org.apache.comet.CometArrowAllocator +import org.apache.comet.tags.ExecTestGroup import org.apache.comet.vector.{CometDecodedVector, CometDictionary, CometDictionaryVector, CometPlainVector, CometStructVector} +@ExecTestGroup class CometArrowPythonRunnerSuite extends AnyFunSuite with Matchers { /**