From 903acc7f9955b7cadad137efc37b4cc0feb91f21 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 20 Nov 2026 09:05:10 -0700 Subject: [PATCH 1/6] ci: run the PySpark runner on Spark 3.5 as well as 4.2 The PySpark runner job now runs once for each supported Spark version, 3.5.9 and 4.2.0. To run on Spark 3.5, the runner: - accepts PySpark 3.5 and 4.2 - rewrites `expr::type` to `CAST(expr AS type)`, because Spark 3.5 has no `::` syntax - skips interval results that PySpark 3.5 cannot read, as it already does on 4.2 - works without `pyspark.logger`, which PySpark 3.5 does not have An `# xfail spark` comment can now take a version condition, such as `# xfail spark<4.0: `, and a record can have several. The new markers cover the records where Spark 3.5 differs from Spark 4: encode charsets, greatest's wider DECIMAL type and to_json's offset seconds. abs's DECIMAL markers now apply only to Spark 4. --- .github/workflows/pyspark-runner.yml | 16 +- crates/sqllogictest/scripts/README.md | 41 ++- .../scripts/pyspark_slt_runner.py | 291 +++++++++++++++--- .../scripts/requirements-spark-3.5.txt | 5 + .../scripts/test_pyspark_slt_runner.py | 65 +++- .../sqllogictest/test_files/json/to_json.slt | 1 + crates/sqllogictest/test_files/math/abs.slt | 4 +- .../sqllogictest/test_files/math/greatest.slt | 2 + .../sqllogictest/test_files/string/encode.slt | 7 + 9 files changed, 380 insertions(+), 52 deletions(-) create mode 100644 crates/sqllogictest/scripts/requirements-spark-3.5.txt diff --git a/.github/workflows/pyspark-runner.yml b/.github/workflows/pyspark-runner.yml index 6733dbba..5c65621c 100644 --- a/.github/workflows/pyspark-runner.yml +++ b/.github/workflows/pyspark-runner.yml @@ -32,8 +32,16 @@ jobs: - run: python3 -m unittest discover -s crates/sqllogictest/scripts spark: - name: Run the sqllogictests on Spark + name: Run the sqllogictests on Spark ${{ matrix.spark }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - spark: "3.5" + requirements: requirements-spark-3.5.txt + - spark: "4.2" + requirements: requirements.txt steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 @@ -44,8 +52,8 @@ jobs: with: python-version: "3.12" cache: pip - cache-dependency-path: crates/sqllogictest/scripts/requirements.txt - - run: pip install -r crates/sqllogictest/scripts/requirements.txt + cache-dependency-path: crates/sqllogictest/scripts/${{ matrix.requirements }} + - run: pip install -r crates/sqllogictest/scripts/${{ matrix.requirements }} # Exits with status 1 if any record differs from Spark without an - # `# xfail spark:` comment, or has one but now passes. + # `# xfail spark:` comment for this Spark version, or has one but now passes. - run: python crates/sqllogictest/scripts/pyspark_slt_runner.py diff --git a/crates/sqllogictest/scripts/README.md b/crates/sqllogictest/scripts/README.md index fcbe30f6..bf2c7e4a 100644 --- a/crates/sqllogictest/scripts/README.md +++ b/crates/sqllogictest/scripts/README.md @@ -8,14 +8,15 @@ SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one `pyspark_slt_runner.py` runs the `.slt` files in [`crates/sqllogictest/test_files`](../test_files) -on Apache Spark 4.2 and reports every query where Spark's result differs from -the expected result in the file. +on Apache Spark and reports every query where Spark's result differs from the +expected result in the file. It supports Spark 3.5 and 4.2. The expected results in those files come from DataFusion, and nothing else checks them against Spark. A difference means that DataFusion and Spark disagree, or that the test expects the wrong value. CI runs the runner on -every pull request that changes an `.slt` file or the runner, and fails if it -reports a difference that has no `# xfail spark:` comment (see +every pull request that changes an `.slt` file or the runner, once for each +supported Spark version, and fails if it reports a difference that has no +`# xfail spark:` comment for that version (see [Known differences](#known-differences)) or a comment on a record that now passes. @@ -29,8 +30,16 @@ python3 -m venv /tmp/pyspark-4.2 /tmp/pyspark-4.2/bin/pip install -r crates/sqllogictest/scripts/requirements.txt ``` -The runner ignores `SPARK_HOME`, so it always uses the Spark bundled with this -PySpark. +For Spark 3.5, install `requirements-spark-3.5.txt` into a second virtual +environment, and run the runner with that environment's Python: + +```shell +python3 -m venv /tmp/pyspark-3.5 +/tmp/pyspark-3.5/bin/pip install -r crates/sqllogictest/scripts/requirements-spark-3.5.txt +``` + +The runner ignores `SPARK_HOME`, so it always uses the Spark bundled with the +installed PySpark, and it checks that this is a supported version. ## Usage @@ -78,6 +87,21 @@ SELECT round(arrow_cast(2.5, 'Decimal32(9, 1)'), 0::int); 3.0 ``` +A comparison after `spark` limits the comment to some Spark versions, which it +compares by major and minor version. Several comparisons separated by commas +must all hold: + +```text +# xfail spark<4.0: intended: Spark 3.5 accepts any Java charset name, which Spark 4 allows only with spark.sql.legacy.javaCharsets +statement error INVALID_PARAMETER_VALUE\.CHARSET +SELECT encode('A'::string, 'utf-32be'::string); +``` + +The operators are `<`, `<=`, `>`, `>=` and `==`, as in `spark>=4.0, <4.2`. A +record can have several comments, one line each, and the first that matches +the Spark version applies. The runner stops with an error if it cannot read a +version condition. + The reason links an issue, or says what differs and why: a bug in datafusion-spark, a difference that comes from how the test is written, or a limitation of this runner. A marked record that differs from Spark in any way @@ -116,6 +140,11 @@ are DataFusion's defaults. These settings in the files are passed on to Spark: Spark 4.2 parses most of the SQL as written, including `::` casts. The runner makes only these changes: +- For Spark 3.5, which has no `::` syntax, `expr::type` becomes + `CAST(expr AS type)`. As in Spark 4, `::` applies to the literal, column, + call, subscript or parenthesized expression before it, and a minus sign + belongs to a number, so `-1::int` becomes `CAST(-1 AS int)`. + - `CREATE TABLE` becomes `CREATE TEMPORARY VIEW`, and `DROP TABLE` becomes `DROP VIEW` - PostgreSQL type names that Spark does not accept are renamed, for example diff --git a/crates/sqllogictest/scripts/pyspark_slt_runner.py b/crates/sqllogictest/scripts/pyspark_slt_runner.py index fbdd4e1f..fd480388 100755 --- a/crates/sqllogictest/scripts/pyspark_slt_runner.py +++ b/crates/sqllogictest/scripts/pyspark_slt_runner.py @@ -3,7 +3,7 @@ # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -"""Run the datafusion-spark sqllogictest files against Apache Spark 4.2. +"""Run the datafusion-spark sqllogictest files against Apache Spark 3.5 or 4.2. Each record in crates/sqllogictest/test_files/**/*.slt is run on a local PySpark session, and Spark's result is compared with the expected result @@ -11,7 +11,7 @@ reported here is a query where DataFusion and Spark disagree, or where the test expects the wrong value. -Running Spark is manual; only the unit tests run in CI. See README.md for setup. +CI runs this on every supported Spark version. See README.md for setup. """ from __future__ import annotations @@ -19,6 +19,7 @@ import argparse import datetime import math +import operator import os import re import struct @@ -32,7 +33,8 @@ from pathlib import Path from typing import Optional -SPARK_VERSION = "4.2" +# The Spark versions this runner supports, as major.minor. CI runs every one. +SUPPORTED_VERSIONS = ("3.5", "4.2") SCRIPT_DIR = Path(__file__).resolve().parent REPO_ROOT = SCRIPT_DIR.parents[2] @@ -44,10 +46,23 @@ # A comment that marks the next record as a known difference from Spark: # # # xfail spark: +# # xfail spark<4.0: +# +# The second form applies only to the Spark versions that match every +# comma-separated condition, such as `>=4.0, <4.2`. A record can have several +# markers, and the first that matches the Spark version applies. # # The Rust sqllogictest parser rejects directives it does not know, so the # marker is a comment, which DataFusion's runner ignores. -_XFAIL_RE = re.compile(rf"#\s*xfail\s+{LABEL}\s*:\s*(.*\S)\s*$") +_XFAIL_RE = re.compile(rf"#\s*xfail\s+{LABEL}\b([^:]*):\s*(.*\S)\s*$") +_VERSION_CONDITION_RE = re.compile(r"\s*(<=|>=|==|<|>)\s*(\d+)\.(\d+)\s*") +_COMPARISONS = { + "<": operator.lt, + "<=": operator.le, + ">": operator.gt, + ">=": operator.ge, + "==": operator.eq, +} # DataFusion runs these files with ANSI mode off and no time zone (UTC), so # every file starts from the same Spark settings. @@ -117,26 +132,75 @@ class Record: sort_mode: str = "nosort" expected: list[str] = field(default_factory=list) skip_reason: Optional[str] = None - # Why Spark is expected to disagree, from an `# xfail spark:` comment. - xfail_reason: Optional[str] = None + # Why Spark is expected to disagree, from `# xfail spark:` comments. + xfails: list[Xfail] = field(default_factory=list) @property def location(self) -> str: return f"{_display_path(self.path)}:{self.line}" + def xfail(self, spark_version: str) -> Optional[Xfail]: + """The first `# xfail spark` marker that applies to this Spark version.""" + version = _version_tuple(spark_version) + return next((x for x in self.xfails if x.applies_to(version)), None) + + +@dataclass +class Xfail: + """An `# xfail spark: ` comment.""" + + # (operator, (major, minor)) pairs that the Spark version must all match. + conditions: list[tuple[str, tuple[int, int]]] + reason: str + # The comment as written, for the report that asks to remove it. + text: str + + def applies_to(self, version: tuple[int, int]) -> bool: + return all(_COMPARISONS[op](version, v) for op, v in self.conditions) + + +def _version_tuple(version: str) -> tuple[int, int]: + """(major, minor) of a version such as `4.2.0`.""" + major, minor = spark_minor_version(version).split(".") + return int(major), int(minor) + + +def parse_xfail(line: str) -> Optional[Xfail]: + """Parse an `# xfail spark` comment, or return None for any other line. + + Raises ValueError for a version condition this runner cannot read.""" + m = _XFAIL_RE.match(line.strip()) + if not m: + return None + spec, reason = m.group(1).strip(), m.group(2) + conditions = [] + if spec: + for part in spec.split(","): + c = _VERSION_CONDITION_RE.fullmatch(part) + if not c: + raise ValueError( + f"cannot read the Spark version condition `{spec}`. Write a " + "comparison such as `<4.0` or `==3.5`, or several separated by commas" + ) + conditions.append((c.group(1), (int(c.group(2)), int(c.group(3))))) + return Xfail(conditions, reason, line.strip()) + def parse_slt(path: Path) -> list[Record]: """Parse the `statement` and `query` records of an .slt file.""" lines = path.read_text().split("\n") records: list[Record] = [] conditions: list[tuple[str, str]] = [] - xfail_reason: Optional[str] = None + xfails: list[Xfail] = [] i = 0 while i < len(lines): words = lines[i].split() - m = _XFAIL_RE.match(lines[i].strip()) - if m: - xfail_reason = m.group(1) + try: + xfail = parse_xfail(lines[i]) + except ValueError as e: + raise SystemExit(f"error: {_display_path(path)}:{i + 1}: {e}") + if xfail: + xfails.append(xfail) i += 1 continue if not words or words[0].startswith("#"): @@ -149,12 +213,12 @@ def parse_slt(path: Path) -> list[Record]: if words[0] not in ("statement", "query"): # Directives such as `include` and `halt`, which these files do not use. conditions = [] - xfail_reason = None + xfails = [] i += 1 continue - record = Record(path=path, line=i + 1, kind=words[0], xfail_reason=xfail_reason) - xfail_reason = None + record = Record(path=path, line=i + 1, kind=words[0], xfails=xfails) + xfails = [] args = words[1:] if args and args[0] == "error": record.expect_error = True @@ -194,10 +258,11 @@ def parse_slt(path: Path) -> list[Record]: # --------------------------------------------------------------------------- # Translating DataFusion SQL to Spark SQL # -# Spark 4.2 parses most of these files as written, including `::` casts. -# Only a few renames are applied here, and `arrow_cast` and `spark_cast` -# become a CAST (see below). Anything else Spark cannot parse, and any function -# Spark does not have (such as `arrow_typeof`), is skipped. +# Spark 4.2 parses most of these files as written, including `::` casts, which +# become a CAST for Spark 3.5 (see rewrite_double_colons). Only a few renames +# are applied here, and `arrow_cast` and `spark_cast` become a CAST (see +# below). Anything else Spark cannot parse, and any function Spark does not +# have (such as `arrow_typeof`), is skipped. # --------------------------------------------------------------------------- _TOKEN_RE = re.compile( @@ -743,8 +808,13 @@ def rewrite_casts(sql: str, time_zone: str = "UTC") -> str: return "".join(out) -def translate(sql: str, time_zone: str = "UTC") -> str: - """Translate the parts of DataFusion's SQL that Spark spells differently.""" +def translate(sql: str, time_zone: str = "UTC", double_colon: bool = True) -> str: + """Translate the parts of DataFusion's SQL that Spark spells differently. + + With `double_colon` False, for Spark 3.5, `expr::type` becomes a CAST. + """ + if not double_colon: + return rewrite_double_colons(translate(sql, time_zone)) tokens = _tokenize(rewrite_casts(sql, time_zone)) out = [] # The function name before each open parenthesis, to find CAST(x AS type). @@ -788,6 +858,132 @@ def translate(sql: str, time_zone: str = "UTC") -> str: return "".join(out) +# Keywords after which a `-` is a sign, not a subtraction, and a `(` does +# not start the arguments of a call. +_KEYWORDS = { + "ALL", "AND", "ANY", "AS", "BETWEEN", "BY", "CASE", "DISTINCT", "ELSE", + "EXISTS", "FROM", "HAVING", "IN", "IS", "JOIN", "LIKE", "LIMIT", "NOT", + "ON", "OR", "RETURN", "SELECT", "THEN", "VALUES", "WHEN", "WHERE", +} + + +def rewrite_double_colons(sql: str) -> str: + """Replace each `expr::type` with `CAST(expr AS type)`. + + Spark 3.5 does not parse `::`. As in Spark 4, `::` applies to the primary + expression before it: a literal, a column, a call, a parenthesized + expression or a subscript. A sign belongs to a number literal, so + `-1::int` is `CAST(-1 AS int)`, but `-(1)::int` is `-CAST((1) AS int)`. + """ + tokens = _tokenize(sql) + out: list[tuple[str, str]] = [] + i = 0 + while i < len(tokens): + if tokens[i][0] != "cast": + out.append(tokens[i]) + i += 1 + continue + start = _operand_start(out) + operand = "".join(t for _, t in out[start:]).strip() + del out[start:] + end = _type_end(tokens, i + 1) + type_sql = "".join(t for _, t in tokens[i + 1 : end]).strip() + if not operand or not type_sql: + raise Skip("a `::` cast this runner cannot translate") + out.append(("operand", f"CAST({operand} AS {type_sql})")) + i = end + return "".join(t for _, t in out) + + +def _significant_before(tokens: list[tuple[str, str]], end: int) -> int: + """The index of the last token before `end` that is not whitespace, or -1.""" + k = end - 1 + while k >= 0 and tokens[k][0] in ("space", "comment"): + k -= 1 + return k + + +def _operand_start(tokens: list[tuple[str, str]]) -> int: + """The index where the primary expression that ends `tokens` starts.""" + k = _significant_before(tokens, len(tokens)) + if k < 0: + raise Skip("a `::` cast this runner cannot translate") + kind, text = tokens[k] + if kind == "other" and text in (")", "]"): + depth = 0 + while k >= 0: + if tokens[k][0] == "other": + depth += _NESTING.get(tokens[k][1], 0) + if depth == 0: + break + k -= 1 + if k < 0: + raise Skip("unbalanced parentheses") + before = _significant_before(tokens, k) + prev = tokens[before] if before >= 0 else ("", "") + if text == ")" and prev[0] == "word" and prev[1].upper() not in _KEYWORDS: + # A call, f(...) + return _name_start(tokens, before) + if text == "]" and (prev[0] in ("word", "quoted", "string") or prev[1] in (")", "]")): + # A subscript, a[...], of the operand that ends before it + return _operand_start(tokens[: before + 1]) + return k + if kind == "string": + before = _significant_before(tokens, k) + if before >= 0 and tokens[before][0] == "word" and tokens[before][1].upper() in ( + "DATE", "TIME", "TIMESTAMP", "TIMESTAMP_NTZ", "TIMESTAMP_LTZ", "INTERVAL", "X", + ): + # A typed literal, such as DATE '2024-01-01'. + return before + return k + if kind == "number": + before = _significant_before(tokens, k) + if before >= 0 and tokens[before][1] == "-": + sign_of = _significant_before(tokens, before) + prev = tokens[sign_of] if sign_of >= 0 else None + if ( + prev is None + or (prev[0] == "other" and prev[1] not in (")", "]")) + or (prev[0] == "word" and prev[1].upper() in _KEYWORDS) + ): + return before + return k + if kind in ("word", "quoted"): + return _name_start(tokens, k) + if kind == "operand": + # A cast that this function already rewrote, as in `x::int::string`. + return k + raise Skip("a `::` cast this runner cannot translate") + + +def _name_start(tokens: list[tuple[str, str]], k: int) -> int: + """The start of a possibly qualified name, such as `t.a`, that ends at tokens[k].""" + while k >= 2 and tokens[k - 1][1] == "." and tokens[k - 2][0] in ("word", "quoted"): + k -= 2 + return k + + +def _type_end(tokens: list[tuple[str, str]], start: int) -> int: + """The index after the type name that starts at tokens[start], such as + `int`, `decimal(10, 2)` or `array>`.""" + k = _next_significant(tokens, start) + if k >= len(tokens) or tokens[k][0] != "word": + raise Skip("a `::` cast this runner cannot translate") + k += 1 + j = _next_significant(tokens, k) + if j < len(tokens) and tokens[j][1] in ("(", "<"): + depth = 0 + while j < len(tokens): + text = tokens[j][1] + if tokens[j][0] == "other": + depth += {"(": 1, "<": 1, ")": -1, ">": -1}.get(text, 0) + if depth == 0: + return j + 1 + j += 1 + raise Skip("unbalanced parentheses") + return k + + _NAME = r"[A-Za-z_][A-Za-z0-9_]*" _CREATE_RE = re.compile( rf"\s*CREATE\s+(OR\s+REPLACE\s+)?(?:TABLE|VIEW)\s+(IF\s+NOT\s+EXISTS\s+)?({_NAME})\s*", @@ -800,14 +996,14 @@ def translate(sql: str, time_zone: str = "UTC") -> str: _AS_RE = re.compile(r"AS\b\s*", re.IGNORECASE) -def translate_statement(sql: str, time_zone: str = "UTC") -> str: +def translate_statement(sql: str, time_zone: str = "UTC", double_colon: bool = True) -> str: """Translate a statement, turning DataFusion tables into Spark temporary views.""" m = _DROP_RE.match(sql) if m: return f"DROP VIEW {'IF EXISTS ' if m.group(1) else ''}{m.group(2)}" m = _CREATE_RE.match(sql) if not m: - return translate(sql, time_zone) + return translate(sql, time_zone, double_colon) if m.group(2): # Spark does not allow IF NOT EXISTS on a temporary view. raise Skip("CREATE TABLE IF NOT EXISTS, which a Spark temporary view cannot express") @@ -824,7 +1020,7 @@ def translate_statement(sql: str, time_zone: str = "UTC") -> str: raise Skip("CREATE TABLE without AS") query = rest[m.end() :].rstrip().rstrip(";") if columns is None: - return translate(f"{create} {name} AS {query}", time_zone) + return translate(f"{create} {name} AS {query}", time_zone, double_colon) # A newline ends a trailing `--` comment, which would hide the `)`. end_comment = "\n" if "--" in query else "" names, casts = [], [] @@ -838,6 +1034,7 @@ def translate_statement(sql: str, time_zone: str = "UTC") -> str: f"{create} {name} AS SELECT {', '.join(casts)} " f"FROM ({query}{end_comment}) AS __slt_values({', '.join(names)})", time_zone, + double_colon, ) @@ -1242,17 +1439,22 @@ def __init__(self): except ImportError: raise SystemExit(f"error: PySpark is not installed. Install it with `{install}`.") - if not pyspark.__version__.startswith(SPARK_VERSION + "."): + if spark_minor_version(pyspark.__version__) not in SUPPORTED_VERSIONS: + supported = ", ".join(SUPPORTED_VERSIONS) raise SystemExit( - f"error: this runner needs PySpark {SPARK_VERSION}, found " + f"error: this runner needs PySpark {supported}, found " f"{pyspark.__version__}. Install it with `{install}`." ) - from pyspark.logger import PySparkLogger - - # PySpark logs every failed query to stderr as JSON. The runner reports - # them itself. - for name in ("SQLQueryContextLogger", "DataFrameQueryContextLogger"): - PySparkLogger.getLogger(name).disabled = True + try: + from pyspark.logger import PySparkLogger + except ImportError: + # PySpark 3.5 has no pyspark.logger, and does not log failed queries as JSON. + pass + else: + # PySpark logs every failed query to stderr as JSON. The runner + # reports them itself. + for name in ("SQLQueryContextLogger", "DataFrameQueryContextLogger"): + PySparkLogger.getLogger(name).disabled = True # Removed when the runner exits. self._warehouse = tempfile.TemporaryDirectory(prefix="pyspark-slt-") self._start() @@ -1277,6 +1479,8 @@ def _start(self) -> None: self.spark = builder.getOrCreate() self.spark.sparkContext.setLogLevel("FATAL") self.version = self.spark.version + # Spark 4.0 added the `expr::type` cast syntax. + self.double_colon = _version_tuple(self.version) >= (4, 0) self.time_zone = SPARK_DEFAULTS["spark.sql.session.timeZone"] # Set when a query runs the JVM out of memory, which leaves it unusable. self.broken = False @@ -1348,6 +1552,9 @@ def run(self, sql: str): if m: raise SparkError("convert", f"Spark returned a {m.group(1)} value") condition = _condition(e) + if condition == "CANNOT_PARSE_DATATYPE" and "interval" in str(e): + # PySpark 3.5 cannot read a schema with a CalendarIntervalType. + raise SparkError("convert", "Spark returned a CalendarIntervalType value") if condition == "UNRESOLVED_ROUTINE": name = _parameter(e, "routineName") or "?" raise SparkError("function", f"Spark has no function {name}") @@ -1472,17 +1679,19 @@ def run_record(runner: SparkRunner, record: Record) -> Outcome: A difference that the comment expects becomes `xfail`, and a record that passes in spite of the comment becomes `xpass`, so a stale comment is - reported. A skipped record stays skipped. + reported. A skipped record stays skipped. Only a comment whose version + condition matches the runner's Spark version applies. """ outcome = _run_record(runner, record) - if record.xfail_reason is None: + xfail = record.xfail(runner.version) + if xfail is None: return outcome if outcome.status in EXPECTED_TO_FAIL: - outcome.detail = f"{DIVERGENT[outcome.status]} ({record.xfail_reason})" + outcome.detail = f"{DIVERGENT[outcome.status]} ({xfail.reason})" outcome.status = "xfail" elif outcome.status == "pass": outcome.status = "xpass" - outcome.detail = record.xfail_reason + outcome.detail = xfail.text return outcome @@ -1506,11 +1715,12 @@ def _run_record(runner: SparkRunner, record: Record) -> Outcome: m = _ENGINE_SPECIFIC_RE.match(record.sql) if m: return Outcome(record, "skip", f"{m.group(1).upper()} output differs between engines") + double_colon = getattr(runner, "double_colon", True) try: if record.kind == "statement": - spark_sql = translate_statement(record.sql, runner.time_zone) + spark_sql = translate_statement(record.sql, runner.time_zone, double_colon) else: - spark_sql = translate(record.sql, runner.time_zone) + spark_sql = translate(record.sql, runner.time_zone, double_colon) except Skip as e: return Outcome(record, "skip", str(e)) if any("" in line for line in record.expected): @@ -1587,7 +1797,7 @@ def report(outcome: Outcome, spark_version: str) -> str: if outcome.status == "xfail": return f"{location}: expected failure: {outcome.detail}" if outcome.status == "xpass": - return f"{location}: {DIVERGENT['xpass']}, remove `# xfail {LABEL}: {outcome.detail}`" + return f"{location}: {DIVERGENT['xpass']}, remove `{outcome.detail}`" parts = [f"{location}: {DIVERGENT[outcome.status]}", _indent(record.sql.split("\n"))] if outcome.spark_sql and outcome.spark_sql != record.sql: parts += [" Spark SQL:", _indent(outcome.spark_sql.split("\n"))] @@ -1601,6 +1811,11 @@ def report(outcome: Outcome, spark_version: str) -> str: return "\n".join(parts) +def spark_minor_version(version: str) -> str: + """The major.minor part of a version such as `4.2.0` or `4.2.0.dev1`.""" + return ".".join(version.split(".")[:2]) + + def discover(paths: list[str]) -> list[Path]: files = [] for name in paths: @@ -1617,7 +1832,7 @@ def discover(paths: list[str]) -> list[Path]: def main(argv: Optional[list[str]] = None) -> int: parser = argparse.ArgumentParser( description=( - f"Run the datafusion-spark .slt files on Spark {SPARK_VERSION} and " + f"Run the datafusion-spark .slt files on Spark ({', '.join(SUPPORTED_VERSIONS)}) and " "report where Spark disagrees with the expected results." ) ) diff --git a/crates/sqllogictest/scripts/requirements-spark-3.5.txt b/crates/sqllogictest/scripts/requirements-spark-3.5.txt new file mode 100644 index 00000000..3e437b01 --- /dev/null +++ b/crates/sqllogictest/scripts/requirements-spark-3.5.txt @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +pyspark==3.5.9 diff --git a/crates/sqllogictest/scripts/test_pyspark_slt_runner.py b/crates/sqllogictest/scripts/test_pyspark_slt_runner.py index feaa748d..db71ab19 100644 --- a/crates/sqllogictest/scripts/test_pyspark_slt_runner.py +++ b/crates/sqllogictest/scripts/test_pyspark_slt_runner.py @@ -95,8 +95,33 @@ def test_xfail(self): "query I\n" "SELECT 4\n" ) - reasons = [record.xfail_reason for record in records] - self.assertEqual(reasons, ["Spark returns NULL", None, "no space after the hash", None]) + reasons = [[x.reason for x in record.xfails] for record in records] + self.assertEqual(reasons, [["Spark returns NULL"], [], ["no space after the hash"], []]) + + def test_xfail_spark_versions(self): + [record] = parse( + "# xfail spark<4.0: Spark 3.5 differs\n" + "# xfail spark >= 4.0, <4.2: Spark 4.0 and 4.1 differ\n" + "# xfail spark==4.2: Spark 4.2 differs\n" + "query I\n" + "SELECT 1\n" + ) + reason = lambda version: getattr(record.xfail(version), "reason", None) # noqa: E731 + self.assertEqual(reason("3.5.9"), "Spark 3.5 differs") + self.assertEqual(reason("4.0.1"), "Spark 4.0 and 4.1 differ") + self.assertEqual(reason("4.1.0"), "Spark 4.0 and 4.1 differ") + self.assertEqual(reason("4.2.0"), "Spark 4.2 differs") + self.assertIsNone(reason("4.3.0")) + + def test_first_matching_xfail_applies(self): + [record] = parse("# xfail spark<4.0: old\n# xfail spark: any\nquery I\nSELECT 1\n") + self.assertEqual(record.xfail("3.5.9").reason, "old") + self.assertEqual(record.xfail("4.2.0").reason, "any") + + def test_xfail_bad_version_condition(self): + for marker in ("# xfail spark 3.5: x", "# xfail spark<4: x", "# xfail spark<4.0;>3.4: x"): + with self.assertRaisesRegex(SystemExit, "version condition"): + parse(f"{marker}\nquery I\nSELECT 1\n") class TranslateTest(unittest.TestCase): @@ -104,6 +129,29 @@ def test_spark_parses_double_colon_casts(self): sql = "SELECT abs(-128::TINYINT), 'a'::STRING" self.assertEqual(r.translate(sql), sql) + def test_double_colon_casts_for_spark_3(self): + cases = { + "SELECT 1::int": "SELECT CAST(1 AS int)", + # A sign belongs to a number literal, as in Spark 4. + "SELECT -128::tinyint, 1 -2::int": "SELECT CAST(-128 AS tinyint), 1 -CAST(2 AS int)", + "SELECT -(1)::double": "SELECT -CAST((1) AS double)", + "SELECT f(a, b)::string FROM t": "SELECT CAST(f(a, b) AS string) FROM t", + "SELECT t.a::bigint": "SELECT CAST(t.a AS bigint)", + "SELECT x::int::string": "SELECT CAST(CAST(x AS int) AS string)", + "SELECT DATE '2020-01-01'::string": "SELECT CAST(DATE '2020-01-01' AS string)", + "SELECT 1.5::decimal(10, 2) + 2": "SELECT CAST(1.5 AS decimal(10, 2)) + 2", + "SELECT a[1]::int, (1+2)::int": "SELECT CAST(a[1] AS int), CAST((1+2) AS int)", + "SELECT NULL::array>": "SELECT CAST(NULL AS array>)", + "SELECT (SELECT 1)::int": "SELECT CAST((SELECT 1) AS int)", + "SELECT 'a::b'::FLOAT8": "SELECT CAST('a::b' AS DOUBLE)", + } + for sql, expected in cases.items(): + self.assertEqual(r.translate(sql, double_colon=False), expected, sql) + self.assertEqual( + r.translate_statement("CREATE TABLE t AS SELECT 1::int AS a", double_colon=False), + "CREATE TEMPORARY VIEW t AS SELECT CAST(1 AS int) AS a", + ) + def test_type_aliases(self): self.assertEqual( r.translate("SELECT 1::FLOAT8, CAST(x AS text), TRY_CAST(y AS Bytea)"), @@ -435,6 +483,7 @@ class FakeRunner: """Stands in for SparkRunner: returns a fixed result or raises a SparkError.""" time_zone = "UTC" + version = "4.2.0" def __init__(self, result=None, set_error=None): self.result = result @@ -543,6 +592,18 @@ def test_xpass(self): self.assertEqual(outcome.status, "xpass") self.assertIn("remove `# xfail spark: a fixed bug`", r.report(outcome, "4.2.0")) + def test_xfail_for_another_spark_version(self): + result = (FakeSchema("IntegerType"), [(1,)]) + differs = "# xfail spark<4.0: only Spark 3.5\nquery I\nSELECT a\n----\n2\n" + self.assertEqual(self.outcome(differs, result).status, "mismatch") + passes = "# xfail spark<4.0: only Spark 3.5\nquery I\nSELECT a\n----\n1\n" + self.assertEqual(self.outcome(passes, result).status, "pass") + + def test_xpass_names_the_versioned_marker(self): + result = (FakeSchema("IntegerType"), [(1,)]) + outcome = self.outcome("# xfail spark>=4.0: fixed\nquery I\nSELECT a\n----\n1\n", result) + self.assertIn("remove `# xfail spark>=4.0: fixed`", r.report(outcome, "4.2.0")) + def test_xfail_skipped_record_stays_skipped(self): parse_error = r.SparkError("parse", "Spark cannot parse it (PARSE_SYNTAX_ERROR)") outcome = self.outcome("# xfail spark: x\nquery ?\nSELECT [1]\n----\n[1]\n", parse_error) diff --git a/crates/sqllogictest/test_files/json/to_json.slt b/crates/sqllogictest/test_files/json/to_json.slt index f011673f..3f68aae9 100644 --- a/crates/sqllogictest/test_files/json/to_json.slt +++ b/crates/sqllogictest/test_files/json/to_json.slt @@ -302,6 +302,7 @@ RESET datafusion.execution.time_zone; statement ok SET datafusion.execution.time_zone = 'America/Los_Angeles'; +# xfail spark<4.0: intended: Spark 3.5 drops the seconds of the offset; datafusion-spark follows Spark 4 query T SELECT to_json(named_struct('t', arrow_cast('1800-01-01T07:52:58Z', 'Timestamp(Microsecond, Some("America/Los_Angeles"))'))); ---- diff --git a/crates/sqllogictest/test_files/math/abs.slt b/crates/sqllogictest/test_files/math/abs.slt index f9cc3ecc..481b9276 100644 --- a/crates/sqllogictest/test_files/math/abs.slt +++ b/crates/sqllogictest/test_files/math/abs.slt @@ -74,7 +74,7 @@ SELECT abs(('-99999999.99')::DECIMAL(10, 2)), abs(0::DECIMAL(10, 2)), abs(NULL:: ---- 99999999.99 0.00 NULL -# xfail spark: intended: Spark's abs rounds DECIMAL to 34 digits and overflows, DataFusion returns the exact value +# xfail spark>=4.0: intended: Spark's abs rounds DECIMAL to 34 digits and overflows, DataFusion returns the exact value query RRR SELECT abs(('-9999999999999999999999999999.9999999999')::DECIMAL(38, 10)), abs(0::DECIMAL(38, 10)), abs(NULL::DECIMAL(38, 10)); ---- @@ -196,7 +196,7 @@ SELECT abs(a) FROM (VALUES (('-99999999.99')::DECIMAL(10, 2)), (0::DECIMAL(10, 2 0.00 NULL -# xfail spark: intended: Spark's abs rounds DECIMAL to 34 digits and overflows, DataFusion returns the exact value +# xfail spark>=4.0: intended: Spark's abs rounds DECIMAL to 34 digits and overflows, DataFusion returns the exact value query R SELECT abs(a) FROM (VALUES (('-9999999999999999999999999999.9999999999')::DECIMAL(38, 10)), (0::DECIMAL(38, 10)), (NULL::DECIMAL(38, 10))) AS t(a); ---- diff --git a/crates/sqllogictest/test_files/math/greatest.slt b/crates/sqllogictest/test_files/math/greatest.slt index e4410a48..818bb07d 100644 --- a/crates/sqllogictest/test_files/math/greatest.slt +++ b/crates/sqllogictest/test_files/math/greatest.slt @@ -93,11 +93,13 @@ SELECT arrow_typeof(greatest(1::decimal(38,10), 1::decimal(38,0))); Decimal128(38, 0) # The fractional digits that do not fit are rounded away +# xfail spark<4.0: intended: Spark 3.5 picks DECIMAL(38, 10) as the wider type and returns NULL for the value that does not fit; datafusion-spark follows Spark 4 query R SELECT greatest(1.5::decimal(38,10), 1::decimal(38,0)); ---- 2 +# xfail spark<4.0: intended: Spark 3.5 picks DECIMAL(38, 10) as the wider type and returns NULL for the value that does not fit; datafusion-spark follows Spark 4 query R SELECT greatest(1::decimal(38,10), '12345678901234567890123456789'::decimal(38,0)); ---- diff --git a/crates/sqllogictest/test_files/string/encode.slt b/crates/sqllogictest/test_files/string/encode.slt index b15f0211..6527fd82 100644 --- a/crates/sqllogictest/test_files/string/encode.slt +++ b/crates/sqllogictest/test_files/string/encode.slt @@ -39,10 +39,12 @@ SELECT encode('Spark SQL'::string, 'utf-16'::string); feff0053007000610072006b002000530051004c # UTF-32BE encoding (alias rejected in Spark 4 strict mode) +# xfail spark<4.0: intended: Spark 3.5 accepts any Java charset name, which Spark 4 allows only with spark.sql.legacy.javaCharsets statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('A'::string, 'utf-32be'::string); # UTF-32LE encoding (alias rejected in Spark 4 strict mode) +# xfail spark<4.0: intended: Spark 3.5 accepts any Java charset name, which Spark 4 allows only with spark.sql.legacy.javaCharsets statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('AB'::string, 'utf-32le'::string); @@ -59,10 +61,12 @@ SELECT encode('hello'::string, 'Utf-8'::string); 68656c6c6f # US-ASCII: unmappable characters fail in Spark 4 strict mode +# xfail spark<4.0: intended: Spark 3.5 replaces unmappable characters with `?`, which Spark 4 does only with spark.sql.legacy.codingErrorAction statement error MALFORMED_CHARACTER_CODING SELECT encode('é'::string, 'us-ascii'::string); # ISO-8859-1: code points above U+00FF fail in Spark 4 strict mode (Ā is U+0100) +# xfail spark<4.0: intended: Spark 3.5 replaces unmappable characters with `?`, which Spark 4 does only with spark.sql.legacy.codingErrorAction statement error MALFORMED_CHARACTER_CODING SELECT encode('Ā'::string, 'iso-8859-1'::string); @@ -87,9 +91,11 @@ SELECT encode('😀'::string, 'utf-16'::string); ---- feffd83dde00 +# xfail spark<4.0: intended: Spark 3.5 accepts any Java charset name, which Spark 4 allows only with spark.sql.legacy.javaCharsets statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('😀'::string, 'utf-32be'::string); +# xfail spark<4.0: intended: Spark 3.5 accepts any Java charset name, which Spark 4 allows only with spark.sql.legacy.javaCharsets statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('😀'::string, 'utf-32le'::string); @@ -129,6 +135,7 @@ SELECT encode(X'FEFF00610073', 'utf-8'::string); efbfbdefbfbd00610073 # Strict mode: binary input with invalid UTF-8 byte 0xFF becomes U+FFFD, which cannot be encoded in US-ASCII +# xfail spark<4.0: intended: Spark 3.5 replaces unmappable characters with `?`, which Spark 4 does only with spark.sql.legacy.codingErrorAction query error \[MALFORMED_CHARACTER_CODING\] Invalid value found when performing `encode` with us-ascii SQLSTATE: 22000 SELECT encode(X'ff', 'us-ascii'::string); From d08877a8597e9e4f2dd978c93297c10b84d6888e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 20 Nov 2026 09:20:25 -0700 Subject: [PATCH 2/6] ci: make the PySpark runner a required check A required check that never runs blocks a pull request, so the PySpark runner workflow, which had `paths` filters, could not be required. The workflow now runs on every pull request. A `changes` job skips the unit-test and Spark jobs when nothing they test changed, and a `PySpark runner` job always reports: it passes when those jobs pass or are skipped. That job is now a required check in .asf.yaml. --- .asf.yaml | 6 +-- .github/workflows/pyspark-runner.yml | 58 ++++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 81241187..40434e19 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -34,9 +34,8 @@ github: main: required_pull_request_reviews: required_approving_review_count: 1 - # Job names from .github/workflows. The PySpark runner jobs are left out - # because they only run when their paths change, and a required check - # that does not run blocks the pull request. + # Job names from .github/workflows. Each of these jobs runs on every + # pull request, because a required check that does not run blocks it. required_status_checks: contexts: - "cargo fmt" @@ -47,6 +46,7 @@ github: - "Run sqllogictests" - "Apache RAT" - "ASF allowlist check" + - "PySpark runner" pull_requests: # enable updating head branches of pull requests allow_update_branch: true diff --git a/.github/workflows/pyspark-runner.yml b/.github/workflows/pyspark-runner.yml index 5c65621c..b5b01c07 100644 --- a/.github/workflows/pyspark-runner.yml +++ b/.github/workflows/pyspark-runner.yml @@ -4,25 +4,47 @@ name: PySpark runner +# The workflow runs on every pull request, so that its `PySpark runner` job +# always reports and can be a required check. A `paths` filter would leave the +# check pending, and block the pull request, when the paths do not change. +# Instead the `changes` job skips the Spark jobs when nothing they test changed. on: push: branches: [main] - paths: - - "crates/sqllogictest/scripts/**" - - "crates/sqllogictest/test_files/**" - - ".github/workflows/pyspark-runner.yml" pull_request: - paths: - - "crates/sqllogictest/scripts/**" - - "crates/sqllogictest/test_files/**" - - ".github/workflows/pyspark-runner.yml" permissions: contents: read jobs: + changes: + name: Check for PySpark runner changes + runs-on: ubuntu-latest + outputs: + runner: ${{ steps.diff.outputs.runner }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: diff + env: + EVENT: ${{ github.event_name }} + BASE: ${{ github.event.pull_request.base.sha }} + run: | + if [ "$EVENT" != pull_request ] || + git diff --name-only "$BASE" HEAD -- \ + crates/sqllogictest/scripts \ + crates/sqllogictest/test_files \ + .github/workflows/pyspark-runner.yml | grep -q .; then + echo runner=true >> "$GITHUB_OUTPUT" + else + echo runner=false >> "$GITHUB_OUTPUT" + fi + unit-tests: name: PySpark runner unit tests + needs: changes + if: needs.changes.outputs.runner == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -33,6 +55,8 @@ jobs: spark: name: Run the sqllogictests on Spark ${{ matrix.spark }} + needs: changes + if: needs.changes.outputs.runner == 'true' runs-on: ubuntu-latest strategy: fail-fast: false @@ -57,3 +81,21 @@ jobs: # Exits with status 1 if any record differs from Spark without an # `# xfail spark:` comment for this Spark version, or has one but now passes. - run: python crates/sqllogictest/scripts/pyspark_slt_runner.py + + # The required check. It passes when the jobs above pass or are skipped. + result: + name: PySpark runner + needs: [changes, unit-tests, spark] + if: always() + runs-on: ubuntu-latest + steps: + - env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + echo "$RESULTS" + for result in $RESULTS; do + case "$result" in + success | skipped) ;; + *) exit 1 ;; + esac + done From 712edef4ce16c850ac724e138e061db185840c0a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 20 Nov 2026 09:22:28 -0700 Subject: [PATCH 3/6] ci: diff only the pull request's own changes, and reword the new xfail reasons --- .github/workflows/pyspark-runner.yml | 4 +++- crates/sqllogictest/scripts/README.md | 1 - crates/sqllogictest/test_files/json/to_json.slt | 2 +- crates/sqllogictest/test_files/math/greatest.slt | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pyspark-runner.yml b/.github/workflows/pyspark-runner.yml index b5b01c07..51689dbb 100644 --- a/.github/workflows/pyspark-runner.yml +++ b/.github/workflows/pyspark-runner.yml @@ -30,9 +30,11 @@ jobs: env: EVENT: ${{ github.event_name }} BASE: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | + # Three dots: the changes in the pull request, not those on main since. if [ "$EVENT" != pull_request ] || - git diff --name-only "$BASE" HEAD -- \ + git diff --name-only "$BASE...$HEAD_SHA" -- \ crates/sqllogictest/scripts \ crates/sqllogictest/test_files \ .github/workflows/pyspark-runner.yml | grep -q .; then diff --git a/crates/sqllogictest/scripts/README.md b/crates/sqllogictest/scripts/README.md index bf2c7e4a..0d6cbb55 100644 --- a/crates/sqllogictest/scripts/README.md +++ b/crates/sqllogictest/scripts/README.md @@ -144,7 +144,6 @@ makes only these changes: `CAST(expr AS type)`. As in Spark 4, `::` applies to the literal, column, call, subscript or parenthesized expression before it, and a minus sign belongs to a number, so `-1::int` becomes `CAST(-1 AS int)`. - - `CREATE TABLE` becomes `CREATE TEMPORARY VIEW`, and `DROP TABLE` becomes `DROP VIEW` - PostgreSQL type names that Spark does not accept are renamed, for example diff --git a/crates/sqllogictest/test_files/json/to_json.slt b/crates/sqllogictest/test_files/json/to_json.slt index 3f68aae9..99da5b66 100644 --- a/crates/sqllogictest/test_files/json/to_json.slt +++ b/crates/sqllogictest/test_files/json/to_json.slt @@ -302,7 +302,7 @@ RESET datafusion.execution.time_zone; statement ok SET datafusion.execution.time_zone = 'America/Los_Angeles'; -# xfail spark<4.0: intended: Spark 3.5 drops the seconds of the offset; datafusion-spark follows Spark 4 +# xfail spark<4.0: intended: Spark 3.5 drops the seconds of the offset, and datafusion-spark follows Spark 4 query T SELECT to_json(named_struct('t', arrow_cast('1800-01-01T07:52:58Z', 'Timestamp(Microsecond, Some("America/Los_Angeles"))'))); ---- diff --git a/crates/sqllogictest/test_files/math/greatest.slt b/crates/sqllogictest/test_files/math/greatest.slt index 818bb07d..8160c7aa 100644 --- a/crates/sqllogictest/test_files/math/greatest.slt +++ b/crates/sqllogictest/test_files/math/greatest.slt @@ -93,13 +93,13 @@ SELECT arrow_typeof(greatest(1::decimal(38,10), 1::decimal(38,0))); Decimal128(38, 0) # The fractional digits that do not fit are rounded away -# xfail spark<4.0: intended: Spark 3.5 picks DECIMAL(38, 10) as the wider type and returns NULL for the value that does not fit; datafusion-spark follows Spark 4 +# xfail spark<4.0: intended: Spark 3.5 picks DECIMAL(38, 10) as the wider type and returns NULL for the value that does not fit, and datafusion-spark follows Spark 4 query R SELECT greatest(1.5::decimal(38,10), 1::decimal(38,0)); ---- 2 -# xfail spark<4.0: intended: Spark 3.5 picks DECIMAL(38, 10) as the wider type and returns NULL for the value that does not fit; datafusion-spark follows Spark 4 +# xfail spark<4.0: intended: Spark 3.5 picks DECIMAL(38, 10) as the wider type and returns NULL for the value that does not fit, and datafusion-spark follows Spark 4 query R SELECT greatest(1::decimal(38,10), '12345678901234567890123456789'::decimal(38,0)); ---- From fa3ddae27416d77bbc470ec8cf5e6383c1c3781f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 20 Nov 2026 09:23:44 -0700 Subject: [PATCH 4/6] fix: report a malformed xfail version condition before Spark starts --- crates/sqllogictest/scripts/pyspark_slt_runner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/sqllogictest/scripts/pyspark_slt_runner.py b/crates/sqllogictest/scripts/pyspark_slt_runner.py index fd480388..7d70cfc7 100755 --- a/crates/sqllogictest/scripts/pyspark_slt_runner.py +++ b/crates/sqllogictest/scripts/pyspark_slt_runner.py @@ -1852,6 +1852,9 @@ def main(argv: Optional[list[str]] = None) -> int: files = discover(args.paths) if not files: raise SystemExit("error: no .slt files found") + # Report a malformed `# xfail spark` comment before Spark starts. + for path in files: + parse_slt(path) # Keep the output complete when it is piped to a file and the run dies. sys.stdout.reconfigure(line_buffering=True) From 87939bb0c14d4661fbe5f525ff47083df2b3eb68 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 20 Nov 2026 09:27:31 -0700 Subject: [PATCH 5/6] refactor: simplify the Spark 3.5 support after review - apply the :: rewrite in _run_record from the Spark version, in place of a double_colon flag passed through translate - keep versions as (major, minor) tuples, with one parser - match brackets with _matching_paren and a backward twin - parse each file once - diff the merge commit against its first parent, and fail the summary job with contains() --- .github/workflows/pyspark-runner.yml | 20 +-- .../scripts/pyspark_slt_runner.py | 129 +++++++++--------- .../scripts/test_pyspark_slt_runner.py | 30 ++-- 3 files changed, 86 insertions(+), 93 deletions(-) diff --git a/.github/workflows/pyspark-runner.yml b/.github/workflows/pyspark-runner.yml index 51689dbb..4584ac52 100644 --- a/.github/workflows/pyspark-runner.yml +++ b/.github/workflows/pyspark-runner.yml @@ -23,18 +23,16 @@ jobs: outputs: runner: ${{ steps.diff.outputs.runner }} steps: + # A pull request checks out a merge commit, whose first parent is main. - uses: actions/checkout@v4 with: - fetch-depth: 0 + fetch-depth: 2 - id: diff env: EVENT: ${{ github.event_name }} - BASE: ${{ github.event.pull_request.base.sha }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - # Three dots: the changes in the pull request, not those on main since. if [ "$EVENT" != pull_request ] || - git diff --name-only "$BASE...$HEAD_SHA" -- \ + git diff --name-only HEAD^1 HEAD -- \ crates/sqllogictest/scripts \ crates/sqllogictest/test_files \ .github/workflows/pyspark-runner.yml | grep -q .; then @@ -91,13 +89,5 @@ jobs: if: always() runs-on: ubuntu-latest steps: - - env: - RESULTS: ${{ join(needs.*.result, ' ') }} - run: | - echo "$RESULTS" - for result in $RESULTS; do - case "$result" in - success | skipped) ;; - *) exit 1 ;; - esac - done + - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: exit 1 diff --git a/crates/sqllogictest/scripts/pyspark_slt_runner.py b/crates/sqllogictest/scripts/pyspark_slt_runner.py index 7d70cfc7..d7ee3846 100755 --- a/crates/sqllogictest/scripts/pyspark_slt_runner.py +++ b/crates/sqllogictest/scripts/pyspark_slt_runner.py @@ -33,8 +33,8 @@ from pathlib import Path from typing import Optional -# The Spark versions this runner supports, as major.minor. CI runs every one. -SUPPORTED_VERSIONS = ("3.5", "4.2") +# The Spark versions this runner supports, as (major, minor). CI runs every one. +SUPPORTED_VERSIONS = ((3, 5), (4, 2)) SCRIPT_DIR = Path(__file__).resolve().parent REPO_ROOT = SCRIPT_DIR.parents[2] @@ -139,9 +139,8 @@ class Record: def location(self) -> str: return f"{_display_path(self.path)}:{self.line}" - def xfail(self, spark_version: str) -> Optional[Xfail]: + def xfail(self, version: tuple[int, int]) -> Optional[Xfail]: """The first `# xfail spark` marker that applies to this Spark version.""" - version = _version_tuple(spark_version) return next((x for x in self.xfails if x.applies_to(version)), None) @@ -159,12 +158,16 @@ def applies_to(self, version: tuple[int, int]) -> bool: return all(_COMPARISONS[op](version, v) for op, v in self.conditions) -def _version_tuple(version: str) -> tuple[int, int]: - """(major, minor) of a version such as `4.2.0`.""" - major, minor = spark_minor_version(version).split(".") +def version_tuple(version: str) -> tuple[int, int]: + """(major, minor) of a version such as `4.2.0` or `4.2.0.dev1`.""" + major, minor = version.split(".")[:2] return int(major), int(minor) +def _version_string(version: tuple[int, int]) -> str: + return ".".join(map(str, version)) + + def parse_xfail(line: str) -> Optional[Xfail]: """Parse an `# xfail spark` comment, or return None for any other line. @@ -692,11 +695,20 @@ def _temporal_name(type_name: str) -> Optional[str]: return "TIMESTAMP" if type_name in _TIMESTAMP_TYPES else None -def _matching_paren(tokens: list[tuple[str, str]], open_index: int) -> int: +def _matching_paren(tokens: list[tuple[str, str]], open_index: int, nesting: dict = _NESTING) -> int: + return _matching(tokens, range(open_index, len(tokens)), nesting) + + +def _matching_open(tokens: list[tuple[str, str]], close_index: int) -> int: + """The index of the bracket that opens the one at tokens[close_index].""" + return _matching(tokens, range(close_index, -1, -1), _NESTING) + + +def _matching(tokens: list[tuple[str, str]], indexes: range, nesting: dict) -> int: depth = 0 - for k in range(open_index, len(tokens)): + for k in indexes: if tokens[k][0] == "other": - depth += _NESTING.get(tokens[k][1], 0) + depth += nesting.get(tokens[k][1], 0) if depth == 0: return k raise Skip("unbalanced parentheses") @@ -808,13 +820,8 @@ def rewrite_casts(sql: str, time_zone: str = "UTC") -> str: return "".join(out) -def translate(sql: str, time_zone: str = "UTC", double_colon: bool = True) -> str: - """Translate the parts of DataFusion's SQL that Spark spells differently. - - With `double_colon` False, for Spark 3.5, `expr::type` becomes a CAST. - """ - if not double_colon: - return rewrite_double_colons(translate(sql, time_zone)) +def translate(sql: str, time_zone: str = "UTC") -> str: + """Translate the parts of DataFusion's SQL that Spark spells differently.""" tokens = _tokenize(rewrite_casts(sql, time_zone)) out = [] # The function name before each open parenthesis, to find CAST(x AS type). @@ -867,6 +874,15 @@ def translate(sql: str, time_zone: str = "UTC", double_colon: bool = True) -> st } +# The type names of typed literals, such as DATE '2024-01-01'. +_TYPED_LITERALS = ("DATE", "TIME", "INTERVAL", "X", *_TIMESTAMP_TYPES) + +# Brackets that nest in a type name, such as `array`. +_TYPE_NESTING = {"(": 1, "<": 1, ")": -1, ">": -1} + +_UNTRANSLATABLE = "a `::` cast this runner cannot translate" + + def rewrite_double_colons(sql: str) -> str: """Replace each `expr::type` with `CAST(expr AS type)`. @@ -889,7 +905,7 @@ def rewrite_double_colons(sql: str) -> str: end = _type_end(tokens, i + 1) type_sql = "".join(t for _, t in tokens[i + 1 : end]).strip() if not operand or not type_sql: - raise Skip("a `::` cast this runner cannot translate") + raise Skip(_UNTRANSLATABLE) out.append(("operand", f"CAST({operand} AS {type_sql})")) i = end return "".join(t for _, t in out) @@ -907,18 +923,10 @@ def _operand_start(tokens: list[tuple[str, str]]) -> int: """The index where the primary expression that ends `tokens` starts.""" k = _significant_before(tokens, len(tokens)) if k < 0: - raise Skip("a `::` cast this runner cannot translate") + raise Skip(_UNTRANSLATABLE) kind, text = tokens[k] if kind == "other" and text in (")", "]"): - depth = 0 - while k >= 0: - if tokens[k][0] == "other": - depth += _NESTING.get(tokens[k][1], 0) - if depth == 0: - break - k -= 1 - if k < 0: - raise Skip("unbalanced parentheses") + k = _matching_open(tokens, k) before = _significant_before(tokens, k) prev = tokens[before] if before >= 0 else ("", "") if text == ")" and prev[0] == "word" and prev[1].upper() not in _KEYWORDS: @@ -930,9 +938,7 @@ def _operand_start(tokens: list[tuple[str, str]]) -> int: return k if kind == "string": before = _significant_before(tokens, k) - if before >= 0 and tokens[before][0] == "word" and tokens[before][1].upper() in ( - "DATE", "TIME", "TIMESTAMP", "TIMESTAMP_NTZ", "TIMESTAMP_LTZ", "INTERVAL", "X", - ): + if before >= 0 and tokens[before][0] == "word" and tokens[before][1].upper() in _TYPED_LITERALS: # A typed literal, such as DATE '2024-01-01'. return before return k @@ -953,7 +959,7 @@ def _operand_start(tokens: list[tuple[str, str]]) -> int: if kind == "operand": # A cast that this function already rewrote, as in `x::int::string`. return k - raise Skip("a `::` cast this runner cannot translate") + raise Skip(_UNTRANSLATABLE) def _name_start(tokens: list[tuple[str, str]], k: int) -> int: @@ -968,19 +974,11 @@ def _type_end(tokens: list[tuple[str, str]], start: int) -> int: `int`, `decimal(10, 2)` or `array>`.""" k = _next_significant(tokens, start) if k >= len(tokens) or tokens[k][0] != "word": - raise Skip("a `::` cast this runner cannot translate") + raise Skip(_UNTRANSLATABLE) k += 1 j = _next_significant(tokens, k) if j < len(tokens) and tokens[j][1] in ("(", "<"): - depth = 0 - while j < len(tokens): - text = tokens[j][1] - if tokens[j][0] == "other": - depth += {"(": 1, "<": 1, ")": -1, ">": -1}.get(text, 0) - if depth == 0: - return j + 1 - j += 1 - raise Skip("unbalanced parentheses") + return _matching_paren(tokens, j, _TYPE_NESTING) + 1 return k @@ -996,14 +994,14 @@ def _type_end(tokens: list[tuple[str, str]], start: int) -> int: _AS_RE = re.compile(r"AS\b\s*", re.IGNORECASE) -def translate_statement(sql: str, time_zone: str = "UTC", double_colon: bool = True) -> str: +def translate_statement(sql: str, time_zone: str = "UTC") -> str: """Translate a statement, turning DataFusion tables into Spark temporary views.""" m = _DROP_RE.match(sql) if m: return f"DROP VIEW {'IF EXISTS ' if m.group(1) else ''}{m.group(2)}" m = _CREATE_RE.match(sql) if not m: - return translate(sql, time_zone, double_colon) + return translate(sql, time_zone) if m.group(2): # Spark does not allow IF NOT EXISTS on a temporary view. raise Skip("CREATE TABLE IF NOT EXISTS, which a Spark temporary view cannot express") @@ -1020,7 +1018,7 @@ def translate_statement(sql: str, time_zone: str = "UTC", double_colon: bool = T raise Skip("CREATE TABLE without AS") query = rest[m.end() :].rstrip().rstrip(";") if columns is None: - return translate(f"{create} {name} AS {query}", time_zone, double_colon) + return translate(f"{create} {name} AS {query}", time_zone) # A newline ends a trailing `--` comment, which would hide the `)`. end_comment = "\n" if "--" in query else "" names, casts = [], [] @@ -1034,7 +1032,6 @@ def translate_statement(sql: str, time_zone: str = "UTC", double_colon: bool = T f"{create} {name} AS SELECT {', '.join(casts)} " f"FROM ({query}{end_comment}) AS __slt_values({', '.join(names)})", time_zone, - double_colon, ) @@ -1439,8 +1436,8 @@ def __init__(self): except ImportError: raise SystemExit(f"error: PySpark is not installed. Install it with `{install}`.") - if spark_minor_version(pyspark.__version__) not in SUPPORTED_VERSIONS: - supported = ", ".join(SUPPORTED_VERSIONS) + if version_tuple(pyspark.__version__) not in SUPPORTED_VERSIONS: + supported = ", ".join(map(_version_string, SUPPORTED_VERSIONS)) raise SystemExit( f"error: this runner needs PySpark {supported}, found " f"{pyspark.__version__}. Install it with `{install}`." @@ -1479,8 +1476,6 @@ def _start(self) -> None: self.spark = builder.getOrCreate() self.spark.sparkContext.setLogLevel("FATAL") self.version = self.spark.version - # Spark 4.0 added the `expr::type` cast syntax. - self.double_colon = _version_tuple(self.version) >= (4, 0) self.time_zone = SPARK_DEFAULTS["spark.sql.session.timeZone"] # Set when a query runs the JVM out of memory, which leaves it unusable. self.broken = False @@ -1683,7 +1678,7 @@ def run_record(runner: SparkRunner, record: Record) -> Outcome: condition matches the runner's Spark version applies. """ outcome = _run_record(runner, record) - xfail = record.xfail(runner.version) + xfail = record.xfail(version_tuple(runner.version)) if xfail is None: return outcome if outcome.status in EXPECTED_TO_FAIL: @@ -1715,12 +1710,14 @@ def _run_record(runner: SparkRunner, record: Record) -> Outcome: m = _ENGINE_SPECIFIC_RE.match(record.sql) if m: return Outcome(record, "skip", f"{m.group(1).upper()} output differs between engines") - double_colon = getattr(runner, "double_colon", True) try: if record.kind == "statement": - spark_sql = translate_statement(record.sql, runner.time_zone, double_colon) + spark_sql = translate_statement(record.sql, runner.time_zone) else: - spark_sql = translate(record.sql, runner.time_zone, double_colon) + spark_sql = translate(record.sql, runner.time_zone) + if version_tuple(runner.version) < (4, 0): + # Spark 4.0 added the `expr::type` syntax. + spark_sql = rewrite_double_colons(spark_sql) except Skip as e: return Outcome(record, "skip", str(e)) if any("" in line for line in record.expected): @@ -1811,11 +1808,6 @@ def report(outcome: Outcome, spark_version: str) -> str: return "\n".join(parts) -def spark_minor_version(version: str) -> str: - """The major.minor part of a version such as `4.2.0` or `4.2.0.dev1`.""" - return ".".join(version.split(".")[:2]) - - def discover(paths: list[str]) -> list[Path]: files = [] for name in paths: @@ -1832,7 +1824,8 @@ def discover(paths: list[str]) -> list[Path]: def main(argv: Optional[list[str]] = None) -> int: parser = argparse.ArgumentParser( description=( - f"Run the datafusion-spark .slt files on Spark ({', '.join(SUPPORTED_VERSIONS)}) and " + f"Run the datafusion-spark .slt files on Spark " + f"({', '.join(map(_version_string, SUPPORTED_VERSIONS))}) and " "report where Spark disagrees with the expected results." ) ) @@ -1852,15 +1845,15 @@ def main(argv: Optional[list[str]] = None) -> int: files = discover(args.paths) if not files: raise SystemExit("error: no .slt files found") - # Report a malformed `# xfail spark` comment before Spark starts. - for path in files: - parse_slt(path) + # Parsed before Spark starts, so a malformed `# xfail spark` comment + # fails the run at once. + parsed = {path: parse_slt(path) for path in files} # Keep the output complete when it is piped to a file and the run dies. sys.stdout.reconfigure(line_buffering=True) runner = SparkRunner() print(f"Running {len(files)} file(s) on Spark {runner.version}\n") - statuses, skip_reasons, stopped = run_files(runner, files, args.verbose) + statuses, skip_reasons, stopped = run_files(runner, parsed, args.verbose) total = sum(statuses.values()) print(f"Spark {runner.version}: {total} records in {len(files)} file(s)") @@ -1886,8 +1879,8 @@ def main(argv: Optional[list[str]] = None) -> int: MAX_RESTARTS = 5 -def run_files(runner, files: list[Path], verbose: bool = False): - """Run every record in `files`, printing each difference. +def run_files(runner, files: dict[Path, list[Record]], verbose: bool = False): + """Run the parsed records of each file, printing each difference. Returns (statuses, skip_reasons, stopped). `stopped` says why the run ended early, or is None. @@ -1911,7 +1904,7 @@ def new_session(path: Path, reason: str) -> Optional[str]: return None broken = False - for path in files: + for path, records in files.items(): stopped = None if broken: stopped = new_session(path, "Spark failed in the previous file") @@ -1923,7 +1916,7 @@ def new_session(path: Path, reason: str) -> Optional[str]: if stopped: return statuses, skip_reasons, stopped broken = False - for record in parse_slt(path): + for record in records: if broken: # A new session would not have this file's views and settings. outcome = Outcome(record, "skip", "Spark failed earlier in the file") diff --git a/crates/sqllogictest/scripts/test_pyspark_slt_runner.py b/crates/sqllogictest/scripts/test_pyspark_slt_runner.py index db71ab19..635a4528 100644 --- a/crates/sqllogictest/scripts/test_pyspark_slt_runner.py +++ b/crates/sqllogictest/scripts/test_pyspark_slt_runner.py @@ -106,7 +106,7 @@ def test_xfail_spark_versions(self): "query I\n" "SELECT 1\n" ) - reason = lambda version: getattr(record.xfail(version), "reason", None) # noqa: E731 + reason = lambda version: getattr(record.xfail(r.version_tuple(version)), "reason", None) # noqa: E731 self.assertEqual(reason("3.5.9"), "Spark 3.5 differs") self.assertEqual(reason("4.0.1"), "Spark 4.0 and 4.1 differ") self.assertEqual(reason("4.1.0"), "Spark 4.0 and 4.1 differ") @@ -115,8 +115,8 @@ def test_xfail_spark_versions(self): def test_first_matching_xfail_applies(self): [record] = parse("# xfail spark<4.0: old\n# xfail spark: any\nquery I\nSELECT 1\n") - self.assertEqual(record.xfail("3.5.9").reason, "old") - self.assertEqual(record.xfail("4.2.0").reason, "any") + self.assertEqual(record.xfail((3, 5)).reason, "old") + self.assertEqual(record.xfail((4, 2)).reason, "any") def test_xfail_bad_version_condition(self): for marker in ("# xfail spark 3.5: x", "# xfail spark<4: x", "# xfail spark<4.0;>3.4: x"): @@ -146,11 +146,7 @@ def test_double_colon_casts_for_spark_3(self): "SELECT 'a::b'::FLOAT8": "SELECT CAST('a::b' AS DOUBLE)", } for sql, expected in cases.items(): - self.assertEqual(r.translate(sql, double_colon=False), expected, sql) - self.assertEqual( - r.translate_statement("CREATE TABLE t AS SELECT 1::int AS a", double_colon=False), - "CREATE TEMPORARY VIEW t AS SELECT CAST(1 AS int) AS a", - ) + self.assertEqual(r.rewrite_double_colons(r.translate(sql)), expected, sql) def test_type_aliases(self): self.assertEqual( @@ -592,6 +588,20 @@ def test_xpass(self): self.assertEqual(outcome.status, "xpass") self.assertIn("remove `# xfail spark: a fixed bug`", r.report(outcome, "4.2.0")) + def test_double_colon_rewritten_only_before_spark_4(self): + result = (FakeSchema("IntegerType"), [(1,)]) + for version, expected in (("3.5.9", "SELECT CAST(1 AS int)"), ("4.2.0", "SELECT 1::int")): + runner = FakeRunner(result) + runner.version = version + [record] = parse("query I\nSELECT 1::int\n----\n1\n") + self.assertEqual(r.run_record(runner, record).status, "pass") + self.assertEqual(runner.sql, expected) + runner = FakeRunner(result) + runner.version = "3.5.9" + [record] = parse("statement ok\nCREATE TABLE t AS SELECT 1::int AS a\n") + r.run_record(runner, record) + self.assertEqual(runner.sql, "CREATE TEMPORARY VIEW t AS SELECT CAST(1 AS int) AS a") + def test_xfail_for_another_spark_version(self): result = (FakeSchema("IntegerType"), [(1,)]) differs = "# xfail spark<4.0: only Spark 3.5\nquery I\nSELECT a\n----\n2\n" @@ -678,7 +688,7 @@ def run_files(self, *contents): paths[-1].write_text("".join(self.QUERY.format(q) for q in queries)) runner = DyingRunner() with contextlib.redirect_stdout(io.StringIO()) as out: - result = r.run_files(runner, paths) + result = r.run_files(runner, {p: r.parse_slt(p) for p in paths}) return runner, result, out.getvalue() def test_new_session_after_the_jvm_dies(self): @@ -710,7 +720,7 @@ def test_new_session_when_reset_fails(self): path = Path(d, "0.slt") path.write_text(self.QUERY.format(1)) with contextlib.redirect_stdout(io.StringIO()) as out: - statuses, _, stopped = r.run_files(runner, [path]) + statuses, _, stopped = r.run_files(runner, {path: r.parse_slt(path)}) self.assertIsNone(stopped) self.assertEqual((runner.restarts, statuses["pass"]), (1, 1)) self.assertIn("Spark failed (An error occurred while calling o49.listTables.)", out.getvalue()) From f4b912895dc356b37c8f92822716fa95f6771dc0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 20 Nov 2026 11:45:55 -0700 Subject: [PATCH 6/6] fix: reject xfail markers that never apply, and fix three :: rewrite cases Reject an xfail marker that no supported Spark version reaches, at parse time. Treat exists and any as functions, keep the base of a field of a call or subscript, and read < after a type other than ARRAY, MAP or STRUCT as a comparison. Correct the reason on one greatest.slt record. --- crates/sqllogictest/scripts/README.md | 5 ++-- .../scripts/pyspark_slt_runner.py | 21 ++++++++++++++--- .../scripts/test_pyspark_slt_runner.py | 23 +++++++++++++++---- .../sqllogictest/test_files/math/greatest.slt | 2 +- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/crates/sqllogictest/scripts/README.md b/crates/sqllogictest/scripts/README.md index 0d6cbb55..640f5b6c 100644 --- a/crates/sqllogictest/scripts/README.md +++ b/crates/sqllogictest/scripts/README.md @@ -97,10 +97,11 @@ statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('A'::string, 'utf-32be'::string); ``` -The operators are `<`, `<=`, `>`, `>=` and `==`, as in `spark>=4.0, <4.2`. A +The operators are `<`, `<=`, `>`, `>=` and `==`, as in `spark>=4.0, <=4.2`. A record can have several comments, one line each, and the first that matches the Spark version applies. The runner stops with an error if it cannot read a -version condition. +version condition, or if a comment applies to none of the Spark versions it +supports because no version matches it before an earlier comment does. The reason links an issue, or says what differs and why: a bug in datafusion-spark, a difference that comes from how the test is written, or a diff --git a/crates/sqllogictest/scripts/pyspark_slt_runner.py b/crates/sqllogictest/scripts/pyspark_slt_runner.py index d7ee3846..dc31bff8 100755 --- a/crates/sqllogictest/scripts/pyspark_slt_runner.py +++ b/crates/sqllogictest/scripts/pyspark_slt_runner.py @@ -221,6 +221,14 @@ def parse_slt(path: Path) -> list[Record]: continue record = Record(path=path, line=i + 1, kind=words[0], xfails=xfails) + for x in xfails: + # A marker that no supported version reaches would never be + # reported as stale, so reject it before Spark starts. + if all(record.xfail(v) is not x for v in SUPPORTED_VERSIONS): + raise SystemExit( + f"error: {_display_path(path)}:{i + 1}: `{x.text}` never applies, because no " + "supported Spark version matches it before an earlier marker does" + ) xfails = [] args = words[1:] if args and args[0] == "error": @@ -868,8 +876,8 @@ def translate(sql: str, time_zone: str = "UTC") -> str: # Keywords after which a `-` is a sign, not a subtraction, and a `(` does # not start the arguments of a call. _KEYWORDS = { - "ALL", "AND", "ANY", "AS", "BETWEEN", "BY", "CASE", "DISTINCT", "ELSE", - "EXISTS", "FROM", "HAVING", "IN", "IS", "JOIN", "LIKE", "LIMIT", "NOT", + "ALL", "AND", "AS", "BETWEEN", "BY", "CASE", "DISTINCT", "ELSE", + "FROM", "HAVING", "IN", "IS", "JOIN", "LIKE", "LIMIT", "NOT", "ON", "OR", "RETURN", "SELECT", "THEN", "VALUES", "WHEN", "WHERE", } @@ -966,6 +974,9 @@ def _name_start(tokens: list[tuple[str, str]], k: int) -> int: """The start of a possibly qualified name, such as `t.a`, that ends at tokens[k].""" while k >= 2 and tokens[k - 1][1] == "." and tokens[k - 2][0] in ("word", "quoted"): k -= 2 + if k >= 2 and tokens[k - 1][1] == "." and tokens[k - 2][1] in (")", "]"): + # A field of a call or subscript, such as `struct(1 AS x).x` + return _operand_start(tokens[: k - 1]) return k @@ -975,9 +986,13 @@ def _type_end(tokens: list[tuple[str, str]], start: int) -> int: k = _next_significant(tokens, start) if k >= len(tokens) or tokens[k][0] != "word": raise Skip(_UNTRANSLATABLE) + name = tokens[k][1].upper() k += 1 j = _next_significant(tokens, k) - if j < len(tokens) and tokens[j][1] in ("(", "<"): + # After any other type, `<` is a comparison, as in `a::int < 5`. + if j < len(tokens) and ( + tokens[j][1] == "(" or (tokens[j][1] == "<" and name in ("ARRAY", "MAP", "STRUCT")) + ): return _matching_paren(tokens, j, _TYPE_NESTING) + 1 return k diff --git a/crates/sqllogictest/scripts/test_pyspark_slt_runner.py b/crates/sqllogictest/scripts/test_pyspark_slt_runner.py index 635a4528..ac142cfa 100644 --- a/crates/sqllogictest/scripts/test_pyspark_slt_runner.py +++ b/crates/sqllogictest/scripts/test_pyspark_slt_runner.py @@ -101,18 +101,22 @@ def test_xfail(self): def test_xfail_spark_versions(self): [record] = parse( "# xfail spark<4.0: Spark 3.5 differs\n" - "# xfail spark >= 4.0, <4.2: Spark 4.0 and 4.1 differ\n" - "# xfail spark==4.2: Spark 4.2 differs\n" + "# xfail spark >= 4.0, <=4.2: Spark 4.0 to 4.2 differ\n" "query I\n" "SELECT 1\n" ) reason = lambda version: getattr(record.xfail(r.version_tuple(version)), "reason", None) # noqa: E731 self.assertEqual(reason("3.5.9"), "Spark 3.5 differs") - self.assertEqual(reason("4.0.1"), "Spark 4.0 and 4.1 differ") - self.assertEqual(reason("4.1.0"), "Spark 4.0 and 4.1 differ") - self.assertEqual(reason("4.2.0"), "Spark 4.2 differs") + self.assertEqual(reason("4.0.1"), "Spark 4.0 to 4.2 differ") + self.assertEqual(reason("4.2.0"), "Spark 4.0 to 4.2 differ") self.assertIsNone(reason("4.3.0")) + def test_xfail_that_never_applies(self): + # No supported version matches the first, and the second is shadowed. + for markers in ("# xfail spark>=4.0, <4.2: x\n", "# xfail spark: x\n# xfail spark<4.0: y\n"): + with self.assertRaisesRegex(SystemExit, "never applies"): + parse(f"{markers}query I\nSELECT 1\n") + def test_first_matching_xfail_applies(self): [record] = parse("# xfail spark<4.0: old\n# xfail spark: any\nquery I\nSELECT 1\n") self.assertEqual(record.xfail((3, 5)).reason, "old") @@ -144,6 +148,15 @@ def test_double_colon_casts_for_spark_3(self): "SELECT NULL::array>": "SELECT CAST(NULL AS array>)", "SELECT (SELECT 1)::int": "SELECT CAST((SELECT 1) AS int)", "SELECT 'a::b'::FLOAT8": "SELECT CAST('a::b' AS DOUBLE)", + # `exists` and `any` are functions here, not keywords. + "SELECT exists(arr, x -> x > 1)::string FROM t": "SELECT CAST(exists(arr, x -> x > 1) AS string) FROM t", + "SELECT any(a > 1)::string FROM t": "SELECT CAST(any(a > 1) AS string) FROM t", + # A field of a call or a subscript + "SELECT struct(1 AS x).x::string": "SELECT CAST(struct(1 AS x).x AS string)", + "SELECT map(1, struct(2 AS y))[1].y::string": "SELECT CAST(map(1, struct(2 AS y))[1].y AS string)", + # After a type other than ARRAY, MAP or STRUCT, `<` is a comparison. + "SELECT a::int < 5 AND b > 3 FROM t": "SELECT CAST(a AS int) < 5 AND b > 3 FROM t", + "SELECT a::int