diff --git a/.asf.yaml b/.asf.yaml index 81241187..40434e19 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -34,9 +34,8 @@ github: main: required_pull_request_reviews: required_approving_review_count: 1 - # Job names from .github/workflows. The PySpark runner jobs are left out - # because they only run when their paths change, and a required check - # that does not run blocks the pull request. + # Job names from .github/workflows. Each of these jobs runs on every + # pull request, because a required check that does not run blocks it. required_status_checks: contexts: - "cargo fmt" @@ -47,6 +46,7 @@ github: - "Run sqllogictests" - "Apache RAT" - "ASF allowlist check" + - "PySpark runner" pull_requests: # enable updating head branches of pull requests allow_update_branch: true diff --git a/.github/workflows/pyspark-runner.yml b/.github/workflows/pyspark-runner.yml index 6733dbba..4584ac52 100644 --- a/.github/workflows/pyspark-runner.yml +++ b/.github/workflows/pyspark-runner.yml @@ -4,25 +4,47 @@ name: PySpark runner +# The workflow runs on every pull request, so that its `PySpark runner` job +# always reports and can be a required check. A `paths` filter would leave the +# check pending, and block the pull request, when the paths do not change. +# Instead the `changes` job skips the Spark jobs when nothing they test changed. on: push: branches: [main] - paths: - - "crates/sqllogictest/scripts/**" - - "crates/sqllogictest/test_files/**" - - ".github/workflows/pyspark-runner.yml" pull_request: - paths: - - "crates/sqllogictest/scripts/**" - - "crates/sqllogictest/test_files/**" - - ".github/workflows/pyspark-runner.yml" permissions: contents: read jobs: + changes: + name: Check for PySpark runner changes + runs-on: ubuntu-latest + outputs: + runner: ${{ steps.diff.outputs.runner }} + steps: + # A pull request checks out a merge commit, whose first parent is main. + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + - id: diff + env: + EVENT: ${{ github.event_name }} + run: | + if [ "$EVENT" != pull_request ] || + git diff --name-only HEAD^1 HEAD -- \ + crates/sqllogictest/scripts \ + crates/sqllogictest/test_files \ + .github/workflows/pyspark-runner.yml | grep -q .; then + echo runner=true >> "$GITHUB_OUTPUT" + else + echo runner=false >> "$GITHUB_OUTPUT" + fi + unit-tests: name: PySpark runner unit tests + needs: changes + if: needs.changes.outputs.runner == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -32,8 +54,18 @@ jobs: - run: python3 -m unittest discover -s crates/sqllogictest/scripts spark: - name: Run the sqllogictests on Spark + name: Run the sqllogictests on Spark ${{ matrix.spark }} + needs: changes + if: needs.changes.outputs.runner == 'true' runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - spark: "3.5" + requirements: requirements-spark-3.5.txt + - spark: "4.2" + requirements: requirements.txt steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 @@ -44,8 +76,18 @@ jobs: with: python-version: "3.12" cache: pip - cache-dependency-path: crates/sqllogictest/scripts/requirements.txt - - run: pip install -r crates/sqllogictest/scripts/requirements.txt + cache-dependency-path: crates/sqllogictest/scripts/${{ matrix.requirements }} + - run: pip install -r crates/sqllogictest/scripts/${{ matrix.requirements }} # Exits with status 1 if any record differs from Spark without an - # `# xfail spark:` comment, or has one but now passes. + # `# xfail spark:` comment for this Spark version, or has one but now passes. - run: python crates/sqllogictest/scripts/pyspark_slt_runner.py + + # The required check. It passes when the jobs above pass or are skipped. + result: + name: PySpark runner + needs: [changes, unit-tests, spark] + if: always() + runs-on: ubuntu-latest + steps: + - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: exit 1 diff --git a/crates/sqllogictest/scripts/README.md b/crates/sqllogictest/scripts/README.md index fcbe30f6..640f5b6c 100644 --- a/crates/sqllogictest/scripts/README.md +++ b/crates/sqllogictest/scripts/README.md @@ -8,14 +8,15 @@ SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one `pyspark_slt_runner.py` runs the `.slt` files in [`crates/sqllogictest/test_files`](../test_files) -on Apache Spark 4.2 and reports every query where Spark's result differs from -the expected result in the file. +on Apache Spark and reports every query where Spark's result differs from the +expected result in the file. It supports Spark 3.5 and 4.2. The expected results in those files come from DataFusion, and nothing else checks them against Spark. A difference means that DataFusion and Spark disagree, or that the test expects the wrong value. CI runs the runner on -every pull request that changes an `.slt` file or the runner, and fails if it -reports a difference that has no `# xfail spark:` comment (see +every pull request that changes an `.slt` file or the runner, once for each +supported Spark version, and fails if it reports a difference that has no +`# xfail spark:` comment for that version (see [Known differences](#known-differences)) or a comment on a record that now passes. @@ -29,8 +30,16 @@ python3 -m venv /tmp/pyspark-4.2 /tmp/pyspark-4.2/bin/pip install -r crates/sqllogictest/scripts/requirements.txt ``` -The runner ignores `SPARK_HOME`, so it always uses the Spark bundled with this -PySpark. +For Spark 3.5, install `requirements-spark-3.5.txt` into a second virtual +environment, and run the runner with that environment's Python: + +```shell +python3 -m venv /tmp/pyspark-3.5 +/tmp/pyspark-3.5/bin/pip install -r crates/sqllogictest/scripts/requirements-spark-3.5.txt +``` + +The runner ignores `SPARK_HOME`, so it always uses the Spark bundled with the +installed PySpark, and it checks that this is a supported version. ## Usage @@ -78,6 +87,22 @@ SELECT round(arrow_cast(2.5, 'Decimal32(9, 1)'), 0::int); 3.0 ``` +A comparison after `spark` limits the comment to some Spark versions, which it +compares by major and minor version. Several comparisons separated by commas +must all hold: + +```text +# xfail spark<4.0: intended: Spark 3.5 accepts any Java charset name, which Spark 4 allows only with spark.sql.legacy.javaCharsets +statement error INVALID_PARAMETER_VALUE\.CHARSET +SELECT encode('A'::string, 'utf-32be'::string); +``` + +The operators are `<`, `<=`, `>`, `>=` and `==`, as in `spark>=4.0, <=4.2`. A +record can have several comments, one line each, and the first that matches +the Spark version applies. The runner stops with an error if it cannot read a +version condition, or if a comment applies to none of the Spark versions it +supports because no version matches it before an earlier comment does. + The reason links an issue, or says what differs and why: a bug in datafusion-spark, a difference that comes from how the test is written, or a limitation of this runner. A marked record that differs from Spark in any way @@ -116,6 +141,10 @@ are DataFusion's defaults. These settings in the files are passed on to Spark: Spark 4.2 parses most of the SQL as written, including `::` casts. The runner makes only these changes: +- For Spark 3.5, which has no `::` syntax, `expr::type` becomes + `CAST(expr AS type)`. As in Spark 4, `::` applies to the literal, column, + call, subscript or parenthesized expression before it, and a minus sign + belongs to a number, so `-1::int` becomes `CAST(-1 AS int)`. - `CREATE TABLE` becomes `CREATE TEMPORARY VIEW`, and `DROP TABLE` becomes `DROP VIEW` - PostgreSQL type names that Spark does not accept are renamed, for example diff --git a/crates/sqllogictest/scripts/pyspark_slt_runner.py b/crates/sqllogictest/scripts/pyspark_slt_runner.py index fbdd4e1f..dc31bff8 100755 --- a/crates/sqllogictest/scripts/pyspark_slt_runner.py +++ b/crates/sqllogictest/scripts/pyspark_slt_runner.py @@ -3,7 +3,7 @@ # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -"""Run the datafusion-spark sqllogictest files against Apache Spark 4.2. +"""Run the datafusion-spark sqllogictest files against Apache Spark 3.5 or 4.2. Each record in crates/sqllogictest/test_files/**/*.slt is run on a local PySpark session, and Spark's result is compared with the expected result @@ -11,7 +11,7 @@ reported here is a query where DataFusion and Spark disagree, or where the test expects the wrong value. -Running Spark is manual; only the unit tests run in CI. See README.md for setup. +CI runs this on every supported Spark version. See README.md for setup. """ from __future__ import annotations @@ -19,6 +19,7 @@ import argparse import datetime import math +import operator import os import re import struct @@ -32,7 +33,8 @@ from pathlib import Path from typing import Optional -SPARK_VERSION = "4.2" +# The Spark versions this runner supports, as (major, minor). CI runs every one. +SUPPORTED_VERSIONS = ((3, 5), (4, 2)) SCRIPT_DIR = Path(__file__).resolve().parent REPO_ROOT = SCRIPT_DIR.parents[2] @@ -44,10 +46,23 @@ # A comment that marks the next record as a known difference from Spark: # # # xfail spark: +# # xfail spark<4.0: +# +# The second form applies only to the Spark versions that match every +# comma-separated condition, such as `>=4.0, <4.2`. A record can have several +# markers, and the first that matches the Spark version applies. # # The Rust sqllogictest parser rejects directives it does not know, so the # marker is a comment, which DataFusion's runner ignores. -_XFAIL_RE = re.compile(rf"#\s*xfail\s+{LABEL}\s*:\s*(.*\S)\s*$") +_XFAIL_RE = re.compile(rf"#\s*xfail\s+{LABEL}\b([^:]*):\s*(.*\S)\s*$") +_VERSION_CONDITION_RE = re.compile(r"\s*(<=|>=|==|<|>)\s*(\d+)\.(\d+)\s*") +_COMPARISONS = { + "<": operator.lt, + "<=": operator.le, + ">": operator.gt, + ">=": operator.ge, + "==": operator.eq, +} # DataFusion runs these files with ANSI mode off and no time zone (UTC), so # every file starts from the same Spark settings. @@ -117,26 +132,78 @@ class Record: sort_mode: str = "nosort" expected: list[str] = field(default_factory=list) skip_reason: Optional[str] = None - # Why Spark is expected to disagree, from an `# xfail spark:` comment. - xfail_reason: Optional[str] = None + # Why Spark is expected to disagree, from `# xfail spark:` comments. + xfails: list[Xfail] = field(default_factory=list) @property def location(self) -> str: return f"{_display_path(self.path)}:{self.line}" + def xfail(self, version: tuple[int, int]) -> Optional[Xfail]: + """The first `# xfail spark` marker that applies to this Spark version.""" + return next((x for x in self.xfails if x.applies_to(version)), None) + + +@dataclass +class Xfail: + """An `# xfail spark: ` comment.""" + + # (operator, (major, minor)) pairs that the Spark version must all match. + conditions: list[tuple[str, tuple[int, int]]] + reason: str + # The comment as written, for the report that asks to remove it. + text: str + + def applies_to(self, version: tuple[int, int]) -> bool: + return all(_COMPARISONS[op](version, v) for op, v in self.conditions) + + +def version_tuple(version: str) -> tuple[int, int]: + """(major, minor) of a version such as `4.2.0` or `4.2.0.dev1`.""" + major, minor = version.split(".")[:2] + return int(major), int(minor) + + +def _version_string(version: tuple[int, int]) -> str: + return ".".join(map(str, version)) + + +def parse_xfail(line: str) -> Optional[Xfail]: + """Parse an `# xfail spark` comment, or return None for any other line. + + Raises ValueError for a version condition this runner cannot read.""" + m = _XFAIL_RE.match(line.strip()) + if not m: + return None + spec, reason = m.group(1).strip(), m.group(2) + conditions = [] + if spec: + for part in spec.split(","): + c = _VERSION_CONDITION_RE.fullmatch(part) + if not c: + raise ValueError( + f"cannot read the Spark version condition `{spec}`. Write a " + "comparison such as `<4.0` or `==3.5`, or several separated by commas" + ) + conditions.append((c.group(1), (int(c.group(2)), int(c.group(3))))) + return Xfail(conditions, reason, line.strip()) + def parse_slt(path: Path) -> list[Record]: """Parse the `statement` and `query` records of an .slt file.""" lines = path.read_text().split("\n") records: list[Record] = [] conditions: list[tuple[str, str]] = [] - xfail_reason: Optional[str] = None + xfails: list[Xfail] = [] i = 0 while i < len(lines): words = lines[i].split() - m = _XFAIL_RE.match(lines[i].strip()) - if m: - xfail_reason = m.group(1) + try: + xfail = parse_xfail(lines[i]) + except ValueError as e: + raise SystemExit(f"error: {_display_path(path)}:{i + 1}: {e}") + if xfail: + xfails.append(xfail) i += 1 continue if not words or words[0].startswith("#"): @@ -149,12 +216,20 @@ def parse_slt(path: Path) -> list[Record]: if words[0] not in ("statement", "query"): # Directives such as `include` and `halt`, which these files do not use. conditions = [] - xfail_reason = None + xfails = [] i += 1 continue - record = Record(path=path, line=i + 1, kind=words[0], xfail_reason=xfail_reason) - xfail_reason = None + record = Record(path=path, line=i + 1, kind=words[0], xfails=xfails) + for x in xfails: + # A marker that no supported version reaches would never be + # reported as stale, so reject it before Spark starts. + if all(record.xfail(v) is not x for v in SUPPORTED_VERSIONS): + raise SystemExit( + f"error: {_display_path(path)}:{i + 1}: `{x.text}` never applies, because no " + "supported Spark version matches it before an earlier marker does" + ) + xfails = [] args = words[1:] if args and args[0] == "error": record.expect_error = True @@ -194,10 +269,11 @@ def parse_slt(path: Path) -> list[Record]: # --------------------------------------------------------------------------- # Translating DataFusion SQL to Spark SQL # -# Spark 4.2 parses most of these files as written, including `::` casts. -# Only a few renames are applied here, and `arrow_cast` and `spark_cast` -# become a CAST (see below). Anything else Spark cannot parse, and any function -# Spark does not have (such as `arrow_typeof`), is skipped. +# Spark 4.2 parses most of these files as written, including `::` casts, which +# become a CAST for Spark 3.5 (see rewrite_double_colons). Only a few renames +# are applied here, and `arrow_cast` and `spark_cast` become a CAST (see +# below). Anything else Spark cannot parse, and any function Spark does not +# have (such as `arrow_typeof`), is skipped. # --------------------------------------------------------------------------- _TOKEN_RE = re.compile( @@ -627,11 +703,20 @@ def _temporal_name(type_name: str) -> Optional[str]: return "TIMESTAMP" if type_name in _TIMESTAMP_TYPES else None -def _matching_paren(tokens: list[tuple[str, str]], open_index: int) -> int: +def _matching_paren(tokens: list[tuple[str, str]], open_index: int, nesting: dict = _NESTING) -> int: + return _matching(tokens, range(open_index, len(tokens)), nesting) + + +def _matching_open(tokens: list[tuple[str, str]], close_index: int) -> int: + """The index of the bracket that opens the one at tokens[close_index].""" + return _matching(tokens, range(close_index, -1, -1), _NESTING) + + +def _matching(tokens: list[tuple[str, str]], indexes: range, nesting: dict) -> int: depth = 0 - for k in range(open_index, len(tokens)): + for k in indexes: if tokens[k][0] == "other": - depth += _NESTING.get(tokens[k][1], 0) + depth += nesting.get(tokens[k][1], 0) if depth == 0: return k raise Skip("unbalanced parentheses") @@ -788,6 +873,130 @@ def translate(sql: str, time_zone: str = "UTC") -> str: return "".join(out) +# Keywords after which a `-` is a sign, not a subtraction, and a `(` does +# not start the arguments of a call. +_KEYWORDS = { + "ALL", "AND", "AS", "BETWEEN", "BY", "CASE", "DISTINCT", "ELSE", + "FROM", "HAVING", "IN", "IS", "JOIN", "LIKE", "LIMIT", "NOT", + "ON", "OR", "RETURN", "SELECT", "THEN", "VALUES", "WHEN", "WHERE", +} + + +# The type names of typed literals, such as DATE '2024-01-01'. +_TYPED_LITERALS = ("DATE", "TIME", "INTERVAL", "X", *_TIMESTAMP_TYPES) + +# Brackets that nest in a type name, such as `array`. +_TYPE_NESTING = {"(": 1, "<": 1, ")": -1, ">": -1} + +_UNTRANSLATABLE = "a `::` cast this runner cannot translate" + + +def rewrite_double_colons(sql: str) -> str: + """Replace each `expr::type` with `CAST(expr AS type)`. + + Spark 3.5 does not parse `::`. As in Spark 4, `::` applies to the primary + expression before it: a literal, a column, a call, a parenthesized + expression or a subscript. A sign belongs to a number literal, so + `-1::int` is `CAST(-1 AS int)`, but `-(1)::int` is `-CAST((1) AS int)`. + """ + tokens = _tokenize(sql) + out: list[tuple[str, str]] = [] + i = 0 + while i < len(tokens): + if tokens[i][0] != "cast": + out.append(tokens[i]) + i += 1 + continue + start = _operand_start(out) + operand = "".join(t for _, t in out[start:]).strip() + del out[start:] + end = _type_end(tokens, i + 1) + type_sql = "".join(t for _, t in tokens[i + 1 : end]).strip() + if not operand or not type_sql: + raise Skip(_UNTRANSLATABLE) + out.append(("operand", f"CAST({operand} AS {type_sql})")) + i = end + return "".join(t for _, t in out) + + +def _significant_before(tokens: list[tuple[str, str]], end: int) -> int: + """The index of the last token before `end` that is not whitespace, or -1.""" + k = end - 1 + while k >= 0 and tokens[k][0] in ("space", "comment"): + k -= 1 + return k + + +def _operand_start(tokens: list[tuple[str, str]]) -> int: + """The index where the primary expression that ends `tokens` starts.""" + k = _significant_before(tokens, len(tokens)) + if k < 0: + raise Skip(_UNTRANSLATABLE) + kind, text = tokens[k] + if kind == "other" and text in (")", "]"): + k = _matching_open(tokens, k) + before = _significant_before(tokens, k) + prev = tokens[before] if before >= 0 else ("", "") + if text == ")" and prev[0] == "word" and prev[1].upper() not in _KEYWORDS: + # A call, f(...) + return _name_start(tokens, before) + if text == "]" and (prev[0] in ("word", "quoted", "string") or prev[1] in (")", "]")): + # A subscript, a[...], of the operand that ends before it + return _operand_start(tokens[: before + 1]) + return k + if kind == "string": + before = _significant_before(tokens, k) + if before >= 0 and tokens[before][0] == "word" and tokens[before][1].upper() in _TYPED_LITERALS: + # A typed literal, such as DATE '2024-01-01'. + return before + return k + if kind == "number": + before = _significant_before(tokens, k) + if before >= 0 and tokens[before][1] == "-": + sign_of = _significant_before(tokens, before) + prev = tokens[sign_of] if sign_of >= 0 else None + if ( + prev is None + or (prev[0] == "other" and prev[1] not in (")", "]")) + or (prev[0] == "word" and prev[1].upper() in _KEYWORDS) + ): + return before + return k + if kind in ("word", "quoted"): + return _name_start(tokens, k) + if kind == "operand": + # A cast that this function already rewrote, as in `x::int::string`. + return k + raise Skip(_UNTRANSLATABLE) + + +def _name_start(tokens: list[tuple[str, str]], k: int) -> int: + """The start of a possibly qualified name, such as `t.a`, that ends at tokens[k].""" + while k >= 2 and tokens[k - 1][1] == "." and tokens[k - 2][0] in ("word", "quoted"): + k -= 2 + if k >= 2 and tokens[k - 1][1] == "." and tokens[k - 2][1] in (")", "]"): + # A field of a call or subscript, such as `struct(1 AS x).x` + return _operand_start(tokens[: k - 1]) + return k + + +def _type_end(tokens: list[tuple[str, str]], start: int) -> int: + """The index after the type name that starts at tokens[start], such as + `int`, `decimal(10, 2)` or `array>`.""" + k = _next_significant(tokens, start) + if k >= len(tokens) or tokens[k][0] != "word": + raise Skip(_UNTRANSLATABLE) + name = tokens[k][1].upper() + k += 1 + j = _next_significant(tokens, k) + # After any other type, `<` is a comparison, as in `a::int < 5`. + if j < len(tokens) and ( + tokens[j][1] == "(" or (tokens[j][1] == "<" and name in ("ARRAY", "MAP", "STRUCT")) + ): + return _matching_paren(tokens, j, _TYPE_NESTING) + 1 + return k + + _NAME = r"[A-Za-z_][A-Za-z0-9_]*" _CREATE_RE = re.compile( rf"\s*CREATE\s+(OR\s+REPLACE\s+)?(?:TABLE|VIEW)\s+(IF\s+NOT\s+EXISTS\s+)?({_NAME})\s*", @@ -1242,17 +1451,22 @@ def __init__(self): except ImportError: raise SystemExit(f"error: PySpark is not installed. Install it with `{install}`.") - if not pyspark.__version__.startswith(SPARK_VERSION + "."): + if version_tuple(pyspark.__version__) not in SUPPORTED_VERSIONS: + supported = ", ".join(map(_version_string, SUPPORTED_VERSIONS)) raise SystemExit( - f"error: this runner needs PySpark {SPARK_VERSION}, found " + f"error: this runner needs PySpark {supported}, found " f"{pyspark.__version__}. Install it with `{install}`." ) - from pyspark.logger import PySparkLogger - - # PySpark logs every failed query to stderr as JSON. The runner reports - # them itself. - for name in ("SQLQueryContextLogger", "DataFrameQueryContextLogger"): - PySparkLogger.getLogger(name).disabled = True + try: + from pyspark.logger import PySparkLogger + except ImportError: + # PySpark 3.5 has no pyspark.logger, and does not log failed queries as JSON. + pass + else: + # PySpark logs every failed query to stderr as JSON. The runner + # reports them itself. + for name in ("SQLQueryContextLogger", "DataFrameQueryContextLogger"): + PySparkLogger.getLogger(name).disabled = True # Removed when the runner exits. self._warehouse = tempfile.TemporaryDirectory(prefix="pyspark-slt-") self._start() @@ -1348,6 +1562,9 @@ def run(self, sql: str): if m: raise SparkError("convert", f"Spark returned a {m.group(1)} value") condition = _condition(e) + if condition == "CANNOT_PARSE_DATATYPE" and "interval" in str(e): + # PySpark 3.5 cannot read a schema with a CalendarIntervalType. + raise SparkError("convert", "Spark returned a CalendarIntervalType value") if condition == "UNRESOLVED_ROUTINE": name = _parameter(e, "routineName") or "?" raise SparkError("function", f"Spark has no function {name}") @@ -1472,17 +1689,19 @@ def run_record(runner: SparkRunner, record: Record) -> Outcome: A difference that the comment expects becomes `xfail`, and a record that passes in spite of the comment becomes `xpass`, so a stale comment is - reported. A skipped record stays skipped. + reported. A skipped record stays skipped. Only a comment whose version + condition matches the runner's Spark version applies. """ outcome = _run_record(runner, record) - if record.xfail_reason is None: + xfail = record.xfail(version_tuple(runner.version)) + if xfail is None: return outcome if outcome.status in EXPECTED_TO_FAIL: - outcome.detail = f"{DIVERGENT[outcome.status]} ({record.xfail_reason})" + outcome.detail = f"{DIVERGENT[outcome.status]} ({xfail.reason})" outcome.status = "xfail" elif outcome.status == "pass": outcome.status = "xpass" - outcome.detail = record.xfail_reason + outcome.detail = xfail.text return outcome @@ -1511,6 +1730,9 @@ def _run_record(runner: SparkRunner, record: Record) -> Outcome: spark_sql = translate_statement(record.sql, runner.time_zone) else: spark_sql = translate(record.sql, runner.time_zone) + if version_tuple(runner.version) < (4, 0): + # Spark 4.0 added the `expr::type` syntax. + spark_sql = rewrite_double_colons(spark_sql) except Skip as e: return Outcome(record, "skip", str(e)) if any("" in line for line in record.expected): @@ -1587,7 +1809,7 @@ def report(outcome: Outcome, spark_version: str) -> str: if outcome.status == "xfail": return f"{location}: expected failure: {outcome.detail}" if outcome.status == "xpass": - return f"{location}: {DIVERGENT['xpass']}, remove `# xfail {LABEL}: {outcome.detail}`" + return f"{location}: {DIVERGENT['xpass']}, remove `{outcome.detail}`" parts = [f"{location}: {DIVERGENT[outcome.status]}", _indent(record.sql.split("\n"))] if outcome.spark_sql and outcome.spark_sql != record.sql: parts += [" Spark SQL:", _indent(outcome.spark_sql.split("\n"))] @@ -1617,7 +1839,8 @@ def discover(paths: list[str]) -> list[Path]: def main(argv: Optional[list[str]] = None) -> int: parser = argparse.ArgumentParser( description=( - f"Run the datafusion-spark .slt files on Spark {SPARK_VERSION} and " + f"Run the datafusion-spark .slt files on Spark " + f"({', '.join(map(_version_string, SUPPORTED_VERSIONS))}) and " "report where Spark disagrees with the expected results." ) ) @@ -1637,12 +1860,15 @@ def main(argv: Optional[list[str]] = None) -> int: files = discover(args.paths) if not files: raise SystemExit("error: no .slt files found") + # Parsed before Spark starts, so a malformed `# xfail spark` comment + # fails the run at once. + parsed = {path: parse_slt(path) for path in files} # Keep the output complete when it is piped to a file and the run dies. sys.stdout.reconfigure(line_buffering=True) runner = SparkRunner() print(f"Running {len(files)} file(s) on Spark {runner.version}\n") - statuses, skip_reasons, stopped = run_files(runner, files, args.verbose) + statuses, skip_reasons, stopped = run_files(runner, parsed, args.verbose) total = sum(statuses.values()) print(f"Spark {runner.version}: {total} records in {len(files)} file(s)") @@ -1668,8 +1894,8 @@ def main(argv: Optional[list[str]] = None) -> int: MAX_RESTARTS = 5 -def run_files(runner, files: list[Path], verbose: bool = False): - """Run every record in `files`, printing each difference. +def run_files(runner, files: dict[Path, list[Record]], verbose: bool = False): + """Run the parsed records of each file, printing each difference. Returns (statuses, skip_reasons, stopped). `stopped` says why the run ended early, or is None. @@ -1693,7 +1919,7 @@ def new_session(path: Path, reason: str) -> Optional[str]: return None broken = False - for path in files: + for path, records in files.items(): stopped = None if broken: stopped = new_session(path, "Spark failed in the previous file") @@ -1705,7 +1931,7 @@ def new_session(path: Path, reason: str) -> Optional[str]: if stopped: return statuses, skip_reasons, stopped broken = False - for record in parse_slt(path): + for record in records: if broken: # A new session would not have this file's views and settings. outcome = Outcome(record, "skip", "Spark failed earlier in the file") diff --git a/crates/sqllogictest/scripts/requirements-spark-3.5.txt b/crates/sqllogictest/scripts/requirements-spark-3.5.txt new file mode 100644 index 00000000..3e437b01 --- /dev/null +++ b/crates/sqllogictest/scripts/requirements-spark-3.5.txt @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +pyspark==3.5.9 diff --git a/crates/sqllogictest/scripts/test_pyspark_slt_runner.py b/crates/sqllogictest/scripts/test_pyspark_slt_runner.py index feaa748d..ac142cfa 100644 --- a/crates/sqllogictest/scripts/test_pyspark_slt_runner.py +++ b/crates/sqllogictest/scripts/test_pyspark_slt_runner.py @@ -95,8 +95,37 @@ def test_xfail(self): "query I\n" "SELECT 4\n" ) - reasons = [record.xfail_reason for record in records] - self.assertEqual(reasons, ["Spark returns NULL", None, "no space after the hash", None]) + reasons = [[x.reason for x in record.xfails] for record in records] + self.assertEqual(reasons, [["Spark returns NULL"], [], ["no space after the hash"], []]) + + def test_xfail_spark_versions(self): + [record] = parse( + "# xfail spark<4.0: Spark 3.5 differs\n" + "# xfail spark >= 4.0, <=4.2: Spark 4.0 to 4.2 differ\n" + "query I\n" + "SELECT 1\n" + ) + reason = lambda version: getattr(record.xfail(r.version_tuple(version)), "reason", None) # noqa: E731 + self.assertEqual(reason("3.5.9"), "Spark 3.5 differs") + self.assertEqual(reason("4.0.1"), "Spark 4.0 to 4.2 differ") + self.assertEqual(reason("4.2.0"), "Spark 4.0 to 4.2 differ") + self.assertIsNone(reason("4.3.0")) + + def test_xfail_that_never_applies(self): + # No supported version matches the first, and the second is shadowed. + for markers in ("# xfail spark>=4.0, <4.2: x\n", "# xfail spark: x\n# xfail spark<4.0: y\n"): + with self.assertRaisesRegex(SystemExit, "never applies"): + parse(f"{markers}query I\nSELECT 1\n") + + def test_first_matching_xfail_applies(self): + [record] = parse("# xfail spark<4.0: old\n# xfail spark: any\nquery I\nSELECT 1\n") + self.assertEqual(record.xfail((3, 5)).reason, "old") + self.assertEqual(record.xfail((4, 2)).reason, "any") + + def test_xfail_bad_version_condition(self): + for marker in ("# xfail spark 3.5: x", "# xfail spark<4: x", "# xfail spark<4.0;>3.4: x"): + with self.assertRaisesRegex(SystemExit, "version condition"): + parse(f"{marker}\nquery I\nSELECT 1\n") class TranslateTest(unittest.TestCase): @@ -104,6 +133,34 @@ def test_spark_parses_double_colon_casts(self): sql = "SELECT abs(-128::TINYINT), 'a'::STRING" self.assertEqual(r.translate(sql), sql) + def test_double_colon_casts_for_spark_3(self): + cases = { + "SELECT 1::int": "SELECT CAST(1 AS int)", + # A sign belongs to a number literal, as in Spark 4. + "SELECT -128::tinyint, 1 -2::int": "SELECT CAST(-128 AS tinyint), 1 -CAST(2 AS int)", + "SELECT -(1)::double": "SELECT -CAST((1) AS double)", + "SELECT f(a, b)::string FROM t": "SELECT CAST(f(a, b) AS string) FROM t", + "SELECT t.a::bigint": "SELECT CAST(t.a AS bigint)", + "SELECT x::int::string": "SELECT CAST(CAST(x AS int) AS string)", + "SELECT DATE '2020-01-01'::string": "SELECT CAST(DATE '2020-01-01' AS string)", + "SELECT 1.5::decimal(10, 2) + 2": "SELECT CAST(1.5 AS decimal(10, 2)) + 2", + "SELECT a[1]::int, (1+2)::int": "SELECT CAST(a[1] AS int), CAST((1+2) AS int)", + "SELECT NULL::array>": "SELECT CAST(NULL AS array>)", + "SELECT (SELECT 1)::int": "SELECT CAST((SELECT 1) AS int)", + "SELECT 'a::b'::FLOAT8": "SELECT CAST('a::b' AS DOUBLE)", + # `exists` and `any` are functions here, not keywords. + "SELECT exists(arr, x -> x > 1)::string FROM t": "SELECT CAST(exists(arr, x -> x > 1) AS string) FROM t", + "SELECT any(a > 1)::string FROM t": "SELECT CAST(any(a > 1) AS string) FROM t", + # A field of a call or a subscript + "SELECT struct(1 AS x).x::string": "SELECT CAST(struct(1 AS x).x AS string)", + "SELECT map(1, struct(2 AS y))[1].y::string": "SELECT CAST(map(1, struct(2 AS y))[1].y AS string)", + # After a type other than ARRAY, MAP or STRUCT, `<` is a comparison. + "SELECT a::int < 5 AND b > 3 FROM t": "SELECT CAST(a AS int) < 5 AND b > 3 FROM t", + "SELECT a::int=4.0: fixed\nquery I\nSELECT a\n----\n1\n", result) + self.assertIn("remove `# xfail spark>=4.0: fixed`", r.report(outcome, "4.2.0")) + def test_xfail_skipped_record_stays_skipped(self): parse_error = r.SparkError("parse", "Spark cannot parse it (PARSE_SYNTAX_ERROR)") outcome = self.outcome("# xfail spark: x\nquery ?\nSELECT [1]\n----\n[1]\n", parse_error) @@ -617,7 +701,7 @@ def run_files(self, *contents): paths[-1].write_text("".join(self.QUERY.format(q) for q in queries)) runner = DyingRunner() with contextlib.redirect_stdout(io.StringIO()) as out: - result = r.run_files(runner, paths) + result = r.run_files(runner, {p: r.parse_slt(p) for p in paths}) return runner, result, out.getvalue() def test_new_session_after_the_jvm_dies(self): @@ -649,7 +733,7 @@ def test_new_session_when_reset_fails(self): path = Path(d, "0.slt") path.write_text(self.QUERY.format(1)) with contextlib.redirect_stdout(io.StringIO()) as out: - statuses, _, stopped = r.run_files(runner, [path]) + statuses, _, stopped = r.run_files(runner, {path: r.parse_slt(path)}) self.assertIsNone(stopped) self.assertEqual((runner.restarts, statuses["pass"]), (1, 1)) self.assertIn("Spark failed (An error occurred while calling o49.listTables.)", out.getvalue()) diff --git a/crates/sqllogictest/test_files/json/to_json.slt b/crates/sqllogictest/test_files/json/to_json.slt index f011673f..99da5b66 100644 --- a/crates/sqllogictest/test_files/json/to_json.slt +++ b/crates/sqllogictest/test_files/json/to_json.slt @@ -302,6 +302,7 @@ RESET datafusion.execution.time_zone; statement ok SET datafusion.execution.time_zone = 'America/Los_Angeles'; +# xfail spark<4.0: intended: Spark 3.5 drops the seconds of the offset, and datafusion-spark follows Spark 4 query T SELECT to_json(named_struct('t', arrow_cast('1800-01-01T07:52:58Z', 'Timestamp(Microsecond, Some("America/Los_Angeles"))'))); ---- diff --git a/crates/sqllogictest/test_files/math/abs.slt b/crates/sqllogictest/test_files/math/abs.slt index f9cc3ecc..481b9276 100644 --- a/crates/sqllogictest/test_files/math/abs.slt +++ b/crates/sqllogictest/test_files/math/abs.slt @@ -74,7 +74,7 @@ SELECT abs(('-99999999.99')::DECIMAL(10, 2)), abs(0::DECIMAL(10, 2)), abs(NULL:: ---- 99999999.99 0.00 NULL -# xfail spark: intended: Spark's abs rounds DECIMAL to 34 digits and overflows, DataFusion returns the exact value +# xfail spark>=4.0: intended: Spark's abs rounds DECIMAL to 34 digits and overflows, DataFusion returns the exact value query RRR SELECT abs(('-9999999999999999999999999999.9999999999')::DECIMAL(38, 10)), abs(0::DECIMAL(38, 10)), abs(NULL::DECIMAL(38, 10)); ---- @@ -196,7 +196,7 @@ SELECT abs(a) FROM (VALUES (('-99999999.99')::DECIMAL(10, 2)), (0::DECIMAL(10, 2 0.00 NULL -# xfail spark: intended: Spark's abs rounds DECIMAL to 34 digits and overflows, DataFusion returns the exact value +# xfail spark>=4.0: intended: Spark's abs rounds DECIMAL to 34 digits and overflows, DataFusion returns the exact value query R SELECT abs(a) FROM (VALUES (('-9999999999999999999999999999.9999999999')::DECIMAL(38, 10)), (0::DECIMAL(38, 10)), (NULL::DECIMAL(38, 10))) AS t(a); ---- diff --git a/crates/sqllogictest/test_files/math/greatest.slt b/crates/sqllogictest/test_files/math/greatest.slt index e4410a48..8d25eff7 100644 --- a/crates/sqllogictest/test_files/math/greatest.slt +++ b/crates/sqllogictest/test_files/math/greatest.slt @@ -93,11 +93,13 @@ SELECT arrow_typeof(greatest(1::decimal(38,10), 1::decimal(38,0))); Decimal128(38, 0) # The fractional digits that do not fit are rounded away +# xfail spark<4.0: intended: Spark 3.5 picks DECIMAL(38, 10) as the wider type and keeps the fraction, and datafusion-spark follows Spark 4 query R SELECT greatest(1.5::decimal(38,10), 1::decimal(38,0)); ---- 2 +# xfail spark<4.0: intended: Spark 3.5 picks DECIMAL(38, 10) as the wider type and returns NULL for the value that does not fit, and datafusion-spark follows Spark 4 query R SELECT greatest(1::decimal(38,10), '12345678901234567890123456789'::decimal(38,0)); ---- diff --git a/crates/sqllogictest/test_files/string/encode.slt b/crates/sqllogictest/test_files/string/encode.slt index b15f0211..6527fd82 100644 --- a/crates/sqllogictest/test_files/string/encode.slt +++ b/crates/sqllogictest/test_files/string/encode.slt @@ -39,10 +39,12 @@ SELECT encode('Spark SQL'::string, 'utf-16'::string); feff0053007000610072006b002000530051004c # UTF-32BE encoding (alias rejected in Spark 4 strict mode) +# xfail spark<4.0: intended: Spark 3.5 accepts any Java charset name, which Spark 4 allows only with spark.sql.legacy.javaCharsets statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('A'::string, 'utf-32be'::string); # UTF-32LE encoding (alias rejected in Spark 4 strict mode) +# xfail spark<4.0: intended: Spark 3.5 accepts any Java charset name, which Spark 4 allows only with spark.sql.legacy.javaCharsets statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('AB'::string, 'utf-32le'::string); @@ -59,10 +61,12 @@ SELECT encode('hello'::string, 'Utf-8'::string); 68656c6c6f # US-ASCII: unmappable characters fail in Spark 4 strict mode +# xfail spark<4.0: intended: Spark 3.5 replaces unmappable characters with `?`, which Spark 4 does only with spark.sql.legacy.codingErrorAction statement error MALFORMED_CHARACTER_CODING SELECT encode('é'::string, 'us-ascii'::string); # ISO-8859-1: code points above U+00FF fail in Spark 4 strict mode (Ā is U+0100) +# xfail spark<4.0: intended: Spark 3.5 replaces unmappable characters with `?`, which Spark 4 does only with spark.sql.legacy.codingErrorAction statement error MALFORMED_CHARACTER_CODING SELECT encode('Ā'::string, 'iso-8859-1'::string); @@ -87,9 +91,11 @@ SELECT encode('😀'::string, 'utf-16'::string); ---- feffd83dde00 +# xfail spark<4.0: intended: Spark 3.5 accepts any Java charset name, which Spark 4 allows only with spark.sql.legacy.javaCharsets statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('😀'::string, 'utf-32be'::string); +# xfail spark<4.0: intended: Spark 3.5 accepts any Java charset name, which Spark 4 allows only with spark.sql.legacy.javaCharsets statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('😀'::string, 'utf-32le'::string); @@ -129,6 +135,7 @@ SELECT encode(X'FEFF00610073', 'utf-8'::string); efbfbdefbfbd00610073 # Strict mode: binary input with invalid UTF-8 byte 0xFF becomes U+FFFD, which cannot be encoded in US-ASCII +# xfail spark<4.0: intended: Spark 3.5 replaces unmappable characters with `?`, which Spark 4 does only with spark.sql.legacy.codingErrorAction query error \[MALFORMED_CHARACTER_CODING\] Invalid value found when performing `encode` with us-ascii SQLSTATE: 22000 SELECT encode(X'ff', 'us-ascii'::string);