From 2d22fc560e007ac81e68522f7fb0e1672c5904a4 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 06:04:13 -0700 Subject: [PATCH 1/3] feat: add datafusion-spark-cli, a SQL shell with the Spark functions apache/datafusion#27337 removed `datafusion-cli --spark`, so there is no longer a way to try the Spark-compatible functions interactively. Add a `datafusion-spark-cli` binary that uses the `datafusion-cli` library for the shell, with the Spark SQL dialect and functions always enabled and ANSI mode on by default, as in Spark 4. The crate is not published, and is not a default workspace member, so a plain `cargo build` or `cargo test` does not build its dependencies. A new workflow builds it and runs a smoke test. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/cli.yml | 35 + AGENTS.md | 2 + Cargo.lock | 1653 +++++++++++++++++++++- Cargo.toml | 5 +- README.md | 2 + crates/cli/Cargo.toml | 31 + crates/cli/src/main.rs | 249 ++++ crates/spark/README.md | 13 +- crates/sqllogictest/test_files/README.md | 7 +- 9 files changed, 1924 insertions(+), 73 deletions(-) create mode 100644 .github/workflows/cli.yml create mode 100644 crates/cli/Cargo.toml create mode 100644 crates/cli/src/main.rs diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml new file mode 100644 index 00000000..892e4fba --- /dev/null +++ b/.github/workflows/cli.yml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +name: CLI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + smoke-test: + name: datafusion-spark-cli smoke test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: rustup toolchain install stable --profile minimal --no-self-update && rustup default stable + - uses: Swatinem/rust-cache@v2 + - run: cargo build -p datafusion-spark-cli + # A Spark function, and ANSI mode on by default and off with `--ansi false`. + - name: Run queries + run: | + set -euo pipefail + cli=target/debug/datafusion-spark-cli + $cli -q --format csv -c "SELECT sha2('Spark', 256) AS h" \ + | grep -x '529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b' + if $cli -q -c "SELECT abs(cast(-2147483648 AS int))"; then + echo "expected an ARITHMETIC_OVERFLOW error with ANSI mode on" && exit 1 + fi + $cli -q --ansi false --format csv -c "SELECT abs(cast(-2147483648 AS int)) AS a" \ + | grep -x -- '-2147483648' diff --git a/AGENTS.md b/AGENTS.md index 48ba53c5..06a5a868 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,6 +10,8 @@ DataFusion expressions that emulate Apache Spark's behavior. - `crates/spark` – the `datafusion-spark` crate - `crates/sqllogictest` – `.slt` tests for the Spark functions +- `crates/cli` – `datafusion-spark-cli`, a SQL shell for trying the functions (not a default + workspace member, so build it with `-p datafusion-spark-cli`) Before opening a PR, run `cargo fmt --all`, `cargo clippy --all-targets --all-features -- -D warnings`, `cargo test -p datafusion-spark` and `cargo test -p datafusion-spark-sqllogictest`. diff --git a/Cargo.lock b/Cargo.lock index 461a5116..1d6d8020 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,7 +112,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -123,7 +123,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -135,6 +135,15 @@ dependencies = [ "object", ] +[[package]] +name = "arc-swap" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b5d51209f6de83d37a0b33d320acb8da88823e42a94c075140708f032a5" +dependencies = [ + "rustversion", +] + [[package]] name = "arrayvec" version = "0.7.8" @@ -436,18 +445,418 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-config" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16ec023cac0d0c0432e606a2d14f04eb70d09c71a739a406141e7be6b923ab95" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http", + "sha1 0.10.7", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576e8da35751dc85409ec1cd15999f692c277a1fbe9336780fe6f15e49dd637d" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bfbe32ac36b3c428696ce315464181bc4029d689fd58e3f2a4e58c252242f52" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "aws-runtime" +version = "1.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de8930c11353c72ae54aa24dc94d692df74d59814839cbaf8069d63dbcb4c519" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http", + "http-body", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.121.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d414ba8f00c2b4b506893ded67827e63be88b5e6cce71b7c0ac8f96ce09bc5" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.123.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db26187b124bcc637641f7963d5200c5ec2409ea5b7730dd7e014ef4e0ab6f19" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "fastrand", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.126.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c667da7839295125cfe23280f798c07ba694a9fed191af804edb4bda09ac7f10" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a4393e6addc3380632887b0998d06ad5f099ab5c478be0a3eeedecf8604161" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http", + "percent-encoding", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de681a9df7ff79823b759ee17061076ec049cd61a8fa28ece4293d89c9f6964e" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.64.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19af609dd5edf8b056cd0bb001c706c8b3b1458055328b334ec13d7f46f6aaa8" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859a81364f8185349c584fb41cd8765a400cf982f447adada590cf008a9f3b85" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2", + "http", + "hyper", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.65.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45f9fc77ffe92c92c341165a01a565bd30c39ecf5aa8c0719b41719009604f9" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a416257d7b1b0f32a517a470a8c1dd354d8c4aa697ccab81a11a1e6cfb7418e3" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "tracing", +] + +[[package]] +name = "aws-smithy-query" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165f6265342a313ad6f7a1ac17a0cdba30c522039d3f41d8d6871ce8a2b34ecd" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bc688673e282f42d3152dcd9cacc1a9a75734d7cd39d02e58989276fc750f3c" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "bytes", + "fastrand", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0467a2414e84b1b01e37452896fb3562b93926e81c619eba6ecf27ce9e682a0e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce237f890a2283c9b328955c3b0918d2d5bf66a1cb6ceccd36a0047246fbc6e6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "aws-smithy-schema" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7ef30d39e131651359b60a1c560de9a9502a4c3754172ab2d6e8734fe21bcba" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http", +] + +[[package]] +name = "aws-smithy-types" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aff4bf014c512840b2bba80a20417b723cbd6a0a65c02e2d81f50b4268aedfc7" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "http", + "http-body", + "http-body-util", + "itoa", + "pin-project-lite", + "ryu", + "serde", + "time", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.64.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "590fdc1a70546b2f7f2061c42eaf173fb6f26d935330d069edc04dbd8f01d924" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0d970d3947bb064d2ae1d823efa186c1dbde88f88afa8f6549946e5d8deee" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "base64" version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "bigdecimal" version = "0.4.11" @@ -486,7 +895,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.3.1", ] [[package]] @@ -540,6 +949,16 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bzip2" version = "0.6.1" @@ -586,7 +1005,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -598,6 +1017,7 @@ checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", + "serde", "windows-link", ] @@ -678,6 +1098,24 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" +[[package]] +name = "clipboard-win" +version = "5.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052af7878466feccda12ba7a6a67d019d712f296809559b47c12c8cc196d3a9a" +dependencies = [ + "error-code", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cmov" version = "0.5.4" @@ -690,6 +1128,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "comfy-table" version = "8.0.1" @@ -730,7 +1178,7 @@ dependencies = [ "encode_unicode", "libc", "unicode-width 0.2.3", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -765,12 +1213,31 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -795,6 +1262,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + [[package]] name = "crc32fast" version = "1.5.5" @@ -936,7 +1413,7 @@ dependencies = [ [[package]] name = "datafusion" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-schema", @@ -989,7 +1466,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-trait", @@ -1013,7 +1490,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-trait", @@ -1034,21 +1511,50 @@ dependencies = [ ] [[package]] -name = "datafusion-common" +name = "datafusion-cli" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", - "arrow-ipc", - "arrow-schema", + "async-trait", + "aws-config", + "aws-credential-types", "chrono", - "foldhash 0.2.0", - "half", - "hashbrown 0.17.1", - "indexmap", - "itertools 0.15.0", - "libc", - "log", + "clap", + "datafusion", + "datafusion-common", + "datafusion-spark 56.0.0 (git+https://github.com/apache/datafusion?branch=branch-56)", + "dirs", + "env_logger", + "futures", + "log", + "mimalloc", + "object_store", + "parking_lot 0.13.0", + "parquet", + "regex", + "rustyline", + "tokio", + "url", +] + +[[package]] +name = "datafusion-common" +version = "56.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "hex", + "indexmap", + "itertools 0.15.0", + "libc", + "log", "num-traits", "object_store", "parquet", @@ -1062,7 +1568,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "futures", "log", @@ -1072,7 +1578,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-compression", @@ -1108,7 +1614,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-ipc", @@ -1131,7 +1637,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-avro", @@ -1154,7 +1660,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-trait", @@ -1176,7 +1682,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-trait", @@ -1198,7 +1704,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-schema", @@ -1229,12 +1735,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" [[package]] name = "datafusion-execution" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-buffer", @@ -1248,6 +1754,7 @@ dependencies = [ "log", "object_store", "parking_lot 0.13.0", + "parquet", "pin-project-lite", "rand 0.9.5", "tempfile", @@ -1259,7 +1766,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-schema", @@ -1282,7 +1789,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "datafusion-common", @@ -1293,7 +1800,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-buffer", @@ -1325,7 +1832,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-select", @@ -1347,7 +1854,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "datafusion-common", @@ -1358,7 +1865,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-ord", @@ -1382,7 +1889,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-trait", @@ -1397,7 +1904,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "datafusion-common", @@ -1413,7 +1920,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1422,7 +1929,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "datafusion-doc", "quote", @@ -1432,7 +1939,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "chrono", @@ -1451,7 +1958,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-schema", @@ -1474,7 +1981,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "datafusion-common", @@ -1488,7 +1995,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "chrono", @@ -1505,7 +2012,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "datafusion-common", @@ -1525,7 +2032,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-data", @@ -1562,7 +2069,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "datafusion-common", @@ -1577,7 +2084,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow-schema", "async-trait", @@ -1615,7 +2122,7 @@ dependencies = [ "percent-encoding", "rand 0.9.5", "serde_json", - "sha1", + "sha1 0.11.0", "sha2", "tokio", "twox-hash", @@ -1625,7 +2132,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "base64", @@ -1646,12 +2153,24 @@ dependencies = [ "percent-encoding", "rand 0.9.5", "serde_json", - "sha1", + "sha1 0.11.0", "sha2", "twox-hash", "url", ] +[[package]] +name = "datafusion-spark-cli" +version = "56.0.0" +dependencies = [ + "clap", + "datafusion", + "datafusion-cli", + "datafusion-spark 56.0.0", + "env_logger", + "tokio", +] + [[package]] name = "datafusion-spark-sqllogictest" version = "56.0.0" @@ -1667,7 +2186,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-schema", @@ -1687,7 +2206,7 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-trait", @@ -1709,6 +2228,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" + [[package]] name = "digest" version = "0.10.7" @@ -1731,6 +2256,27 @@ dependencies = [ "ctutils", ] +[[package]] +name = "dirs" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d57d423b3c82e89b9a24ca3091fee61f456a26edbd28d26c65906f4bc1dcd8f" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.7" @@ -1742,6 +2288,12 @@ dependencies = [ "syn 3.0.10", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "educe" version = "0.6.0" @@ -1766,6 +2318,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "endian-type" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "869b0adbda23651a9c5c0c3d270aac9fcb52e8622a8f2b17e57802d7791962f2" + [[package]] name = "enum-ordinalize" version = "4.4.2" @@ -1786,6 +2344,29 @@ dependencies = [ "syn 3.0.10", ] +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.3" @@ -1799,9 +2380,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] +[[package]] +name = "error-code" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5343afd4a8365a643ac588dab4cf234a190c7f6c88c9f6dd6ffe00837661b7" + [[package]] name = "escape8259" version = "0.5.4" @@ -1847,6 +2434,12 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -1877,6 +2470,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.34" @@ -1982,8 +2581,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -2005,9 +2606,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -2016,6 +2619,25 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" @@ -2066,6 +2688,24 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "http" version = "1.5.0" @@ -2076,6 +2716,35 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "humantime" version = "2.4.0" @@ -2091,6 +2760,67 @@ dependencies = [ "typenum", ] +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -2242,6 +2972,12 @@ dependencies = [ "web-time", ] +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -2272,6 +3008,90 @@ version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4" +[[package]] +name = "jiff" +version = "0.2.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175dc66334db3b9ab9985bf8b61cfed50dc16d20964bb47f9e479adcae984a7f" +dependencies = [ + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e599b80292e597019b8b3e392de3f4bea2959ddfb8c7dff60e035e9e311a2c1b" +dependencies = [ + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7b1af7547d47354bc438d797e3854601bde8f8d185780c5ba4ab27afa54f1ba" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "jobserver" version = "0.1.35" @@ -2388,6 +3208,24 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + [[package]] name = "libtest-mimic" version = "0.8.2" @@ -2436,6 +3274,12 @@ version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + [[package]] name = "lz4_flex" version = "0.14.0" @@ -2472,15 +3316,44 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] -name = "miniz_oxide" -version = "0.9.1" +name = "mimalloc" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" dependencies = [ - "adler2", + "libmimalloc-sys", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", "simd-adler32", ] +[[package]] +name = "mio" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20521a903f02b0a573cd6840e9d0a5a0cc043d361fe924092cd70f8556cc9c4a" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nix" version = "0.31.3" @@ -2522,6 +3395,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-integer" version = "0.1.47" @@ -2557,17 +3436,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" dependencies = [ "async-trait", + "aws-lc-rs", + "base64", "bytes", "chrono", + "crc-fast", + "form_urlencoded", "futures-channel", "futures-core", "futures-util", "http", + "http-body-util", "humantime", + "hyper", "itertools 0.15.0", + "md-5 0.11.0", "nix", "parking_lot 0.12.5", "percent-encoding", + "quick-xml", + "rand 0.10.3", + "reqwest", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", "thiserror", "tokio", "tracing", @@ -2575,7 +3468,7 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2596,6 +3489,24 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "openssl-probe" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "owo-colors" version = "4.4.0" @@ -2683,6 +3594,7 @@ dependencies = [ "num-integer", "num-traits", "object_store", + "ring", "seq-macro", "simdutf8", "snap", @@ -2793,6 +3705,15 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.6" @@ -2802,6 +3723,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2830,6 +3757,73 @@ dependencies = [ "cc", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.47" @@ -2851,6 +3845,16 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radix_trie" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b4431027dcd37fc2a73ef740b5f233aa805897935b8bce0195e41bbf9a3289a" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.8.8" @@ -2927,6 +3931,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rayon" version = "1.12.0" @@ -2985,6 +3998,16 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" +dependencies = [ + "libredox", + "thiserror", +] + [[package]] name = "regex" version = "1.13.1" @@ -3008,12 +4031,77 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -3033,7 +4121,82 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "102f35f7de7b06d8c61f1cff6592f5253e0dea431c74867aad5dfd70f44663fb" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a0760a887c14823fca5a1c970b01d47c328c696c0c64829d1df50e8f853fd5" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -3042,6 +4205,26 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "rustyline" +version = "19.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e1d6bd81c2d5feafd2419138df9183c45a10a492c4e873b8b0acb8eb2b549e" +dependencies = [ + "bitflags", + "clipboard-win", + "home", + "libc", + "log", + "memchr", + "nix", + "radix_trie", + "unicode-segmentation", + "unicode-width 0.2.3", + "utf8parse", + "windows-sys 0.61.2", +] + [[package]] name = "ryu" version = "1.0.23" @@ -3057,12 +4240,44 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.28" @@ -3118,6 +4333,29 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha1" version = "0.11.0" @@ -3125,7 +4363,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "digest 0.11.3", ] @@ -3136,7 +4374,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "digest 0.11.3", ] @@ -3146,12 +4384,32 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "simd-adler32" version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -3188,6 +4446,22 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "sqllogictest" version = "0.29.1" @@ -3251,7 +4525,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3282,6 +4556,12 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.119" @@ -3304,6 +4584,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.14.0" @@ -3325,7 +4614,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3348,6 +4637,36 @@ dependencies = [ "syn 3.0.10", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -3377,6 +4696,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + [[package]] name = "tokio" version = "1.54.1" @@ -3384,8 +4709,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", + "libc", + "mio", + "parking_lot 0.12.5", "pin-project-lite", + "signal-hook-registry", + "socket2", "tokio-macros", + "windows-sys 0.61.2", ] [[package]] @@ -3399,6 +4730,16 @@ dependencies = [ "syn 3.0.10", ] +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.19" @@ -3420,10 +4761,56 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", "pin-project-lite", + "sync_wrapper", "tokio", + "tower-layer", + "tower-service", ] +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.44" @@ -3455,6 +4842,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "twox-hash" version = "2.1.5" @@ -3500,6 +4893,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -3512,6 +4911,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.1.0" @@ -3550,6 +4955,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -3560,6 +4971,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -3630,6 +5050,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.107" @@ -3650,6 +5083,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "293198dcebfb821155b5092ab94b5a82ff180eddd7241df3bd6e66a4ffd1a585" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -3672,7 +5114,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3740,6 +5182,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -3749,6 +5200,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wit-bindgen" version = "0.57.1" @@ -3761,6 +5276,12 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yoke" version = "0.8.4" @@ -3825,6 +5346,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + [[package]] name = "zerotrie" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index 8428bc24..76f0d58f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,10 @@ [workspace] resolver = "2" -members = ["crates/spark", "crates/sqllogictest"] +members = ["crates/cli", "crates/spark", "crates/sqllogictest"] +# The CLI pulls in datafusion-cli and its object store dependencies, so a plain +# `cargo build` or `cargo test` at the root skips it. `--workspace` includes it. +default-members = ["crates/spark", "crates/sqllogictest"] [workspace.package] authors = ["Apache DataFusion "] diff --git a/README.md b/README.md index 08ecc021..dabfa799 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ for more. - `crates/spark` – the `datafusion-spark` crate - `crates/sqllogictest` – SQL logic tests for the Spark functions +- `crates/cli` – `datafusion-spark-cli`, a SQL shell with the Spark functions registered + (`cargo run -p datafusion-spark-cli`) - `.ai/skills` – agent skills for working on this repository (see `AGENTS.md`) ## Building and testing diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml new file mode 100644 index 00000000..9792126d --- /dev/null +++ b/crates/cli/Cargo.toml @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +[package] +name = "datafusion-spark-cli" +description = "Command line SQL shell for DataFusion with the Spark-compatible functions and SQL dialect" +version = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } +edition = { workspace = true } +rust-version = { workspace = true } +# Not published to crates.io yet. +publish = false + +[lints] +workspace = true + +[[bin]] +name = "datafusion-spark-cli" +path = "src/main.rs" + +[dependencies] +clap = { version = "4.5.60", features = ["derive"] } +datafusion = { workspace = true, features = ["sql", "parquet", "nested_expressions"] } +datafusion-cli = { git = "https://github.com/apache/datafusion", branch = "branch-56" } +datafusion-spark = { path = "../spark", features = ["core"] } +env_logger = "0.11" +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs new file mode 100644 index 00000000..bc8c75e4 --- /dev/null +++ b/crates/cli/src/main.rs @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! `datafusion-spark-cli`: the DataFusion CLI with the Spark-compatible functions +//! registered and the Spark SQL dialect enabled. +//! +//! The shell itself (the REPL, running files and commands, output formats and +//! object stores) comes from the `datafusion-cli` library. + +use std::env; +use std::path::Path; +use std::process::ExitCode; +use std::sync::Arc; + +use clap::{ArgAction, Parser}; +use datafusion::common::config::{ConfigNonZeroUsize, Dialect}; +use datafusion::common::{DataFusionError, Result}; +use datafusion::config::ConfigOptions; +use datafusion::execution::SessionStateBuilder; +use datafusion::execution::runtime_env::RuntimeEnvBuilder; +use datafusion::logical_expr::ExplainFormat; +use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion_cli::catalog::DynamicObjectStoreCatalog; +use datafusion_cli::exec; +use datafusion_cli::object_storage::instrumented::{ + InstrumentedObjectStoreMode, InstrumentedObjectStoreRegistry, +}; +use datafusion_cli::object_storage::{StdinCarriesCommands, is_stdin_location}; +use datafusion_cli::print_format::PrintFormat; +use datafusion_cli::print_options::{MaxRows, PrintOptions}; +use datafusion_spark::SessionStateBuilderSpark; + +#[derive(Debug, Parser)] +#[clap(version, about, long_about = None)] +struct Args { + #[clap( + short = 'c', + long, + num_args = 0.., + help = "Execute the given command string(s), then exit", + value_parser(parse_command) + )] + command: Vec, + + #[clap( + short, + long, + num_args = 0.., + help = "Execute commands from file(s), then exit", + value_parser(parse_valid_file) + )] + file: Vec, + + #[clap( + short = 'r', + long, + num_args = 0.., + help = "Run the provided files on startup, before the REPL", + value_parser(parse_valid_file), + conflicts_with = "file" + )] + rc: Vec, + + #[clap( + short = 'p', + long, + help = "Path to your data, default to current directory", + value_parser(parse_valid_data_dir) + )] + data_path: Option, + + #[clap( + short = 'b', + long, + help = "The batch size of each query, or use DataFusion default", + value_parser(parse_batch_size) + )] + batch_size: Option, + + #[clap( + long, + default_value_t = true, + action = ArgAction::Set, + help = "Evaluate with ANSI mode on, like Spark 4's default \ + `spark.sql.ansi.enabled = true`. Pass `--ansi false` for Spark's legacy behavior" + )] + ansi: bool, + + #[clap(long, value_enum, default_value_t = PrintFormat::Automatic)] + format: PrintFormat, + + #[clap( + short, + long, + help = "Reduce printing other than the results and work quietly" + )] + quiet: bool, + + #[clap( + long, + help = "The max number of rows to display for 'Table' format\n[possible values: numbers(0/10/...), inf(no limit)]", + default_value = "40" + )] + maxrows: MaxRows, + + #[clap(long, help = "Enables console syntax highlighting")] + color: bool, +} + +impl Args { + /// Without `-c` or `-f` the CLI enters the REPL, which reads SQL from stdin. + fn repl_mode(&self) -> bool { + self.command.is_empty() && self.file.is_empty() + } + + /// Whether stdin carries the session's SQL, so it cannot also back a + /// `LOCATION '/dev/stdin'` table. + fn reads_sql_from_stdin(&self) -> bool { + self.repl_mode() || self.file.iter().any(|f| is_stdin_location(f)) + } +} + +#[tokio::main] +async fn main() -> ExitCode { + if let Err(e) = main_inner().await { + println!("Error: {e}"); + return ExitCode::FAILURE; + } + ExitCode::SUCCESS +} + +async fn main_inner() -> Result<()> { + env_logger::init(); + let args = Args::parse(); + + if !args.quiet { + println!( + "DataFusion Spark CLI v{} (ANSI mode {})", + env!("CARGO_PKG_VERSION"), + if args.ansi { "on" } else { "off" } + ); + } + + if let Some(path) = &args.data_path { + env::set_current_dir(Path::new(path))?; + } + + let instrumented_registry = Arc::new( + InstrumentedObjectStoreRegistry::new() + .with_profile_mode(InstrumentedObjectStoreMode::Disabled), + ); + let runtime_env = RuntimeEnvBuilder::new() + .with_object_store_registry(Arc::clone(&instrumented_registry) as _) + .build_arc()?; + let state = SessionStateBuilder::new() + .with_config(session_config(&args)?) + .with_runtime_env(runtime_env) + .with_default_features() + .with_spark_features() + .build(); + + // Query files by their path or URL, such as `SELECT * FROM 'data.parquet'`, as + // `datafusion-cli` does. + let ctx = SessionContext::new_with_state(state).enable_url_table(); + ctx.refresh_catalogs().await?; + ctx.register_catalog_list(Arc::new(DynamicObjectStoreCatalog::new( + ctx.state().catalog_list().clone(), + ctx.state_weak_ref(), + ))); + + let mut print_options = PrintOptions { + format: args.format, + quiet: args.quiet, + maxrows: args.maxrows, + color: args.color, + instrumented_registry, + }; + + if args.repl_mode() { + if !args.rc.is_empty() { + exec::exec_from_files(&ctx, args.rc, &print_options).await?; + } + return exec::exec_from_repl(&ctx, &mut print_options) + .await + .map_err(|e| DataFusionError::External(Box::new(e))); + } + if !args.file.is_empty() { + exec::exec_from_files(&ctx, args.file.clone(), &print_options).await?; + } + if !args.command.is_empty() { + exec::exec_from_commands(&ctx, args.command.clone(), &print_options).await?; + } + Ok(()) +} + +fn session_config(args: &Args) -> Result { + // Read `DATAFUSION_*` environment variables, then apply the command line. + let mut options = ConfigOptions::from_env()?; + options.sql_parser.dialect = Dialect::Spark; + options.execution.enable_ansi_mode = args.ansi; + if let Some(batch_size) = args.batch_size { + options.execution.batch_size = ConfigNonZeroUsize::try_new(batch_size)?; + } + // Defaults that `datafusion-cli` also uses, unless set in the environment. + if env::var_os("DATAFUSION_EXPLAIN_FORMAT").is_none() { + options.explain.format = ExplainFormat::Tree; + } + if env::var_os("DATAFUSION_FORMAT_NULL").is_none() { + options.format.null = String::from("NULL"); + } + + let mut config = SessionConfig::from(options).with_information_schema(true); + if args.reads_sql_from_stdin() { + config = config.with_extension(Arc::new(StdinCarriesCommands)); + } + Ok(config) +} + +fn parse_valid_file(path: &str) -> Result { + if Path::new(path).is_file() { + Ok(path.to_string()) + } else { + Err(format!("Invalid file '{path}'")) + } +} + +fn parse_valid_data_dir(dir: &str) -> Result { + if Path::new(dir).is_dir() { + Ok(dir.to_string()) + } else { + Err(format!("Invalid data directory '{dir}'")) + } +} + +fn parse_batch_size(size: &str) -> Result { + match size.parse::() { + Ok(size) if size > 0 => Ok(size), + _ => Err(format!("Invalid batch size '{size}'")), + } +} + +fn parse_command(command: &str) -> Result { + if command.is_empty() { + Err("-c flag expects only non empty commands".to_string()) + } else { + Ok(command.to_string()) + } +} diff --git a/crates/spark/README.md b/crates/spark/README.md index f0a24128..914d5ed5 100644 --- a/crates/spark/README.md +++ b/crates/spark/README.md @@ -66,17 +66,18 @@ use datafusion_spark::expr_fn::sha2; let expr = sha2(col("my_data"), lit(256)); ``` -## Using the DataFusion CLI +## Trying the functions in a SQL shell -Pass `--spark` to `datafusion-cli` to enable the Spark SQL dialect and register -the Spark-compatible scalar, aggregate, and window functions for the session: +`datafusion-spark-cli` is the DataFusion CLI with the Spark SQL dialect enabled +and the Spark-compatible scalar, aggregate, and window functions registered. It +lives in [`crates/cli`](../cli) in this repository and is not published yet: ```shell -datafusion-cli --spark +cargo run -p datafusion-spark-cli ``` -The option is disabled by default because Spark-compatible functions can -override DataFusion functions with the same name. +The Spark functions override DataFusion functions with the same name. ANSI mode +is on by default, as in Spark 4. Pass `--ansi false` for Spark's legacy behavior. ## Things to know diff --git a/crates/sqllogictest/test_files/README.md b/crates/sqllogictest/test_files/README.md index f06180e1..f1684c67 100644 --- a/crates/sqllogictest/test_files/README.md +++ b/crates/sqllogictest/test_files/README.md @@ -39,9 +39,10 @@ statement ok set datafusion.execution.enable_ansi_mode = false; ``` -Run `datafusion-cli --spark` to explore Spark-compatible functions -interactively. This enables the Spark SQL dialect and registers the -Spark-compatible function library for the CLI session. +Run `cargo run -p datafusion-spark-cli` to explore Spark-compatible functions +interactively. The CLI enables the Spark SQL dialect and registers the +Spark-compatible function library. Unlike these test files, it starts with +ANSI mode on, as Spark 4 does. Pass `--ansi false` to turn it off. ### Finding Test Cases From f0cf6de160bd2df74d3109fd640996202f7d6183 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 19 Nov 2026 05:51:04 -0700 Subject: [PATCH 2/3] fix(cli): resolve -f and -r paths before changing to the data path With -p the CLI changed directory before opening the -f and -r files, so a relative path ran the file of that name in the data directory, or panicked if there was none. Make those paths absolute first, open the files ourselves so a failure is an error rather than a panic, and accept -f /dev/stdin. CI now also runs the REPL from stdin with a Spark-only type, checks the ARITHMETIC_OVERFLOW message, and runs -f and -r with a data path. The READMEs show that --ansi false goes after -- with cargo run. --- .github/workflows/cli.yml | 20 ++++++++++- crates/cli/src/main.rs | 46 +++++++++++++++++++++--- crates/spark/README.md | 7 +++- crates/sqllogictest/test_files/README.md | 3 +- 4 files changed, 68 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 892e4fba..074770bf 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -28,8 +28,26 @@ jobs: cli=target/debug/datafusion-spark-cli $cli -q --format csv -c "SELECT sha2('Spark', 256) AS h" \ | grep -x '529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b' - if $cli -q -c "SELECT abs(cast(-2147483648 AS int))"; then + if out=$($cli -q -c "SELECT abs(cast(-2147483648 AS int))" 2>&1); then echo "expected an ARITHMETIC_OVERFLOW error with ANSI mode on" && exit 1 fi + grep -F '[ARITHMETIC_OVERFLOW]' <<< "$out" $cli -q --ansi false --format csv -c "SELECT abs(cast(-2147483648 AS int)) AS a" \ | grep -x -- '-2147483648' + # SQL read from stdin by the REPL. `LONG` is a type in the Spark dialect only. + - name: Run the REPL from stdin + run: | + set -euo pipefail + echo "SELECT CAST(1 AS LONG) AS l;" \ + | target/debug/datafusion-spark-cli -q --format csv | grep -x '1' + # `-f` and `-r` paths are relative to the working directory, not to `-p`. + - name: Run files with a data path + run: | + set -euo pipefail + cli=$PWD/target/debug/datafusion-spark-cli + dir=$(mktemp -d) && cd "$dir" && mkdir data + echo "SELECT 'cwd' AS s;" > q.sql + echo "SELECT 'data' AS s;" > data/q.sql + $cli -q --format csv -p data -f q.sql | grep -x 'cwd' + echo "" | $cli -q --format csv -p data -r q.sql | grep -x 'cwd' + echo "SELECT 'stdin' AS s;" | $cli -q --format csv -p data -f /dev/stdin | grep -x 'stdin' diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index bc8c75e4..e2f0772f 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -9,7 +9,9 @@ //! object stores) comes from the `datafusion-cli` library. use std::env; -use std::path::Path; +use std::fs::File; +use std::io::BufReader; +use std::path::{Path, absolute}; use std::process::ExitCode; use std::sync::Arc; @@ -132,7 +134,7 @@ async fn main() -> ExitCode { async fn main_inner() -> Result<()> { env_logger::init(); - let args = Args::parse(); + let mut args = Args::parse(); if !args.quiet { println!( @@ -143,6 +145,10 @@ async fn main_inner() -> Result<()> { } if let Some(path) = &args.data_path { + // `-f` and `-r` paths are relative to where the CLI was started, not to + // the data directory. + args.file = absolute_paths(&args.file)?; + args.rc = absolute_paths(&args.rc)?; env::set_current_dir(Path::new(path))?; } @@ -179,14 +185,14 @@ async fn main_inner() -> Result<()> { if args.repl_mode() { if !args.rc.is_empty() { - exec::exec_from_files(&ctx, args.rc, &print_options).await?; + exec_files(&ctx, &args.rc, &print_options).await?; } return exec::exec_from_repl(&ctx, &mut print_options) .await .map_err(|e| DataFusionError::External(Box::new(e))); } if !args.file.is_empty() { - exec::exec_from_files(&ctx, args.file.clone(), &print_options).await?; + exec_files(&ctx, &args.file, &print_options).await?; } if !args.command.is_empty() { exec::exec_from_commands(&ctx, args.command.clone(), &print_options).await?; @@ -194,6 +200,35 @@ async fn main_inner() -> Result<()> { Ok(()) } +/// Makes each path absolute, leaving the names of stdin as they are. +fn absolute_paths(paths: &[String]) -> Result> { + paths + .iter() + .map(|path| { + if is_stdin_location(path) { + return Ok(path.clone()); + } + Ok(absolute(path)?.to_string_lossy().into_owned()) + }) + .collect() +} + +/// Runs the SQL in each file in turn. Unlike `exec::exec_from_files`, which panics, +/// a file that cannot be opened is an error. +async fn exec_files( + ctx: &SessionContext, + files: &[String], + print_options: &PrintOptions, +) -> Result<()> { + for path in files { + let file = File::open(path).map_err(|e| { + DataFusionError::Execution(format!("Cannot open file '{path}': {e}")) + })?; + exec::exec_from_lines(ctx, &mut BufReader::new(file), print_options).await?; + } + Ok(()) +} + fn session_config(args: &Args) -> Result { // Read `DATAFUSION_*` environment variables, then apply the command line. let mut options = ConfigOptions::from_env()?; @@ -218,7 +253,8 @@ fn session_config(args: &Args) -> Result { } fn parse_valid_file(path: &str) -> Result { - if Path::new(path).is_file() { + // stdin is a pipe or a terminal, not a regular file. + if is_stdin_location(path) || Path::new(path).is_file() { Ok(path.to_string()) } else { Err(format!("Invalid file '{path}'")) diff --git a/crates/spark/README.md b/crates/spark/README.md index be2f195f..592f4c2d 100644 --- a/crates/spark/README.md +++ b/crates/spark/README.md @@ -84,7 +84,12 @@ cargo run -p datafusion-spark-cli ``` The Spark functions override DataFusion functions with the same name. ANSI mode -is on by default, as in Spark 4. Pass `--ansi false` for Spark's legacy behavior. +is on by default, as in Spark 4. Pass `--ansi false` for Spark's legacy behavior, +after `--` so that `cargo` passes it to the CLI: + +```shell +cargo run -p datafusion-spark-cli -- --ansi false +``` ## Things to know diff --git a/crates/sqllogictest/test_files/README.md b/crates/sqllogictest/test_files/README.md index 305932b1..1b1e5a0c 100644 --- a/crates/sqllogictest/test_files/README.md +++ b/crates/sqllogictest/test_files/README.md @@ -42,7 +42,8 @@ set datafusion.execution.enable_ansi_mode = false; Run `cargo run -p datafusion-spark-cli` to explore Spark-compatible functions interactively. The CLI enables the Spark SQL dialect and registers the Spark-compatible function library. Unlike these test files, it starts with -ANSI mode on, as Spark 4 does. Pass `--ansi false` to turn it off. +ANSI mode on, as Spark 4 does. To turn it off, pass `--ansi false` after `--`, +so that `cargo` passes it to the CLI: `cargo run -p datafusion-spark-cli -- --ansi false`. ### Finding Test Cases From 224e924342254f36b3e1bae7ddfd9c2f593ef710 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 20 Nov 2026 06:18:58 -0700 Subject: [PATCH 3/3] fix(cli): open every script before running any of them A file that cannot be opened now fails before an earlier -f or -r script has run. Opening the files before changing to the -p data directory also replaces the conversion to absolute paths, which went through to_string_lossy. --- .github/workflows/cli.yml | 25 ++++++++++++++++++++- crates/cli/src/main.rs | 46 ++++++++++++++++++--------------------- 2 files changed, 45 insertions(+), 26 deletions(-) diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 074770bf..8bed6bee 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -40,7 +40,8 @@ jobs: set -euo pipefail echo "SELECT CAST(1 AS LONG) AS l;" \ | target/debug/datafusion-spark-cli -q --format csv | grep -x '1' - # `-f` and `-r` paths are relative to the working directory, not to `-p`. + # `-f` and `-r` paths are relative to the working directory, not to `-p`, while + # paths in the SQL are relative to `-p`. - name: Run files with a data path run: | set -euo pipefail @@ -51,3 +52,25 @@ jobs: $cli -q --format csv -p data -f q.sql | grep -x 'cwd' echo "" | $cli -q --format csv -p data -r q.sql | grep -x 'cwd' echo "SELECT 'stdin' AS s;" | $cli -q --format csv -p data -f /dev/stdin | grep -x 'stdin' + printf 's\ncwd_csv\n' > t.csv + printf 's\ndata_csv\n' > data/t.csv + echo "SELECT s FROM 't.csv';" > csv.sql + echo "SELECT 'wrong script' AS s;" > data/csv.sql + out=$($cli -q --format csv -p data -f csv.sql) + grep -x 'data_csv' <<< "$out" + if grep -e 'cwd_csv' -e 'wrong script' <<< "$out"; then exit 1; fi + # Every file is opened before any runs, so a file that cannot be opened fails + # before an earlier file has written anything. + - name: Fail before running when a file cannot be opened + run: | + set -euo pipefail + cli=$PWD/target/debug/datafusion-spark-cli + dir=$(mktemp -d) && cd "$dir" + echo "COPY (SELECT 1 AS a) TO 'out.csv';" > first.sql + echo "SELECT 2 AS b;" > second.sql + chmod 000 second.sql + if out=$($cli -q -f first.sql second.sql 2>&1); then + echo "expected an error for the unreadable file" && exit 1 + fi + grep -F "Cannot open file 'second.sql'" <<< "$out" + test ! -e out.csv diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index e2f0772f..d34f2e16 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -11,7 +11,7 @@ use std::env; use std::fs::File; use std::io::BufReader; -use std::path::{Path, absolute}; +use std::path::Path; use std::process::ExitCode; use std::sync::Arc; @@ -134,7 +134,7 @@ async fn main() -> ExitCode { async fn main_inner() -> Result<()> { env_logger::init(); - let mut args = Args::parse(); + let args = Args::parse(); if !args.quiet { println!( @@ -144,11 +144,16 @@ async fn main_inner() -> Result<()> { ); } + // Open every script before running any of them, so a file that cannot be opened + // fails before an earlier script has run. Opening them before changing to the + // data directory also keeps `-f` and `-r` paths relative to where the CLI was + // started. + let scripts = open_files(if args.repl_mode() { + &args.rc + } else { + &args.file + })?; if let Some(path) = &args.data_path { - // `-f` and `-r` paths are relative to where the CLI was started, not to - // the data directory. - args.file = absolute_paths(&args.file)?; - args.rc = absolute_paths(&args.rc)?; env::set_current_dir(Path::new(path))?; } @@ -183,47 +188,38 @@ async fn main_inner() -> Result<()> { instrumented_registry, }; + exec_files(&ctx, scripts, &print_options).await?; if args.repl_mode() { - if !args.rc.is_empty() { - exec_files(&ctx, &args.rc, &print_options).await?; - } return exec::exec_from_repl(&ctx, &mut print_options) .await .map_err(|e| DataFusionError::External(Box::new(e))); } - if !args.file.is_empty() { - exec_files(&ctx, &args.file, &print_options).await?; - } if !args.command.is_empty() { exec::exec_from_commands(&ctx, args.command.clone(), &print_options).await?; } Ok(()) } -/// Makes each path absolute, leaving the names of stdin as they are. -fn absolute_paths(paths: &[String]) -> Result> { +/// Opens each file. Unlike `exec::exec_from_files`, which panics, a file that cannot +/// be opened is an error. +fn open_files(paths: &[String]) -> Result> { paths .iter() .map(|path| { - if is_stdin_location(path) { - return Ok(path.clone()); - } - Ok(absolute(path)?.to_string_lossy().into_owned()) + File::open(path).map_err(|e| { + DataFusionError::Execution(format!("Cannot open file '{path}': {e}")) + }) }) .collect() } -/// Runs the SQL in each file in turn. Unlike `exec::exec_from_files`, which panics, -/// a file that cannot be opened is an error. +/// Runs the SQL in each file in turn. async fn exec_files( ctx: &SessionContext, - files: &[String], + files: Vec, print_options: &PrintOptions, ) -> Result<()> { - for path in files { - let file = File::open(path).map_err(|e| { - DataFusionError::Execution(format!("Cannot open file '{path}': {e}")) - })?; + for file in files { exec::exec_from_lines(ctx, &mut BufReader::new(file), print_options).await?; } Ok(())