diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 7f017ad7..5afc7c0d 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -226,7 +226,9 @@ jobs: - name: Measure Python coverage with offline test suite run: | - cd python && PYTHONPATH=$(pwd) python -m coverage run --rcfile=.coveragerc -m pytest tests/test_dropin_offline.py + cd python + PYTHONPATH=$(pwd) python -m coverage run --rcfile=.coveragerc -m pytest \ + tests/test_dropin_offline.py tests/test_arrow_handoff.py PYTHONPATH=$(pwd) python -m coverage json --rcfile=.coveragerc -o coverage-python.json - name: Parse Python coverage and create badge JSON diff --git a/.github/workflows/python-arrow.yml b/.github/workflows/python-arrow.yml new file mode 100644 index 00000000..c4bde403 --- /dev/null +++ b/.github/workflows/python-arrow.yml @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Python Arrow export + +on: + push: + branches: [master, "branch-*"] + pull_request: + branches: [master, "branch-*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + converter: + name: Python Arrow export + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install Rust + run: | + rustup toolchain install stable --profile minimal --component clippy + rustup default stable + - name: Install PyArrow + run: python -m pip install "pyarrow==26.0.0" + - name: Test Arrow table export + # Test separately from pyspark-rs: extension-module feature unification + # would disable libpython linking for the embedded Python tests. + run: | + cargo test -p spark-python-arrow --locked --features pyo3/abi3-py39 + - name: Check converter Clippy + run: | + cargo clippy -p spark-python-arrow \ + --locked --all-targets --features pyo3/abi3-py39 -- -D warnings diff --git a/Cargo.lock b/Cargo.lock index 39d506da..7dfe0a04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -168,6 +168,7 @@ dependencies = [ "arrow-ipc", "arrow-json", "arrow-ord", + "arrow-pyarrow", "arrow-row", "arrow-schema", "arrow-select", @@ -322,6 +323,18 @@ dependencies = [ "arrow-select", ] +[[package]] +name = "arrow-pyarrow" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29abdf672a81c1aeb57fd2661457f9918964d49aed0e9f18932535f2a9e49ce" +dependencies = [ + "arrow-array", + "arrow-data", + "arrow-schema", + "pyo3", +] + [[package]] name = "arrow-row" version = "58.3.0" @@ -341,6 +354,7 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" dependencies = [ + "bitflags", "serde_core", "serde_json", ] @@ -3395,6 +3409,7 @@ dependencies = [ "prost", "pyo3", "serde_json", + "spark-python-arrow", "tokio", "tonic", ] @@ -3996,6 +4011,14 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spark-python-arrow" +version = "0.1.0" +dependencies = [ + "arrow", + "pyo3", +] + [[package]] name = "sqlparser" version = "0.60.0" diff --git a/Cargo.toml b/Cargo.toml index 5fe753a8..31fe8090 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ members = [ "crates/spark-connect-macros", "crates/spark-connect-build", "crates/pyspark-rs", + "crates/spark-python-arrow", "examples", ] exclude = [ @@ -34,6 +35,8 @@ exclude = [ # Rust-native `examples` crate both link the pure-Rust core directly; both are # workspace members but excluded from `default-members` so `cargo test`/`cargo build` # stay clean. Build them explicitly (`cargo build -p pyspark-rs` / `-p examples`). +# `spark-python-arrow` is also explicit: its tests embed Python and require PyArrow. +# Test it separately to avoid extension-module feature unification disabling libpython linking. # # The `examples/wasm-udfs` and `examples/wasm-udf-inline` crates are excluded from # the workspace entirely because they require the `wasm32-unknown-unknown` target, diff --git a/crates/pyspark-rs/Cargo.toml b/crates/pyspark-rs/Cargo.toml index 16148d41..05278bad 100644 --- a/crates/pyspark-rs/Cargo.toml +++ b/crates/pyspark-rs/Cargo.toml @@ -22,6 +22,7 @@ non_snake_case = "allow" spark-connect = { package = "apache-spark-connect", path = "../spark-connect" } spark-connect-core = { package = "apache-spark-connect-core", path = "../spark-connect-core" } spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto" } +spark-python-arrow = { path = "../spark-python-arrow" } pyo3.workspace = true arrow.workspace = true tokio.workspace = true diff --git a/crates/pyspark-rs/src/dataframe.rs b/crates/pyspark-rs/src/dataframe.rs index 7101aadf..25529563 100644 --- a/crates/pyspark-rs/src/dataframe.rs +++ b/crates/pyspark-rs/src/dataframe.rs @@ -1166,19 +1166,15 @@ impl PyDataFrame { /// Collect the DataFrame into a `pyarrow.Table`. /// /// Mirrors `pyspark.sql.DataFrame.toArrow()`: the result is decoded from the - /// server's Arrow IPC stream and handed to `pyarrow` (no row-by-row copy). + /// server's Arrow IPC stream and handed to `pyarrow` through the shared + /// converter, sharing compatible decoded Arrow value buffers. Bitmap alignment + /// and schemas requiring the IPC compatibility fallback can still copy. #[pyo3(name = "toArrow")] fn to_arrow(&self, py: Python<'_>) -> PyResult> { - use pyo3::types::PyBytes; - // `DataFrame::to_arrow` produces Arrow IPC *file* bytes (FileWriter). - let bytes = py.detach(|| self.dataframe.to_arrow()).to_pyerr()?; - let pa = py.import("pyarrow")?; - let py_bytes = PyBytes::new(py, &bytes); - let table = pa - .getattr("ipc")? - .call_method1("open_file", (py_bytes,))? - .call_method0("read_all")?; - Ok(table.unbind()) + let batches = py + .detach(|| self.dataframe.collect_record_batches()) + .to_pyerr()?; + spark_python_arrow::bridge::record_batches_to_pyarrow(py, batches) } /// Collect the DataFrame into a pandas DataFrame. diff --git a/crates/spark-python-arrow/Cargo.toml b/crates/spark-python-arrow/Cargo.toml new file mode 100644 index 00000000..563d00c4 --- /dev/null +++ b/crates/spark-python-arrow/Cargo.toml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +[package] +name = "spark-python-arrow" +version = "0.1.0" +edition = "2021" +rust-version = "1.85" +license = "Apache-2.0" +publish = false +description = "Arrow record batch export to PyArrow for Spark clients" +repository = "https://github.com/apache/spark-connect-rust" + +[dependencies] +arrow = { version = "58", default-features = false, features = ["ffi", "ipc", "pyarrow"] } +# The consumer chooses the Python ABI floor and embedding/extension-module linking. +pyo3 = { version = "0.28" } + +[lints.rust] +unsafe_op_in_unsafe_fn = "deny" diff --git a/crates/spark-python-arrow/README.md b/crates/spark-python-arrow/README.md new file mode 100644 index 00000000..44c438c4 --- /dev/null +++ b/crates/spark-python-arrow/README.md @@ -0,0 +1,121 @@ + + +# Arrow table export to PyArrow + +`spark-python-arrow` exports decoded Rust Arrow record batches to PyArrow for +`pyspark-rs`. It has one public entry point, +`bridge::record_batches_to_pyarrow`, used by `DataFrame.toArrow()`. It does not +depend on Spark Connect transport, generated protobufs, or an async runtime. + +Python-to-Rust Arrow imports, standalone array/type exports, and worker-side UDF +conversion are outside this change. Scalar input conversion and row +materialization remain in `pyspark-rs` until they have another concrete consumer. + +## Export contract + +The bridge uses `arrow::pyarrow::Table` to export standard Spark result batches. +It preserves batch boundaries, schema and field metadata, and zero-column row +counts, sharing compatible Arrow value and offset buffers with PyArrow. The +caller supplies valid Rust record batches and owns its Python interpreter; the +library does not initialize Python or retain session state. + +`toArrow()` collects all decoded batches into a Rust vector before the handoff. +In Arrow 58.3.0, the table helper imports each batch through its own native +C Stream reader and calls `read_next_batch()`, then builds a Python list and +calls `pyarrow.Table.from_batches()`. It does not export the whole result through +one reader or stream network results into Python. C Stream transfers buffer +descriptors and ownership, not serialized IPC bytes. + +The contract covers Spark's ordinary map representation (`keys_sorted=false`). +Sorted-map flags and dictionary-encoded arrays are outside this Spark result +contract; the bridge does not add compatibility fixes for those generic Arrow +inputs. + +PyArrow consumes the exported descriptors and retains their release callbacks, +so the data survives after Rust's temporary wrappers have been dropped. If an +importer raises before consuming a descriptor, Rust releases it. PyArrow and +Python code in the same interpreter are trusted, including the public aliases +used by the export path. + +Field names or timezone strings containing NUL use an IPC compatibility fallback +over the already collected batches, without another RPC. Length-prefixed schema +metadata can contain NUL without this fallback. The bridge preflights +C-compatible schemas through PyArrow's standalone schema importer +before the table helper imports any batches. This lets older PyArrow versions +reject unsupported schemas before entering the C Stream path. + +`Table::try_new` validates batch schema consistency, including field metadata, +before either export path. A mismatch raises `ValueError`. This is stricter than +the previous IPC export path, which wrote batches under the first batch's schema. +An empty batch vector still produces an empty table with an empty schema; +preserving schemas from schema-only IPC streams requires a separate change to +the decoding path. + +Zero-copy describes compatible Arrow **value and offset buffers**, not schema +descriptors, Python scalar/row objects, pandas materialization, IPC decoding, or +the IPC fallback. Sliced arrays whose value and null offsets differ can require +copying the validity bitmap during C Data export. + +## Integration + +This is an unpublished workspace implementation crate (`publish = false`), not +a separately released package. `pyspark-rs` uses a path dependency. The current +package name and `0.1.0` version do not identify a crates.io release; independent +publication would require Apache package naming and release-flow integration. +Source-based consumers can pin a repository revision and validate it in their +own dependency graph. The repository's root `LICENSE` covers this crate. + +The library does not enable PyO3's `extension-module` feature or select a minimum Python ABI; +`pyspark-rs` selects these features for the Python extension. + +PyO3 must resolve to one compatible version in each consuming dependency graph. +The manifest uses the same PyO3 0.28 line as the Python extension, with 0.28.3 +pinned in the workspace lockfile. This change does not upgrade PyO3. +This crate declares Rust 1.85, matching Arrow 58.3.0's declared minimum. +The existing workspace default of 1.82 is unchanged; it is not a claim that +these Python/Arrow packages build on 1.82. The minimum compiler has not been +independently tested locally. + +The Python Arrow export CI job tests this crate with the workspace lockfile. +These tests run separately from the extension build and are not repeated in coverage. + +## Tests + +With PyArrow installed in the Python environment used by PyO3: + +```sh +cargo test -p spark-python-arrow --locked --features pyo3/abi3-py39 +``` + +For an offline build with cached dependencies, prefix the command with +`CARGO_NET_OFFLINE=true CARGO_REGISTRIES_CRATES_IO_PROTOCOL=git`. + +Run this package separately from the extension's test/build command: Cargo +feature unification with an extension-module consumer can disable libpython +linking required by the embedded test executable. For a virtual environment, +set `PYO3_PYTHON` to its interpreter and expose its site-packages to the embedded +interpreter with `PYTHONPATH` when necessary. + +The standalone tests exercise the table export entry point: buffer identity and +lifetime, schema metadata, slices, chunks, empty shapes, schema mismatch errors, +and NUL fallback. The Python client's `test_arrow_handoff.py` exercises the public +`toArrow()` and `toPandas()` paths, including malformed IPC responses. Scalar regressions +live with the client tests. These are not substitutes for Spark's complete +Connect parity suite. diff --git a/crates/spark-python-arrow/src/bridge.rs b/crates/spark-python-arrow/src/bridge.rs new file mode 100644 index 00000000..bd28c624 --- /dev/null +++ b/crates/spark-python-arrow/src/bridge.rs @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Export Spark result batches through arrow-rs's PyArrow table conversion. +//! +//! The caller supplies valid Rust record batches and owns the Python interpreter. +//! PyArrow's importers consume the exported descriptors and retain their buffers. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::error::ArrowError; +use arrow::ffi::FFI_ArrowSchema; +use arrow::ipc::writer::FileWriter; +use arrow::pyarrow::{IntoPyArrow, Table}; +use arrow::record_batch::RecordBatch; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::PyBytes; + +// Called after table export has routed NUL names and timezones to the IPC fallback. +fn validate_pyarrow_schema(py: Python<'_>, schema: &Schema) -> PyResult<()> { + let mut schema = FFI_ArrowSchema::try_from(schema).map_err(arrow_error)?; + let address = &mut schema as *mut FFI_ArrowSchema as usize; + py.import("pyarrow")? + .getattr("Schema")? + .call_method1("_import_from_c", (address,))?; + Ok(()) +} + +/// Build a PyArrow table from Spark result batches, sharing compatible buffers. +/// +/// Batch boundaries, schema metadata, and zero-column row counts are preserved. +/// An empty batch vector produces an empty table with an empty schema. +/// Value and offset buffers are shared; sliced validity bitmaps may be realigned. +/// Inconsistent batch schemas, including metadata, raise `ValueError`. +/// Sorted-map flags and dictionary encoding are outside the Spark result contract. +pub fn record_batches_to_pyarrow(py: Python<'_>, batches: Vec) -> PyResult> { + let schema = batches + .first() + .map(|batch| batch.schema()) + .unwrap_or_else(|| Arc::new(Schema::empty())); + // Validate before either export path, including the NUL compatibility path. + let table = Table::try_new(batches, schema.clone()) + .map_err(|_| PyValueError::new_err("Arrow record batches have inconsistent schemas"))?; + let pa = py.import("pyarrow")?; + if schema.fields().iter().any(|field| field_needs_ipc(field)) { + // C schema strings cannot contain NUL. Use the already collected + // batches for this rare compatibility path without repeating any RPC. + let bytes = py + .detach(|| -> arrow::error::Result> { + let mut bytes = Vec::new(); + let mut writer = FileWriter::try_new(&mut bytes, &schema)?; + for batch in table.record_batches() { + writer.write(batch)?; + } + writer.finish()?; + drop(writer); + Ok(bytes) + }) + .map_err(arrow_error)?; + return Ok(pa + .getattr("ipc")? + .call_method1("open_file", (PyBytes::new(py, &bytes),))? + .call_method0("read_all")? + .unbind()); + } + + // Arrow's table helper imports batches before its final schema. Preflight + // first so older PyArrow versions can reject unsupported stream schemas. + validate_pyarrow_schema(py, &schema)?; + Ok(table.into_pyarrow(py)?.unbind()) +} + +fn arrow_error(error: ArrowError) -> PyErr { + PyRuntimeError::new_err(error.to_string()) +} + +fn field_needs_ipc(field: &Field) -> bool { + field.name().contains('\0') || data_type_needs_ipc(field.data_type()) +} + +fn data_type_needs_ipc(data_type: &DataType) -> bool { + match data_type { + DataType::Timestamp(_, Some(timezone)) => timezone.contains('\0'), + DataType::List(field) + | DataType::LargeList(field) + | DataType::ListView(field) + | DataType::LargeListView(field) + | DataType::FixedSizeList(field, _) + | DataType::Map(field, _) => field_needs_ipc(field), + DataType::Struct(fields) => fields.iter().any(|field| field_needs_ipc(field)), + DataType::Union(fields, _) => fields.iter().any(|(_, field)| field_needs_ipc(field)), + DataType::RunEndEncoded(run_ends, values) => { + field_needs_ipc(run_ends) || field_needs_ipc(values) + } + DataType::Dictionary(keys, values) => { + data_type_needs_ipc(keys) || data_type_needs_ipc(values) + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use arrow::datatypes::TimeUnit; + + use super::*; + + #[test] + fn invalid_time_type_is_rejected_without_panicking() { + Python::initialize(); + Python::attach(|py| { + let schema = Schema::new(vec![Field::new( + "invalid", + DataType::Time32(TimeUnit::Nanosecond), + true, + )]); + let error = validate_pyarrow_schema(py, &schema).unwrap_err(); + assert!(error.is_instance_of::(py)); + }); + } +} diff --git a/crates/spark-python-arrow/src/lib.rs b/crates/spark-python-arrow/src/lib.rs new file mode 100644 index 00000000..60c5d084 --- /dev/null +++ b/crates/spark-python-arrow/src/lib.rs @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Arrow record batch export to PyArrow without a dependency on Spark transport. +//! +//! The bridge shares compatible Arrow value buffers. Scalar conversion and row materialization +//! remain in the client binding. The caller owns its Python interpreter. + +pub mod bridge; diff --git a/crates/spark-python-arrow/tests/bridge_regressions.rs b/crates/spark-python-arrow/tests/bridge_regressions.rs new file mode 100644 index 00000000..fb92ca11 --- /dev/null +++ b/crates/spark-python-arrow/tests/bridge_regressions.rs @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::ffi::CStr; +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, Int64Array, ListArray, MapArray, StringArray, StructArray}; +use arrow::buffer::{NullBuffer, OffsetBuffer}; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::record_batch::RecordBatch; +use pyo3::prelude::*; +use pyo3::types::PyDict; +use spark_python_arrow::bridge::record_batches_to_pyarrow; + +fn assert_export_equal(py: Python<'_>, batch: RecordBatch, expected_code: &CStr) -> PyResult<()> { + let locals = PyDict::new(py); + locals.set_item("pa", py.import("pyarrow")?)?; + py.run(expected_code, Some(&locals), Some(&locals))?; + let expected = locals.get_item("expected")?.expect("expected table"); + let actual = record_batches_to_pyarrow(py, vec![batch])?; + assert!(actual + .bind(py) + .call_method1("equals", (&expected,))? + .is_truthy()?); + assert!(actual + .bind(py) + .getattr("schema")? + .call_method1("equals", (expected.getattr("schema")?, true))? + .is_truthy()?); + Ok(()) +} + +#[test] +fn table_export_preserves_spark_maps_in_nested_fields() { + Python::initialize(); + Python::attach(|py| -> PyResult<()> { + let entries = StructArray::new( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, true), + ] + .into(), + vec![ + Arc::new(StringArray::from(vec!["b", "a"])), + Arc::new(Int64Array::from(vec![1, 2])), + ], + None, + ); + let maps: ArrayRef = Arc::new(MapArray::new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + OffsetBuffer::new(vec![0, 2, 2, 2].into()), + entries, + Some(NullBuffer::from(vec![true, false, true])), + false, + )); + let lists: ArrayRef = Arc::new(ListArray::new( + Arc::new(Field::new("item", maps.data_type().clone(), true)), + OffsetBuffer::new(vec![0, 1, 2, 3].into()), + maps.clone(), + None, + )); + let nested: ArrayRef = Arc::new(StructArray::new( + vec![Field::new("maps", lists.data_type().clone(), true)].into(), + vec![lists], + None, + )); + let schema = Arc::new(Schema::new(vec![ + Field::new("maps", maps.data_type().clone(), true), + Field::new("nested", nested.data_type().clone(), true), + ])); + let batch = RecordBatch::try_new(schema, vec![maps, nested]).unwrap(); + assert_export_equal( + py, + batch, + c"map_type = pa.map_(pa.string(), pa.int64(), keys_sorted=False) +maps = pa.array([[('b', 1), ('a', 2)], None, []], type=map_type) +nested_type = pa.struct([pa.field('maps', pa.list_(map_type))]) +nested = pa.array([ + {'maps': [[('b', 1), ('a', 2)]]}, {'maps': [None]}, {'maps': [[]]} +], type=nested_type) +expected = pa.Table.from_arrays([maps, nested], names=['maps', 'nested'])", + ) + }) + .unwrap(); +} + +#[test] +fn sliced_validity_shares_values_without_requiring_bitmap_identity() { + Python::initialize(); + Python::attach(|py| -> PyResult<()> { + let original = Int64Array::from(vec![Some(1), Some(2), None, Some(4), None]); + let slice = original.slice(1, 3); + assert_eq!(slice.nulls().unwrap().offset(), 1); + assert_eq!(slice.to_data().offset(), 0); + let address = slice.values().as_ptr() as usize; + let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(slice)]).unwrap(); + let table = record_batches_to_pyarrow(py, vec![batch])?; + drop(original); + py.import("gc")?.call_method0("collect")?; + let array = table + .bind(py) + .call_method1("column", (0,))? + .call_method1("chunk", (0,))?; + assert_eq!( + array + .call_method0("buffers")? + .get_item(1)? + .getattr("address")? + .extract::()?, + address + ); + // Arrow may realign validity to the value buffer's offset. The bitmap's + // identity is not part of the value-buffer sharing promise. + assert_eq!(array.getattr("offset")?.extract::()?, 0); + assert_eq!( + array + .call_method0("to_pylist")? + .extract::>>()?, + vec![Some(2), None, Some(4)] + ); + Ok(()) + }) + .unwrap(); +} diff --git a/crates/spark-python-arrow/tests/conversion.rs b/crates/spark-python-arrow/tests/conversion.rs new file mode 100644 index 00000000..12c83536 --- /dev/null +++ b/crates/spark-python-arrow/tests/conversion.rs @@ -0,0 +1,340 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Table export checks against the installed PyArrow. No Spark server is needed. + +use std::collections::HashMap; +use std::ffi::CStr; +use std::sync::Arc; + +use arrow::array::{Array, Int64Array, StringArray, StructArray}; +use arrow::datatypes::{DataType, Field, Schema, TimeUnit}; +use arrow::record_batch::{RecordBatch, RecordBatchOptions}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyDict; +use spark_python_arrow::bridge::record_batches_to_pyarrow; + +fn evaluate<'py>(py: Python<'py>, expression: &CStr) -> PyResult> { + let locals = PyDict::new(py); + locals.set_item("pa", py.import("pyarrow")?)?; + py.eval(expression, Some(&locals), Some(&locals)) +} + +fn assert_schema_equal(actual: &Bound<'_, PyAny>, expected: &Bound<'_, PyAny>) -> PyResult<()> { + assert!(actual + .call_method1("equals", (expected, true))? + .is_truthy()?); + Ok(()) +} + +#[test] +fn table_export_shares_value_and_offset_buffers_after_rust_wrappers_drop() { + Python::initialize(); + Python::attach(|py| -> PyResult<()> { + let numbers = Int64Array::from(vec![Some(12), None, Some(34)]); + let strings = StringArray::from(vec![Some("first"), None, Some("last")]); + let addresses = [ + (0, 1, numbers.values().as_ptr() as usize), + (1, 1, strings.value_offsets().as_ptr() as usize), + (1, 2, strings.value_data().as_ptr() as usize), + ]; + let schema = Arc::new(Schema::new_with_metadata( + vec![ + Field::new("number", DataType::Int64, true), + Field::new("text", DataType::Utf8, true), + ], + HashMap::from([("buffer\0key".into(), "shared\0values".into())]), + )); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(numbers), Arc::new(strings)]).unwrap(); + // Export consumes the batch; only PyArrow owns the buffers afterward. + let table = record_batches_to_pyarrow(py, vec![batch])?; + py.import("gc")?.call_method0("collect")?; + for (column, buffer, address) in addresses { + let chunk = table + .bind(py) + .call_method1("column", (column,))? + .call_method1("chunk", (0,))?; + assert_eq!( + chunk + .call_method0("buffers")? + .get_item(buffer)? + .getattr("address")? + .extract::()?, + address + ); + } + let expected = evaluate( + py, + c"pa.table({'number': pa.array([12, None, 34], type=pa.int64()), + 'text': ['first', None, 'last']})", + )?; + assert!(table + .bind(py) + .call_method1("equals", (expected,))? + .is_truthy()?); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn table_export_preserves_chunks_and_shares_each_buffer() { + Python::initialize(); + Python::attach(|py| -> PyResult<()> { + let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)])); + let arrays = [ + Arc::new(Int64Array::from(vec![Some(1), None])), + Arc::new(Int64Array::from(vec![Some(3)])), + ]; + let addresses: Vec<_> = arrays + .iter() + .map(|array| array.values().as_ptr() as usize) + .collect(); + let batches = arrays + .into_iter() + .map(|array| RecordBatch::try_new(schema.clone(), vec![array]).unwrap()) + .collect(); + let table = record_batches_to_pyarrow(py, batches)?; + let column = table.bind(py).call_method1("column", (0,))?; + assert_eq!(column.getattr("num_chunks")?.extract::()?, 2); + for (index, address) in addresses.iter().enumerate() { + let chunk = column.call_method1("chunk", (index,))?; + assert_eq!( + chunk + .call_method0("buffers")? + .get_item(1)? + .getattr("address")? + .extract::()?, + *address + ); + } + assert_eq!( + column + .call_method0("to_pylist")? + .extract::>>()?, + vec![Some(1), None, Some(3)] + ); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn table_export_preserves_nested_values_and_nul_metadata() { + Python::initialize(); + Python::attach(|py| -> PyResult<()> { + let field = Arc::new( + Field::new("value", DataType::Int64, true) + .with_metadata(HashMap::from([("field\0key".into(), "a\0b".into())])), + ); + let nested = StructArray::new( + vec![field].into(), + vec![Arc::new(Int64Array::from(vec![Some(1), None, Some(3)]))], + None, + ); + let schema = Arc::new(Schema::new_with_metadata( + vec![Field::new("nested", nested.data_type().clone(), true)], + HashMap::from([("schema_key".into(), "c\0d".into())]), + )); + let batch = RecordBatch::try_new(schema, vec![Arc::new(nested)]).unwrap(); + let table = record_batches_to_pyarrow(py, vec![batch])?; + let expected_schema = evaluate( + py, + c"pa.schema([pa.field('nested', pa.struct([ + pa.field('value', pa.int64(), metadata={b'field\\x00key': b'a\\x00b'}) + ]))], metadata={b'schema_key': b'c\\x00d'})", + )?; + assert_schema_equal(&table.bind(py).getattr("schema")?, &expected_schema)?; + let actual = table.bind(py).call_method0("to_pylist")?; + let expected = evaluate( + py, + c"[{'nested': {'value': 1}}, {'nested': {'value': None}}, + {'nested': {'value': 3}}]", + )?; + assert!(actual.eq(expected)?); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn table_export_preserves_empty_schemas_and_zero_column_rows() { + Python::initialize(); + Python::attach(|py| -> PyResult<()> { + let empty = record_batches_to_pyarrow(py, vec![])?; + assert_eq!(empty.bind(py).getattr("num_rows")?.extract::()?, 0); + assert_eq!( + empty.bind(py).getattr("num_columns")?.extract::()?, + 0 + ); + assert_schema_equal( + &empty.bind(py).getattr("schema")?, + &evaluate(py, c"pa.schema([])")?, + )?; + + let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, true)])); + let empty = record_batches_to_pyarrow(py, vec![RecordBatch::new_empty(schema)])?; + assert_eq!(empty.bind(py).getattr("num_rows")?.extract::()?, 0); + assert_schema_equal( + &empty.bind(py).getattr("schema")?, + &evaluate(py, c"pa.schema([('x', pa.int64())])")?, + )?; + + for rows in [0, 4] { + let schema = + Schema::empty().with_metadata(HashMap::from([("empty".into(), "metadata".into())])); + let options = RecordBatchOptions::new().with_row_count(Some(rows)); + let batch = + RecordBatch::try_new_with_options(Arc::new(schema), vec![], &options).unwrap(); + let table = record_batches_to_pyarrow(py, vec![batch])?; + assert_eq!( + table.bind(py).getattr("num_rows")?.extract::()?, + rows + ); + assert_eq!( + table.bind(py).getattr("num_columns")?.extract::()?, + 0 + ); + assert_schema_equal( + &table.bind(py).getattr("schema")?, + &evaluate(py, c"pa.schema([], metadata={b'empty': b'metadata'})")?, + )?; + } + Ok(()) + }) + .unwrap(); +} + +#[test] +fn table_export_rejects_inconsistent_fields_and_metadata() { + Python::initialize(); + Python::attach(|py| { + let field = Field::new("x", DataType::Int64, true); + let schema = Schema::new(vec![field.clone()]); + for other in [ + Schema::new(vec![Field::new("other", DataType::Int64, true)]), + schema + .clone() + .with_metadata(HashMap::from([("schema".into(), "other".into())])), + Schema::new(vec![field + .clone() + .with_metadata(HashMap::from([("field".into(), "other".into())]))]), + ] { + let batches = vec![ + RecordBatch::new_empty(Arc::new(schema.clone())), + RecordBatch::new_empty(Arc::new(other)), + ]; + let error = record_batches_to_pyarrow(py, batches).unwrap_err(); + assert!(error.is_instance_of::(py)); + assert!(error.to_string().contains("inconsistent schemas")); + } + }); +} + +#[test] +fn table_export_falls_back_to_ipc_for_nul_names_and_timezones() { + Python::initialize(); + Python::attach(|py| -> PyResult<()> { + let cases: [(Field, &CStr); 3] = [ + ( + Field::new("bad\0name", DataType::Int64, true), + c"pa.schema([('bad\\x00name', pa.int64())])", + ), + ( + Field::new( + "nested", + DataType::Struct(vec![Field::new("bad\0name", DataType::Int64, true)].into()), + true, + ), + c"pa.schema([('nested', pa.struct([('bad\\x00name', pa.int64())]))])", + ), + ( + Field::new( + "time", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC\0suffix".into())), + true, + ), + c"pa.schema([('time', pa.timestamp('us', tz='UTC\\x00suffix'))])", + ), + ]; + for (field, expected) in cases { + let batch = RecordBatch::new_empty(Arc::new(Schema::new(vec![field]))); + let table = record_batches_to_pyarrow(py, vec![batch])?; + assert_schema_equal(&table.bind(py).getattr("schema")?, &evaluate(py, expected)?)?; + } + Ok(()) + }) + .unwrap(); +} + +#[test] +fn nul_fallback_preserves_nonempty_batches() { + Python::initialize(); + Python::attach(|py| -> PyResult<()> { + let schema = Arc::new(Schema::new(vec![Field::new( + "bad\0name", + DataType::Int64, + true, + )])); + let batches = [ + Int64Array::from(vec![Some(1), None]), + Int64Array::from(vec![Some(3), Some(4)]), + ] + .into_iter() + .map(|array| RecordBatch::try_new(schema.clone(), vec![Arc::new(array)]).unwrap()) + .collect(); + let table = record_batches_to_pyarrow(py, batches)?; + let column = table.bind(py).call_method1("column", (0,))?; + assert_eq!(column.getattr("num_chunks")?.extract::()?, 2); + assert_eq!( + column + .call_method0("to_pylist")? + .extract::>>()?, + vec![Some(1), None, Some(3), Some(4)] + ); + assert_schema_equal( + &table.bind(py).getattr("schema")?, + &evaluate(py, c"pa.schema([('bad\\x00name', pa.int64())])")?, + )?; + Ok(()) + }) + .unwrap(); +} + +#[test] +fn nul_fallback_rejects_mismatched_schemas_first() { + Python::initialize(); + Python::attach(|py| { + let field = Field::new("bad\0name", DataType::Int64, true); + let schema = Schema::new(vec![field]); + let other = schema + .clone() + .with_metadata(HashMap::from([("changed".into(), "metadata".into())])); + let batches = [schema, other] + .into_iter() + .map(|schema| { + RecordBatch::try_new(Arc::new(schema), vec![Arc::new(Int64Array::from(vec![1]))]) + .unwrap() + }) + .collect(); + let error = record_batches_to_pyarrow(py, batches).unwrap_err(); + assert!(error.is_instance_of::(py)); + assert!(error.to_string().contains("inconsistent schemas")); + }); +} diff --git a/dev/design/architecture.md b/dev/design/architecture.md index 87303768..e1ebbf9d 100644 --- a/dev/design/architecture.md +++ b/dev/design/architecture.md @@ -47,6 +47,7 @@ crates/ column, plan, dataframe, functions, session, readwriter, catalog, group, window, udf/udtf, streaming, observation pyspark-rs PyO3 bindings -> native module `_pyspark` + spark-python-arrow Spark result export via arrow::pyarrow::Table; no transport python/ pyspark/… thin drop-in package; public modules re-export from `_pyspark` so `from pyspark.sql import SparkSession` works @@ -67,9 +68,26 @@ Server-streaming RPCs (`ExecutePlan`) are consumed into Arrow record batches. ## Data path -Results arrive as Arrow IPC streams. `arrow-rs` decodes them; hand-off to Python -uses the **Arrow C Data Interface** (zero-copy to `pyarrow`), which is also how -we build `toPandas()` / `createDataFrame(pandas)` without re-serializing. +Results arrive as Arrow IPC streams. `arrow-rs` decodes them; `toArrow()` collects +all batches into a Rust vector and hands them to `spark-python-arrow` through +`arrow::pyarrow::Table`. In Arrow 58.3.0, that helper imports a separate C Stream +reader for each batch, calls `read_next_batch()`, and passes a Python list to +`pyarrow.Table.from_batches()`. This shares compatible value and offset buffers; +it does not provide network streaming or bounded-memory collection. Sliced +validity bitmaps may need realignment. + +The helper validates batch schema consistency, including metadata, before export. +A standalone PyArrow schema import rejects unsupported C-compatible schemas +before any batches are imported. Embedded NUL field names or timezones use an +IPC compatibility fallback without another RPC. The contract covers standard +Spark result types, not sorted-map flags or dictionary encoding. `toPandas()` +then calls PyArrow's pandas conversion, which may copy. + +Scalar input conversion and row materialization stay in `pyspark-rs`. The export +crate does not own schema inference, transport, or worker execution. `collect()` +still uses the native client's owned rows; `createDataFrame(pandas)` still +materializes Python rows. Neither path is claimed to be zero-copy. See the +[table export contract](../../crates/spark-python-arrow/README.md). ## UDF strategy (full compatibility) diff --git a/docs/architecture.md b/docs/architecture.md index f7fdb920..fa253cb6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -63,6 +63,7 @@ A Cargo workspace of library crates, plus a Python skin: | `apache-spark-connect-core` | `crates/spark-connect-core` | Transport: channel, retries, reattach, artifacts, errors | | `apache-spark-connect` | `crates/spark-connect` | The DataFrame API: session, dataframe, column, functions, plan, group, catalog, window, readwriter, streaming, types | | `pyspark-rs` | `crates/pyspark-rs` | PyO3 bindings - builds the `_pyspark` extension module | +| `spark-python-arrow` (unpublished) | `crates/spark-python-arrow` | Spark result export through Arrow's PyArrow table helper | | Python skin | `python/pyspark` | The drop-in `pyspark` package (+ vendored `pyspark.pandas`, `cloudpickle`, `pyspark.testing`) | ```mermaid @@ -70,6 +71,7 @@ flowchart LR PROTO["spark-connect-proto"] --> CORE["spark-connect-core"] CORE --> SC["spark-connect (DataFrame API)"] SC --> PYO3["pyspark-rs (PyO3)"] + PYARROW["spark-python-arrow (unpublished)"] --> PYO3 PYO3 --> SKIN["python/pyspark skin"] SC --> RUSTAPP["your Rust app"] ``` @@ -102,6 +104,24 @@ triggers `ExecutePlan`. If the response stream breaks mid-flight, the core's reattachable-execute iterator resumes from the last response it saw, so long results survive transient disconnects. +### Python Arrow table handoff + +`DataFrame.toArrow()` collects decoded Rust batches, then uses +`arrow::pyarrow::Table` to build a PyArrow table. In Arrow 58.3.0, the helper +imports each batch through a separate C Stream reader, reads that batch, and +passes a Python list to `pyarrow.Table.from_batches()`. Compatible value and +offset buffers are shared, but all batches are collected upfront. This is not +network streaming or bounded-memory collection. IPC decoding, sliced validity +bitmap realignment, and pandas conversion can still allocate or copy. + +The bridge checks batch schema consistency and preflights C-compatible schemas +with PyArrow before importing batches. Field names or timezones containing NUL +use an IPC fallback over the collected batches, without another RPC. Schema +metadata is preserved. The contract covers standard Spark results, including +maps with `keys_sorted=false`, not generic sorted maps or dictionary encoding. +An empty batch vector still produces an empty schema. Row materialization in +`collect()` is unchanged, and this change does not upgrade PyO3. + ## Two front-ends, one core === "Rust (native)" @@ -152,7 +172,7 @@ See [Rust UDFs via WebAssembly](udfs.md) for the full story. 3. **Correctness is gated in CI** - golden-proto tests guard plan building; the official Apache Spark Connect test suite guards transport and Arrow paths (see the CI notes under `dev/design/`). -4. **Version = Spark version** - the crate/wheel version tracks the Spark release +4. **Version = Spark version** - the published crate/wheel version tracks the Spark release it targets, so the number tells you which Spark it speaks. For the deeper design write-ups (design notes, CI, the official test suite, and diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py new file mode 100644 index 00000000..c9188ccd --- /dev/null +++ b/python/tests/test_arrow_handoff.py @@ -0,0 +1,345 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Check public Arrow handoff behavior against a deterministic mock RPC service. + +These tests preserve Spark-output values, schemas, chunks, and lifetimes without +depending on private PyArrow importers. Rust tests verify decoded buffer identity +at the handoff: server-side Python addresses cannot establish that property. +""" + +import gc +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from datetime import datetime, timezone +from decimal import Decimal + +import grpc +import pandas as pd +import pyarrow as pa +import pytest +from pyspark.sql import SparkSession + + +def _varint(value): + encoded = bytearray() + while value > 127: + encoded.append((value & 127) | 128) + value >>= 7 + encoded.append(value) + return bytes(encoded) + + +def _bytes_field(number, value): + return _varint((number << 3) | 2) + _varint(len(value)) + value + + +def _request_fields(data): + """Read only the protobuf envelope, not the plan or other nested messages.""" + position = 0 + + def varint(): + nonlocal position + value = shift = 0 + while True: + byte = data[position] + position += 1 + value |= (byte & 127) << shift + if byte < 128: + return value + shift += 7 + + fields = {} + while position < len(data): + tag = varint() + wire_type = tag & 7 + if wire_type == 2: + length = varint() + fields[tag >> 3] = data[position : position + length] + position += length + elif wire_type == 0: + varint() + elif wire_type in (1, 5): + position += 8 if wire_type == 1 else 4 + else: + raise AssertionError(f"Unexpected protobuf wire type: {wire_type}") + return fields + + +@contextmanager +def _dataframe(batches, *, payloads=None, calls=None): + """Serve just ExecutePlan and release RPCs; unexpected RPCs fail closed. + + Field numbers are from spark/connect/base.proto. Using the wire envelope + avoids depending on official pyspark's generated protobuf package, which + cannot be imported alongside the Rust-backed drop-in under the same name. + """ + if payloads is None: + payloads = [] + for batch in batches: + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, batch.schema) as writer: + writer.write_batch(batch) + payloads.append((batch.num_rows, sink.getvalue().to_pybytes())) + server_id = b"00000000-0000-0000-0000-000000000001" + + def execute(request, context): + if calls is not None: + calls["execute"] = calls.get("execute", 0) + 1 + fields = _request_fields(request) + header = ( + _bytes_field(1, fields[1]) + _bytes_field(12, fields[6]) + _bytes_field(15, server_id) + ) + for index, (rows, payload) in enumerate(payloads): + arrow_batch = _varint(8) + _varint(rows) + _bytes_field(2, payload) + yield (header + _bytes_field(13, str(index).encode()) + _bytes_field(2, arrow_batch)) + yield header + _bytes_field(13, b"complete") + _bytes_field(14, b"") + + def release_execute(request, context): + fields = _request_fields(request) + return _bytes_field(1, fields[1]) + _bytes_field(2, fields[3]) + _bytes_field(3, server_id) + + def release_session(request, context): + return _bytes_field(1, _request_fields(request)[1]) + _bytes_field(2, server_id) + + with ThreadPoolExecutor(max_workers=2) as executor: + server = grpc.server(executor) + server.add_generic_rpc_handlers( + [ + grpc.method_handlers_generic_handler( + "spark.connect.SparkConnectService", + { + "ExecutePlan": grpc.unary_stream_rpc_method_handler(execute), + "ReleaseExecute": grpc.unary_unary_rpc_method_handler(release_execute), + "ReleaseSession": grpc.unary_unary_rpc_method_handler(release_session), + }, + ) + ] + ) + port = server.add_insecure_port("127.0.0.1:0") + assert port != 0 + server.start() + session = None + try: + session = SparkSession.builder.remote(f"sc://127.0.0.1:{port}").getOrCreate() + yield session.range(1), session + finally: + try: + if session is not None: + session.stop() + finally: + server.stop(0).wait(timeout=5) + + +def _mixed_batch(): + nested_type = pa.struct( + [ + pa.field("items", pa.list_(pa.int64()), metadata={b"nested": b"metadata"}), + pa.field("labels", pa.map_(pa.string(), pa.int64())), + ] + ) + arrays = [ + pa.array([1, None, -3, 4], type=pa.int64()), + pa.array([True, None, False, True]), + pa.array(["alpha", None, "", "omega"]), + pa.array([b"a", None, b"", b"xyz"]), + pa.array( + [Decimal("1.23"), None, Decimal("-4.56"), Decimal("0.00")], type=pa.decimal128(12, 2) + ), + pa.array( + [ + datetime(2020, 1, 1, tzinfo=timezone.utc), + None, + datetime(2020, 1, 2, tzinfo=timezone.utc), + datetime(2020, 1, 3, tzinfo=timezone.utc), + ], + type=pa.timestamp("us", tz="UTC"), + ), + pa.array( + [ + {"items": [1, None], "labels": [("a", 1)]}, + None, + {"items": [], "labels": []}, + {"items": None, "labels": [("b", None)]}, + ], + type=nested_type, + ), + pa.nulls(4), + ] + fields = [ + pa.field(f"c{index}", array.type, metadata={b"field": str(index).encode()}) + for index, array in enumerate(arrays) + ] + schema = pa.schema(fields, metadata={b"schema": b"preserved"}) + return pa.RecordBatch.from_arrays(arrays, schema=schema) + + +def test_handoff_preserves_values_metadata_and_chunks(): + batch = _mixed_batch() + batches = [batch.slice(1, 2), batch.slice(0, 1), batch.slice(3, 1)] + expected = pa.Table.from_batches(batches) + calls = {} + with _dataframe(batches, calls=calls) as (frame, _): + actual = frame.toArrow() + assert actual.equals(expected, check_metadata=True) + for column in actual.columns: + assert [len(chunk) for chunk in column.chunks] == [2, 1, 1] + assert calls == {"execute": 1} + + +def test_handoff_buffers_outlive_dataframe_session_and_table(): + batch = _mixed_batch() + expected = batch.to_pydict() + calls = {} + + def collect(): + with _dataframe([batch], calls=calls) as (frame, session): + table = frame.toArrow() + session.stop() + return table + + table = collect() + gc.collect() + assert table.to_pydict() == expected + numeric = table.column(0).chunk(0).slice(1, 3) + strings = table.column(2).chunk(0).slice(1, 3) + addresses = [buffer.address for buffer in (numeric.buffers()[1], strings.buffers()[2])] + del table + gc.collect() + assert numeric.to_pylist() == [None, -3, 4] + assert strings.to_pylist() == [None, "", "omega"] + assert numeric.buffers()[1].address == addresses[0] + assert strings.buffers()[2].address == addresses[1] + assert calls == {"execute": 1} + + +@pytest.mark.parametrize("kind", ["no-batches", "empty-typed", "zero-columns"]) +def test_handoff_empty_and_zero_column_results(kind): + if kind == "no-batches": + batches = [] + expected = pa.table({}) + elif kind == "empty-typed": + batches = [_mixed_batch().slice(0, 0)] + expected = pa.Table.from_batches(batches) + else: + structs = pa.array([{}] * 5, type=pa.struct([])) + batch = pa.RecordBatch.from_struct_array(structs) + batch = batch.replace_schema_metadata({b"zero-column": b"preserved"}) + batches = [batch.slice(0, 3), batch.slice(3, 2)] + expected = pa.Table.from_batches(batches) + assert expected.num_rows == 5 + calls = {} + with _dataframe(batches, calls=calls) as (frame, _): + actual = frame.toArrow() + assert actual.num_rows == expected.num_rows + assert actual.num_columns == expected.num_columns + assert actual.equals(expected, check_metadata=True) + assert calls == {"execute": 1} + + +def test_to_pandas_uses_the_same_handoff(): + batch = _mixed_batch() + indices = [0, 2, 4, 5] + schema = pa.schema( + [batch.schema.field(index) for index in indices], metadata=batch.schema.metadata + ) + arrays = [batch.column(index) for index in indices] + selected = pa.RecordBatch.from_arrays(arrays, schema=schema) + batches = [selected.slice(0, 2), selected.slice(2, 2)] + expected = pa.Table.from_batches(batches).to_pandas() + calls = {} + with _dataframe(batches, calls=calls) as (frame, _): + actual = frame.toPandas() + pd.testing.assert_frame_equal(actual, expected) + assert calls == {"execute": 1} + + +@pytest.mark.parametrize("kind", ["field", "nested-field", "timezone"]) +def test_nul_schema_strings_fall_back_without_executing_again(kind): + if kind == "field": + name = "bad\x00name" + values = pa.array([1, None, -3, 4]) + elif kind == "nested-field": + name = "nested" + data_type = pa.struct([pa.field("bad\x00name", pa.int64())]) + values = pa.array( + [{"bad\x00name": 1}, None, {"bad\x00name": None}, {"bad\x00name": 4}], + type=data_type, + ) + else: + name = "timestamp" + values = pa.array([0, None, 2, 3], type=pa.timestamp("us", tz="UTC\x00suffix")) + schema = pa.schema( + [pa.field(name, values.type, metadata={b"field": b"preserved"})], + metadata={b"schema": b"preserved"}, + ) + batch = pa.RecordBatch.from_arrays([values], schema=schema) + batches = [batch.slice(0, 2), batch.slice(2, 2)] + expected = pa.Table.from_batches(batches) + calls = {} + with _dataframe(batches, calls=calls) as (frame, _): + actual = frame.toArrow() + # Compare raw Arrow data, not Python datetimes: the deliberately unusual + # timezone string is preserved but is not a Python timezone identifier. + assert actual.equals(expected, check_metadata=True) + assert [len(chunk) for chunk in actual.column(0).chunks] == [2, 2] + assert calls == {"execute": 1} + + +def test_nul_schema_and_field_metadata_are_preserved(): + field = pa.field("value", pa.int64(), metadata={b"field\x00key": b"field\x00value"}) + schema = pa.schema([field], metadata={b"schema\x00key": b"schema\x00value"}) + batch = pa.RecordBatch.from_arrays([pa.array([1, None, 3])], schema=schema) + calls = {} + with _dataframe([batch], calls=calls) as (frame, _): + actual = frame.toArrow() + assert actual.equals(pa.Table.from_batches([batch]), check_metadata=True) + assert calls == {"execute": 1} + + +@pytest.mark.parametrize("difference", ["type", "name", "schema-metadata", "field-metadata"]) +@pytest.mark.parametrize("nul_name", [False, True]) +def test_mismatched_batch_schemas_are_rejected_before_fallback(difference, nul_name): + name = "bad\x00name" if nul_name else "value" + first_field = pa.field(name, pa.int64(), metadata={b"field": b"original"}) + first_schema = pa.schema([first_field], metadata={b"schema": b"original"}) + second_values = pa.array(["text", "other"]) if difference == "type" else pa.array([3, 4]) + second_name = name + "-different" if difference == "name" else name + field_metadata = {b"field": b"different"} if difference == "field-metadata" else None + schema_metadata = {b"schema": b"different"} if difference == "schema-metadata" else None + second_field = pa.field( + second_name, second_values.type, metadata=field_metadata or first_field.metadata + ) + second_schema = pa.schema( + [second_field], metadata=schema_metadata or first_schema.metadata + ) + batches = [ + pa.RecordBatch.from_arrays([pa.array([1, 2])], schema=first_schema), + pa.RecordBatch.from_arrays([second_values], schema=second_schema), + ] + calls = {} + with _dataframe(batches, calls=calls) as (frame, _): + with pytest.raises(ValueError, match="inconsistent schemas"): + frame.toArrow() + assert calls == {"execute": 1} + + +def test_malformed_ipc_is_reported(): + calls = {} + with _dataframe([], payloads=[(1, b"not arrow ipc")], calls=calls) as (frame, _): + with pytest.raises(RuntimeError, match="Arrow"): + frame.toArrow() + assert calls == {"execute": 1}