Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .github/workflows/build_python_connect.yml
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,13 @@ jobs:
--exclude "test_artifact.py,test_artifact_localcluster.py,test_parity_resources.py" \
2>&1 | tee /tmp/parity_ours.log

- name: Native Arrow handoff smoke tests (against the live server)
working-directory: python
env:
SPARK_REMOTE: sc://localhost:15002
PYTHONPATH: ${{ github.workspace }}/python
run: python -m pytest tests/test_arrow_e2e.py -q

- name: Restart Connect server in local-cluster mode
# Resource profiles and artifact distribution across executors can only be
# exercised against a real multi-executor cluster, so - mirroring Apache's
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/coverage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,9 @@ jobs:

- name: Measure Python coverage with offline test suite
run: |
cd python && PYTHONPATH=$(pwd) python -m coverage run --rcfile=.coveragerc -m pytest tests/test_dropin_offline.py
cd python
PYTHONPATH=$(pwd) python -m coverage run --rcfile=.coveragerc -m pytest \
tests/test_dropin_offline.py tests/test_arrow_handoff.py
Comment on lines +230 to +231

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Carrying over from #103: pyproject.toml still declares pyarrow>=10.0.0 (L30), but this step is the only CI run of the handoff tests, and it gets the latest PyArrow from L93 (26.0.0 in this head's run). The preflight for older PyArrow is gone, and toArrow() now goes through PyArrow's private RecordBatchReader._import_from_c (arrow-pyarrow lib.rs#L460-L466) instead of the public pa.ipc.open_file, so the local PyArrow 10 run is the only evidence for the floor. Could CI add a leg pinned to pyarrow==10.0.0? If that isn't practical, could the docs state which PyArrow versions are tested?

PYTHONPATH=$(pwd) python -m coverage json --rcfile=.coveragerc -o coverage-python.json

- name: Parse Python coverage and create badge JSON
Expand Down
14 changes: 14 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion crates/pyspark-rs/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
name = "pyspark-rs"
version.workspace = true
edition.workspace = true
# Arrow 58.3.0, including arrow-pyarrow, requires Rust 1.85.
rust-version = "1.85"
license.workspace = true
description = "PyO3 bindings exposing the Rust Spark Connect client as a drop-in pyspark package"
# The Python extension (cdylib) is distributed as a PyPI wheel, not on crates.io.
Expand All @@ -23,7 +25,7 @@ spark-connect = { package = "apache-spark-connect", path = "../spark-connect" }
spark-connect-core = { package = "apache-spark-connect-core", path = "../spark-connect-core" }
spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto" }
pyo3.workspace = true
arrow.workspace = true
arrow = { workspace = true, features = ["pyarrow"] }
tokio.workspace = true
prost.workspace = true
tonic.workspace = true
Expand Down
18 changes: 7 additions & 11 deletions crates/pyspark-rs/src/dataframe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1166,19 +1166,15 @@ impl PyDataFrame {
/// Collect the DataFrame into a `pyarrow.Table`.
///
/// Mirrors `pyspark.sql.DataFrame.toArrow()`: the result is decoded from the
/// server's Arrow IPC stream and handed to `pyarrow` (no row-by-row copy).
/// server's Arrow IPC stream and handed to `pyarrow` through arrow-rs's
/// reader conversion, sharing compatible decoded Arrow value buffers. Bitmap alignment
/// and schemas requiring the IPC compatibility fallback can still copy.
#[pyo3(name = "toArrow")]
fn to_arrow(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
use pyo3::types::PyBytes;
// `DataFrame::to_arrow` produces Arrow IPC *file* bytes (FileWriter).
let bytes = py.detach(|| self.dataframe.to_arrow()).to_pyerr()?;
let pa = py.import("pyarrow")?;
let py_bytes = PyBytes::new(py, &bytes);
let table = pa
.getattr("ipc")?
.call_method1("open_file", (py_bytes,))?
.call_method0("read_all")?;
Ok(table.unbind())
let batches = py
.detach(|| self.dataframe.collect_record_batches())
.to_pyerr()?;
crate::pyarrow_export::record_batches_to_pyarrow(py, batches)
}

/// Collect the DataFrame into a pandas DataFrame.
Expand Down
1 change: 1 addition & 0 deletions crates/pyspark-rs/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ mod ml;
mod observation;
mod pipelines;
mod profiler;
mod pyarrow_export;
mod readwriter;
mod resource;
mod row;
Expand Down
108 changes: 108 additions & 0 deletions crates/pyspark-rs/src/pyarrow_export.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

//! Export decoded Spark results using arrow-rs's PyArrow reader conversion.

use std::sync::Arc;

use arrow::datatypes::{DataType, Field, Schema};
use arrow::ipc::writer::FileWriter;
use arrow::pyarrow::IntoPyArrow;
use arrow::record_batch::{RecordBatch, RecordBatchIterator, RecordBatchReader};
use pyo3::exceptions::{PyRuntimeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::PyBytes;

/// Share compatible value and offset buffers with PyArrow in one reader import.
///
/// Batches have already been collected; this does not stream the network response.
/// Spark result maps are unsorted and result fields are not dictionary-encoded.
/// An empty vector retains the existing empty-schema behavior.
pub(crate) fn record_batches_to_pyarrow(
py: Python<'_>,
batches: Vec<RecordBatch>,
) -> PyResult<Py<PyAny>> {
let schema = batches
.first()
.map(|batch| batch.schema())
.unwrap_or_else(|| Arc::new(Schema::empty()));
// RecordBatchIterator does not validate schemas. Check before either path,
// including metadata, so later batches are not relabeled with the first schema.
if batches.iter().any(|batch| batch.schema_ref() != &schema) {
return Err(PyValueError::new_err(
"Arrow record batches have inconsistent schemas",
));
}
if schema.fields().iter().any(|field| field_needs_ipc(field)) {
// Arrow 58.3 panics on NUL-containing C strings inside the C stream
// `get_schema` callback. Unwinding across that extern "C" boundary
// aborts Python instead of raising an exception. Check before export,
// then reuse the collected batches for IPC without another request.
// CString::new(...).unwrap() in FFI_ArrowSchema::try_new / with_name:
// https://github.com/apache/arrow-rs/blob/58.3.0/arrow-schema/src/ffi.rs#L148
// https://github.com/apache/arrow-rs/blob/58.3.0/arrow-schema/src/ffi.rs#L174
let bytes = py
.detach(|| -> arrow::error::Result<Vec<u8>> {
let mut bytes = Vec::new();
let mut writer = FileWriter::try_new(&mut bytes, &schema)?;
for batch in &batches {
writer.write(batch)?;
}
writer.finish()?;
drop(writer);
Ok(bytes)
})
.map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
return Ok(py
.import("pyarrow")?
.getattr("ipc")?
.call_method1("open_file", (PyBytes::new(py, &bytes),))?
.call_method0("read_all")?
.unbind());
}

let reader: Box<dyn RecordBatchReader + Send> = Box::new(RecordBatchIterator::new(
batches.into_iter().map(Ok),
schema,
));
Ok(reader.into_pyarrow(py)?.call_method0("read_all")?.unbind())
}

fn field_needs_ipc(field: &Field) -> bool {
field.name().contains('\0') || data_type_needs_ipc(field.data_type())
}

fn data_type_needs_ipc(data_type: &DataType) -> bool {
match data_type {
DataType::Timestamp(_, Some(timezone)) => timezone.contains('\0'),
DataType::List(field)
| DataType::LargeList(field)
| DataType::ListView(field)
| DataType::LargeListView(field)
| DataType::FixedSizeList(field, _)
| DataType::Map(field, _) => field_needs_ipc(field),
DataType::Struct(fields) => fields.iter().any(|field| field_needs_ipc(field)),
DataType::Union(fields, _) => fields.iter().any(|(_, field)| field_needs_ipc(field)),
DataType::RunEndEncoded(run_ends, values) => {
field_needs_ipc(run_ends) || field_needs_ipc(values)
}
DataType::Dictionary(keys, values) => {
data_type_needs_ipc(keys) || data_type_needs_ipc(values)
}
_ => false,
}
}
30 changes: 27 additions & 3 deletions dev/design/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,33 @@ Server-streaming RPCs (`ExecutePlan`) are consumed into Arrow record batches.

## Data path

Results arrive as Arrow IPC streams. `arrow-rs` decodes them; hand-off to Python
uses the **Arrow C Data Interface** (zero-copy to `pyarrow`), which is also how
we build `toPandas()` / `createDataFrame(pandas)` without re-serializing.
Results arrive as Arrow IPC streams. `arrow-rs` decodes them; `toArrow()` collects
all batches into a Rust vector. `pyspark-rs` uses arrow-rs's `IntoPyArrow` for
one boxed `RecordBatchReader`, importing it as a PyArrow
`RecordBatchReader`, and calls `read_all()`. PyArrow pulls batches through native
callbacks that do not call Python, avoiding a per-batch Python import loop.
This is an in-process buffer handoff, not network streaming or bounded-memory
result collection.

Compatible Arrow value and offset buffers are shared with PyArrow; sliced
validity bitmaps may need realignment. The bridge validates batch schema
consistency, including metadata. Schemas with embedded
NUL field names or timezones use an IPC compatibility fallback, without another
RPC. Spark results use unsorted maps and are not dictionary-encoded; this path
does not repair generic Arrow ordering flags. `toPandas()` then calls
PyArrow's pandas conversion, which may copy.

Scalar input conversion and row materialization stay in `pyspark-rs`. `collect()`
still uses the native client's owned rows; `createDataFrame(pandas)` still
materializes Python rows. Neither path is claimed to be zero-copy.

The Arrow handoff has been tested locally with PyArrow 10.0.0 (Python 3.9)
and 26.0.0 (Python 3.12). CI installs the latest PyArrow; it does not pin or
separately test the declared PyArrow 10 minimum. The native live-server smoke
tests in `python/tests/test_arrow_e2e.py` run in the Connect parity CI job
against its live server and built native extension. Official parity tests
exercise the Rust transport with official PySpark; the separate native smoke
step covers the drop-in `toArrow()` path.

## UDF strategy (full compatibility)

Expand Down
9 changes: 9 additions & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,15 @@ triggers `ExecutePlan`. If the response stream breaks mid-flight, the core's
reattachable-execute iterator resumes from the last response it saw, so long
results survive transient disconnects.

### Python Arrow table handoff

`toArrow()` collects decoded batches and uses arrow-rs's `IntoPyArrow` for one
boxed `RecordBatchReader`. Compatible value and offset buffers are shared;
network decoding, sliced validity bitmaps, and pandas conversion may still copy.
Batch schema consistency is validated before export. Field names or timezones
containing NUL use IPC over the collected batches, without another RPC. Empty
batch vectors retain the empty-schema behavior.

## Two front-ends, one core

=== "Rust (native)"
Expand Down
67 changes: 67 additions & 0 deletions python/tests/test_arrow_e2e.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Native toArrow smoke tests against a live Spark Connect server."""

import os
from datetime import date, datetime, timezone

import pyarrow as pa
import pytest
from pyspark.sql import SparkSession


@pytest.fixture
def session():
remote = os.environ.get("SPARK_REMOTE")
if not remote:
pytest.skip("SPARK_REMOTE is required for live-server tests")
session = SparkSession.builder.remote(remote).getOrCreate()
try:
yield session
finally:
session.stop()


def test_to_arrow_nested_null_binary_and_temporal_values(session):
table = session.sql("""
SELECT array(1, CAST(NULL AS INT), 3) AS items,
named_struct('name', 'spark', 'value', CAST(NULL AS BIGINT)) AS nested,
X'00FF' AS binary_value, DATE '2020-01-02' AS day,
TIMESTAMP '2020-01-02 03:04:05+00:00' AS instant
""").toArrow()
assert isinstance(table, pa.Table)
values = table.to_pylist()
assert len(values) == 1
assert values[0] == {
"items": [1, None, 3],
"nested": {"name": "spark", "value": None},
"binary_value": b"\x00\xff",
"day": date(2020, 1, 2),
"instant": datetime(2020, 1, 2, 3, 4, 5, tzinfo=timezone.utc),
}


def test_to_arrow_numeric_range(session):
table = session.range(10000).toArrow()
assert table.num_rows == 10000
assert table.column("id").to_pylist() == list(range(10000))


def test_to_arrow_empty_result(session):
table = session.range(0).toArrow()
assert isinstance(table, pa.Table)
assert table.num_rows == 0
Loading
Loading