diff --git a/.github/workflows/build_python_connect.yml b/.github/workflows/build_python_connect.yml index 993eccf3..b520e281 100644 --- a/.github/workflows/build_python_connect.yml +++ b/.github/workflows/build_python_connect.yml @@ -213,6 +213,13 @@ jobs: --exclude "test_artifact.py,test_artifact_localcluster.py,test_parity_resources.py" \ 2>&1 | tee /tmp/parity_ours.log + - name: Native Arrow handoff smoke tests (against the live server) + working-directory: python + env: + SPARK_REMOTE: sc://localhost:15002 + PYTHONPATH: ${{ github.workspace }}/python + run: python -m pytest tests/test_arrow_e2e.py -q + - name: Restart Connect server in local-cluster mode # Resource profiles and artifact distribution across executors can only be # exercised against a real multi-executor cluster, so - mirroring Apache's diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 7f017ad7..5afc7c0d 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -226,7 +226,9 @@ jobs: - name: Measure Python coverage with offline test suite run: | - cd python && PYTHONPATH=$(pwd) python -m coverage run --rcfile=.coveragerc -m pytest tests/test_dropin_offline.py + cd python + PYTHONPATH=$(pwd) python -m coverage run --rcfile=.coveragerc -m pytest \ + tests/test_dropin_offline.py tests/test_arrow_handoff.py PYTHONPATH=$(pwd) python -m coverage json --rcfile=.coveragerc -o coverage-python.json - name: Parse Python coverage and create badge JSON diff --git a/Cargo.lock b/Cargo.lock index 39d506da..29e7d3fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -168,6 +168,7 @@ dependencies = [ "arrow-ipc", "arrow-json", "arrow-ord", + "arrow-pyarrow", "arrow-row", "arrow-schema", "arrow-select", @@ -322,6 +323,18 @@ dependencies = [ "arrow-select", ] +[[package]] +name = "arrow-pyarrow" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29abdf672a81c1aeb57fd2661457f9918964d49aed0e9f18932535f2a9e49ce" +dependencies = [ + "arrow-array", + "arrow-data", + "arrow-schema", + "pyo3", +] + [[package]] name = "arrow-row" version = "58.3.0" @@ -341,6 +354,7 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" dependencies = [ + "bitflags", "serde_core", "serde_json", ] diff --git a/crates/pyspark-rs/Cargo.toml b/crates/pyspark-rs/Cargo.toml index 16148d41..24171693 100644 --- a/crates/pyspark-rs/Cargo.toml +++ b/crates/pyspark-rs/Cargo.toml @@ -2,6 +2,8 @@ name = "pyspark-rs" version.workspace = true edition.workspace = true +# Arrow 58.3.0, including arrow-pyarrow, requires Rust 1.85. +rust-version = "1.85" license.workspace = true description = "PyO3 bindings exposing the Rust Spark Connect client as a drop-in pyspark package" # The Python extension (cdylib) is distributed as a PyPI wheel, not on crates.io. @@ -23,7 +25,7 @@ spark-connect = { package = "apache-spark-connect", path = "../spark-connect" } spark-connect-core = { package = "apache-spark-connect-core", path = "../spark-connect-core" } spark-connect-proto = { package = "apache-spark-connect-proto", path = "../spark-connect-proto" } pyo3.workspace = true -arrow.workspace = true +arrow = { workspace = true, features = ["pyarrow"] } tokio.workspace = true prost.workspace = true tonic.workspace = true diff --git a/crates/pyspark-rs/src/dataframe.rs b/crates/pyspark-rs/src/dataframe.rs index 7101aadf..5ae82b0c 100644 --- a/crates/pyspark-rs/src/dataframe.rs +++ b/crates/pyspark-rs/src/dataframe.rs @@ -1166,19 +1166,15 @@ impl PyDataFrame { /// Collect the DataFrame into a `pyarrow.Table`. /// /// Mirrors `pyspark.sql.DataFrame.toArrow()`: the result is decoded from the - /// server's Arrow IPC stream and handed to `pyarrow` (no row-by-row copy). + /// server's Arrow IPC stream and handed to `pyarrow` through arrow-rs's + /// reader conversion, sharing compatible decoded Arrow value buffers. Bitmap alignment + /// and schemas requiring the IPC compatibility fallback can still copy. #[pyo3(name = "toArrow")] fn to_arrow(&self, py: Python<'_>) -> PyResult> { - use pyo3::types::PyBytes; - // `DataFrame::to_arrow` produces Arrow IPC *file* bytes (FileWriter). - let bytes = py.detach(|| self.dataframe.to_arrow()).to_pyerr()?; - let pa = py.import("pyarrow")?; - let py_bytes = PyBytes::new(py, &bytes); - let table = pa - .getattr("ipc")? - .call_method1("open_file", (py_bytes,))? - .call_method0("read_all")?; - Ok(table.unbind()) + let batches = py + .detach(|| self.dataframe.collect_record_batches()) + .to_pyerr()?; + crate::pyarrow_export::record_batches_to_pyarrow(py, batches) } /// Collect the DataFrame into a pandas DataFrame. diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs index 43a850f4..fec1a7d0 100644 --- a/crates/pyspark-rs/src/lib.rs +++ b/crates/pyspark-rs/src/lib.rs @@ -37,6 +37,7 @@ mod ml; mod observation; mod pipelines; mod profiler; +mod pyarrow_export; mod readwriter; mod resource; mod row; diff --git a/crates/pyspark-rs/src/pyarrow_export.rs b/crates/pyspark-rs/src/pyarrow_export.rs new file mode 100644 index 00000000..0a9418d0 --- /dev/null +++ b/crates/pyspark-rs/src/pyarrow_export.rs @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Export decoded Spark results using arrow-rs's PyArrow reader conversion. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::ipc::writer::FileWriter; +use arrow::pyarrow::IntoPyArrow; +use arrow::record_batch::{RecordBatch, RecordBatchIterator, RecordBatchReader}; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::PyBytes; + +/// Share compatible value and offset buffers with PyArrow in one reader import. +/// +/// Batches have already been collected; this does not stream the network response. +/// Spark result maps are unsorted and result fields are not dictionary-encoded. +/// An empty vector retains the existing empty-schema behavior. +pub(crate) fn record_batches_to_pyarrow( + py: Python<'_>, + batches: Vec, +) -> PyResult> { + let schema = batches + .first() + .map(|batch| batch.schema()) + .unwrap_or_else(|| Arc::new(Schema::empty())); + // RecordBatchIterator does not validate schemas. Check before either path, + // including metadata, so later batches are not relabeled with the first schema. + if batches.iter().any(|batch| batch.schema_ref() != &schema) { + return Err(PyValueError::new_err( + "Arrow record batches have inconsistent schemas", + )); + } + if schema.fields().iter().any(|field| field_needs_ipc(field)) { + // Arrow 58.3 panics on NUL-containing C strings inside the C stream + // `get_schema` callback. Unwinding across that extern "C" boundary + // aborts Python instead of raising an exception. Check before export, + // then reuse the collected batches for IPC without another request. + // CString::new(...).unwrap() in FFI_ArrowSchema::try_new / with_name: + // https://github.com/apache/arrow-rs/blob/58.3.0/arrow-schema/src/ffi.rs#L148 + // https://github.com/apache/arrow-rs/blob/58.3.0/arrow-schema/src/ffi.rs#L174 + let bytes = py + .detach(|| -> arrow::error::Result> { + let mut bytes = Vec::new(); + let mut writer = FileWriter::try_new(&mut bytes, &schema)?; + for batch in &batches { + writer.write(batch)?; + } + writer.finish()?; + drop(writer); + Ok(bytes) + }) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + return Ok(py + .import("pyarrow")? + .getattr("ipc")? + .call_method1("open_file", (PyBytes::new(py, &bytes),))? + .call_method0("read_all")? + .unbind()); + } + + let reader: Box = Box::new(RecordBatchIterator::new( + batches.into_iter().map(Ok), + schema, + )); + Ok(reader.into_pyarrow(py)?.call_method0("read_all")?.unbind()) +} + +fn field_needs_ipc(field: &Field) -> bool { + field.name().contains('\0') || data_type_needs_ipc(field.data_type()) +} + +fn data_type_needs_ipc(data_type: &DataType) -> bool { + match data_type { + DataType::Timestamp(_, Some(timezone)) => timezone.contains('\0'), + DataType::List(field) + | DataType::LargeList(field) + | DataType::ListView(field) + | DataType::LargeListView(field) + | DataType::FixedSizeList(field, _) + | DataType::Map(field, _) => field_needs_ipc(field), + DataType::Struct(fields) => fields.iter().any(|field| field_needs_ipc(field)), + DataType::Union(fields, _) => fields.iter().any(|(_, field)| field_needs_ipc(field)), + DataType::RunEndEncoded(run_ends, values) => { + field_needs_ipc(run_ends) || field_needs_ipc(values) + } + DataType::Dictionary(keys, values) => { + data_type_needs_ipc(keys) || data_type_needs_ipc(values) + } + _ => false, + } +} diff --git a/dev/design/architecture.md b/dev/design/architecture.md index 87303768..ead757c1 100644 --- a/dev/design/architecture.md +++ b/dev/design/architecture.md @@ -67,9 +67,33 @@ Server-streaming RPCs (`ExecutePlan`) are consumed into Arrow record batches. ## Data path -Results arrive as Arrow IPC streams. `arrow-rs` decodes them; hand-off to Python -uses the **Arrow C Data Interface** (zero-copy to `pyarrow`), which is also how -we build `toPandas()` / `createDataFrame(pandas)` without re-serializing. +Results arrive as Arrow IPC streams. `arrow-rs` decodes them; `toArrow()` collects +all batches into a Rust vector. `pyspark-rs` uses arrow-rs's `IntoPyArrow` for +one boxed `RecordBatchReader`, importing it as a PyArrow +`RecordBatchReader`, and calls `read_all()`. PyArrow pulls batches through native +callbacks that do not call Python, avoiding a per-batch Python import loop. +This is an in-process buffer handoff, not network streaming or bounded-memory +result collection. + +Compatible Arrow value and offset buffers are shared with PyArrow; sliced +validity bitmaps may need realignment. The bridge validates batch schema +consistency, including metadata. Schemas with embedded +NUL field names or timezones use an IPC compatibility fallback, without another +RPC. Spark results use unsorted maps and are not dictionary-encoded; this path +does not repair generic Arrow ordering flags. `toPandas()` then calls +PyArrow's pandas conversion, which may copy. + +Scalar input conversion and row materialization stay in `pyspark-rs`. `collect()` +still uses the native client's owned rows; `createDataFrame(pandas)` still +materializes Python rows. Neither path is claimed to be zero-copy. + +The Arrow handoff has been tested locally with PyArrow 10.0.0 (Python 3.9) +and 26.0.0 (Python 3.12). CI installs the latest PyArrow; it does not pin or +separately test the declared PyArrow 10 minimum. The native live-server smoke +tests in `python/tests/test_arrow_e2e.py` run in the Connect parity CI job +against its live server and built native extension. Official parity tests +exercise the Rust transport with official PySpark; the separate native smoke +step covers the drop-in `toArrow()` path. ## UDF strategy (full compatibility) diff --git a/docs/architecture.md b/docs/architecture.md index f7fdb920..17559ed9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -102,6 +102,15 @@ triggers `ExecutePlan`. If the response stream breaks mid-flight, the core's reattachable-execute iterator resumes from the last response it saw, so long results survive transient disconnects. +### Python Arrow table handoff + +`toArrow()` collects decoded batches and uses arrow-rs's `IntoPyArrow` for one +boxed `RecordBatchReader`. Compatible value and offset buffers are shared; +network decoding, sliced validity bitmaps, and pandas conversion may still copy. +Batch schema consistency is validated before export. Field names or timezones +containing NUL use IPC over the collected batches, without another RPC. Empty +batch vectors retain the empty-schema behavior. + ## Two front-ends, one core === "Rust (native)" diff --git a/python/tests/test_arrow_e2e.py b/python/tests/test_arrow_e2e.py new file mode 100644 index 00000000..dc76866d --- /dev/null +++ b/python/tests/test_arrow_e2e.py @@ -0,0 +1,67 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Native toArrow smoke tests against a live Spark Connect server.""" + +import os +from datetime import date, datetime, timezone + +import pyarrow as pa +import pytest +from pyspark.sql import SparkSession + + +@pytest.fixture +def session(): + remote = os.environ.get("SPARK_REMOTE") + if not remote: + pytest.skip("SPARK_REMOTE is required for live-server tests") + session = SparkSession.builder.remote(remote).getOrCreate() + try: + yield session + finally: + session.stop() + + +def test_to_arrow_nested_null_binary_and_temporal_values(session): + table = session.sql(""" + SELECT array(1, CAST(NULL AS INT), 3) AS items, + named_struct('name', 'spark', 'value', CAST(NULL AS BIGINT)) AS nested, + X'00FF' AS binary_value, DATE '2020-01-02' AS day, + TIMESTAMP '2020-01-02 03:04:05+00:00' AS instant + """).toArrow() + assert isinstance(table, pa.Table) + values = table.to_pylist() + assert len(values) == 1 + assert values[0] == { + "items": [1, None, 3], + "nested": {"name": "spark", "value": None}, + "binary_value": b"\x00\xff", + "day": date(2020, 1, 2), + "instant": datetime(2020, 1, 2, 3, 4, 5, tzinfo=timezone.utc), + } + + +def test_to_arrow_numeric_range(session): + table = session.range(10000).toArrow() + assert table.num_rows == 10000 + assert table.column("id").to_pylist() == list(range(10000)) + + +def test_to_arrow_empty_result(session): + table = session.range(0).toArrow() + assert isinstance(table, pa.Table) + assert table.num_rows == 0 diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py new file mode 100644 index 00000000..214a09a0 --- /dev/null +++ b/python/tests/test_arrow_handoff.py @@ -0,0 +1,397 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Check public Arrow handoff behavior against a deterministic mock RPC service. + +Cases cover Spark-output values, schemas, chunks, and lifetimes through public +APIs. Server-side Python addresses cannot prove Rust-to-Python buffer sharing. +""" + +import gc +import os +import shutil +import subprocess +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from datetime import datetime, timezone +from decimal import Decimal +from pathlib import Path + +import grpc +import pandas as pd +import pyarrow as pa +import pytest +from google.protobuf import descriptor_pb2, descriptor_pool, message_factory +from pyspark.sql import SparkSession + + +@pytest.fixture(scope="module") +def protocol(tmp_path_factory): + """Compile the repository protocol instead of hard-coding protobuf field numbers.""" + protoc = shutil.which(os.environ.get("PROTOC", "protoc")) + if protoc is None: + message = "Arrow handoff tests require protoc; install it or set PROTOC" + if os.environ.get("CI"): + pytest.fail(message) + pytest.skip(message) + proto_root = Path(__file__).resolve().parents[2] / "crates/spark-connect-proto/proto" + descriptor = tmp_path_factory.mktemp("connect-proto") / "connect.pb" + command = [protoc, f"-I{proto_root}"] + if include := os.environ.get("PROTOC_INCLUDE"): + command.append(f"-I{include}") + subprocess.run( + command + + [ + "--include_imports", + f"--descriptor_set_out={descriptor}", + str(proto_root / "spark/connect/base.proto"), + ], + check=True, + ) + pool = descriptor_pool.DescriptorPool() + for file in descriptor_pb2.FileDescriptorSet.FromString(descriptor.read_bytes()).file: + pool.Add(file) + + def message(name, **fields): + return message_factory.GetMessageClass(pool.FindMessageTypeByName(f"spark.connect.{name}"))( + **fields + ) + + return message + + +@pytest.mark.parametrize("in_ci", [False, True]) +def test_missing_protoc_fails_in_ci_and_skips_locally(monkeypatch, tmp_path_factory, in_ci): + monkeypatch.setattr(shutil, "which", lambda _: None) + if in_ci: + monkeypatch.setenv("CI", "true") + else: + monkeypatch.delenv("CI", raising=False) + error = pytest.fail.Exception if in_ci else pytest.skip.Exception + with pytest.raises(error, match="require protoc"): + protocol.__wrapped__(tmp_path_factory) + + +@contextmanager +def _dataframe(batches, *, protocol, payloads=None, calls=None): + """Serve Arrow results using the repository's protobuf message definitions.""" + if payloads is None: + payloads = [] + for batch in batches: + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, batch.schema) as writer: + writer.write_batch(batch) + payloads.append((batch.num_rows, sink.getvalue().to_pybytes())) + server_id = "00000000-0000-0000-0000-000000000001" + + def execute(request, context): + if calls is not None: + calls["execute"] = calls.get("execute", 0) + 1 + request = protocol("ExecutePlanRequest").FromString(request) + header = dict( + session_id=request.session_id, + operation_id=request.operation_id, + server_side_session_id=server_id, + ) + for index, (rows, payload) in enumerate(payloads): + yield protocol( + "ExecutePlanResponse", + **header, + response_id=str(index), + arrow_batch=dict(row_count=rows, data=payload), + ).SerializeToString() + yield protocol( + "ExecutePlanResponse", **header, response_id="complete", result_complete={} + ).SerializeToString() + + def release_execute(request, context): + request = protocol("ReleaseExecuteRequest").FromString(request) + return protocol( + "ReleaseExecuteResponse", + session_id=request.session_id, + operation_id=request.operation_id, + server_side_session_id=server_id, + ).SerializeToString() + + def release_session(request, context): + request = protocol("ReleaseSessionRequest").FromString(request) + return protocol( + "ReleaseSessionResponse", + session_id=request.session_id, + server_side_session_id=server_id, + ).SerializeToString() + + with ThreadPoolExecutor(max_workers=2) as executor: + server = grpc.server(executor) + server.add_generic_rpc_handlers( + [ + grpc.method_handlers_generic_handler( + "spark.connect.SparkConnectService", + { + "ExecutePlan": grpc.unary_stream_rpc_method_handler(execute), + "ReleaseExecute": grpc.unary_unary_rpc_method_handler(release_execute), + "ReleaseSession": grpc.unary_unary_rpc_method_handler(release_session), + }, + ) + ] + ) + port = server.add_insecure_port("127.0.0.1:0") + assert port != 0 + server.start() + session = None + try: + session = SparkSession.builder.remote(f"sc://127.0.0.1:{port}").getOrCreate() + yield session.range(1), session + finally: + try: + if session is not None: + session.stop() + finally: + server.stop(0).wait(timeout=5) + + +def _mixed_batch(): + nested_type = pa.struct( + [ + pa.field("items", pa.list_(pa.int64()), metadata={b"nested": b"metadata"}), + pa.field("labels", pa.map_(pa.string(), pa.int64())), + ] + ) + arrays = [ + pa.array([1, None, -3, 4], type=pa.int64()), + pa.array([True, None, False, True]), + pa.array(["alpha", None, "", "omega"]), + pa.array([b"a", None, b"", b"xyz"]), + pa.array( + [Decimal("1.23"), None, Decimal("-4.56"), Decimal("0.00")], type=pa.decimal128(12, 2) + ), + pa.array( + [ + datetime(2020, 1, 1, tzinfo=timezone.utc), + None, + datetime(2020, 1, 2, tzinfo=timezone.utc), + datetime(2020, 1, 3, tzinfo=timezone.utc), + ], + type=pa.timestamp("us", tz="UTC"), + ), + pa.array( + [ + {"items": [1, None], "labels": [("a", 1)]}, + None, + {"items": [], "labels": []}, + {"items": None, "labels": [("b", None)]}, + ], + type=nested_type, + ), + pa.nulls(4), + ] + fields = [ + pa.field(f"c{index}", array.type, metadata={b"field": str(index).encode()}) + for index, array in enumerate(arrays) + ] + schema = pa.schema(fields, metadata={b"schema": b"preserved"}) + return pa.RecordBatch.from_arrays(arrays, schema=schema) + + +def test_handoff_preserves_values_metadata_and_chunks(protocol): + batch = _mixed_batch() + batches = [batch.slice(1, 2), batch.slice(0, 1), batch.slice(3, 1)] + expected = pa.Table.from_batches(batches) + calls = {} + with _dataframe(protocol=protocol, batches=batches, calls=calls) as (frame, _): + actual = frame.toArrow() + assert actual.equals(expected, check_metadata=True) + for column in actual.columns: + assert [len(chunk) for chunk in column.chunks] == [2, 1, 1] + assert calls == {"execute": 1} + + +def test_normal_handoff_does_not_reopen_ipc(protocol, monkeypatch): + def unexpected_ipc(*args, **kwargs): + pytest.fail("Normal toArrow handoff must use the C Stream importer") + + monkeypatch.setattr(pa.ipc, "open_file", unexpected_ipc) + batch = _mixed_batch() + with _dataframe(protocol=protocol, batches=[batch]) as (frame, _): + assert frame.toArrow().equals(pa.Table.from_batches([batch]), check_metadata=True) + + +def test_handoff_buffers_outlive_dataframe_session_and_table(protocol): + batch = _mixed_batch() + expected = batch.to_pydict() + calls = {} + + def collect(): + with _dataframe(protocol=protocol, batches=[batch], calls=calls) as (frame, session): + table = frame.toArrow() + session.stop() + return table + + table = collect() + gc.collect() + assert table.to_pydict() == expected + numeric = table.column(0).chunk(0).slice(1, 3) + strings = table.column(2).chunk(0).slice(1, 3) + del table + gc.collect() + assert numeric.to_pylist() == [None, -3, 4] + assert strings.to_pylist() == [None, "", "omega"] + assert calls == {"execute": 1} + + +@pytest.mark.parametrize("kind", ["no-batches", "empty-typed", "zero-columns"]) +def test_handoff_empty_and_zero_column_results(protocol, kind): + if kind == "no-batches": + batches = [] + expected = pa.table({}) + elif kind == "empty-typed": + batches = [_mixed_batch().slice(0, 0)] + expected = pa.Table.from_batches(batches) + else: + structs = pa.array([{}] * 5, type=pa.struct([])) + batch = pa.RecordBatch.from_struct_array(structs) + batch = batch.replace_schema_metadata({b"zero-column": b"preserved"}) + batches = [batch.slice(0, 3), batch.slice(3, 2)] + expected = pa.Table.from_batches(batches) + assert expected.num_rows == 5 + calls = {} + with _dataframe(protocol=protocol, batches=batches, calls=calls) as (frame, _): + actual = frame.toArrow() + assert actual.num_rows == expected.num_rows + assert actual.num_columns == expected.num_columns + assert actual.equals(expected, check_metadata=True) + assert calls == {"execute": 1} + + +def test_to_pandas_uses_the_same_handoff(protocol): + batch = _mixed_batch() + indices = [0, 2, 4, 5] + schema = pa.schema( + [batch.schema.field(index) for index in indices], metadata=batch.schema.metadata + ) + arrays = [batch.column(index) for index in indices] + selected = pa.RecordBatch.from_arrays(arrays, schema=schema) + batches = [selected.slice(0, 2), selected.slice(2, 2)] + expected = pa.Table.from_batches(batches).to_pandas() + calls = {} + with _dataframe(protocol=protocol, batches=batches, calls=calls) as (frame, _): + actual = frame.toPandas() + pd.testing.assert_frame_equal(actual, expected) + assert calls == {"execute": 1} + + +@pytest.mark.parametrize( + "kind", ["field", "nested-field", "list-struct", "map-value-struct", "timezone"] +) +def test_nul_schema_strings_fall_back_without_executing_again(protocol, kind, monkeypatch): + open_file = pa.ipc.open_file + ipc_calls = [] + + def record_ipc(*args, **kwargs): + ipc_calls.append(True) + return open_file(*args, **kwargs) + + monkeypatch.setattr(pa.ipc, "open_file", record_ipc) + if kind == "field": + name = "bad\x00name" + values = pa.array([1, None, -3, 4]) + elif kind == "nested-field": + name = "nested" + data_type = pa.struct([pa.field("bad\x00name", pa.int64())]) + values = pa.array( + [{"bad\x00name": 1}, None, {"bad\x00name": None}, {"bad\x00name": 4}], + type=data_type, + ) + elif kind == "list-struct": + name = "items" + data_type = pa.list_(pa.struct([pa.field("bad\x00name", pa.int64())])) + values = pa.array([[{"bad\x00name": 1}], None, [], [{"bad\x00name": None}]], type=data_type) + elif kind == "map-value-struct": + name = "mapping" + data_type = pa.map_(pa.string(), pa.struct([pa.field("bad\x00name", pa.int64())])) + values = pa.array( + [[("a", {"bad\x00name": 1})], None, [], [("b", {"bad\x00name": None})]], type=data_type + ) + else: + name = "timestamp" + values = pa.array([0, None, 2, 3], type=pa.timestamp("us", tz="UTC\x00suffix")) + schema = pa.schema( + [pa.field(name, values.type, metadata={b"field": b"preserved"})], + metadata={b"schema": b"preserved"}, + ) + batch = pa.RecordBatch.from_arrays([values], schema=schema) + batches = [batch.slice(0, 2), batch.slice(2, 2)] + expected = pa.Table.from_batches(batches) + calls = {} + with _dataframe(protocol=protocol, batches=batches, calls=calls) as (frame, _): + actual = frame.toArrow() + # Compare raw Arrow data, not Python datetimes: the deliberately unusual + # timezone string is preserved but is not a Python timezone identifier. + assert actual.equals(expected, check_metadata=True) + assert [len(chunk) for chunk in actual.column(0).chunks] == [2, 2] + assert ipc_calls == [True] + assert calls == {"execute": 1} + + +def test_nul_schema_and_field_metadata_are_preserved(protocol): + field = pa.field("value", pa.int64(), metadata={b"field\x00key": b"field\x00value"}) + schema = pa.schema([field], metadata={b"schema\x00key": b"schema\x00value"}) + batch = pa.RecordBatch.from_arrays([pa.array([1, None, 3])], schema=schema) + calls = {} + with _dataframe(protocol=protocol, batches=[batch], calls=calls) as (frame, _): + actual = frame.toArrow() + assert actual.equals(pa.Table.from_batches([batch]), check_metadata=True) + assert calls == {"execute": 1} + + +@pytest.mark.parametrize( + "difference", ["type", "name", "schema-metadata", "field-metadata", "nullability"] +) +@pytest.mark.parametrize("nul_name", [False, True]) +def test_mismatched_batch_schemas_are_rejected_before_fallback(protocol, difference, nul_name): + name = "bad\x00name" if nul_name else "value" + first_field = pa.field(name, pa.int64(), metadata={b"field": b"original"}) + first_schema = pa.schema([first_field], metadata={b"schema": b"original"}) + second_values = pa.array(["text", "other"]) if difference == "type" else pa.array([3, 4]) + second_name = name + "-different" if difference == "name" else name + field_metadata = {b"field": b"different"} if difference == "field-metadata" else None + schema_metadata = {b"schema": b"different"} if difference == "schema-metadata" else None + second_field = pa.field( + second_name, + second_values.type, + nullable=difference != "nullability", + metadata=field_metadata or first_field.metadata, + ) + second_schema = pa.schema([second_field], metadata=schema_metadata or first_schema.metadata) + batches = [ + pa.RecordBatch.from_arrays([pa.array([1, 2])], schema=first_schema), + pa.RecordBatch.from_arrays([second_values], schema=second_schema), + ] + calls = {} + with _dataframe(protocol=protocol, batches=batches, calls=calls) as (frame, _): + with pytest.raises(ValueError, match="inconsistent schemas"): + frame.toArrow() + assert calls == {"execute": 1} + + +def test_malformed_ipc_is_reported(protocol): + calls = {} + with _dataframe( + protocol=protocol, batches=[], payloads=[(1, b"not arrow ipc")], calls=calls + ) as (frame, _): + with pytest.raises(RuntimeError, match="Arrow"): + frame.toArrow() + assert calls == {"execute": 1}