diff --git a/Cargo.lock b/Cargo.lock index 29e7d3fe..6238cb96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,6 +87,7 @@ dependencies = [ "prost-types", "regex", "serde_json", + "sha2", "tempfile", "thiserror", "tokio", diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs index 0394c642..29584def 100644 --- a/crates/pyspark-rs/src/lib.rs +++ b/crates/pyspark-rs/src/lib.rs @@ -38,6 +38,7 @@ mod observation; mod pipelines; mod profiler; mod pyarrow_export; +mod pyarrow_import; mod readwriter; mod resource; mod row; diff --git a/crates/pyspark-rs/src/pyarrow_import.rs b/crates/pyspark-rs/src/pyarrow_import.rs new file mode 100644 index 00000000..eb39cfbf --- /dev/null +++ b/crates/pyspark-rs/src/pyarrow_import.rs @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! PyArrow Table input through arrow-rs's owned C Stream importer. + +use arrow::pyarrow::{FromPyArrow, Table}; +use arrow::record_batch::RecordBatch; +use pyo3::prelude::*; +use spark_connect::session::SparkSession; + +use crate::dataframe::PyDataFrame; +use crate::errors::ResultExt; + +pub(crate) fn create_dataframe( + py: Python<'_>, + session: &SparkSession, + table: &Bound<'_, PyAny>, + schema: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let helper = py.import("pyspark.sql._arrow_input")?; + // Resolve DDL with the same converter as the existing local-data path. + let schema = match schema { + Some(s) if s.extract::().is_ok() => Some(crate::types::data_type_to_py( + py, + &crate::types::py_to_data_type(s)?, + )?), + Some(s) => Some(s.clone().unbind()), + None => None, + }; + let schema = schema.unwrap_or_else(|| py.None()); + // Empty input needs only the explicit schema, and no server configuration. + let empty: bool = table.call_method0("__len__")?.extract::()? == 0; + let options = if empty { + None + } else { + Some(py.detach(|| session.arrow_input_options().to_pyerr())?) + }; + let result = helper.getattr("prepare_table")?.call1(( + table, + schema, + options.as_ref().and_then(|o| o.timestamp_type()), + options.as_ref().and_then(|o| o.timezone()), + options.as_ref().and_then(|o| o.use_large_var_types()), + ))?; + let normalized = result.get_item(0)?; + let schema = result.get_item(1)?; + let spark_schema = if schema.is_none() { + None + } else { + Some(crate::types::py_to_data_type(&schema)?) + }; + let names: Option> = result.get_item(2)?.extract()?; + let batches = if empty { + Vec::new() + } else { + // PyArrow 10 readers support _export_to_c, even without PyCapsules. + let (mut batches, arrow_schema) = + Table::from_pyarrow_bound(&normalized.call_method0("to_reader")?)?.into_inner(); + // The cast drops the rows of a zero-column table, leaving no batches. Keep its + // schema, as PySpark's IPC stream does, so the relation still carries one. + if batches.is_empty() { + batches.push(RecordBatch::new_empty(arrow_schema)); + } + batches + }; + let dataframe = py.detach(|| { + match options.as_ref() { + Some(options) => { + session.create_dataframe_from_arrow_with_options(batches, spark_schema, options) + } + None => session.create_dataframe_from_arrow(batches, spark_schema), + } + .to_pyerr() + })?; + // As in PySpark, column names are applied with toDF only when there are any. + let dataframe = match names { + Some(names) if !names.is_empty() => { + dataframe.to_df(names.iter().map(String::as_str).collect()) + } + _ => dataframe, + }; + Ok(PyDataFrame::new(dataframe)) +} diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs index 89dd7ce9..b8f6d5fd 100644 --- a/crates/pyspark-rs/src/session.rs +++ b/crates/pyspark-rs/src/session.rs @@ -278,118 +278,11 @@ impl PySparkSession { pub fn new(session: SparkSession) -> Self { PySparkSession { session } } - - /// `createDataFrame` of a pyarrow Table, as the reference client does it: the column - /// names come from a list `schema` (extra columns are named `_N`), or else from the table; - /// the Spark types come from a `StructType` (or DDL) `schema`, or else from the Arrow - /// types. The table is cast to the Arrow form of those types, with its timestamps - /// localized to the session time zone, and sent as an Arrow IPC stream. - fn create_dataframe_from_arrow( - &self, - py: Python<'_>, - table: &Bound<'_, PyAny>, - schema: Option<&Bound<'_, PyAny>>, - ) -> PyResult { - let mut names: Option> = None; - let mut user_schema: Option = None; - match schema { - None => names = Some(table.getattr("column_names")?.extract()?), - Some(s) => { - if let Ok(mut given) = s.extract::>() { - let num_columns: usize = table.getattr("num_columns")?.extract()?; - given.extend((given.len()..num_columns).map(|i| format!("_{}", i + 1))); - names = Some(given); - } else { - user_schema = Some(crate::types::py_to_data_type(s)?); - } - } - } - // As the reference does for any empty input: a relation of the user's schema, which - // a list of names does not give. - if table.len()? == 0 { - return match user_schema { - Some(dt) => Ok(PyDataFrame::new( - py.detach(|| self.session.create_dataframe(vec![], dt)) - .to_pyerr()?, - )), - None => Err(crate::column::raise_pyspark( - py, - "PySparkValueError", - "CANNOT_INFER_EMPTY_SCHEMA", - &[], - )), - }; - } - - let conf = self.session.conf(); - let configs = py - .detach(|| { - conf.get_many(&[ - "spark.sql.timestampType", - "spark.sql.session.timeZone", - "spark.sql.execution.arrow.useLargeVarTypes", - ]) - }) - .to_pyerr()?; - let config = |i: usize| configs.get(i).cloned().flatten(); - let prefer_timestamp_ntz = config(0).as_deref() == Some("TIMESTAMP_NTZ"); - let timezone = config(1); - let prefers_large_types = config(2).as_deref() == Some("true"); - - let types = py.import("pyspark.sql.pandas.types")?; - let spark_schema = match &user_schema { - Some(dt @ DataType::Struct { .. }) => { - crate::types::data_type_to_py(py, dt)?.into_bound(py) - } - _ => types.call_method1( - "from_arrow_schema", - (table.getattr("schema")?, prefer_timestamp_ntz), - )?, - }; - let kwargs = PyDict::new(py); - kwargs.set_item("error_on_duplicated_field_names_in_struct", true)?; - kwargs.set_item("timezone", "UTC")?; - kwargs.set_item("prefers_large_types", prefers_large_types)?; - let arrow_schema = types.call_method("to_arrow_schema", (&spark_schema,), Some(&kwargs))?; - let converted = types - .call_method1( - "_check_arrow_table_timestamps_localize", - (table, &spark_schema, true, timezone), - )? - .call_method1("cast", (arrow_schema,))? - .call_method1("rename_columns", (spark_schema.getattr("names")?,))?; - - let pa = py.import("pyarrow")?; - let sink = pa.call_method0("BufferOutputStream")?; - let writer = pa - .getattr("ipc")? - .call_method1("new_stream", (&sink, converted.getattr("schema")?))?; - writer.call_method1("write_table", (&converted,))?; - writer.call_method0("close")?; - let stream = sink.call_method0("getvalue")?.call_method0("to_pybytes")?; - let data = stream.cast::()?.as_bytes().to_vec(); - - // The reference sends the user's schema, or none for the server to read the Arrow - // one; the plan always carries a schema, so the Arrow one is sent explicitly. - let relation_schema = match user_schema { - Some(dt) => dt, - None => crate::types::py_to_data_type(&spark_schema)?, - }; - let df = self - .session - .create_dataframe_from_arrow(data, relation_schema); - // Like the reference, apply the column names with toDF. - Ok(PyDataFrame::new(match names { - Some(names) if !names.is_empty() => { - df.to_df(names.iter().map(String::as_str).collect()) - } - _ => df, - })) - } } /// Whether `obj` is an instance of `module.class`, without importing `module` when it is not -/// loaded yet: an instance can only exist once its module was imported. +/// loaded yet: an instance can only exist once its module was imported. A `None` entry in +/// `sys.modules` blocks the import, so it counts as not loaded. fn is_loaded_instance( py: Python<'_>, obj: &Bound<'_, PyAny>, @@ -398,8 +291,8 @@ fn is_loaded_instance( ) -> PyResult { let modules = py.import("sys")?.getattr("modules")?; match modules.cast::()?.get_item(module)? { - Some(m) => obj.is_instance(&m.getattr(class)?), - None => Ok(false), + Some(m) if !m.is_none() => obj.is_instance(&m.getattr(class)?), + _ => Ok(false), } } @@ -508,8 +401,9 @@ impl PySparkSession { // signature parity. See PARITY_GAP_ANALYSIS.md. let _ = (samplingRatio, verifySchema); + // A caller holding a Table has already imported PyArrow. if is_loaded_instance(py, data, "pyarrow", "Table")? { - return self.create_dataframe_from_arrow(py, data, schema); + return crate::pyarrow_import::create_dataframe(py, &self.session, data, schema); } // Accept a pandas DataFrame (as pyspark does): convert it to a list of row lists diff --git a/crates/spark-connect-core/Cargo.toml b/crates/spark-connect-core/Cargo.toml index c6a0c088..6cdf0e01 100644 --- a/crates/spark-connect-core/Cargo.toml +++ b/crates/spark-connect-core/Cargo.toml @@ -26,6 +26,7 @@ serde_json = "1" regex = "1" prost-types.workspace = true crc32fast = "1" +sha2 = "0.10" [dev-dependencies] tempfile = "3" diff --git a/crates/spark-connect-core/src/client.rs b/crates/spark-connect-core/src/client.rs index 948a0a52..8534be34 100644 --- a/crates/spark-connect-core/src/client.rs +++ b/crates/spark-connect-core/src/client.rs @@ -744,33 +744,7 @@ impl SparkConnectClient { // Build request stream with proper chunking/batching let requests = build_artifact_request_stream(self.session_id.clone(), artifacts)?; - - // Create a futures stream from the requests - let request_stream = futures::stream::iter(requests); - - // Call the AddArtifacts streaming RPC - let mut req = Request::new(request_stream); - self._attach_metadata(&mut req); - - let response = self - .stub - .clone() - .add_artifacts(req) - .await - .map_err(SparkError::from_grpc_status)? - .into_inner(); - - // Check that all artifacts were successfully received - for summary in response.artifacts { - if !summary.is_crc_successful { - return Err(SparkError::connect_msg(format!( - "CRC check failed for artifact: {}", - summary.name - ))); - } - } - - Ok(()) + self.upload_artifact_requests(requests, None).await } /// Upload a single local file as an artifact under an explicit artifact name @@ -784,8 +758,61 @@ impl SparkConnectClient { Box::new(FileArtifact::new(local_path)), )]; let requests = build_artifact_request_stream(self.session_id.clone(), artifacts)?; - let request_stream = futures::stream::iter(requests); - let mut req = Request::new(request_stream); + self.upload_artifact_requests(requests, None).await + } + + /// Upload session-scoped cache blobs, returning SHA-256 hashes in input order. + /// Existing artifacts and duplicate blobs are uploaded only once. + pub async fn cache_artifacts(&self, blobs: Vec>) -> Result> { + use crate::artifact::InMemoryArtifact; + use sha2::{Digest, Sha256}; + use std::collections::HashSet; + + let hashes: Vec = blobs + .iter() + .map(|blob| { + Sha256::digest(blob) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + }) + .collect(); + let names: Vec = hashes.iter().map(|hash| format!("cache/{hash}")).collect(); + let unique: HashSet<&str> = names.iter().map(String::as_str).collect(); + if unique.is_empty() { + return Ok(hashes); + } + let cached = self + .artifact_status(&unique.into_iter().collect::>()) + .await?; + let mut seen = HashSet::new(); + let mut artifacts = Vec::new(); + for (blob, name) in blobs.into_iter().zip(names) { + if !cached.get(&name).copied().unwrap_or(false) && seen.insert(name.clone()) { + artifacts.push(NamedArtifact::new( + name, + Box::new(InMemoryArtifact::new(blob)), + )); + } + } + if artifacts.is_empty() { + return Ok(hashes); + } + let expected: HashSet = artifacts.iter().map(|a| a.name.clone()).collect(); + let requests = build_artifact_request_stream(self.session_id.clone(), artifacts)?; + self.upload_artifact_requests(requests, Some(&expected)) + .await?; + Ok(hashes) + } + + /// Send artifact frames and validate the response. + /// Cache uploads also require an acknowledgement for every requested artifact. + async fn upload_artifact_requests( + &self, + requests: Vec, + expected_cache_names: Option<&std::collections::HashSet>, + ) -> Result<()> { + let mut req = Request::new(futures::stream::iter(requests)); self._attach_metadata(&mut req); let response = self .stub @@ -794,6 +821,7 @@ impl SparkConnectClient { .await .map_err(SparkError::from_grpc_status)? .into_inner(); + let mut received = std::collections::HashSet::new(); for summary in response.artifacts { if !summary.is_crc_successful { return Err(SparkError::connect_msg(format!( @@ -801,6 +829,16 @@ impl SparkConnectClient { summary.name ))); } + if expected_cache_names.is_some() { + received.insert(summary.name); + } + } + if let Some(expected) = expected_cache_names { + if !expected.is_subset(&received) { + return Err(SparkError::connect_msg( + "Missing cache artifact upload acknowledgement", + )); + } } Ok(()) } diff --git a/crates/spark-connect/src/arrow_input.rs b/crates/spark-connect/src/arrow_input.rs new file mode 100644 index 00000000..1592468a --- /dev/null +++ b/crates/spark-connect/src/arrow_input.rs @@ -0,0 +1,388 @@ +//! Arrow local relations and session-scoped chunked uploads. + +use arrow::datatypes::SchemaRef; +use arrow::ipc::writer::StreamWriter; +use arrow::record_batch::RecordBatch; +use spark_connect_core::client::SparkConnectClient; +use spark_connect_core::error::{Result, SparkError, SparkErrorKind}; +use spark_connect_core::runtime::block_on; +use std::collections::HashMap; + +use crate::plan::LogicalPlan; +use crate::types::DataType; + +/// Internal settings snapshot shared with the Python Arrow importer; not a stable API. +#[doc(hidden)] +pub struct ArrowInputOptions { + timestamp_type: String, + timezone: String, + use_large_var_types: bool, + cache_threshold: usize, + upload: Options, +} + +impl ArrowInputOptions { + const KEYS: [&'static str; 8] = [ + "spark.sql.timestampType", + "spark.sql.session.timeZone", + "spark.sql.execution.arrow.useLargeVarTypes", + "spark.sql.session.localRelationCacheThreshold", + "spark.sql.session.localRelationSizeLimit", + "spark.sql.session.localRelationChunkSizeRows", + "spark.sql.session.localRelationChunkSizeBytes", + "spark.sql.session.localRelationBatchOfChunksSizeBytes", + ]; + pub(crate) fn from_client(client: &SparkConnectClient) -> Result { + let values = block_on(client.get_configs(&Self::KEYS))?; + let values = Self::KEYS + .into_iter() + .zip(values) + .filter_map(|(key, value)| value.map(|value| (key, value))) + .collect(); + Self::from_values(values) + } + + fn from_values(values: HashMap<&str, String>) -> Result { + let required = |key: &str| { + values.get(key).map(String::as_str).ok_or_else(|| { + SparkError::classed( + SparkErrorKind::SparkNoSuchElement, + "SQL_CONF_NOT_FOUND", + &[("sqlConf", key)], + ) + }) + }; + let invalid = |key: &str| SparkError::connect_msg(format!("Invalid configuration: {key}")); + let integer = |key: &str, allow_zero: bool| { + required(key)? + .parse::() + .ok() + .filter(|value| allow_zero || *value > 0) + .ok_or_else(|| invalid(key)) + }; + let chunk_bytes = integer("spark.sql.session.localRelationChunkSizeBytes", false)?; + let upload_batch_bytes = integer( + "spark.sql.session.localRelationBatchOfChunksSizeBytes", + false, + )?; + Ok(Self { + timestamp_type: required("spark.sql.timestampType")?.to_owned(), + timezone: required("spark.sql.session.timeZone")?.to_owned(), + use_large_var_types: required("spark.sql.execution.arrow.useLargeVarTypes")? == "true", + cache_threshold: integer("spark.sql.session.localRelationCacheThreshold", true)?, + upload: Options { + size_limit: integer("spark.sql.session.localRelationSizeLimit", false)?, + chunk_rows: integer("spark.sql.session.localRelationChunkSizeRows", false)?, + // Match upstream's min(chunk size, upload batch size). + chunk_bytes: chunk_bytes.min(upload_batch_bytes), + upload_batch_bytes, + }, + }) + } + + /// Server timestamp type, including its registered default. + pub fn timestamp_type(&self) -> Option<&str> { + Some(&self.timestamp_type) + } + + /// Server session timezone; do not substitute a client-local timezone. + pub fn timezone(&self) -> Option<&str> { + Some(&self.timezone) + } + + /// Whether the server prefers large Arrow string and binary offsets. + pub fn use_large_var_types(&self) -> Option<&str> { + Some(if self.use_large_var_types { + "true" + } else { + "false" + }) + } +} + +struct Options { + size_limit: usize, + chunk_rows: usize, + chunk_bytes: usize, + upload_batch_bytes: usize, +} + +fn arrow_error(error: arrow::error::ArrowError) -> SparkError { + SparkError::connect_msg(error.to_string()) +} + +fn validate(batches: &[RecordBatch]) -> Result<()> { + if let Some(first) = batches.first() { + if batches.iter().any(|batch| batch.schema() != first.schema()) { + return Err(SparkError::connect_msg("Arrow batch schemas must match")); + } + } + Ok(()) +} + +fn batch_data_size(batch: &RecordBatch) -> usize { + batch + .columns() + .iter() + .map(|array| { + array + .to_data() + .get_slice_memory_size() + .unwrap_or_else(|_| array.get_buffer_memory_size()) + }) + .sum() +} + +pub(crate) fn inline_relation( + batches: &[RecordBatch], + schema: Option, +) -> Result { + validate(batches)?; + let data = if let Some(first) = batches.first() { + let mut writer = StreamWriter::try_new(Vec::new(), &first.schema()).map_err(arrow_error)?; + for batch in batches { + writer.write(batch).map_err(arrow_error)?; + } + Some(writer.into_inner().map_err(arrow_error)?) + } else { + None + }; + Ok(LogicalPlan::LocalRelation { schema, data }) +} + +pub(crate) fn create_relation( + client: &SparkConnectClient, + batches: Vec, + schema: Option, + options: Option<&ArrowInputOptions>, +) -> Result { + // Validate before any RPC, and preserve the offline empty-input path. + validate(&batches)?; + if batches.iter().all(|batch| batch.num_rows() == 0) { + return inline_relation(&batches, schema); + } + let fetched; + let options = match options { + Some(options) => options, + None => { + fetched = ArrowInputOptions::from_client(client)?; + &fetched + } + }; + // Count the logical slices rather than their potentially much larger backing buffers. + let bytes: usize = batches.iter().map(batch_data_size).sum(); + if bytes < options.cache_threshold { + return inline_relation(&batches, schema); + } + let options = &options.upload; + + let mut pending = Vec::new(); + let mut pending_size = 0; + let mut total_size = 0; + let mut hashes = Vec::new(); + let mut enqueue = |chunk: Vec| -> Result<()> { + total_size += chunk.len(); + if total_size > options.size_limit { + return Err(SparkError::classed( + SparkErrorKind::Analysis, + "LOCAL_RELATION_SIZE_LIMIT_EXCEEDED", + &[ + ("actualSize", &total_size.to_string()), + ("sizeLimit", &options.size_limit.to_string()), + ], + )); + } + if !pending.is_empty() && pending_size + chunk.len() > options.upload_batch_bytes { + hashes.extend(block_on( + client.cache_artifacts(std::mem::take(&mut pending)), + )?); + pending_size = 0; + } + pending_size += chunk.len(); + pending.push(chunk); + Ok(()) + }; + if let Some(schema) = &schema { + enqueue(schema.json().into_bytes())?; + } + serialize_chunks(&batches, options, &mut enqueue)?; + hashes.extend(block_on(client.cache_artifacts(pending))?); + let schema_hash = schema.map(|_| hashes.remove(0)); + Ok(LogicalPlan::ChunkedCachedLocalRelation { + data_hashes: hashes, + schema_hash, + }) +} + +fn new_writer(schema: &SchemaRef) -> Result>> { + StreamWriter::try_new(Vec::new(), schema).map_err(arrow_error) +} + +/// Match PySpark's pre-write estimate: stream schema plus each batch's data size. +/// The row setting caps individual record batches, not the complete IPC stream. +fn serialize_chunks( + batches: &[RecordBatch], + options: &Options, + emit: &mut impl FnMut(Vec) -> Result<()>, +) -> Result<()> { + let schema = batches[0].schema(); + let schema_size = new_writer(&schema)? + .into_inner() + .map_err(arrow_error)? + .len(); + let mut writer = new_writer(&schema)?; + let mut current_size = schema_size; + let mut has_data = false; + for batch in batches { + let mut offset = 0; + while offset < batch.num_rows() { + let length = (batch.num_rows() - offset) + .min(1024) + .min(options.chunk_rows); + let slice = batch.slice(offset, length); + let batch_size = batch_data_size(&slice); + if has_data && current_size + batch_size > options.chunk_bytes { + emit(writer.into_inner().map_err(arrow_error)?)?; + writer = new_writer(&schema)?; + current_size = schema_size; + } + writer.write(&slice).map_err(arrow_error)?; + current_size += batch_size; + has_data = true; + offset += length; + } + } + if has_data { + emit(writer.into_inner().map_err(arrow_error)?)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::Int64Array; + use arrow::datatypes::{DataType as ArrowType, Field, Schema}; + use arrow::ipc::reader::StreamReader; + use std::io::Cursor; + use std::ops::Range; + use std::sync::Arc; + + fn batch(values: Range) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![Field::new("id", ArrowType::Int64, true)])); + RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from_iter_values(values))]).unwrap() + } + + fn chunks(batch: &RecordBatch, chunk_rows: usize, chunk_bytes: usize) -> Vec> { + let options = Options { + size_limit: usize::MAX, + chunk_rows, + chunk_bytes, + upload_batch_bytes: usize::MAX, + }; + let mut chunks = Vec::new(); + serialize_chunks(std::slice::from_ref(batch), &options, &mut |chunk| { + chunks.push(chunk); + Ok(()) + }) + .unwrap(); + chunks + .into_iter() + .map(|chunk| { + let reader = StreamReader::try_new(Cursor::new(chunk), None).unwrap(); + assert_eq!(reader.schema(), batch.schema()); + reader.collect::, _>>().unwrap() + }) + .collect() + } + + fn chunk_row_counts(chunks: &[Vec]) -> Vec { + chunks + .iter() + .map(|chunk| chunk.iter().map(RecordBatch::num_rows).sum()) + .collect() + } + + #[test] + fn row_setting_limits_batches_without_splitting_the_stream() { + let chunks = chunks(&batch(0..2500), 700, usize::MAX); + assert_eq!(chunks.len(), 1); + assert_eq!( + chunks[0] + .iter() + .map(RecordBatch::num_rows) + .collect::>(), + vec![700, 700, 700, 400] + ); + } + + #[test] + fn chunks_flush_before_a_batch_exceeds_the_data_size_estimate() { + let batch = batch(0..25); + let schema_size = new_writer(&batch.schema()) + .unwrap() + .into_inner() + .unwrap() + .len(); + // Two four-row Int64 batches fit exactly, despite IPC message overhead. + let chunks = chunks(&batch, 4, schema_size + 8 * 8); + assert_eq!(chunk_row_counts(&chunks), vec![8, 8, 8, 1]); + let actual: Vec<_> = chunks + .iter() + .flatten() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .iter() + .copied() + }) + .collect(); + assert_eq!(actual, (0..25).collect::>()); + } + + #[test] + fn an_oversized_batch_does_not_emit_an_empty_stream() { + let batch = batch(0..10); + let schema_size = new_writer(&batch.schema()) + .unwrap() + .into_inner() + .unwrap() + .len(); + let chunks = chunks(&batch, 4, schema_size + 4 * 8 - 1); + assert_eq!(chunk_row_counts(&chunks), vec![4, 4, 2]); + } + + #[test] + fn inline_schema_is_sent_only_when_explicit() { + let batch = batch(0..2); + let explicit = DataType::Struct { + fields: vec![crate::types::StructField { + name: "id".to_owned(), + data_type: DataType::Long, + nullable: true, + metadata: Default::default(), + }], + }; + for schema in [None, Some(explicit)] { + let relation = inline_relation(std::slice::from_ref(&batch), schema.clone()) + .unwrap() + .to_proto(); + let Some(spark_connect_proto::relation::RelType::LocalRelation(local)) = + relation.rel_type + else { + panic!("expected inline relation"); + }; + assert_eq!(local.schema, schema.as_ref().map(DataType::json)); + let data = local.data.unwrap(); + let decoded = StreamReader::try_new(Cursor::new(data), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(decoded, vec![batch.clone()]); + } + } +} diff --git a/crates/spark-connect/src/conf.rs b/crates/spark-connect/src/conf.rs index d90907cf..d43d611f 100644 --- a/crates/spark-connect/src/conf.rs +++ b/crates/spark-connect/src/conf.rs @@ -70,12 +70,6 @@ impl RuntimeConf { Ok(results.into_iter().next().flatten()) } - /// Get several configuration values in one request, in the order of `keys`; an unknown - /// key errors as in `get`. Mirrors the reference client's `get_config_dict`. - pub fn get_many(&self, keys: &[&str]) -> Result>> { - block_on(self.client.get_configs(keys)) - } - /// Get all configuration values as a HashMap. /// /// # Example diff --git a/crates/spark-connect/src/dataframe.rs b/crates/spark-connect/src/dataframe.rs index e74798c6..3c5b0305 100644 --- a/crates/spark-connect/src/dataframe.rs +++ b/crates/spark-connect/src/dataframe.rs @@ -1693,8 +1693,15 @@ impl DataFrame { } /// Check if this DataFrame is local (collected). + /// + /// Mirrors the server's `Dataset.isLocal`: only a local relation itself is local. The + /// server resolves `toDF`, which Arrow input with inferred or listed column names + /// applies, into a `Project`, which is not. pub fn is_local(&self) -> bool { - matches!(self.plan.as_ref(), LogicalPlan::LocalRelation { .. }) + matches!( + self.plan.as_ref(), + LogicalPlan::LocalRelation { .. } | LogicalPlan::ChunkedCachedLocalRelation { .. } + ) } /// Check if this DataFrame is streaming. @@ -3321,6 +3328,25 @@ mod plan_construction_tests { .expect("failed to build session") } + #[test] + fn is_local_only_for_a_local_relation_itself() { + let spark = session(); + let local = spark.empty_data_frame().unwrap(); + let cached = DataFrame::new( + spark.clone(), + LogicalPlan::ChunkedCachedLocalRelation { + data_hashes: vec!["hash".to_string()], + schema_hash: None, + }, + ); + assert!(local.is_local()); + assert!(cached.is_local()); + // As in PySpark, where the server resolves toDF into a Project. + assert!(!local.to_df(vec![]).is_local()); + assert!(!cached.to_df(vec!["x"]).is_local()); + assert!(!spark.range(1).unwrap().is_local()); + } + #[test] fn with_watermark_plan() { let spark = session(); diff --git a/crates/spark-connect/src/lib.rs b/crates/spark-connect/src/lib.rs index d6a6e3fd..2b2c8708 100644 --- a/crates/spark-connect/src/lib.rs +++ b/crates/spark-connect/src/lib.rs @@ -5,6 +5,7 @@ #![allow(clippy::result_large_err)] // SparkError is large; changing the type would break the API +mod arrow_input; pub mod catalog; pub mod column; pub mod conf; @@ -35,6 +36,8 @@ pub mod wasm_udf; pub mod window; // Re-export commonly used types +#[doc(hidden)] +pub use arrow_input::ArrowInputOptions; pub use column::{col, lit, lit_boolean, lit_double, lit_string, Column}; pub use dataframe::{DataFrame, LocalRowIterator}; pub use datasource::{CommonInlineUserDefinedDataSourceExpression, PythonDataSourcePayload}; diff --git a/crates/spark-connect/src/plan.rs b/crates/spark-connect/src/plan.rs index 7d5274a5..7167eb6e 100644 --- a/crates/spark-connect/src/plan.rs +++ b/crates/spark-connect/src/plan.rs @@ -225,9 +225,14 @@ pub enum LogicalPlan { }, /// LocalRelation: data supplied locally. LocalRelation { - schema: DataType, + schema: Option, data: Option>, }, + /// Local Arrow data uploaded as session-scoped cache artifacts. + ChunkedCachedLocalRelation { + data_hashes: Vec, + schema_hash: Option, + }, /// CachedRemoteRelation: a reference to a cached remote relation. CachedRemoteRelation { relation_id: String }, /// Read: `spark.read.format(...).load(...)` or `spark.read.table(...)`. @@ -944,14 +949,25 @@ impl LogicalPlan { Some(proto::relation::RelType::SubqueryAlias(Box::new(sq_alias))); } + LogicalPlan::ChunkedCachedLocalRelation { + data_hashes, + schema_hash, + } => { + relation.rel_type = Some(proto::relation::RelType::ChunkedCachedLocalRelation( + proto::ChunkedCachedLocalRelation { + data_hashes: data_hashes.clone(), + schema_hash: schema_hash.clone(), + }, + )); + } LogicalPlan::LocalRelation { schema, data } => { let mut local = proto::LocalRelation::default(); if let Some(d) = data { local.data = Some(d.clone().into()); } - // Carry the schema (JSON) so the server applies the user's column - // names/types; required when no Arrow `data` is provided (emptyDataFrame). - local.schema = Some(schema.json()); + // Explicit schemas apply server-side casts. Inferred Arrow input + // omits this to avoid a redundant projection on every query. + local.schema = schema.as_ref().map(DataType::json); relation.rel_type = Some(proto::relation::RelType::LocalRelation(local)); } @@ -1744,7 +1760,10 @@ pub fn subquery_alias(input: impl Into>, alias: impl Into>) -> LogicalPlan { - LogicalPlan::LocalRelation { schema, data } + LogicalPlan::LocalRelation { + schema: Some(schema), + data, + } } /// Create a CachedRemoteRelation plan. diff --git a/crates/spark-connect/src/session.rs b/crates/spark-connect/src/session.rs index 7791e3c8..1493ae03 100644 --- a/crates/spark-connect/src/session.rs +++ b/crates/spark-connect/src/session.rs @@ -291,21 +291,50 @@ impl SparkSession { } else { Some(rows_to_arrow_ipc(&rows, &schema)?) }; - let plan = LogicalPlan::LocalRelation { schema, data }; + let plan = LogicalPlan::LocalRelation { + schema: Some(schema), + data, + }; Ok(DataFrame::new(self.clone(), plan)) } - /// Create a DataFrame from rows that are already encoded as an Arrow IPC stream, whose - /// Arrow schema matches `schema`. + /// Create a local DataFrame from Arrow batches without materializing rows. /// - /// Mirrors `pyspark.sql.SparkSession.createDataFrame(pyarrow.Table)`, which sends the - /// table as an Arrow IPC stream in a `LocalRelation`. - pub fn create_dataframe_from_arrow(&self, data: Vec, schema: DataType) -> DataFrame { - let plan = LogicalPlan::LocalRelation { - schema, - data: Some(data), - }; - DataFrame::new(self.clone(), plan) + /// `schema` optionally supplies the Spark schema applied by the server. With + /// `None`, Spark infers it from Arrow. All batches must share the same Arrow + /// schema; IPC serialization is still required for transport. + /// Non-empty inputs read the server's local-relation configuration. Large inputs + /// are uploaded as cache artifacts and referenced by hash in the resulting plan. + pub fn create_dataframe_from_arrow( + &self, + batches: Vec, + schema: impl Into>, + ) -> Result { + let plan = crate::arrow_input::create_relation(&self.client, batches, schema.into(), None)?; + Ok(DataFrame::new(self.clone(), plan)) + } + + /// Internal settings handoff for the Python Arrow importer; not a stable API. + #[doc(hidden)] + pub fn arrow_input_options(&self) -> Result { + crate::ArrowInputOptions::from_client(&self.client) + } + + /// Internal Arrow import entry point using the normalization settings snapshot. + #[doc(hidden)] + pub fn create_dataframe_from_arrow_with_options( + &self, + batches: Vec, + schema: impl Into>, + options: &crate::ArrowInputOptions, + ) -> Result { + let plan = crate::arrow_input::create_relation( + &self.client, + batches, + schema.into(), + Some(options), + )?; + Ok(DataFrame::new(self.clone(), plan)) } /// Create a DataFrameReader for reading data from various sources. @@ -421,7 +450,7 @@ impl SparkSession { /// `SparkSession.emptyDataFrame`. pub fn empty_data_frame(&self) -> Result { let plan = LogicalPlan::LocalRelation { - schema: DataType::Struct { fields: vec![] }, + schema: Some(DataType::Struct { fields: vec![] }), data: None, }; Ok(DataFrame::new(self.clone(), plan)) @@ -1271,6 +1300,55 @@ mod tests { .expect("failed to build session") } + #[test] + fn arrow_local_relation_preserves_batches_and_rejects_schema_mismatch() { + use arrow::array::Array; + use arrow::datatypes::{DataType as ArrowType, Field, Schema}; + let arrow_schema = Arc::new(Schema::new(vec![Field::new("x", ArrowType::Int32, true)])); + let batch = RecordBatch::try_new( + arrow_schema, + vec![Arc::new(Int32Array::from(vec![Some(1), None]))], + ) + .unwrap(); + let schema = DataType::Struct { + fields: vec![struct_field("x", DataType::Integer)], + }; + let spark = session(); + let plan = crate::arrow_input::inline_relation( + &[batch.clone(), batch.clone()], + Some(schema.clone()), + ) + .unwrap(); + let LogicalPlan::LocalRelation { + data: Some(data), .. + } = &plan + else { + panic!("expected Arrow local relation") + }; + let batches = StreamReader::try_new(Cursor::new(data), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(batches.len(), 2); + assert_eq!(batches[0], batch); + assert!(batches[1].column(0).is_null(1)); + let other = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("y", ArrowType::Int32, true)])), + batch.columns().to_vec(), + ) + .unwrap(); + assert!(spark + .create_dataframe_from_arrow(vec![batch, other], schema.clone()) + .is_err()); + assert!(matches!( + spark + .create_dataframe_from_arrow(vec![], schema) + .unwrap() + .plan(), + LogicalPlan::LocalRelation { data: None, .. } + )); + } + #[test] fn stop_is_idempotent_and_marks_stopped() { // Port 1: nothing listens, so the ReleaseSession RPC fails fast. The session is @@ -1293,21 +1371,6 @@ mod tests { assert!(!spark.is_stopped()); } - #[test] - fn create_dataframe_from_arrow_sends_the_stream_and_schema() { - let schema = DataType::Struct { - fields: vec![struct_field("id", DataType::Long)], - }; - let df = session().create_dataframe_from_arrow(vec![1, 2, 3], schema.clone()); - match df.plan().to_proto().rel_type { - Some(proto::relation::RelType::LocalRelation(local)) => { - assert_eq!(local.data.as_deref(), Some(&[1u8, 2, 3][..])); - assert_eq!(local.schema, Some(schema.json())); - } - other => panic!("expected LocalRelation, got {other:?}"), - } - } - #[test] fn session_id_is_set() { let spark = session(); diff --git a/dev/design/architecture.md b/dev/design/architecture.md index c7e6f979..cd625c30 100644 --- a/dev/design/architecture.md +++ b/dev/design/architecture.md @@ -86,9 +86,12 @@ PyArrow's pandas conversion, which may copy. Scalar input conversion and row materialization stay in `pyspark-rs`. `collect()` still uses the native client's owned rows; `createDataFrame(pandas)` still materializes Python rows. Neither path is claimed to be zero-copy. -`createDataFrame(pyarrow.Table)` converts the table with PyArrow as the reference -client does (timestamp localization, a cast to the target schema) and sends it as an -Arrow IPC stream. +`createDataFrame(pyarrow.Table)` normalizes the table with PySpark's helpers as the +reference client does (timestamp localization, a cast to the target schema), imports +it into arrow-rs through the C Stream interface, and serializes the batches to Arrow +IPC in Rust: inline in a `LocalRelation`, or, at or above +`spark.sql.session.localRelationCacheThreshold`, as session-scoped cache artifacts +referenced by a `ChunkedCachedLocalRelation` (see `docs/architecture.md`). The Arrow handoff has been tested locally with PyArrow 10.0.0 (Python 3.9) and 26.0.0 (Python 3.12). CI installs the latest PyArrow; it does not pin or diff --git a/docs/architecture.md b/docs/architecture.md index 020361d9..32d6168d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -119,6 +119,21 @@ Batch schema consistency is validated before export. Field names or timezones containing NUL use IPC over the collected batches, without another RPC. Empty batch vectors retain the empty-schema behavior. +`createDataFrame(pyarrow_table)` uses the reverse boundary: Spark's schema and +timestamp conversion helpers normalize the table, then arrow-rs imports its +`RecordBatchReader` through the C Stream interface. The Rust core serializes +those batches to IPC for the local relation sent to Spark. No Python rows are +materialized; casts, buffer alignment, and IPC serialization may still copy. +Empty tables require an explicit schema, matching PySpark Connect. +Inputs at or above `spark.sql.session.localRelationCacheThreshold` are serialized +into independent IPC streams and uploaded as session-scoped cache artifacts. +The plan carries `ChunkedCachedLocalRelation` hashes instead of embedding the +entire payload. Spark infers omitted schemas from the Arrow streams; explicit +schemas travel in the local relation or a separate schema artifact. +Chunk sizes, upload batch sizes, and the total relation size +limit follow the server's local-relation settings; artifact RPC messages use +32 KiB pieces, with SHA-256 deduplication and CRC checks. + ## Two front-ends, one core === "Rust (native)" diff --git a/python/pyspark/sql/_arrow_input.py b/python/pyspark/sql/_arrow_input.py new file mode 100644 index 00000000..0c5b337f --- /dev/null +++ b/python/pyspark/sql/_arrow_input.py @@ -0,0 +1,93 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +"""Schema normalization for the native Arrow Table input path.""" + +from pyspark.errors import PySparkTypeError, PySparkValueError +from pyspark.sql.pandas.types import ( + _check_arrow_table_timestamps_localize, + from_arrow_schema, + to_arrow_schema, +) +from pyspark.sql.types import AtomicType, StructType + + +def prepare_table(table, schema, timestamp_type, timezone, large_types): + """Match Connect's Arrow casts without converting any column to Python rows.""" + if schema is not None and not isinstance(schema, (AtomicType, StructType, list, tuple)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "arg_name": "schema", + "expected_type": "list, None or StructType", + "arg_type": type(schema).__name__, + }, + ) + names = list(schema) if isinstance(schema, (list, tuple)) else None + explicit = schema is not None and names is None + if len(table) == 0: + if not explicit: + raise PySparkValueError(errorClass="CANNOT_INFER_EMPTY_SCHEMA", messageParameters={}) + return table, schema, None + + expected_columns = None + if isinstance(schema, StructType): + expected_columns = len(schema.fields) + elif isinstance(schema, AtomicType): + expected_columns = 1 + if expected_columns is not None and expected_columns != table.num_columns: + raise PySparkValueError( + errorClass="AXIS_LENGTH_MISMATCH", + messageParameters={ + "expected_length": str(expected_columns), + "actual_length": str(table.num_columns), + }, + ) + + if names is not None: + names.extend(f"_{i + 1}" for i in range(len(names), table.num_columns)) + inferred = ( + schema + if isinstance(schema, StructType) + else from_arrow_schema(table.schema, prefer_timestamp_ntz=timestamp_type == "TIMESTAMP_NTZ") + ) + normalized = ( + _check_arrow_table_timestamps_localize(table, inferred, True, timezone) + .cast( + to_arrow_schema( + inferred, + error_on_duplicated_field_names_in_struct=True, + timezone="UTC", + prefers_large_types=large_types == "true", + ) + ) + .rename_columns(inferred.names) + ) + if names is not None and len(names) != normalized.num_columns: + raise PySparkValueError( + errorClass="AXIS_LENGTH_MISMATCH", + messageParameters={ + "expected_length": str(len(names)), + "actual_length": str(normalized.num_columns), + }, + ) + # Match Connect: inferred Arrow schemas travel in IPC, without an extra + # Spark schema that would trigger server-side casts on every query. + if not explicit and names is None: + names = table.column_names + return normalized, schema if explicit else None, names diff --git a/python/tests/test_arrow_e2e.py b/python/tests/test_arrow_e2e.py index dc76866d..eeec709e 100644 --- a/python/tests/test_arrow_e2e.py +++ b/python/tests/test_arrow_e2e.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Native toArrow smoke tests against a live Spark Connect server.""" +"""Native Arrow input and output checks against a live Spark Connect server.""" import os from datetime import date, datetime, timezone @@ -65,3 +65,67 @@ def test_to_arrow_empty_result(session): table = session.range(0).toArrow() assert isinstance(table, pa.Table) assert table.num_rows == 0 + + +def test_create_dataframe_arrow_cached_upload(session): + """Materialize cached Arrow input larger than 1 MiB on a real server.""" + import numpy as np + + rows = 262144 + table = pa.table({"id": np.arange(rows, dtype=np.int64)}) + assert table.nbytes == 2 * 1024**2 + isolated = session.newSession() + try: + isolated.conf.set("spark.sql.session.localRelationCacheThreshold", str(1024**2)) + frame = isolated.createDataFrame(table) + actual = frame.selectExpr("count(*) AS rows", "sum(id) AS checksum").collect()[0] + assert actual[0] == rows + assert actual[1] == rows * (rows - 1) // 2 + # As in PySpark, inferred column names are applied with toDF, which the server + # resolves into a Project, so the frame is not local. + assert not frame.isLocal() + finally: + isolated.stop() + + +@pytest.mark.parametrize("schema", [None, []], ids=["inferred", "empty-list"]) +def test_create_dataframe_arrow_zero_columns(session, schema): + table = pa.table({"a": [1, 2]}).select([]) + frame = session.createDataFrame(table, schema) + # As in PySpark, no toDF is applied without column names, so the frame is local. + assert frame.isLocal() + assert frame.columns == [] + # Both clients first cast the input to its Spark schema, and for a zero-column table + # that cast decides the row count (PyArrow drops the rows). + assert frame.count() == table.cast(pa.schema([])).num_rows + + +@pytest.mark.parametrize("cached", [False, True], ids=["inline", "cached"]) +@pytest.mark.parametrize("timestamp_type", ["TIMESTAMP_LTZ", "TIMESTAMP_NTZ"]) +def test_create_dataframe_arrow_inferred_timestamp(session, cached, timestamp_type): + value = datetime(2020, 1, 2, 3, 4, 5) + table = pa.table({"t": pa.array([value], type=pa.timestamp("ns"))}) + isolated = session.newSession() + try: + isolated.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + isolated.conf.set("spark.sql.timestampType", timestamp_type) + isolated.conf.set("spark.sql.session.localRelationCacheThreshold", + "0" if cached else str(1024**2)) + frame = isolated.createDataFrame(table) + spark_type = "timestamp" if timestamp_type == "TIMESTAMP_LTZ" else "timestamp_ntz" + assert frame.schema.simpleString() == f"struct" + assert frame.selectExpr("CAST(t AS STRING)").collect()[0][0] == "2020-01-02 03:04:05" + + result = frame.toArrow() + if timestamp_type == "TIMESTAMP_LTZ": + assert result.column("t").type.unit == "us" + assert result.column("t").type.tz is not None + # The native Arrow result carries the session timezone; compare instants. + assert result.column("t").cast(pa.timestamp("us", tz="UTC")).to_pylist() == [ + datetime(2020, 1, 2, 11, 4, 5, tzinfo=timezone.utc) + ] + else: + assert result.column("t").type == pa.timestamp("us") + assert result.column("t").to_pylist() == [value] + finally: + isolated.stop() diff --git a/python/tests/test_arrow_handoff.py b/python/tests/test_arrow_handoff.py index 214a09a0..f38e3e55 100644 --- a/python/tests/test_arrow_handoff.py +++ b/python/tests/test_arrow_handoff.py @@ -395,3 +395,377 @@ def test_malformed_ipc_is_reported(protocol): with pytest.raises(RuntimeError, match="Arrow"): frame.toArrow() assert calls == {"execute": 1} + + +@contextmanager +def _arrow_input_service(protocol, *, upload_error=None, **overrides): + """Resolve cache artifacts at a mock service with a 4 MiB receive limit.""" + import hashlib + import zlib + + config = { + "spark.sql.timestampType": "TIMESTAMP_LTZ", + "spark.sql.session.timeZone": "UTC", + "spark.sql.execution.arrow.useLargeVarTypes": "false", + "spark.sql.session.localRelationCacheThreshold": "1048576", + "spark.sql.session.localRelationSizeLimit": str(3 * 1024**3), + "spark.sql.session.localRelationChunkSizeRows": "10000", + "spark.sql.session.localRelationChunkSizeBytes": str(16 * 1024**2), + "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(1024**3), + } + config.update(overrides) + state = {"cache": {}, "uploads": 0, "relations": [], "message_sizes": [], + "config_requests": []} + server_id = "00000000-0000-0000-0000-000000000001" + + def configure(raw, context): + request = protocol("ConfigRequest").FromString(raw) + operation = request.operation.WhichOneof("op_type") + keys = getattr(request.operation, operation).keys + state["config_requests"].append((operation, list(keys))) + return protocol("ConfigResponse", session_id=request.session_id, + pairs=[dict(key=k, value=config[k]) for k in keys + if config.get(k) is not None] + ).SerializeToString() + + def statuses(raw, context): + request = protocol("ArtifactStatusesRequest").FromString(raw) + return protocol("ArtifactStatusesResponse", session_id=request.session_id, + statuses={name: dict(exists=name in state["cache"]) + for name in request.names}).SerializeToString() + + def upload(requests, context): + summaries = [] + name, expected, buffer = None, 0, bytearray() + session_id = None + + def save(name, data): + assert name == "cache/" + hashlib.sha256(data).hexdigest() + assert name not in state["cache"], "cached artifacts should not be uploaded again" + state["cache"][name] = bytes(data) + summaries.append(dict(name=name, is_crc_successful=True)) + + def checked(chunk): + assert zlib.crc32(chunk.data) == chunk.crc + return chunk.data + + for raw in requests: + state["message_sizes"].append(len(raw)) + request = protocol("AddArtifactsRequest").FromString(raw) + session_id = request.session_id + if request.HasField("batch"): + for artifact in request.batch.artifacts: + save(artifact.name, checked(artifact.data)) + elif request.HasField("begin_chunk"): + begin = request.begin_chunk + name, expected = begin.name, begin.total_bytes + buffer = bytearray(checked(begin.initial_chunk)) + else: + buffer.extend(checked(request.chunk)) + if name is not None and len(buffer) == expected: + save(name, buffer) + name, buffer = None, bytearray() + assert name is None + state["uploads"] += 1 + if upload_error == "crc": + summaries[0]["is_crc_successful"] = False + elif upload_error == "missing": + summaries = [] + return protocol("AddArtifactsResponse", session_id=session_id, + artifacts=summaries).SerializeToString() + + def execute(raw, context): + request = protocol("ExecutePlanRequest").FromString(raw) + relation = request.plan.root + while relation.HasField("to_df"): + state["column_names"] = list(relation.to_df.column_names) + relation = relation.to_df.input + state["relations"].append(relation) + if relation.HasField("chunked_cached_local_relation"): + cached = relation.chunked_cached_local_relation + payloads = [state["cache"]["cache/" + h] for h in cached.dataHashes] + state["spark_schema"] = ( + state["cache"]["cache/" + cached.schemaHash].decode() + if cached.HasField("schemaHash") else None + ) + else: + payloads = [relation.local_relation.data] + state["spark_schema"] = ( + relation.local_relation.schema + if relation.local_relation.HasField("schema") else None + ) + rows, first, last = 0, None, None + state["arrow_schemas"], state["batch_rows"] = [], [] + for payload in payloads: + reader = pa.ipc.open_stream(payload) + state["arrow_schemas"].append(reader.schema) + stream_rows = [] + state["batch_rows"].append(stream_rows) + for batch in reader: + stream_rows.append(batch.num_rows) + rows += batch.num_rows + if batch.num_rows: + if first is None: + first = batch.column(0)[0].as_py() + last = batch.column(0)[-1].as_py() + state["result"] = (rows, first, last) + result = pa.table({"rows": [rows]}) + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, result.schema) as writer: + writer.write_table(result) + header = dict(session_id=request.session_id, operation_id=request.operation_id, + server_side_session_id=server_id) + yield protocol("ExecutePlanResponse", **header, response_id="0", + arrow_batch=dict(row_count=1, data=sink.getvalue().to_pybytes()) + ).SerializeToString() + yield protocol("ExecutePlanResponse", **header, response_id="1", + result_complete={}).SerializeToString() + + def release(raw, context): + request = protocol("ReleaseExecuteRequest").FromString(raw) + return protocol("ReleaseExecuteResponse", session_id=request.session_id, + operation_id=request.operation_id, + server_side_session_id=server_id).SerializeToString() + + def stop(raw, context): + request = protocol("ReleaseSessionRequest").FromString(raw) + return protocol("ReleaseSessionResponse", session_id=request.session_id, + server_side_session_id=server_id).SerializeToString() + + with ThreadPoolExecutor(max_workers=2) as executor: + server = grpc.server(executor, options=[("grpc.max_receive_message_length", 4 * 1024**2)]) + server.add_generic_rpc_handlers([grpc.method_handlers_generic_handler( + "spark.connect.SparkConnectService", { + "Config": grpc.unary_unary_rpc_method_handler(configure), + "ArtifactStatus": grpc.unary_unary_rpc_method_handler(statuses), + "AddArtifacts": grpc.stream_unary_rpc_method_handler(upload), + "ExecutePlan": grpc.unary_stream_rpc_method_handler(execute), + "ReleaseExecute": grpc.unary_unary_rpc_method_handler(release), + "ReleaseSession": grpc.unary_unary_rpc_method_handler(stop), + })]) + port = server.add_insecure_port("127.0.0.1:0") + server.start() + session = SparkSession.builder.remote(f"sc://127.0.0.1:{port}").getOrCreate() + try: + yield session, state + finally: + session.stop() + server.stop(0).wait(timeout=5) + + +@pytest.mark.parametrize("cached", [False, True], ids=["inline", "cached"]) +@pytest.mark.parametrize("explicit", [False, True], ids=["inferred", "explicit"]) +def test_arrow_input_schema_handoff(protocol, cached, explicit): + import json + from pyspark.sql import types as T + + table = pa.table({"id": [1, 2]}) + schema = T.StructType([T.StructField("id", T.LongType())]) if explicit else None + with _arrow_input_service(protocol, **{ + "spark.sql.session.localRelationCacheThreshold": "0" if cached else "1048576", + }) as (spark, state): + frame = spark.createDataFrame(table, schema) + # As in PySpark: inferred column names are applied with toDF, which is not local. + assert frame.isLocal() == explicit + assert frame.toArrow().column(0).to_pylist() == [2] + assert state["result"] == (2, 1, 2) + assert state["arrow_schemas"] == [table.schema] + assert len(state["config_requests"]) == 1 + assert state["config_requests"][0][0] == "get" + assert len(state["config_requests"][0][1]) == 8 + relation = state["relations"][0] + if cached: + local = relation.chunked_cached_local_relation + assert local.HasField("schemaHash") == explicit + assert len(state["cache"]) == (2 if explicit else 1) + uploads = state["uploads"] + spark.createDataFrame(table, schema).toArrow() + assert state["uploads"] == uploads + else: + assert state["uploads"] == 0 + assert relation.local_relation.HasField("schema") == explicit + if explicit: + assert json.loads(state["spark_schema"]) == json.loads(schema.json()) + else: + assert state["spark_schema"] is None + assert state["column_names"] == ["id"] + + +def test_arrow_input_upload_above_grpc_limit(protocol): + import numpy as np + + rows = 128 * 1024**2 // 8 + 1 + table = pa.table({"id": np.arange(rows, dtype=np.int64)}) + assert table.nbytes > 128 * 1024**2 + with _arrow_input_service(protocol) as (spark, state): + df = spark.createDataFrame(table) + assert not df.isLocal() + assert df.toArrow().column(0).to_pylist() == [rows] + assert state["result"] == (rows, 0, rows - 1) + assert state["relations"][0].HasField("chunked_cached_local_relation") + assert state["relations"][0].ByteSize() < 1024**2 + assert max(state["message_sizes"]) < 33 * 1024 + assert state["spark_schema"] is None + assert all(schema == table.schema for schema in state["arrow_schemas"]) + + +@pytest.mark.parametrize("kind", ["inferred", "empty-list", "struct"]) +def test_arrow_input_zero_column_table(protocol, kind): + from pyspark.sql import types as T + + # The input cast drops the rows of a zero-column table, leaving no batches. + table = pa.table({"a": [1, 2]}).select([]) + schema = {"inferred": None, "empty-list": [], "struct": T.StructType([])}[kind] + explicit = kind == "struct" + with _arrow_input_service(protocol) as (spark, state): + frame = spark.createDataFrame(table, schema) + # As in PySpark, no toDF is applied without column names, so the frame is local. + assert frame.isLocal() + assert frame.toArrow().column(0).to_pylist() == [0] + assert "column_names" not in state + # The relation still carries the Arrow schema, as PySpark's IPC stream does. + assert state["arrow_schemas"] == [pa.schema([])] + assert state["relations"][0].local_relation.HasField("schema") == explicit + + +@pytest.mark.parametrize("schema,expected_width", [ + ("x long", "1"), + ("x long, y long, z long", "3"), + ("long", "1"), +]) +def test_arrow_input_ddl_width_errors(protocol, schema, expected_width): + from pyspark.errors import PySparkValueError + + with _arrow_input_service(protocol) as (spark, state): + with pytest.raises(PySparkValueError) as error: + spark.createDataFrame(pa.table({"x": [1], "y": [2]}), schema) + assert error.value.getCondition() == "AXIS_LENGTH_MISMATCH" + assert error.value.getMessageParameters() == { + "expected_length": expected_width, "actual_length": "2", + } + assert not state["relations"] + assert state["uploads"] == 0 + + +@pytest.mark.parametrize("schema,expected_type", [ + ("array", "ArrayType"), + ("map", "MapType"), +]) +def test_arrow_input_unsupported_ddl_schema(protocol, schema, expected_type): + from pyspark.errors import PySparkTypeError + + with _arrow_input_service(protocol) as (spark, state): + with pytest.raises(PySparkTypeError) as error: + spark.createDataFrame(pa.table({"x": [1]}), schema) + assert error.value.getCondition() == "NOT_EXPECTED_TYPE" + assert error.value.getMessageParameters() == { + "arg_name": "schema", "expected_type": "list, None or StructType", + "arg_type": expected_type, + } + assert not state["relations"] + assert state["uploads"] == 0 + + +def test_arrow_input_upload_limits_and_chunk_order(protocol): + import numpy as np + from pyspark.errors import AnalysisException + table = pa.table({"id": np.arange(100, dtype=np.int64)}) + options = { + "spark.sql.session.localRelationCacheThreshold": "0", + "spark.sql.session.localRelationChunkSizeRows": "7", + "spark.sql.session.localRelationChunkSizeBytes": "500", + "spark.sql.session.localRelationBatchOfChunksSizeBytes": "1000", + } + with _arrow_input_service(protocol, **options) as (spark, state): + df = spark.createDataFrame(table) + df.toArrow() + assert state["result"] == (100, 0, 99) + assert state["uploads"] > 1 + cached = state["relations"][0].chunked_cached_local_relation + assert len(cached.dataHashes) > 1 + assert all(rows <= 7 for stream in state["batch_rows"] for rows in stream) + assert any(sum(stream) > 7 for stream in state["batch_rows"]) + actual = [ + value + for hash in cached.dataHashes + for batch in pa.ipc.open_stream(state["cache"]["cache/" + hash]) + for value in batch.column(0).to_pylist() + ] + assert actual == list(range(100)) + + options["spark.sql.session.localRelationSizeLimit"] = "10" + with _arrow_input_service(protocol, **options) as (spark, state): + with pytest.raises(AnalysisException, match="LOCAL_RELATION_SIZE_LIMIT_EXCEEDED"): + spark.createDataFrame(table) + assert not state["relations"] + assert state["uploads"] == 0 + + +@pytest.mark.parametrize("setting,value", [ + ("SizeLimit", "0"), + ("SizeLimit", "-1"), + ("SizeLimit", "18446744073709551616"), + ("ChunkSizeRows", "0"), +]) +def test_arrow_input_rejects_invalid_upload_sizes(protocol, setting, value): + key = f"spark.sql.session.localRelation{setting}" + with _arrow_input_service(protocol, **{ + "spark.sql.session.localRelationCacheThreshold": "0", key: value, + }) as (spark, state): + with pytest.raises(Exception, match=f"Invalid configuration: {key}"): + spark.createDataFrame(pa.table({"id": [1]})) + assert state["uploads"] == 0 + + +@pytest.mark.parametrize("failure,match", [("crc", "CRC check failed"), + ("missing", "Missing cache artifact")]) +@pytest.mark.parametrize("rows", [2, 4000000]) +def test_arrow_input_upload_failure(protocol, failure, match, rows): + import numpy as np + + # The larger input spans multiple upload batches. + with _arrow_input_service(protocol, upload_error=failure, + **{"spark.sql.session.localRelationCacheThreshold": "0", + "spark.sql.session.localRelationBatchOfChunksSizeBytes": + str(8 * 1024**2)}) as (spark, state): + with pytest.raises(Exception, match=match): + spark.createDataFrame(pa.table({"id": np.arange(rows, dtype=np.int64)})) + assert not state["relations"] + + +@pytest.mark.parametrize("batch_bytes", [500, 1024**3]) +def test_arrow_input_duplicate_chunks_preserve_multiplicity(protocol, batch_bytes): + options = {"spark.sql.session.localRelationCacheThreshold": "0", + "spark.sql.session.localRelationChunkSizeRows": "5", + "spark.sql.session.localRelationChunkSizeBytes": "1", + "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(batch_bytes)} + with _arrow_input_service(protocol, **options) as (spark, state): + spark.createDataFrame(pa.table({"id": [1] * 20})).toArrow() + cached = state["relations"][0].chunked_cached_local_relation + assert len(cached.dataHashes) == 4 + assert len(set(cached.dataHashes)) == 1 + assert not cached.HasField("schemaHash") + assert len(state["cache"]) == 1 + assert state["result"] == (20, 1, 1) + + +@pytest.mark.parametrize("data", [[[1]], pd.DataFrame({"id": [1]})]) +def test_non_arrow_input_does_not_import_pyarrow(protocol, monkeypatch, data): + import builtins + import sys + + imported = [] + original_import = builtins.__import__ + + def import_module(name, *args, **kwargs): + if name == "pyarrow": + imported.append(name) + raise ImportError("PyArrow is unavailable") + return original_import(name, *args, **kwargs) + + with _arrow_input_service(protocol) as (spark, state): + monkeypatch.setitem(sys.modules, "pyarrow", None) + monkeypatch.setattr(builtins, "__import__", import_module) + spark.createDataFrame(data) + assert imported == [] + assert state["config_requests"] == [] diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 209a4b64..e2a26278 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -3236,6 +3236,250 @@ def allow_response_after_request(): assert service.calls == 1 +@pytest.mark.parametrize("names,expected", [(None, ["x", "y"]), (["a"], ["a", "_2"]), ([], ["_1", "_2"])]) +def test_arrow_input_normalization(names, expected): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({"x": pa.chunked_array([[1], [2]]), "y": ["a", None]}) + normalized, schema, columns = prepare_table(table, names, "TIMESTAMP_LTZ", "UTC", "false") + assert normalized.equals(table) + assert schema is None + assert columns == expected + + +def test_arrow_input_empty_and_width_errors(): + pa = pytest.importorskip("pyarrow") + from pyspark.errors import PySparkValueError + from pyspark.sql._arrow_input import prepare_table + + empty = pa.table({"x": pa.array([], type=pa.int64())}) + with pytest.raises(PySparkValueError, match="CANNOT_INFER_EMPTY_SCHEMA"): + prepare_table(empty, None, None, None, None) + schema = T.StructType([T.StructField("x", T.LongType())]) + assert prepare_table(empty, schema, None, None, None)[1] is schema + with pytest.raises(PySparkValueError, match="AXIS_LENGTH_MISMATCH"): + prepare_table(pa.table({"x": [1]}), ["a", "b"], "TIMESTAMP_LTZ", "UTC", "false") + + +@pytest.mark.parametrize("schema,expected_type", [ + pytest.param(T.ArrayType(T.LongType()), "ArrayType", id="array"), + pytest.param(T.MapType(T.StringType(), T.LongType()), "MapType", id="map"), +]) +@pytest.mark.parametrize("empty", [False, True], ids=["nonempty", "empty"]) +def test_arrow_input_unsupported_schema_types(schema, expected_type, empty): + pa = pytest.importorskip("pyarrow") + from pyspark.errors import PySparkTypeError + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({"x": pa.array([] if empty else [1], type=pa.int64())}) + with pytest.raises(PySparkTypeError) as error: + prepare_table(table, schema, "TIMESTAMP_LTZ", "UTC", "false") + assert error.value.getCondition() == "NOT_EXPECTED_TYPE" + assert error.value.getMessageParameters() == { + "arg_name": "schema", "expected_type": "list, None or StructType", + "arg_type": expected_type, + } + + +@pytest.mark.parametrize("schema", [ + None, T.LongType(), T.StructType([T.StructField("x", T.LongType())]), ["x"], ("x",), +]) +def test_arrow_input_supported_schema_types(schema): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({"x": [1]}) + normalized, result, names = prepare_table(table, schema, "TIMESTAMP_LTZ", "UTC", "false") + assert normalized.equals(table) + if isinstance(schema, (T.AtomicType, T.StructType)): + assert result is schema + else: + assert result is None + assert names == (None if isinstance(schema, (T.AtomicType, T.StructType)) else ["x"]) + + +def test_arrow_input_array_and_map_struct_fields_allowed(): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({ + "items": pa.array([[1, None]], type=pa.list_(pa.int64())), + "lookup": pa.array([[("a", 1)]], type=pa.map_(pa.string(), pa.int64())), + }) + schema = T.StructType([ + T.StructField("items", T.ArrayType(T.LongType())), + T.StructField("lookup", T.MapType(T.StringType(), T.LongType())), + ]) + normalized, result, names = prepare_table(table, schema, "TIMESTAMP_LTZ", "UTC", "false") + assert normalized.equals(table) + assert result is schema + assert names is None + + +@pytest.mark.parametrize("schema,expected_length", [ + pytest.param(T.StructType([T.StructField("x", T.IntegerType())]), "1", id="too-few-fields"), + pytest.param(T.StructType([ + T.StructField("x", T.LongType()), T.StructField("y", T.LongType()), + T.StructField("z", T.LongType()), + ]), "3", id="too-many-fields"), + pytest.param(T.LongType(), "1", id="atomic-multiple-columns"), +]) +def test_arrow_input_explicit_schema_width_errors(schema, expected_length): + pa = pytest.importorskip("pyarrow") + from pyspark.errors import PySparkValueError + from pyspark.sql._arrow_input import prepare_table + + # The first value would overflow the one-field IntegerType schema. Width + # validation must produce the structured Spark error before an Arrow cast. + table = pa.table({"x": [2**40], "y": [2]}) + with pytest.raises(PySparkValueError) as error: + prepare_table(table, schema, "TIMESTAMP_LTZ", "UTC", "false") + assert error.value.getCondition() == "AXIS_LENGTH_MISMATCH" + assert error.value.getMessageParameters() == { + "expected_length": expected_length, "actual_length": "2", + } + + +@pytest.mark.parametrize("schema,values", [ + pytest.param(T.StructType([T.StructField("x", T.LongType())]), + {"x": [1, 2]}, id="single-field"), + pytest.param(T.StructType([ + T.StructField("x", T.LongType()), T.StructField("y", T.LongType()), + ]), {"x": [1, 2], "y": [3, 4]}, id="multiple-fields"), + pytest.param(T.LongType(), {"x": [1, 2]}, id="atomic-single-column"), +]) +def test_arrow_input_explicit_schema_matching_width(schema, values): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table(values) + normalized, result, names = prepare_table(table, schema, "TIMESTAMP_LTZ", "UTC", "false") + assert normalized.equals(table) + assert result is schema + assert names is None + + +@pytest.mark.parametrize("schema", [ + T.StructType([]), + T.StructType([T.StructField("x", T.LongType()), T.StructField("y", T.LongType())]), + T.LongType(), +]) +def test_arrow_input_empty_explicit_schema_skips_width_validation(schema): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({"x": pa.array([], type=pa.int64())}) + normalized, result, names = prepare_table(table, schema, None, None, None) + assert normalized is table + assert result is schema + assert names is None + + +@pytest.mark.parametrize("timestamp_type,expected", [("TIMESTAMP_NTZ", T.TimestampNTZType), ("TIMESTAMP_LTZ", T.TimestampType)]) +def test_arrow_input_timestamps(timestamp_type, expected): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({"t": pa.array([1001], type=pa.timestamp("ns"))}) + normalized, schema, _ = prepare_table(table, None, timestamp_type, "UTC", "false") + assert schema is None + assert normalized.column(0).type.tz == ("UTC" if expected is T.TimestampType else None) + assert normalized.column(0).type.unit == "us" + assert normalized.column(0).cast(pa.int64()).to_pylist() == [1] + + +def test_arrow_input_explicit_cast_and_nested_values(): + pa = pytest.importorskip("pyarrow") + from pyspark.sql._arrow_input import prepare_table + + table = pa.table({"x": [1, 2], "nested": [[1, None], []]}) + schema = T.StructType([T.StructField("x", T.IntegerType()), + T.StructField("nested", T.ArrayType(T.LongType()))]) + normalized, result, _ = prepare_table(table, schema, "TIMESTAMP_LTZ", "UTC", "false") + assert result is schema + assert normalized.column(0).type == pa.int32() + assert normalized.to_pylist() == table.to_pylist() + with pytest.raises(pa.ArrowInvalid): + prepare_table(pa.table({"x": [2**40]}), T.StructType([T.StructField("x", T.IntegerType())]), + "TIMESTAMP_LTZ", "UTC", "false") + + +def test_arrow_input_roundtrip_live_server(): + """Run explicitly with SPARK_REMOTE; also checks imported buffers own their lifetime.""" + import gc + import os + pa = pytest.importorskip("pyarrow") + remote = os.environ.get("SPARK_REMOTE") + if not remote: + pytest.skip("requires SPARK_REMOTE") + from pyspark.sql import SparkSession + + spark = SparkSession.builder.remote(remote).getOrCreate() + try: + table = pa.table({"x": pa.chunked_array([[1, 2], [3]]), "nested": [[1], None, []]}) + df = spark.createDataFrame(table, ["id", "items"]) + del table + gc.collect() + assert df.columns == ["id", "items"] + assert [(r[0], r[1]) for r in df.collect()] == [(1, [1]), (2, None), (3, [])] + empty = spark.createDataFrame(pa.table({"x": pa.array([], type=pa.int64())}), + T.StructType([T.StructField("x", T.LongType())])) + assert empty.count() == 0 + assert empty.columns == ["x"] + finally: + spark.stop() + + +@pytest.mark.parametrize("kind", ["slice", "dictionary", "decimal", "null", "struct", "nul"]) +def test_arrow_input_types_live_server(kind): + import os + from decimal import Decimal + pa = pytest.importorskip("pyarrow") + remote = os.environ.get("SPARK_REMOTE") + if not remote: + pytest.skip("requires SPARK_REMOTE") + from pyspark.sql import SparkSession + + tables = { + "slice": pa.table({"x": pa.array([0, 1, 2, 3]).slice(1, 2)}), + "dictionary": pa.table({"x": pa.array(["a", None, "b", "a"]).dictionary_encode()}), + "decimal": pa.table({"x": pa.array([Decimal("1.23"), None], type=pa.decimal128(10, 2))}), + "null": pa.table({"x": pa.nulls(2)}), + "struct": pa.table({"x": [{"a": [1, None]}, None]}), + "nul": pa.table({"a\0b": [1, 2]}), + } + spark = SparkSession.builder.remote(remote).getOrCreate() + try: + table = tables[kind] + df = spark.createDataFrame(table) + assert df.columns == table.column_names + assert df.toArrow().to_pylist() == table.to_pylist() + finally: + spark.stop() + + +def test_arrow_input_large_upload_live_server(): + """A table larger than one gRPC message must use uploaded cache artifacts.""" + import os + pa = pytest.importorskip("pyarrow") + remote = os.environ.get("SPARK_REMOTE") + if not remote: + pytest.skip("requires SPARK_REMOTE") + from pyspark.sql import SparkSession + + rows = 140000 + table = pa.table({"payload": [b"x" * 1024] * rows}) + assert table.nbytes > 128 * 1024**2 + spark = SparkSession.builder.remote(remote).getOrCreate() + try: + df = spark.createDataFrame(table) + assert df.count() == rows + assert df.selectExpr("sum(length(payload)) AS total").collect()[0][0] == rows * 1024 + finally: + spark.stop() + + # --------------------------------------------------------------------------- 4.3.0 gaps