Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
dc29a19
[SPARK-60676][CONNECT] Use Arrow columns for signed int64 pandas inputs
dbtsai Oct 29, 2026
3e79f91
[SPARK-60676][CONNECT] Validate integer upload scales against mock RPC
dbtsai Oct 29, 2026
8c705c0
[SPARK-60676][CONNECT] Cover leading null inference and non-nullable …
dbtsai Oct 29, 2026
fb996d9
[SPARK-60676][CONNECT] Exercise named schemas through mock rename rel…
dbtsai Oct 29, 2026
79d3faf
[SPARK-60676][CONNECT] Avoid Arrow configuration overhead for small N…
dbtsai Oct 29, 2026
e266e8e
[SPARK-60676][CONNECT] Retain IPC row fallback for NUL field names
dbtsai Oct 29, 2026
9942f21
[SPARK-60676][CONNECT] Include pandas regressions in the existing Arr…
dbtsai Oct 29, 2026
5edd700
[SPARK-60676][CONNECT] Preserve integer pandas schema and fallback se…
dbtsai Oct 29, 2026
080aa16
[SPARK-60676][TESTS] Preserve subclass column labels on pandas 3
dbtsai Oct 30, 2026
abbc23a
[SPARK-60676][CONNECT] Preserve customized pandas columns hooks
dbtsai Oct 30, 2026
7961bde
[SPARK-60676][CONNECT] Preserve rebound pandas index storage ordering
dbtsai Oct 30, 2026
3e3a8f5
Preserve integer pandas fallback with missing Arrow aliases
dbtsai Oct 31, 2026
f058ac9
[SPARK-60676][TESTS] Retain current master coverage after rebase
dbtsai Nov 5, 2026
121a141
Preserve current pandas label iteration and fallback order after rebase
dbtsai Nov 5, 2026
aca1d02
Keep canonical string storage guard module identity
dbtsai Nov 5, 2026
ecdd67a
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
e7ae71a
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
40e5266
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
280bdee
[SPARK-60727][CONNECT] Keep canonical double input on inline transport
dbtsai Oct 30, 2026
11ff3b7
[SPARK-60727][CONNECT] Normalize Float helper terminal newline
dbtsai Oct 30, 2026
b9498d1
Reconcile current master interfaces and formatting after rebase
dbtsai Nov 5, 2026
2bd4638
Align float label regression tests with current native iteration
dbtsai Nov 5, 2026
cfde2b6
[CONNECT] Reconcile remaining change with current inference architecture
dbtsai Nov 6, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions crates/pyspark-rs/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -765,6 +765,54 @@ impl PySparkSession {
}
let spec = schema_spec(py, schema)?;
if is_loaded_instance(py, data, "pandas", "DataFrame")? {
// Guarded integer columns share the existing Arrow importer. Schema
// validation above and unsupported pandas conversions keep master's
// normal ordering and row fallback.
let table = py
.import("pyspark.sql._pandas_input")?
.getattr("prepare_int64_table")?
.call1((data, schema))?;
if !table.is_none() {
let frame =
crate::pyarrow_import::create_dataframe(py, &self.session, &table, schema)?;
// pandas applies explicit field names with toDF even though the
// local relation already carries them. Empty input stays local.
return Ok(match &spec {
Spec::Struct(fields) if !fields.is_empty() && table.len()? > 0 => {
PyDataFrame::new(
frame
.dataframe
.to_df(fields.iter().map(|field| field.name.as_str()).collect()),
)
}
_ => frame,
});
}
let ipc = py
.import("pyspark.sql._pandas_input")?
.getattr("prepare_float64_input")?
.call1((data, schema))?;
if !ipc.is_none() {
let fields = match &spec {
Spec::Struct(fields) => fields,
_ => {
return Err(pyo3::exceptions::PyValueError::new_err(
"float64 input requires a struct schema",
));
}
};
let schema = DataType::Struct {
fields: fields.clone(),
};
let frame = self
.session
.create_dataframe_from_inline_arrow_ipc(ipc.extract()?, schema);
// Match pandas row input: explicit nonempty field names travel
// through toDF as well as the local relation's logical schema.
return Ok(PyDataFrame::new(
frame.to_df(fields.iter().map(|field| field.name.as_str()).collect()),
));
}
return self.create_from_pandas(py, data, spec);
}
self.create_from_local(py, data, spec)
Expand Down
17 changes: 17 additions & 0 deletions crates/spark-connect/src/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,23 @@ impl SparkSession {
Ok(DataFrame::new(self.clone(), plan))
}

/// Internal binding bridge for Arrow IPC already validated and encoded from
/// local rows or columns. Retains the row path's inline transport policy.
#[doc(hidden)]
pub fn create_dataframe_from_inline_arrow_ipc(
&self,
data: Vec<u8>,
schema: DataType,
) -> DataFrame {
DataFrame::new(
self.clone(),
LogicalPlan::LocalRelation {
schema: Some(schema),
data: Some(data),
},
)
}

/// Create a local DataFrame from Arrow batches without materializing rows.
///
/// `schema` optionally supplies the Spark schema applied by the server. With
Expand Down
10 changes: 10 additions & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,16 @@ timestamp conversion helpers normalize the table, then arrow-rs imports its
those batches to IPC for the local relation sent to Spark. No Python rows are
materialized; casts, buffer alignment, and IPC serialization may still copy.
Empty tables require an explicit schema, matching PySpark Connect.
Pandas inputs with only NumPy `int64` or pandas nullable `Int64` columns also
use this importer, preserving integer precision and null masks without Python
rows. Small frames (fewer than 1,000 rows) with only NumPy columns retain the
row path to avoid configuration RPC overhead. Nullable `Int64` columns always
use Arrow so schema inference does not depend on the first value being non-null.
This initial path accepts inferred schemas, complete string column names,
and explicit `StructType` schemas whose fields are all `LongType`. Other dtypes,
DDL/atomic schemas, schema conversions, NUL names, and duplicate/non-string input names
retain the existing row conversion path. `verifySchema` remains ignored for
pandas Arrow input, matching Spark Connect.
Inputs at or above `spark.sql.session.localRelationCacheThreshold` are serialized
into independent IPC streams and uploaded as session-scoped cache artifacts.
The plan carries `ChunkedCachedLocalRelation` hashes instead of embedding the
Expand Down
Loading
Loading