Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 68 additions & 10 deletions crates/pyspark-rs/src/catalog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -200,10 +200,13 @@ fn named_tuples<T: NamedTuple>(py: Python<'_>, results: Vec<T>) -> PyResult<Vec<
}

/// Python wrapper for Spark Catalog. It has the default repr, as PySpark Connect's Catalog, which
/// defines none: `<pyspark.sql.connect.catalog.Catalog object at 0x...>`.
/// defines none: `<pyspark.sql.connect.catalog.Catalog object at 0x...>`. As PySpark's plain class,
/// it is subclassed: a subclass constructs of the arguments of its own `__init__`, and its copies
/// and pickles are of its class.
#[pyclass(
name = "Catalog",
module = "pyspark.sql.connect.catalog",
subclass,
weakref,
dict
)]
Expand All @@ -229,32 +232,87 @@ impl PyCatalog {

#[pymethods]
impl PyCatalog {
/// `Catalog(sparkSession)`, as PySpark Connect's: a catalog of the session, another object
/// than `spark.catalog`, which the session keeps.
/// A catalog, as the `object.__new__` of PySpark's plain class: of any arguments, which
/// `__init__` takes, so that a subclass of another `__init__` is built as it is in PySpark. It
/// is of the first of them that is a session, or else of the session of no server until
/// `__init__` gives it one (see `session::constructor_session`). The class shows the signature
/// of `__init__`, as `inspect.signature` of PySpark's class does.
#[new]
#[pyo3(signature = (*args, **kwargs), text_signature = "(sparkSession)")]
fn py_new(
args: &Bound<'_, pyo3::types::PyTuple>,
kwargs: Option<&Bound<'_, pyo3::types::PyDict>>,
) -> PyResult<Self> {
let session = crate::session::constructor_session(args, kwargs, "sparkSession")?;
Ok(PyCatalog::new(session.catalog()))
}

/// `Catalog(sparkSession)`, as PySpark Connect's `__init__`: a catalog of the session, another
/// object than `spark.catalog`, which the session keeps.
#[allow(non_snake_case)]
fn py_new(py: Python<'_>, sparkSession: Py<crate::session::PySparkSession>) -> Self {
PyCatalog::new(sparkSession.borrow(py).session.catalog())
fn __init__(slf: &Bound<'_, Self>, sparkSession: PyRef<'_, crate::session::PySparkSession>) {
let catalog = sparkSession.session.catalog();
// The catalog that it replaces is dropped once the object is not borrowed, as the drop of
// its session may run Python code.
let replaced = std::mem::replace(&mut slf.borrow_mut().catalog, catalog);
drop(replaced);
}

/// The shallow copy of the catalog, as PySpark's `copy.copy`: of the same session.
/// The shallow copy of the catalog, as PySpark's `copy.copy`: of the same session, and of the
/// class of this one, as the copy of PySpark's of a subclass is.
fn __copy__<'py>(slf: &Bound<'py, Self>) -> PyResult<Bound<'py, Self>> {
let copy = Bound::new(slf.py(), PyCatalog::new(slf.borrow().catalog.clone()))?;
let catalog = PyCatalog::new(slf.borrow().catalog.clone());
let copy = crate::reduce::new_of_class(&slf.get_type(), catalog)?;
crate::group::copy_instance_state(copy.as_any(), slf.as_any())?;
Ok(copy)
}

/// The reduce of the catalog, as PySpark's pickle of a plain object of its session: a new
/// `Catalog` of its session, which unpickles with it, and not the catalog that the session
/// keeps (`spark.catalog`), whose attributes a copy or an unpickle would otherwise change.
/// keeps (`spark.catalog`), whose attributes a copy or an unpickle would otherwise change. A
/// catalog of a subclass is made of its class as an unpickle of PySpark's makes it, by
/// `copyreg.__newobj__`, without its `__init__`, which may take other arguments.
fn __reduce__<'py>(slf: &Bound<'py, Self>) -> PyResult<Bound<'py, PyAny>> {
use pyo3::types::PyTuple;
let py = slf.py();
let session = crate::session::session_object(py, slf.borrow().catalog.spark_session())?;
let args = pyo3::types::PyTuple::new(py, [session.into_bound(py)])?;
let reduce = pyo3::types::PyTuple::new(py, [slf.get_type().into_any(), args.into_any()])?;
let session = session.into_bound(py).into_any();
let cls = slf.get_type();
let reduce = if cls.is(py.get_type::<Self>()) {
let args = PyTuple::new(py, [session])?;
PyTuple::new(py, [cls.into_any(), args.into_any()])?
} else {
let newobj = py.import("copyreg")?.getattr("__newobj__")?;
let args = PyTuple::new(py, [cls.into_any(), session])?;
PyTuple::new(py, [newobj, args.into_any()])?
};
crate::group::reduce_with_state(reduce.as_any(), slf.as_any())
}

/// The state of a pickle of a catalog, which pickle sets on the catalog that `__new__` made
/// of no arguments: of PySpark's pickle, the `__dict__` of its plain object, whose
/// `_sparkSession` is the session of the catalog, and the other entries its instance
/// attributes, which are all that the state of the drop-in's pickle holds.
fn __setstate__(slf: &Bound<'_, Self>, state: &Bound<'_, pyo3::types::PyDict>) -> PyResult<()> {
let py = slf.py();
let state = state.copy()?;
if let Some(session) = state.get_item("_sparkSession")? {
if let Ok(session) = session.cast::<crate::session::PySparkSession>() {
Self::__init__(slf, session.borrow());
state.del_item("_sparkSession")?;
}
}
slf.getattr(pyo3::intern!(py, "__dict__"))?
.call_method1(pyo3::intern!(py, "update"), (state,))?;
Ok(())
}

/// The session of the catalog, as PySpark Connect's `Catalog._sparkSession`.
#[getter]
fn _sparkSession(&self, py: Python<'_>) -> PyResult<Py<crate::session::PySparkSession>> {
crate::session::session_object(py, self.catalog.spark_session())
}

/// Get the current catalog.
fn currentCatalog(&self, py: Python<'_>) -> PyResult<String> {
self.detached(py, |catalog| catalog.current_catalog().map_err(Box::new))
Expand Down
119 changes: 91 additions & 28 deletions crates/pyspark-rs/src/datasource.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,79 +14,143 @@ use crate::session::PySparkSession;
/// Exposed as `SparkSession.dataSource` to register custom data sources. It is PySpark Connect's
/// class, `pyspark.sql.connect.datasource.DataSourceRegistration`, which that module re-exports:
/// `pyspark.sql.datasource.DataSourceRegistration` is PySpark's classic class, of which it is not.
/// As PySpark's plain class, it is subclassed: a subclass constructs of the arguments of its own
/// `__init__`, and its copies and pickles are of its class.
#[pyclass(
name = "DataSourceRegistration",
module = "pyspark.sql.connect.datasource",
subclass,
weakref,
dict
)]
pub struct PyDataSourceRegistration {
pub(crate) session: Py<PySparkSession>,
/// Set once the value is written, as `__traverse__` reads `session`, which is not valid while
/// the object is created (see [`crate::gc::GcReady`]).
gc_ready: crate::gc::GcReady,
/// The session of the registration, as PySpark's `sparkSession`, which its `__init__` sets:
/// None before it, as the registration that `__new__` makes has none.
pub(crate) session: Option<Py<PySparkSession>>,
}

impl PyDataSourceRegistration {
pub fn new(session: Py<PySparkSession>) -> Self {
PyDataSourceRegistration {
session,
gc_ready: crate::gc::GcReady::new(),
session: Some(session),
}
}

/// The session of the registration, or the AttributeError of PySpark's registration whose
/// `__init__` did not run, which has no `sparkSession`.
fn session_of(slf: &Bound<'_, Self>) -> PyResult<Py<PySparkSession>> {
let py = slf.py();
match &slf.borrow().session {
Some(session) => Ok(session.clone_ref(py)),
None => Err(pyo3::exceptions::PyAttributeError::new_err(format!(
"'{}' object has no attribute 'sparkSession'",
slf.get_type().name()?
))),
}
}
}

#[pymethods]
impl PyDataSourceRegistration {
/// `DataSourceRegistration(sparkSession)`, as PySpark Connect's: the registration of the data
/// sources of the session.
/// A registration of no session, as the `object.__new__` of PySpark's plain class: of any
/// arguments, which `__init__` takes, so that a subclass of another `__init__` is built as it
/// is in PySpark, and an unpickle makes one of none. The class shows the signature of
/// `__init__`, as `inspect.signature` of PySpark's class does.
#[new]
#[pyo3(signature = (*_args, **_kwargs), text_signature = "(sparkSession)")]
fn py_new(
_args: &Bound<'_, pyo3::types::PyTuple>,
_kwargs: Option<&Bound<'_, pyo3::types::PyDict>>,
) -> Self {
PyDataSourceRegistration { session: None }
}

/// `DataSourceRegistration(sparkSession)`, as PySpark Connect's `__init__`: the registration
/// of the data sources of the session.
#[allow(non_snake_case)]
fn py_new(sparkSession: Py<PySparkSession>) -> Self {
PyDataSourceRegistration::new(sparkSession)
fn __init__(slf: &Bound<'_, Self>, sparkSession: Py<PySparkSession>) {
let replaced = slf.borrow_mut().session.replace(sparkSession);
drop(replaced);
}

/// The shallow copy of the data source registration, as PySpark's `copy.copy`: of the same
/// session, with the instance attributes of this one.
fn __copy__(slf: &Bound<'_, Self>) -> PyResult<Py<Self>> {
/// session, and of the class of this one, with the instance attributes of this one.
fn __copy__<'py>(slf: &Bound<'py, Self>) -> PyResult<Bound<'py, Self>> {
let py = slf.py();
let session = slf.borrow().session.clone_ref(py);
let copy = Py::new(py, PyDataSourceRegistration::new(session))?;
crate::reduce::copy_instance_dict(slf.as_any(), copy.bind(py).as_any())?;
let session = slf.borrow().session.as_ref().map(|s| s.clone_ref(py));
let copy =
crate::reduce::new_of_class(&slf.get_type(), PyDataSourceRegistration { session })?;
crate::reduce::copy_instance_dict(slf.as_any(), copy.as_any())?;
Ok(copy)
}

/// The reduce of the data source registration, as PySpark's: `dataSource` of its session,
/// which unpickles with it, and its instance attributes.
/// The reduce of the data source registration, as PySpark's pickle of its plain object:
/// `copyreg.__newobj__` of its class, of the state of its `__dict__`, its `sparkSession`, which
/// unpickles with it, and its instance attributes (see `__setstate__`). So PySpark loads the
/// pickle as a registration of its own, and PySpark's pickle loads here.
fn __reduce__<'py>(slf: &Bound<'py, Self>) -> PyResult<Bound<'py, PyAny>> {
use pyo3::types::{PyDict, PyTuple};
let py = slf.py();
let state = PyDict::new(py);
if let Some(session) = &slf.borrow().session {
state.set_item("sparkSession", session.clone_ref(py))?;
}
state.call_method1("update", (slf.getattr(pyo3::intern!(py, "__dict__"))?,))?;
let newobj = py.import("copyreg")?.getattr("__newobj__")?;
let args = PyTuple::new(py, [slf.get_type()])?;
let reduced = match state.is_empty() {
true => PyTuple::new(py, [newobj, args.into_any()])?,
false => PyTuple::new(py, [newobj, args.into_any(), state.into_any()])?,
};
Ok(reduced.into_any())
}

/// The state of the pickle of a registration, as pickle sets the `__dict__` of PySpark's: its
/// session, of `sparkSession`, and its instance attributes, of the other entries.
fn __setstate__(slf: &Bound<'_, Self>, state: &Bound<'_, pyo3::types::PyDict>) -> PyResult<()> {
let py = slf.py();
let session = slf.borrow().session.clone_ref(py).into_bound(py).into_any();
let reduced = crate::reduce::getattr_reduce(py, session, "dataSource")?;
crate::reduce::reduce_with_instance_dict(slf.as_any(), reduced.into_any())
let state = state.copy()?;
if let Some(session) = state.get_item("sparkSession")? {
Self::__init__(slf, session.extract()?);
state.del_item("sparkSession")?;
}
slf.getattr(pyo3::intern!(py, "__dict__"))?
.call_method1(pyo3::intern!(py, "update"), (state,))?;
Ok(())
}

/// The session of the registration, as PySpark's `DataSourceRegistration.sparkSession`.
#[getter(sparkSession)]
fn spark_session(&self, py: Python<'_>) -> Py<PySparkSession> {
self.session.clone_ref(py)
fn spark_session(slf: &Bound<'_, Self>) -> PyResult<Py<PySparkSession>> {
Self::session_of(slf)
}

/// As PySpark's registration, a Python object that holds its session, which Python's garbage
/// collector sees: the session is reported, so a reference cycle through it, such as a session
/// that keeps its `dataSource`, is freed. Nothing of a registration that is being created.
/// that keeps its `dataSource`, is freed. Only an `Option<Py<_>>` field is read, which is
/// `None` while the object is created, so no [`crate::gc::GcReady`] is needed.
fn __traverse__(&self, visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> {
if !self.gc_ready.get() {
return Ok(());
if let Some(session) = &self.session {
visit.call(session)?;
}
visit.call(&self.session)
Ok(())
}

/// Break a reference cycle that the garbage collector found unreachable.
fn __clear__(&mut self) {
self.session = None;
}

/// Register a custom data source class.
///
/// Mirrors `pyspark.sql.SparkSession.dataSource.register(dataSource)`, of PySpark's parameter.
/// The data source class is cloudpickled and sent to the server.
#[allow(non_snake_case)]
fn register(&self, py: Python<'_>, dataSource: &Bound<'_, PyAny>) -> PyResult<()> {
fn register(slf: &Bound<'_, Self>, dataSource: &Bound<'_, PyAny>) -> PyResult<()> {
let py = slf.py();
// As PySpark's `self.sparkSession._client.register_data_source(dataSource)`, which reads
// the session first.
let session = Self::session_of(slf)?.borrow(py).session.clone();
// The name of the classmethod `name()`, as PySpark Connect's: the AttributeError of a
// class without one. Its `CommonInlineUserDefinedDataSource` sets the name in the string
// field of its command, as protobuf converts it, before it pickles the class: bytes of
Expand All @@ -102,7 +166,6 @@ impl PyDataSourceRegistration {
let expr = CommonInlineUserDefinedDataSourceExpression::new(name, payload);

// Register with the session (release GIL during the RPC)
let session = self.session.borrow(py).session.clone();
interruptible(py, &session, || {
session.register_data_source(expr).map_err(Box::new)
})?;
Expand Down
2 changes: 1 addition & 1 deletion crates/pyspark-rs/src/gc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ impl GcReady {
}

/// The flag of a clone of a written value, which is written too: for a class whose value is
/// cloned, such as the session builder, of which PyO3 extracts a clone.
/// cloned.
impl Clone for GcReady {
fn clone(&self) -> Self {
GcReady::new()
Expand Down
58 changes: 53 additions & 5 deletions crates/pyspark-rs/src/readwriter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -680,17 +680,16 @@ impl PyDataFrameReader {
/// `DataFrameReader(client)`, as PySpark Connect's, of the session `client`: the reader that
/// `spark.read` gives. As the `object.__new__` of PySpark's plain class, it takes the
/// arguments of the `__init__` of a subclass too, of which the session is the first that is
/// a session (see `session::constructor_session`); `__init__` checks them.
/// a session, and none, as an unpickle gives none (see `session::constructor_session`);
/// `__init__` checks them.
#[new]
#[classmethod]
#[pyo3(signature = (*args, **kwargs), text_signature = "(client)")]
fn py_new(
cls: &Bound<'_, pyo3::types::PyType>,
args: &Bound<'_, pyo3::types::PyTuple>,
kwargs: Option<&Bound<'_, pyo3::types::PyDict>>,
) -> PyResult<Self> {
let session = crate::session::constructor_session(cls, args, kwargs, "client")?;
Ok(PyDataFrameReader::new(session.borrow().session.read()))
let session = crate::session::constructor_session(args, kwargs, "client")?;
Ok(PyDataFrameReader::new(session.read()))
}

/// As PySpark's `__init__(client)`, which makes the reader of the session anew: of no
Expand Down Expand Up @@ -755,6 +754,48 @@ impl PyDataFrameReader {
crate::reduce::reduce_with_instance_dict(slf.as_any(), reduced)
}

/// The state of a pickle of a reader, which pickle sets on the reader that `__new__` made of
/// no arguments. Of PySpark's pickle, the state is the `__dict__` of its plain object, which
/// holds `_options`: so the reader is of the session `_client`, and of the format, the schema
/// and the options of `_format`, `_schema` and `_options`, which it takes as `format`,
/// `schema` and `option` take them, and the other entries are its instance attributes. The
/// state of the drop-in's pickle holds the instance attributes alone (see `__reduce__`), as a
/// reader's `__dict__` cannot hold `_options`, a property of the reader.
fn __setstate__(slf: &Bound<'_, Self>, state: &Bound<'_, PyDict>) -> PyResult<()> {
let py = slf.py();
let state = state.copy()?;
if state.contains("_options")? {
let take = |key: &str| -> PyResult<Option<Bound<'_, PyAny>>> {
let value = state.get_item(key)?;
if value.is_some() {
state.del_item(key)?;
}
Ok(value)
};
if let Some(client) = take("_client")? {
let session = client.cast_into::<crate::session::PySparkSession>()?;
let reader = PyDataFrameReader::new(session.borrow().session.read());
let replaced = std::mem::replace(&mut *slf.borrow_mut(), reader);
drop(replaced);
}
if let Some(format) = take("_format")? {
let source = (!format.is_none()).then_some(&format);
Self::format(slf.borrow_mut(), source);
}
if let Some(schema) = take("_schema")? {
Self::schema(slf.borrow_mut(), &schema)?;
}
if let Some(options) = take("_options")? {
for (key, value) in options.cast_into::<PyDict>()?.iter() {
Self::option(slf.borrow_mut(), &key, &value)?;
}
}
}
slf.getattr(pyo3::intern!(py, "__dict__"))?
.call_method1(pyo3::intern!(py, "update"), (state,))?;
Ok(())
}

/// A reader of its state, as `__reduce__` gives it, of the class `cls`: of the session
/// `session`, the format, the schema, the options, the format that is kept (`unsent`), and
/// the schema and the options that UTF-8 cannot encode (`unencodable`, as
Expand Down Expand Up @@ -883,6 +924,13 @@ impl PyDataFrameReader {
self.unsent_format = None;
}

/// The session of the reader, as PySpark Connect's `DataFrameReader._client`, which its
/// pickle keeps.
#[getter]
fn _client(&self, py: Python<'_>) -> PyResult<Py<crate::session::PySparkSession>> {
crate::session::session_object(py, &self.inner.spark_session())
}

/// The configured source format (PySpark Connect's `DataFrameReader._format`), which is the
/// object itself when it is not a str that the core reader takes, as PySpark keeps it.
#[getter]
Expand Down
Loading
Loading