From b67b2812cc53cf28127e7b6b280105be8315bbcd Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Wed, 18 Nov 2026 13:24:53 +0000 Subject: [PATCH] [SPARK-62017][CONNECT][PYTHON] Report the types, flags, metadata and fields that the array, map, field and struct types hold to Python's garbage collector, as PySpark's __dict__ holds them PySpark's ArrayType, MapType, StructField and StructType are plain Python classes whose __dict__ holds the type objects that they are made of, their flags, the metadata dict of a field and the fields and names lists of a struct. So a reference cycle through them is collected: - T.ArrayType(t) of a DecimalType t with t.job = job and job.held = the array; - a map of such a key or value; - a field of such a type, or of a metadata dict that holds job; - a struct of such a field, also of add. The drop-in held them in Rust (KeptType, Py, Py, Py), and the four classes had no __traverse__, so the collector did not see them and the cycle was never collected, as SPARK-61971 found of the other holders of Python objects. Now the four classes report them in __traverse__ and drop the kept type objects, and the name of a field that is not a str, in __clear__. Their lists, dicts and flags are cleared by their own types. The classes have a __dict__, so the collector can traverse an object of them while it is created, before its value is written: as the other such classes, each has a GcReady that __traverse__ checks first. A KeptType reports its object only if its lock is free, and drops it after the lock is released, as the drop may run Python code. Co-authored-by: Isaac --- crates/pyspark-rs/src/types.rs | 117 ++++++++++++++++++++++++++++ dev/parity/known-differences.md | 27 +++---- docs/compatibility.md | 23 +++--- python/tests/test_dropin_offline.py | 58 +++++++++++++- 4 files changed, 200 insertions(+), 25 deletions(-) diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs index 13c2e786..f7dab459 100644 --- a/crates/pyspark-rs/src/types.rs +++ b/crates/pyspark-rs/src/types.rs @@ -762,6 +762,25 @@ impl KeptType { } } + /// Report the object of the child, if one is kept, to Python's garbage collector, as PySpark's + /// type holds it in its `__dict__`. A child that another thread reads at the time is not + /// reported, which keeps it alive as a reference that the collector does not know of. + fn traverse(&self, visit: &pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { + if let Ok(kept) = self.0.try_lock() { + if let Some(object) = kept.as_ref() { + visit.call(object)?; + } + } + Ok(()) + } + + /// Drop the object of the child, to break a reference cycle through it that the garbage + /// collector found unreachable: after the lock is released, as its drop may run Python code. + fn clear(&self) { + let taken = self.0.lock().unwrap_or_else(|e| e.into_inner()).take(); + drop(taken); + } + /// The core type of the child as it is now, as [`KeptType::live`] reads it, or else the type /// that stands for a child that does not lower now (see [`lenient_type`]). fn live_or_stand_in(&self, py: Python<'_>, snapshot: &DataType) -> PyResult { @@ -866,6 +885,7 @@ fn struct_parts(py: Python<'_>, fields: Bound<'_, PyList>) -> PyResult<(DataType fields: fields.unbind(), names: names.unbind(), need_conversion, + gc_ready: crate::gc::GcReady::new(), }; Ok(( DataType::Struct { @@ -3666,6 +3686,9 @@ pub struct PyArrayType { element_object: KeptType, /// The `containsNull` as it was given (mirrors PySpark, which keeps any value). contains_null_object: Py, + /// Set once the value is written, as `__traverse__` reads `contains_null_object`, which is + /// not valid while the object is created (see [`crate::gc::GcReady`]). + gc_ready: crate::gc::GcReady, } #[pymethods] @@ -3796,6 +3819,7 @@ impl PyArrayType { contains_null: true, element_object: KeptType::default(), contains_null_object: flag_of_bool(py, true), + gc_ready: crate::gc::GcReady::new(), }, [], value @@ -3828,6 +3852,25 @@ impl PyArrayType { fn __setstate__(slf: &Bound<'_, Self>, state: &Bound<'_, PyAny>) -> PyResult<()> { set_pickled_state(slf.as_any(), state, |_, _| Ok(false)) } + /// As PySpark's type, a Python object of a `__dict__` of its `elementType` and `containsNull`, + /// which Python's garbage collector sees: a reference cycle through them is collected, as + /// PySpark collects it. + fn __traverse__(&self, visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { + if !self.gc_ready.get() { + return Ok(()); + } + self.element_object.traverse(&visit)?; + visit.call(&self.contains_null_object) + } + + /// Break a reference cycle that the garbage collector found unreachable: the type drops the + /// object of its element type. + fn __clear__(&mut self) { + if self.gc_ready.get() { + self.element_object.clear(); + } + } + fn __repr__(&self, py: Python<'_>) -> PyResult { // Mirror pyspark: ArrayType(, ). let el = self.element_object.object(py, &self.element_type)?; @@ -3883,6 +3926,9 @@ pub struct PyMapType { value_object: KeptType, /// The `valueContainsNull` as it was given (mirrors PySpark, which keeps any value). value_contains_null_object: Py, + /// Set once the value is written, as `__traverse__` reads the flag, which is not valid while + /// the object is created (see [`crate::gc::GcReady`]). + gc_ready: crate::gc::GcReady, } #[pymethods] @@ -4046,6 +4092,7 @@ impl PyMapType { key_object: KeptType::default(), value_object: KeptType::default(), value_contains_null_object: flag_of_bool(py, true), + gc_ready: crate::gc::GcReady::new(), }, [], value @@ -4083,6 +4130,27 @@ impl PyMapType { fn __setstate__(slf: &Bound<'_, Self>, state: &Bound<'_, PyAny>) -> PyResult<()> { set_pickled_state(slf.as_any(), state, |_, _| Ok(false)) } + /// As PySpark's type, a Python object of a `__dict__` of its `keyType`, `valueType` and + /// `valueContainsNull`, which Python's garbage collector sees: a reference cycle through them + /// is collected, as PySpark collects it. + fn __traverse__(&self, visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { + if !self.gc_ready.get() { + return Ok(()); + } + self.key_object.traverse(&visit)?; + self.value_object.traverse(&visit)?; + visit.call(&self.value_contains_null_object) + } + + /// Break a reference cycle that the garbage collector found unreachable: the type drops the + /// objects of its key and value types. + fn __clear__(&mut self) { + if self.gc_ready.get() { + self.key_object.clear(); + self.value_object.clear(); + } + } + fn __repr__(&self, py: Python<'_>) -> PyResult { // Mirror pyspark: MapType(, , ). let k: String = self @@ -4158,6 +4226,9 @@ pub struct PyStructField { /// not change with its parameters, which may be set, so `add` need not read it again for each /// field. Reset by a set `dataType`. need_conversion_of_type: std::sync::OnceLock, + /// Set once the value is written, as `__traverse__` reads the flag and the metadata, which + /// are not valid while the object is created (see [`crate::gc::GcReady`]). + gc_ready: crate::gc::GcReady, } impl PyStructField { @@ -4171,6 +4242,7 @@ impl PyStructField { metadata, data_type_object: KeptType::default(), need_conversion_of_type: std::sync::OnceLock::new(), + gc_ready: crate::gc::GcReady::new(), }) } @@ -4195,6 +4267,7 @@ impl PyStructField { metadata, data_type_object, need_conversion_of_type: std::sync::OnceLock::new(), + gc_ready: crate::gc::GcReady::new(), }) } @@ -4532,6 +4605,30 @@ impl PyStructField { self.metadata = metadata.clone().unbind(); Ok(()) } + /// As PySpark's field, a Python object of a `__dict__` of its `name`, `dataType`, `nullable` + /// and `metadata`, which Python's garbage collector sees: a reference cycle through them, the + /// metadata dict too, is collected, as PySpark collects it. + fn __traverse__(&self, visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { + if !self.gc_ready.get() { + return Ok(()); + } + self.data_type_object.traverse(&visit)?; + if let Some(name) = &self.name_object { + visit.call(name)?; + } + visit.call(&self.nullable)?; + visit.call(&self.metadata) + } + + /// Break a reference cycle that the garbage collector found unreachable: the field drops the + /// object of its type and the name that is not a str. + fn __clear__(&mut self) { + if self.gc_ready.get() { + self.data_type_object.clear(); + self.name_object = None; + } + } + fn __repr__(&self, py: Python<'_>) -> PyResult { // Mirror pyspark: StructField('name', , nullable). let dt = self.data_type_object.object(py, &self.field.data_type)?; @@ -4660,6 +4757,9 @@ pub struct PyStructType { /// The `needConversion` of each field, as PySpark's `_needConversion`. Like PySpark's, it is /// computed by `__init__` and `add`, so an edit of a field's type does not change it. need_conversion: Vec, + /// Set once the value is written, as `__traverse__` reads `fields` and `names`, which are not + /// valid while the object is created (see [`crate::gc::GcReady`]). + gc_ready: crate::gc::GcReady, } impl PyStructType { @@ -5222,6 +5322,18 @@ impl PyStructType { .unbind()) } + /// As PySpark's struct, a Python object of a `__dict__` of its `fields` and `names` lists, + /// which Python's garbage collector sees: a reference cycle through them is collected, as + /// PySpark collects it. The lists are the struct's own, which a collection of the cycle + /// clears. + fn __traverse__(&self, visit: pyo3::PyVisit<'_>) -> Result<(), pyo3::PyTraverseError> { + if !self.gc_ready.get() { + return Ok(()); + } + visit.call(&self.fields)?; + visit.call(&self.names) + } + fn __repr__(&self, py: Python<'_>) -> PyResult { // Mirror pyspark: StructType([str(field), ...]), each field as its own repr shows it. let mut parts = Vec::with_capacity(self.fields.bind(py).len()); @@ -6938,6 +7050,7 @@ pub(crate) fn data_type_to_py(py: Python<'_>, dt: &DataType) -> PyResult, dt: &DataType) -> PyResult, dt: &DataType) -> PyResult> contains_null: *contains_null, element_object: KeptType::of(element.bind(py)), contains_null_object: flag_of_bool(py, *contains_null), + gc_ready: crate::gc::GcReady::new(), }; Py::new(py, init_chain!(dt.clone(), array, [], value))?.into_any() } @@ -7159,6 +7274,7 @@ pub(crate) fn schema_to_py(py: Python<'_>, dt: &DataType) -> PyResult> key_object: KeptType::of(key.bind(py)), value_object: KeptType::of(value.bind(py)), value_contains_null_object: flag_of_bool(py, *value_contains_null), + gc_ready: crate::gc::GcReady::new(), }; Py::new(py, init_chain!(dt.clone(), map, [], value))?.into_any() } @@ -7179,6 +7295,7 @@ pub(crate) fn schema_to_py(py: Python<'_>, dt: &DataType) -> PyResult> fields: objects.unbind(), names: names.unbind(), need_conversion, + gc_ready: crate::gc::GcReady::new(), }; Py::new(py, init_chain!(dt.clone(), parts, [], value))?.into_any() } diff --git a/dev/parity/known-differences.md b/dev/parity/known-differences.md index 920c9f1e..6333b708 100644 --- a/dev/parity/known-differences.md +++ b/dev/parity/known-differences.md @@ -352,19 +352,20 @@ each UDF, UDTF and map or group-map function of its plan when it builds the plan whose command carries the second pickle; PySpark's carries the first. - **Garbage collection** (tracked). A DataFrame or a Column reports the functions of its UDFs that it alone holds to Python's garbage collector, and so do grouped data and their cogroups, a table - argument, a window specification, the writers, `na`, `stat`, a merge builder and its clauses, - the rows of `toLocalIterator`, and the expression of a Column and of a table argument (`_expr`, - `_subquery_expr`) and its copies, of the Columns and the plans that they hold. Those that hold - their DataFrame report it, which reports the functions of the plan that they share - (`crate::gc::Holders`). A function that a Column and a DataFrame of it share, or two DataFrames, - or a DataFrame and a table argument of it, which holds the plan and not the DataFrame, is - reported by none, so that no reference is counted twice: a reference cycle through it, such as - through a bound method of an object that keeps both the Column and the DataFrame, is not - collected, where PySpark collects it. A stream writer holds its DataFrame, where PySpark's holds - the plan, so a DataFrame that keeps its stream writer is freed by the collector rather than at - its last reference. The session builder reports its channel builder and its hook factories, but - not one that a copy of it shares, so a cycle through one of those is not collected while the - builder and its copy live, where PySpark collects it. + argument, a window specification, the writers, `na`, `stat`, a merge builder and its clauses, the + rows of `toLocalIterator`, and the expression of a Column and of a table argument (`_expr`, + `_subquery_expr`) and its copies, of the Columns and the plans that they hold, and an `ArrayType`, + a `MapType`, a `StructField` and a `StructType`, of the types, flags, metadata and fields that + they hold, as PySpark's `__dict__` holds them. Those that hold their DataFrame report it, which + reports the functions of the plan that they share (`crate::gc::Holders`). A function that a Column + and a DataFrame of it share, or two DataFrames, or a DataFrame and a table argument of it, which + holds the plan and not the DataFrame, is reported by none, so that no reference is counted twice: + a reference cycle through it, such as through a bound method of an object that keeps both the + Column and the DataFrame, is not collected, where PySpark collects it. A stream writer holds its + DataFrame, where PySpark's holds the plan, so a DataFrame that keeps its stream writer is freed by + the collector rather than at its last reference. The session builder reports its channel builder + and its hook factories, but not one that a copy of it shares, so a cycle through one of those is + not collected while the builder and its copy live, where PySpark collects it. ## Session and client (Connect) diff --git a/docs/compatibility.md b/docs/compatibility.md index 98824b18..99415230 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -2446,20 +2446,23 @@ DataFrame or the Column that holds it, such as through a bound method of an obje DataFrame, is collected. So do the objects that hold such Columns, or the plan of a DataFrame, as PySpark's do: grouped data and their cogroups, a table argument, a window specification, the writers of `write`, `writeTo` and `writeStream`, `na` and `stat`, a merge builder and its clauses, -the generator of `toLocalIterator`, which holds its DataFrame until it ends, as PySpark's -generator holds `self`, and the expression of a Column (`Column._expr`) and of a table argument -(`TableArg._subquery_expr`), which `copy.copy` and `copy.deepcopy` copy as they copy a Column. -Those that hold their DataFrame report it, and the DataFrame reports the functions of the plan -that they share, so a cycle through the DataFrame alone, such as `df.back = df.groupBy("id")`, is -collected too. A function that a Column and a DataFrame of it share, or two DataFrames, or a -DataFrame and a table argument of it, which holds its plan and not the DataFrame, as PySpark's +the generator of `toLocalIterator`, which holds its DataFrame until it ends, as PySpark's generator +holds `self`, and the expression of a Column (`Column._expr`) and of a table argument +(`TableArg._subquery_expr`), which `copy.copy` and `copy.deepcopy` copy as they copy a Column. Those +that hold their DataFrame report it, and the DataFrame reports the functions of the plan that they +share, so a cycle through the DataFrame alone, such as `df.back = df.groupBy("id")`, is collected +too. An `ArrayType`, a `MapType`, a `StructField` and a `StructType` report the type objects, the +flags and the metadata dict that they hold, and the lists of a struct, as PySpark's `__dict__` holds +them, so a cycle through a type that one holds, such as a `DecimalType` with an attribute that keeps +the struct, is collected. A function that a Column and a DataFrame of it share, or two DataFrames, +or a DataFrame and a table argument of it, which holds its plan and not the DataFrame, as PySpark's does, is reported by neither, so that no reference is counted twice: a cycle through it is freed only when it is broken, where PySpark collects it. A stream writer holds its DataFrame, where PySpark's holds the plan, so a DataFrame that keeps its stream writer is freed by the collector, where PySpark frees it with its last reference. The functions of `foreachBatch` and `foreach` are -kept as their pickles, as PySpark's stream writer keeps them, so a function that keeps its writer -is freed with its last reference, and a copy or a pickle of the writer carries the pickle of the -call. The registration of `dataSource` reports its session, and the builder of +kept as their pickles, as PySpark's stream writer keeps them, so a function that keeps its writer is +freed with its last reference, and a copy or a pickle of the writer carries the pickle of the call. +The registration of `dataSource` reports its session, and the builder of `pyspark.sql.connect.session.SparkSession` its channel builder and its hook factories, as PySpark's plain objects hold them, so a session that keeps its `dataSource`, or a channel builder or a hook factory that keeps its builder, is collected. One that a copy of the builder shares is reported by diff --git a/python/tests/test_dropin_offline.py b/python/tests/test_dropin_offline.py index 84ac2cde..481ec6e1 100644 --- a/python/tests/test_dropin_offline.py +++ b/python/tests/test_dropin_offline.py @@ -12770,6 +12770,51 @@ def test_cycles_through_instance_dicts_are_collected_as_pyspark(mock_connect): assert freed() is None +def test_cycles_through_the_types_that_a_type_holds_are_collected_as_pyspark(): + # As PySpark's ArrayType, MapType, StructField and StructType, plain Python classes whose + # __dict__ holds the types that they are made of, the metadata dict of a field and the fields + # of a struct: a reference cycle through them is collected. The drop-in's held them in Rust, + # which the garbage collector did not see, so such a cycle was never collected. A child that + # is not a singleton holds the cycle, as PySpark's LongType() is one, which keeps it in both. + import gc + import weakref + + class Job: + pass + + def tagged(data_type, job): + data_type.job = job + return data_type + + def metadata_later(job): + metadata = {"k": 1} + field = T.StructField("a", T.LongType(), True, metadata) + metadata["job"] = job + return field + + def metadata_set(job): + field = T.StructField("a", T.LongType(), True, {"k": 1}) + field.metadata["job"] = job + return field + + for make in [ + lambda job: T.ArrayType(tagged(T.DecimalType(10, 2), job)), + lambda job: T.MapType(T.StringType(), tagged(T.ArrayType(T.LongType()), job)), + lambda job: T.MapType(tagged(T.ArrayType(T.LongType()), job), T.LongType()), + lambda job: T.StructField("a", tagged(T.DecimalType(10, 2), job)), + lambda job: T.StructType([tagged(T.StructField("a", T.LongType()), job)]), + lambda job: T.StructType().add(tagged(T.StructField("a", T.LongType()), job)), + metadata_later, + metadata_set, + ]: + job = Job() + job.held = make(job) + freed = weakref.ref(job) + del job + gc.collect() + assert freed() is None, make + + def test_cycles_through_the_functions_of_udfs_are_collected_as_pyspark(mock_connect): # As PySpark's plans and expressions, of Python objects, which hold the function of a UDF # until a request pickles it: a reference cycle through the function and the DataFrame or the @@ -22080,8 +22125,9 @@ def test_objects_traversed_while_they_are_created_do_not_crash(mock_connect): # the allocations before each object shifted by one, meets each kind of object there: also the # grouped data, the cogroups, the table arguments, the window specifications, the null and # statistics functions, the merge clauses, the stream writers, the data source registrations - # and the session builders, and their copies, the rows of toLocalIterator, and the expressions - # of Columns and table arguments, and their copies. + # and the session builders, and their copies, the rows of toLocalIterator, the expressions + # of Columns and table arguments, and their copies, and the array, map, field and struct + # types, which a schema of the server makes too. import copy import gc @@ -22102,6 +22148,9 @@ def test_objects_traversed_while_they_are_created_do_not_crash(mock_connect): builder = ConnectSession.builder._registerHook(lambda session: None).channelBuilder( DefaultChannelBuilder("sc://localhost")) column = F.col("id") + 1 + array, kv = T.ArrayType(T.DecimalType(10, 2)), T.MapType(T.StringType(), T.LongType()) + field = T.StructField("a", T.DecimalType(10, 2)) + struct = T.StructType([field]) makers = [ lambda: df.write, lambda: copy.copy(writer), lambda: df.writeTo("t"), lambda: copy.copy(writer_v2), lambda: df.mergeInto("t", F.lit(True)), @@ -22121,6 +22170,11 @@ def test_objects_traversed_while_they_are_created_do_not_crash(mock_connect): lambda: copy.copy(builder), lambda: df.toLocalIterator(), lambda: F.col("id")._expr, lambda: copy.copy(column._expr), lambda: copy.deepcopy(column._expr), lambda: table._subquery_expr, + lambda: T.ArrayType(T.DecimalType(10, 2)), + lambda: copy.deepcopy(array), lambda: T.MapType(T.StringType(), T.LongType()), + lambda: copy.deepcopy(kv), lambda: T.StructField("a", T.DecimalType(10, 2)), + lambda: copy.deepcopy(field), lambda: T.StructType([T.StructField("a", T.LongType())]), + lambda: copy.deepcopy(struct), lambda: struct.toNullable(), lambda: df.schema, ] thresholds = gc.get_threshold() gc.collect()