From f34f5fadd1dde72ced7bccb09aa9bd0ba38f5bb8 Mon Sep 17 00:00:00 2001 From: Sergey Zhukov Date: Fri, 20 Nov 2026 11:05:17 +0100 Subject: [PATCH 1/7] accept map and array input in to_json's planning check --- crates/spark/src/function/json/to_json.rs | 120 +++++++++++++++--- .../sqllogictest/test_files/json/to_json.slt | 2 +- 2 files changed, 105 insertions(+), 17 deletions(-) diff --git a/crates/spark/src/function/json/to_json.rs b/crates/spark/src/function/json/to_json.rs index 488b8e56..89910c2d 100644 --- a/crates/spark/src/function/json/to_json.rs +++ b/crates/spark/src/function/json/to_json.rs @@ -104,13 +104,10 @@ impl ScalarUDFImpl for SparkToJson { fn return_type(&self, arg_types: &[DataType]) -> Result { match arg_types { - [DataType::Struct(fields)] => { - for field in fields { - check_spark_json_type(field.data_type(), field.name())?; - } + [input] => { + check_input_type(input)?; Ok(DataType::Utf8) } - [other] => plan_err!("to_json expects a struct argument, got {other}"), _ => plan_err!( "to_json expects exactly one argument, got {}", arg_types.len() @@ -145,8 +142,28 @@ impl ScalarUDFImpl for SparkToJson { } } +/// Checks the type of the value passed to to_json. Spark accepts a struct, a +/// map or an array; their fields, keys, values and elements are checked with +/// `check_spark_json_type`. +fn check_input_type(data_type: &DataType) -> Result<()> { + use DataType::*; + match data_type { + // each field is reported under its own name + Struct(fields) => fields.iter().try_for_each(|f| { + check_spark_json_type(f.data_type(), &format!("field {}", f.name())) + }), + // a map or an array has no field name, so its path starts at `input` + Map(..) | List(_) | LargeList(_) | FixedSizeList(..) | ListView(_) + | LargeListView(_) => check_spark_json_type(data_type, "input"), + other => { + plan_err!("to_json expects a struct, map or array argument, got {other}") + } + } +} + /// Rejects types that have no faithful Spark JSON text. `path` names the -/// field in the error (`a.b[]` for the elements of list `b` in struct `a`). +/// value in the error: `field a.b[]` for the elements of list `b` in struct +/// field `a`, `input[]` for the elements of an array passed to to_json. fn check_spark_json_type(data_type: &DataType, path: &str) -> Result<()> { use DataType::*; match data_type { @@ -166,11 +183,11 @@ fn check_spark_json_type(data_type: &DataType, path: &str) -> Result<()> { }), Map(entries, _) => { let Struct(kv) = entries.data_type() else { - return plan_err!("to_json: field {path} is a map without entries"); + return plan_err!("to_json: {path} is a map without entries"); }; if !is_spark_map_key_type(kv[0].data_type()) { return plan_err!( - "to_json: field {path} has map key type {}, which is not supported", + "to_json: {path} has map key type {}, which is not supported", kv[0].data_type() ); } @@ -183,7 +200,7 @@ fn check_spark_json_type(data_type: &DataType, path: &str) -> Result<()> { // '1' DAY vs INTERVAL '1 00:00:00' DAY TO SECOND), which Arrow does not // record; durations, times, Float16 and unions have no Spark type other => plan_err!( - "to_json: field {path} has type {other}, which has no Spark JSON representation" + "to_json: {path} has type {other}, which has no Spark JSON representation" ), } } @@ -1157,16 +1174,86 @@ mod tests { } #[test] - fn test_return_type_rejects_non_struct() { + fn test_return_type_rejects_unsupported_input() { let err = SparkToJson::new() .return_type(&[DataType::Utf8]) - .unwrap_err(); - assert!(err.to_string().contains("expects a struct argument")); + .unwrap_err() + .to_string(); + assert!( + err.contains("expects a struct, map or array argument"), + "{err}" + ); let err = SparkToJson::new().return_type(&[]).unwrap_err(); assert!(err.to_string().contains("exactly one argument")); } + #[test] + fn test_return_type_accepts_map_and_array_input() { + let func = SparkToJson::new(); + let item = |t: DataType| Arc::new(Field::new("item", t, true)); + let entries = Fields::from(vec![ + Field::new("keys", DataType::Utf8, false), + Field::new("values", DataType::Int32, true), + ]); + let entries = Arc::new(Field::new("entries", DataType::Struct(entries), false)); + let point = Fields::from(vec![Field::new("x", DataType::Int32, true)]); + + for t in [ + DataType::Map(entries, false), + DataType::List(item(DataType::Int32)), + DataType::LargeList(item(DataType::Utf8)), + DataType::FixedSizeList(item(DataType::Float64), 2), + DataType::ListView(item(DataType::Date32)), + DataType::LargeListView(item(DataType::Boolean)), + DataType::List(item(DataType::Struct(point))), + ] { + assert_eq!(func.return_type(&[t]).unwrap(), DataType::Utf8); + } + } + + #[test] + fn test_return_type_checks_map_and_array_input() { + let return_type = |t: DataType| { + SparkToJson::new() + .return_type(&[t]) + .unwrap_err() + .to_string() + }; + let item = |t: DataType| Arc::new(Field::new("item", t, true)); + let map_of = |k: DataType, v: DataType| { + let entries = Fields::from(vec![ + Field::new("keys", k, false), + Field::new("values", v, true), + ]); + DataType::Map( + Arc::new(Field::new("entries", DataType::Struct(entries), false)), + false, + ) + }; + let interval = DataType::Interval(IntervalUnit::DayTime); + + // the input has no field name, so the path starts at `input` + let err = return_type(DataType::List(item(interval.clone()))); + assert!(err.contains("to_json: input[] has type"), "{err}"); + + let inner = Fields::from(vec![Field::new("i", interval.clone(), true)]); + let err = return_type(DataType::List(item(DataType::Struct(inner)))); + assert!(err.contains("to_json: input[].i has type"), "{err}"); + + let err = return_type(map_of(DataType::Utf8, interval)); + assert!(err.contains("to_json: input[] has type"), "{err}"); + + // the key type of a map input is checked too + let list_key = DataType::List(item(DataType::Int32)); + let err = return_type(map_of(list_key, DataType::Int32)); + assert!( + err.contains("to_json: input has map key type List(") + && err.contains("not supported"), + "{err}" + ); + } + #[test] fn test_array_input() { let input = sample_struct(); @@ -2078,12 +2165,13 @@ mod dataframe_tests { batches_to_sort_string(&df_results) ); - // check that a non-struct argument is rejected when the plan is built + // check that an argument that is not a struct, map or array is rejected + // when the plan is built let err = df .with_column("bad", to_json(col("foo"))) .unwrap_err() .to_string(); - assert_contains!(err, "to_json expects a struct argument"); + assert_contains!(err, "to_json expects a struct, map or array argument"); // check that the same result is produced through SQL: to_json is a // regular scalar UDF, so once registered it works on any struct @@ -2105,13 +2193,13 @@ mod dataframe_tests { batches_to_sort_string(&df_results) ); - // the SQL path rejects a non-struct argument at plan time as well + // the SQL path rejects such an argument at plan time as well let err = ctx .sql(&format!("SELECT to_json(foo) FROM {SAMPLE_TABLE_SQL}")) .await .unwrap_err() .to_string(); - assert_contains!(err, "to_json expects a struct argument"); + assert_contains!(err, "to_json expects a struct, map or array argument"); Ok(()) } diff --git a/crates/sqllogictest/test_files/json/to_json.slt b/crates/sqllogictest/test_files/json/to_json.slt index f011673f..484458e6 100644 --- a/crates/sqllogictest/test_files/json/to_json.slt +++ b/crates/sqllogictest/test_files/json/to_json.slt @@ -353,7 +353,7 @@ SELECT to_json(named_struct('s', arrow_cast('a', 'Dictionary(Int32, Utf8)'))); ## Errors -query error Error during planning: to_json expects a struct argument, got Utf8 +query error Error during planning: to_json expects a struct, map or array argument, got Utf8 SELECT to_json('abc'::STRING); query error 'to_json' does not support zero arguments From 9f22f285c70cd8522e553fd29253feb1fd996489 Mon Sep 17 00:00:00 2001 From: Sergey Zhukov Date: Fri, 20 Nov 2026 12:24:40 +0100 Subject: [PATCH 2/7] test to_json with map and array input at execution --- crates/spark/src/function/json/to_json.rs | 199 ++++++++++++++++++++-- 1 file changed, 184 insertions(+), 15 deletions(-) diff --git a/crates/spark/src/function/json/to_json.rs b/crates/spark/src/function/json/to_json.rs index 89910c2d..8a53cb4b 100644 --- a/crates/spark/src/function/json/to_json.rs +++ b/crates/spark/src/function/json/to_json.rs @@ -126,14 +126,14 @@ impl ScalarUDFImpl for SparkToJson { let [field] = take_function_args(self.name(), arg_fields)?; match input { - ColumnarValue::Array(array) => Ok(ColumnarValue::Array(struct_to_json( + ColumnarValue::Array(array) => Ok(ColumnarValue::Array(to_json( &field, &array, &config_options, )?)), ColumnarValue::Scalar(scalar) => { let array = scalar.to_array()?; - let result = struct_to_json(&field, &array, &config_options)?; + let result = to_json(&field, &array, &config_options)?; Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( &result, 0, )?)) @@ -205,14 +205,15 @@ fn check_spark_json_type(data_type: &DataType, path: &str) -> Result<()> { } } -/// Serializes every row of a struct array as a JSON object string; a NULL -/// struct row produces NULL. `field` describes `array` and supplies the JSON -/// keys of nested fields. +/// Serializes every row of a struct, map or list array as a JSON string: an +/// object for a struct or a map, an array for a list. A NULL row produces +/// NULL. `field` describes `array` and supplies the JSON keys of nested +/// fields. /// /// The arrow-json row encoder is built once per batch and each row is written /// straight into a `StringBuilder`; the Spark-specific rules are supplied by /// `SparkJsonEncoderFactory`. -fn struct_to_json( +fn to_json( field: &FieldRef, array: &ArrayRef, config: &ConfigOptions, @@ -945,11 +946,11 @@ mod tests { use arrow::array::{ BinaryArray, BinaryBuilder, BinaryViewArray, BooleanArray, Date32Array, Date64Array, Decimal32Array, Decimal128Array, Decimal256Array, DictionaryArray, - FixedSizeBinaryArray, Float32Array, Float64Array, Float64Builder, Int32Array, - Int32Builder, Int64Array, LargeBinaryArray, LargeStringArray, ListArray, - ListBuilder, MapBuilder, StringArray, StringViewArray, StructArray, - TimestampMicrosecondArray, TimestampNanosecondArray, TimestampSecondArray, - UInt8Array, + FixedSizeBinaryArray, FixedSizeListArray, Float32Array, Float64Array, + Float64Builder, Int32Array, Int32Builder, Int64Array, LargeBinaryArray, + LargeListArray, LargeStringArray, ListArray, ListBuilder, MapBuilder, + StringArray, StringViewArray, StructArray, TimestampMicrosecondArray, + TimestampNanosecondArray, TimestampSecondArray, UInt8Array, }; use arrow::buffer::NullBuffer; use arrow::datatypes::{ @@ -1000,10 +1001,36 @@ mod tests { StructArray::new(fields, vec![a, b, c], Some(nulls)) } - fn field_of(array: &StructArray) -> FieldRef { + fn field_of(array: &dyn Array) -> FieldRef { Field::new("s", array.data_type().clone(), true).into() } + /// Map rows: {"a":1,"b":null}, {}, NULL, {"c":3} + fn sample_map() -> MapArray { + let mut mb = MapBuilder::new(None, StringBuilder::new(), Int32Builder::new()); + mb.keys().append_value("a"); + mb.values().append_value(1); + mb.keys().append_value("b"); + mb.values().append_null(); + mb.append(true).unwrap(); + mb.append(true).unwrap(); + mb.append(false).unwrap(); + mb.keys().append_value("c"); + mb.values().append_value(3); + mb.append(true).unwrap(); + mb.finish() + } + + /// List rows: [1,null,3], [], NULL, [4] + fn sample_list() -> ListArray { + ListArray::from_iter_primitive::(vec![ + Some(vec![Some(1), None, Some(3)]), + Some(vec![]), + None, + Some(vec![Some(4)]), + ]) + } + #[test] fn test_return_type() { let func = SparkToJson::new(); @@ -1255,7 +1282,7 @@ mod tests { } #[test] - fn test_array_input() { + fn test_struct_array_input() { let input = sample_struct(); let field = field_of(&input); let out = run(ColumnarValue::Array(Arc::new(input)), field).unwrap(); @@ -1270,7 +1297,7 @@ mod tests { } #[test] - fn test_scalar_input() { + fn test_struct_scalar_input() { let input = sample_struct(); let field = field_of(&input); let scalar = ScalarValue::Struct(Arc::new(input.slice(0, 1))); @@ -1292,6 +1319,145 @@ mod tests { assert!(out.is_null(0)); } + #[test] + fn test_map_input() { + let input = sample_map(); + let field = field_of(&input); + let out = run(ColumnarValue::Array(Arc::new(input)), field).unwrap(); + let out = as_string_array(&out).unwrap(); + + assert_eq!(out.len(), 4); + // NULL map values are kept, as in a nested map + assert_eq!(out.value(0), r#"{"a":1,"b":null}"#); + assert_eq!(out.value(1), "{}"); + // NULL map row -> NULL + assert!(out.is_null(2)); + assert_eq!(out.value(3), r#"{"c":3}"#); + } + + #[test] + fn test_list_input() { + let input = sample_list(); + let field = field_of(&input); + let out = run(ColumnarValue::Array(Arc::new(input)), field).unwrap(); + let out = as_string_array(&out).unwrap(); + + assert_eq!(out.len(), 4); + // NULL elements are written as null + assert_eq!(out.value(0), "[1,null,3]"); + assert_eq!(out.value(1), "[]"); + // NULL list row -> NULL + assert!(out.is_null(2)); + assert_eq!(out.value(3), "[4]"); + } + + #[test] + fn test_map_scalar_input() { + let input = sample_map(); + let field = field_of(&input); + + let scalar = ScalarValue::Map(Arc::new(input.slice(0, 1))); + let out = run(ColumnarValue::Scalar(scalar), Arc::clone(&field)).unwrap(); + let out = as_string_array(&out).unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out.value(0), r#"{"a":1,"b":null}"#); + + // a NULL map scalar produces NULL + let scalar = ScalarValue::Map(Arc::new(input.slice(2, 1))); + let out = run(ColumnarValue::Scalar(scalar), field).unwrap(); + assert_eq!(out.len(), 1); + assert!(out.is_null(0)); + } + + #[test] + fn test_list_scalar_input() { + let input = sample_list(); + let field = field_of(&input); + + let scalar = ScalarValue::List(Arc::new(input.slice(0, 1))); + let out = run(ColumnarValue::Scalar(scalar), Arc::clone(&field)).unwrap(); + let out = as_string_array(&out).unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out.value(0), "[1,null,3]"); + + // a NULL list scalar produces NULL + let scalar = ScalarValue::List(Arc::new(input.slice(2, 1))); + let out = run(ColumnarValue::Scalar(scalar), field).unwrap(); + assert_eq!(out.len(), 1); + assert!(out.is_null(0)); + } + + #[test] + fn test_sliced_map_and_list_input() { + // rows 1..4 of each sample, through a non-zero-offset slice + let map = sample_map(); + let field = field_of(&map); + let sliced = map.slice(1, 3); + let out = run(ColumnarValue::Array(Arc::new(sliced)), field).unwrap(); + let out = as_string_array(&out).unwrap(); + + assert_eq!(out.len(), 3); + assert_eq!(out.value(0), "{}"); + assert!(out.is_null(1)); + assert_eq!(out.value(2), r#"{"c":3}"#); + + let list = sample_list(); + let field = field_of(&list); + let sliced = list.slice(1, 3); + let out = run(ColumnarValue::Array(Arc::new(sliced)), field).unwrap(); + let out = as_string_array(&out).unwrap(); + + assert_eq!(out.len(), 3); + assert_eq!(out.value(0), "[]"); + assert!(out.is_null(1)); + assert_eq!(out.value(2), "[4]"); + } + + #[test] + fn test_list_of_structs_input() { + // one row: [{a: 1}, {a: NULL}, NULL] + let a = Arc::new(Int32Array::from(vec![Some(1), None, None])) as ArrayRef; + let fields = Fields::from(vec![Field::new("a", DataType::Int32, true)]); + let nulls = NullBuffer::from(vec![true, true, false]); + let structs = StructArray::new(fields, vec![a], Some(nulls)); + let item = Arc::new(Field::new("item", structs.data_type().clone(), true)); + let offsets = OffsetBuffer::new(vec![0, 3].into()); + let input = ListArray::new(item, offsets, Arc::new(structs), None); + let field = field_of(&input); + + let out = run(ColumnarValue::Array(Arc::new(input)), field).unwrap(); + let out = as_string_array(&out).unwrap(); + + // NULL struct fields are omitted inside an array as well, while a NULL + // element is written as null + assert_eq!(out.value(0), r#"[{"a":1},{},null]"#); + } + + #[test] + fn test_large_and_fixed_size_list_input() { + // the same rows as two other Arrow list types + let rows = vec![ + Some(vec![Some(1), None]), + None, + Some(vec![Some(3), Some(4)]), + Some(vec![Some(5), Some(6)]), + ]; + let large = LargeListArray::from_iter_primitive::(rows.clone()); + let fixed = FixedSizeListArray::from_iter_primitive::(rows, 2); + + for input in [Arc::new(large) as ArrayRef, Arc::new(fixed) as ArrayRef] { + let field = field_of(&input); + let out = run(ColumnarValue::Array(input), field).unwrap(); + let out = as_string_array(&out).unwrap(); + + assert_eq!(out.len(), 4); + assert_eq!(out.value(0), "[1,null]"); + assert!(out.is_null(1)); + assert_eq!(out.value(2), "[3,4]"); + assert_eq!(out.value(3), "[5,6]"); + } + } + #[test] fn test_nested_struct_and_list() { // {inner: {x: Int64}, tags: List} @@ -1781,8 +1947,9 @@ mod tests { Field::new("ltz", ltz.data_type().clone(), true), Field::new("ntz", ntz.data_type().clone(), true), ]); - let input = Arc::new(StructArray::new(fields, vec![ltz, ntz], None)); + let input = StructArray::new(fields, vec![ltz, ntz], None); let field = field_of(&input); + let input = Arc::new(input); // default session time zone (UTC): milliseconds, offset written as `Z` let out = run( @@ -2067,6 +2234,8 @@ mod dataframe_tests { use datafusion_common::{assert_batches_sorted_eq, assert_contains}; use datafusion_functions_nested::expr_fn::make_array; + // the expression builder; it takes precedence over the `to_json` helper + // that `super::*` brings in use crate::function::json::expr_fn::to_json; /// id foo bar From 6b333c586819eba6b958d64a38d73235ee992711 Mon Sep 17 00:00:00 2001 From: Sergey Zhukov Date: Fri, 20 Nov 2026 16:16:50 +0100 Subject: [PATCH 3/7] test to_json with map and array input in sqllogictest --- .../sqllogictest/test_files/json/to_json.slt | 200 +++++++++++++++++- 1 file changed, 199 insertions(+), 1 deletion(-) diff --git a/crates/sqllogictest/test_files/json/to_json.slt b/crates/sqllogictest/test_files/json/to_json.slt index 484458e6..c0c46e30 100644 --- a/crates/sqllogictest/test_files/json/to_json.slt +++ b/crates/sqllogictest/test_files/json/to_json.slt @@ -38,7 +38,7 @@ SELECT to_json(named_struct('b', true, 'i', 42::BIGINT, 's', 'text'::STRING)); ---- {"b":true,"i":42,"s":"text"} -# array input +# column input query T rowsort SELECT to_json(named_struct('id', id, 'name', name)) FROM (VALUES (1::INT, 'a'::STRING), (2::INT, NULL::STRING), (NULL::INT, 'c'::STRING)) AS t(id, name); @@ -351,6 +351,191 @@ SELECT to_json(named_struct('s', arrow_cast('a', 'Dictionary(Int32, Utf8)'))); ---- {"s":"a"} +## Map and array input + +# Spark's to_json also takes a map or an array as the value to serialize. These +# queries build their values with array() and map_from_arrays() rather than +# `[...]` literals, which Spark does not parse, so that the PySpark runner +# checks them. + +query T +SELECT to_json(map('a', 1::INT)); +---- +{"a":1} + +query T +SELECT to_json(map_from_arrays(array('a', 'b'), array(1::INT, 2::INT))); +---- +{"a":1,"b":2} + +query T +SELECT to_json(array(1::INT, 2::INT, 3::INT)); +---- +[1,2,3] + +query T +SELECT to_json(array(named_struct('a', 1::INT, 'b', 2::INT))); +---- +[{"a":1,"b":2}] + +query T +SELECT to_json(array(map_from_arrays(array('a'), array(1::INT)))); +---- +[{"a":1}] + +query T +SELECT to_json(map_from_arrays(array('a'), array(named_struct('b', 1::INT)))); +---- +{"a":{"b":1}} + +query T +SELECT to_json(map_from_arrays(array('a'), array(array(1::INT, 2::INT)))); +---- +{"a":[1,2]} + +query T +SELECT to_json(array(array(1::INT, 2::INT), array(3::INT))); +---- +[[1,2],[3]] + +# empty values +query T +SELECT to_json(array()); +---- +[] + +query T +SELECT to_json(cast(array() as array)); +---- +[] + +query T +SELECT to_json(map_from_arrays(cast(array() as array), cast(array() as array))); +---- +{} + +# NULL elements and NULL map values are written as null +query T +SELECT to_json(array(1::INT, NULL, 3::INT)); +---- +[1,null,3] + +query T +SELECT to_json(map_from_arrays(array('a', 'b'), array(1::INT, NULL))); +---- +{"a":1,"b":null} + +# a NULL map or array produces NULL +query T +SELECT to_json(CASE WHEN false THEN map_from_arrays(array('a'), array(1::INT)) END); +---- +NULL + +query T +SELECT to_json(CASE WHEN false THEN array(1::INT) END); +---- +NULL + +# NULL struct fields are omitted inside an array or a map as well +query T +SELECT to_json(array(named_struct('a', 1::INT, 'b', NULL::STRING), named_struct('a', NULL::INT, 'b', NULL::STRING))); +---- +[{"a":1},{}] + +query T +SELECT to_json(map_from_arrays(array('k'), array(named_struct('a', NULL::INT)))); +---- +{"k":{}} + +# a NULL struct element is written as null +query T +SELECT to_json(array(named_struct('a', 1::INT), CASE WHEN false THEN named_struct('a', 1::INT) END)); +---- +[{"a":1},null] + +# map keys are written as strings, with the text of Spark's internal value +query T +SELECT to_json(map_from_arrays(array(1::INT), array('v'))); +---- +{"1":"v"} + +query T +SELECT to_json(map_from_arrays(array(true), array('v'))); +---- +{"true":"v"} + +query T +SELECT to_json(map_from_arrays(array(DATE '2024-01-01'), array(1::INT))); +---- +{"19723":1} + +query T +SELECT to_json(map_from_arrays(array(TIMESTAMP '2024-01-01 01:02:03.123456'), array(1::INT))); +---- +{"1704070923123456":1} + +query T +SELECT to_json(map_from_arrays(array(1e10::DOUBLE), array(1::INT))); +---- +{"1.0E10":1} + +query T +SELECT to_json(map_from_arrays(array(0.0000001::DECIMAL(10, 8)), array(1::INT))); +---- +{"1.0E-7":1} + +# elements and values are formatted as they are inside a struct +query T +SELECT to_json(array(1e-5::DOUBLE, 1.0::DOUBLE, 'NaN'::DOUBLE)); +---- +[1.0E-5,1.0,"NaN"] + +query T +SELECT to_json(array(1.5::DECIMAL(10, 2), NULL)); +---- +[1.50,null] + +query T +SELECT to_json(array(DATE '2024-01-01')); +---- +["2024-01-01"] + +query T +SELECT to_json(array('a"b')); +---- +["a\"b"] + +query T +SELECT to_json(map_from_arrays(array('b'), array(X'0001FF'))); +---- +{"b":"AAH/"} + +# the other Arrow list types are arrays too +query T +SELECT to_json(arrow_cast(array(1::INT, 2::INT), 'LargeList(Int32)')); +---- +[1,2] + +query T +SELECT to_json(arrow_cast(array(1::INT, 2::INT), 'FixedSizeList(2, Int32)')); +---- +[1,2] + +# column input +query T rowsort +SELECT to_json(array(a, b)) +FROM (VALUES (1::INT, 2::INT), (3::INT, NULL::INT)) AS t(a, b); +---- +[1,2] +[3,null] + +query T rowsort +SELECT to_json(map_from_arrays(array(k), array(v))) +FROM (VALUES ('a'::STRING, 1::INT), ('b'::STRING, NULL::INT)) AS t(k, v); +---- +{"a":1} +{"b":null} + ## Errors query error Error during planning: to_json expects a struct, map or array argument, got Utf8 @@ -372,3 +557,16 @@ SELECT to_json(named_struct('s', named_struct('l', [INTERVAL '1' DAY]))); query error Error during planning: to_json: field m has map key type List\(.*\), which is not supported SELECT to_json(named_struct('m', map([[1::INT]], [1::INT]))); + +# the same checks apply to the elements, keys and values of a map or array input +# xfail spark: intended: Arrow intervals do not record Spark's start and end fields, so to_json rejects them +query error Error during planning: to_json: input\[\] has type Interval\(MonthDayNano\), which has no Spark JSON representation +SELECT to_json(array(INTERVAL '1' DAY)); + +# xfail spark: intended: Spark writes an array key as the toString of its internal value, which to_json does not reproduce +query error Error during planning: to_json: input has map key type List\(.*\), which is not supported +SELECT to_json(map_from_arrays(array(array(1::INT)), array(1::INT))); + +# an untyped NULL is not a struct, map or array +query error Error during planning: to_json expects a struct, map or array argument, got Null +SELECT to_json(NULL); From 843cf9c92dae51e411f1ffe1cd9f092e0374384e Mon Sep 17 00:00:00 2001 From: Sergey Zhukov Date: Fri, 20 Nov 2026 16:35:05 +0100 Subject: [PATCH 4/7] docs: describe map and array input for to_json --- crates/spark/src/function/json/mod.rs | 2 +- crates/spark/src/function/json/to_json.rs | 23 ++++++++++++++--------- docs/functions/scalar/json.md | 10 +++++++--- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/crates/spark/src/function/json/mod.rs b/crates/spark/src/function/json/mod.rs index af9db406..1b52197c 100644 --- a/crates/spark/src/function/json/mod.rs +++ b/crates/spark/src/function/json/mod.rs @@ -31,7 +31,7 @@ pub mod expr_fn { )); export_functions!(( to_json, - "Serializes a struct value to a JSON string, following Spark's `to_json`; NULL fields are omitted and a NULL struct produces NULL.", + "Serializes a struct, map or array value to a JSON string, following Spark's `to_json`; NULL struct fields are omitted and a NULL input produces NULL.", input )); } diff --git a/crates/spark/src/function/json/to_json.rs b/crates/spark/src/function/json/to_json.rs index 8a53cb4b..f111e58f 100644 --- a/crates/spark/src/function/json/to_json.rs +++ b/crates/spark/src/function/json/to_json.rs @@ -42,10 +42,11 @@ use datafusion_macros::user_doc; /// #[user_doc( doc_section(label = "JSON Functions"), - description = "Serializes a struct to a JSON string, one object per row, as Spark's `to_json` does with its \ - default options. DataFusion has no built-in `to_json`. \ - A NULL struct returns NULL. NULL struct fields are omitted (`ignoreNullFields`), while NULL map entries \ - are kept as `\"key\":null`. Nested structs, lists and maps are serialized recursively. \ + description = "Serializes a struct, map or array to a JSON string, one per row, as Spark's `to_json` does with \ + its default options: a JSON object for a struct or a map, a JSON array for an array. \ + A NULL input returns NULL. NULL struct fields are omitted (`ignoreNullFields`), while NULL map entries \ + are kept as `\"key\":null` and NULL array elements are written as `null`. \ + Nested structs, lists and maps are serialized recursively. \ Floats are written like Java's `Double.toString` (`1.0`, `1.0E-5`; `NaN` and `Infinity` as strings), \ decimals like `BigDecimal.toString` (`1.50`, `0E-10`), binary as Base64, dates as `yyyy-MM-dd`, \ timestamps as `yyyy-MM-dd'T'HH:mm:ss.SSS` with the session time zone offset \ @@ -54,8 +55,8 @@ use datafusion_macros::user_doc; timestamps as microseconds, numbers as above. \ Types with no Spark JSON representation (intervals, durations, times, unions, map keys other than \ strings, booleans, numbers, dates and timestamps) are rejected when the query is planned. \ - The options argument and map or array input are not supported.", - syntax_example = "to_json(struct_expr)", + The options argument is not supported.", + syntax_example = "to_json(expr)", sql_example = r#"```sql > SELECT to_json(named_struct('a', 1, 'b', 'x', 'c', CAST(NULL AS INT))); {"a":1,"b":"x"} @@ -63,11 +64,15 @@ use datafusion_macros::user_doc; {"t":"2024-01-01T01:02:03.123","d":1.0E-5} > SELECT to_json(named_struct('m', map([DATE '2024-01-01'], [1]))); {"m":{"19723":1}} +> SELECT to_json(map('a', 1)); +{"a":1} +> SELECT to_json(array(1, NULL, 3)); +[1,null,3] ```"#, argument( - name = "struct_expr", - description = "Struct expression to serialize, e.g. a struct column or a `named_struct` call. \ - The field names become the JSON keys. NULL returns NULL." + name = "expr", + description = "Struct, map or array expression to serialize, e.g. a struct column, a `named_struct` \ + call, a map or an array. Struct field names and map keys become the JSON keys. NULL returns NULL." ) )] #[derive(Debug, PartialEq, Eq, Hash)] diff --git a/docs/functions/scalar/json.md b/docs/functions/scalar/json.md index a66d3651..756beeee 100644 --- a/docs/functions/scalar/json.md +++ b/docs/functions/scalar/json.md @@ -67,15 +67,15 @@ NULL ## `to_json` -Serializes a struct to a JSON string, one object per row, as Spark's `to_json` does with its default options. DataFusion has no built-in `to_json`. A NULL struct returns NULL. NULL struct fields are omitted (`ignoreNullFields`), while NULL map entries are kept as `"key":null`. Nested structs, lists and maps are serialized recursively. Floats are written like Java's `Double.toString` (`1.0`, `1.0E-5`; `NaN` and `Infinity` as strings), decimals like `BigDecimal.toString` (`1.50`, `0E-10`), binary as Base64, dates as `yyyy-MM-dd`, timestamps as `yyyy-MM-dd'T'HH:mm:ss.SSS` with the session time zone offset (`datafusion.execution.time_zone`, `Z` for UTC) and without offset for a timestamp without time zone. Map keys are written as strings holding Spark's internal value: dates as days since the epoch, timestamps as microseconds, numbers as above. Types with no Spark JSON representation (intervals, durations, times, unions, map keys other than strings, booleans, numbers, dates and timestamps) are rejected when the query is planned. The options argument and map or array input are not supported. +Serializes a struct, map or array to a JSON string, one per row, as Spark's `to_json` does with its default options: a JSON object for a struct or a map, a JSON array for an array. A NULL input returns NULL. NULL struct fields are omitted (`ignoreNullFields`), while NULL map entries are kept as `"key":null` and NULL array elements are written as `null`. Nested structs, lists and maps are serialized recursively. Floats are written like Java's `Double.toString` (`1.0`, `1.0E-5`; `NaN` and `Infinity` as strings), decimals like `BigDecimal.toString` (`1.50`, `0E-10`), binary as Base64, dates as `yyyy-MM-dd`, timestamps as `yyyy-MM-dd'T'HH:mm:ss.SSS` with the session time zone offset (`datafusion.execution.time_zone`, `Z` for UTC) and without offset for a timestamp without time zone. Map keys are written as strings holding Spark's internal value: dates as days since the epoch, timestamps as microseconds, numbers as above. Types with no Spark JSON representation (intervals, durations, times, unions, map keys other than strings, booleans, numbers, dates and timestamps) are rejected when the query is planned. The options argument is not supported. ```sql -to_json(struct_expr) +to_json(expr) ``` ### Arguments -- **struct_expr**: Struct expression to serialize, e.g. a struct column or a `named_struct` call. The field names become the JSON keys. NULL returns NULL. +- **expr**: Struct, map or array expression to serialize, e.g. a struct column, a `named_struct` call, a map or an array. Struct field names and map keys become the JSON keys. NULL returns NULL. ### Example @@ -86,4 +86,8 @@ to_json(struct_expr) {"t":"2024-01-01T01:02:03.123","d":1.0E-5} > SELECT to_json(named_struct('m', map([DATE '2024-01-01'], [1]))); {"m":{"19723":1}} +> SELECT to_json(map('a', 1)); +{"a":1} +> SELECT to_json(array(1, NULL, 3)); +[1,null,3] ``` From 3520b5f109610d965f72713b2a725324fbb2e8b1 Mon Sep 17 00:00:00 2001 From: Sergey Zhukov Date: Sat, 21 Nov 2026 16:42:15 +0100 Subject: [PATCH 5/7] support the options argument of to_json, with ignoreNullFields --- crates/spark/src/function/json/to_json.rs | 447 ++++++++++++++++-- .../sqllogictest/test_files/json/to_json.slt | 73 ++- 2 files changed, 487 insertions(+), 33 deletions(-) diff --git a/crates/spark/src/function/json/to_json.rs b/crates/spark/src/function/json/to_json.rs index f111e58f..5d45bb3f 100644 --- a/crates/spark/src/function/json/to_json.rs +++ b/crates/spark/src/function/json/to_json.rs @@ -29,11 +29,12 @@ use arrow::temporal_conversions::{ use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use datafusion_common::config::ConfigOptions; -use datafusion_common::utils::take_function_args; -use datafusion_common::{Result, ScalarValue, exec_datafusion_err, plan_err}; +use datafusion_common::{ + Result, ScalarValue, exec_datafusion_err, internal_err, plan_err, +}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + TypeSignature, Volatility, }; use datafusion_macros::user_doc; @@ -89,7 +90,10 @@ impl Default for SparkToJson { impl SparkToJson { pub fn new() -> Self { Self { - signature: Signature::any(1, Volatility::Immutable), + signature: Signature::one_of( + vec![TypeSignature::Any(1), TypeSignature::Any(2)], + Volatility::Immutable, + ), } } } @@ -109,15 +113,19 @@ impl ScalarUDFImpl for SparkToJson { fn return_type(&self, arg_types: &[DataType]) -> Result { match arg_types { - [input] => { + [input] => check_input_type(input)?, + [input, options] => { check_input_type(input)?; - Ok(DataType::Utf8) + check_options_type(options)?; + } + _ => { + return plan_err!( + "to_json expects one or two arguments, got {}", + arg_types.len() + ); } - _ => plan_err!( - "to_json expects exactly one argument, got {}", - arg_types.len() - ), } + Ok(DataType::Utf8) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { @@ -127,18 +135,19 @@ impl ScalarUDFImpl for SparkToJson { config_options, .. } = args; - let [input] = take_function_args(self.name(), args)?; - let [field] = take_function_args(self.name(), arg_fields)?; + let (Some(input), Some(field)) = (args.first(), arg_fields.first()) else { + return internal_err!("to_json expects one or two arguments"); + }; + let options = ToJsonOptions::try_new(args.get(1))?; match input { - ColumnarValue::Array(array) => Ok(ColumnarValue::Array(to_json( - &field, - &array, - &config_options, - )?)), + ColumnarValue::Array(array) => { + let result = to_json(field, array, &config_options, &options)?; + Ok(ColumnarValue::Array(result)) + } ColumnarValue::Scalar(scalar) => { let array = scalar.to_array()?; - let result = to_json(&field, &array, &config_options)?; + let result = to_json(field, &array, &config_options, &options)?; Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( &result, 0, )?)) @@ -147,6 +156,146 @@ impl ScalarUDFImpl for SparkToJson { } } +/// The options of Spark's to_json that are supported. +/// +/// The names and what they mean are Spark's: to_json takes the options of the +/// JSON data source, which are listed in +/// +/// +/// Supporting another of those options starts with a variant here. A name that +/// is not a variant is an error: ignoring an option that Spark applies would +/// give a different result without saying so. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ToJsonOption { + /// `ignoreNullFields`: whether NULL struct fields are omitted + IgnoreNullFields, +} + +impl ToJsonOption { + /// Every supported option. + const ALL: [Self; 1] = [Self::IgnoreNullFields]; + + /// The option's name in Spark. + fn name(self) -> &'static str { + match self { + Self::IgnoreNullFields => "ignoreNullFields", + } + } + + /// Finds the option called `name`. Spark's option names are + /// case-insensitive. + fn parse(name: &str) -> Option { + Self::ALL + .into_iter() + .find(|option| option.name().eq_ignore_ascii_case(name)) + } +} + +/// The values of the supported options (see `ToJsonOption`), with Spark's +/// defaults. +#[derive(Debug, Clone, PartialEq)] +struct ToJsonOptions { + /// `ignoreNullFields`: NULL struct fields are omitted rather than written + /// as `null`. It is on by default, as Spark's + /// `spark.sql.jsonGenerator.ignoreNullFields` is. + ignore_null_fields: bool, +} + +impl Default for ToJsonOptions { + fn default() -> Self { + Self { + ignore_null_fields: true, + } + } +} + +impl ToJsonOptions { + /// Reads the options argument, if there is one. Spark only takes a + /// `map(...)` call of strings there, so the argument has to be a constant. + fn try_new(options: Option<&ColumnarValue>) -> Result { + let mut parsed = Self::default(); + let Some(options) = options else { + return Ok(parsed); + }; + let ColumnarValue::Scalar(ScalarValue::Map(map)) = options else { + return plan_err!( + "to_json: the options argument must be a constant map of strings" + ); + }; + check_options_type(map.data_type())?; + if map.len() != 1 || map.is_null(0) { + return plan_err!("to_json: the options argument must not be NULL"); + } + + let entries = map.value(0); + let keys = cast(entries.column(0), &DataType::Utf8)?; + let values = cast(entries.column(1), &DataType::Utf8)?; + let (keys, values) = (keys.as_string::(), values.as_string::()); + for i in 0..keys.len() { + // Spark turns a NULL value into the string "null" + let value = if values.is_null(i) { + "null" + } else { + values.value(i) + }; + parsed.set(keys.value(i), value)?; + } + Ok(parsed) + } + + /// Applies one entry of the options map. + fn set(&mut self, key: &str, value: &str) -> Result<()> { + match ToJsonOption::parse(key) { + Some(ToJsonOption::IgnoreNullFields) => { + self.ignore_null_fields = parse_bool(key, value)?; + } + None => { + let supported = ToJsonOption::ALL.map(ToJsonOption::name).join(", "); + return plan_err!( + "to_json: unsupported option '{key}', supported options: {supported}" + ); + } + } + Ok(()) + } +} + +/// Parses a boolean option like Scala's `String.toBoolean`: `true` or `false` +/// in any case, and an error for anything else. +fn parse_bool(key: &str, value: &str) -> Result { + if value.eq_ignore_ascii_case("true") { + Ok(true) + } else if value.eq_ignore_ascii_case("false") { + Ok(false) + } else { + plan_err!("to_json: the {key} option must be true or false, got '{value}'") + } +} + +/// Whether `data_type` is one of Arrow's string types. +fn is_string_type(data_type: &DataType) -> bool { + use DataType::*; + matches!(data_type, Utf8 | LargeUtf8 | Utf8View) +} + +/// Checks the type of the options argument: a map whose keys and values are +/// strings. +fn check_options_type(data_type: &DataType) -> Result<()> { + let strings = match data_type { + DataType::Map(entries, _) => match entries.data_type() { + DataType::Struct(kv) => kv.iter().all(|f| is_string_type(f.data_type())), + _ => false, + }, + _ => false, + }; + if !strings { + return plan_err!( + "to_json: the options argument must be a map of strings, got {data_type}" + ); + } + Ok(()) +} + /// Checks the type of the value passed to to_json. Spark accepts a struct, a /// map or an array; their fields, keys, values and elements are checked with /// `check_spark_json_type`. @@ -222,6 +371,7 @@ fn to_json( field: &FieldRef, array: &ArrayRef, config: &ConfigOptions, + options: &ToJsonOptions, ) -> Result { // Spark writes TIMESTAMP values in the session time zone let session_tz: Tz = config @@ -231,12 +381,13 @@ fn to_json( .map(|tz| tz.as_str()) .unwrap_or("+00:00") .parse()?; - let options = EncoderOptions::default() - // Spark default: ignoreNullFields = true -> NULL struct fields are omitted - .with_explicit_nulls(false) + let encoder_options = EncoderOptions::default() + // ignoreNullFields: NULL struct fields are omitted (Spark's default) or + // written as null + .with_explicit_nulls(!options.ignore_null_fields) .with_encoder_factory(Arc::new(SparkJsonEncoderFactory { session_tz })); - let mut encoder = make_encoder(field, array.as_ref(), &options)?; + let mut encoder = make_encoder(field, array.as_ref(), &encoder_options)?; let mut builder = StringBuilder::with_capacity(array.len(), array.len() * 32); let mut buf: Vec = Vec::with_capacity(128); for row in 0..array.len() { @@ -755,7 +906,7 @@ impl<'a> SparkMapEncoder<'a> { ) -> Result { let keys = MapKeyEncoder::try_new(array.keys())?; // The value encoder is built with the same options, so nested structs - // keep omitting NULL fields and nested maps use this encoder. + // treat NULL fields the same way and nested maps use this encoder. let values = make_encoder(field, array.values(), options)?; Ok(Self { @@ -992,6 +1143,57 @@ mod tests { .to_array(number_rows) } + /// A `map(...)` literal of string options, as the function receives it. + fn options_scalar(options: &[(&str, Option<&str>)]) -> ColumnarValue { + let mut mb = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); + for (key, value) in options { + mb.keys().append_value(key); + mb.values().append_option(*value); + } + mb.append(true).unwrap(); + ColumnarValue::Scalar(ScalarValue::Map(Arc::new(mb.finish()))) + } + + /// Parses a `map(...)` literal of options. + fn parse_options(options: &[(&str, Option<&str>)]) -> Result { + ToJsonOptions::try_new(Some(&options_scalar(options))) + } + + /// Like `run`, with a `map(...)` literal of options as the second argument. + fn run_with_options( + input: ColumnarValue, + field: FieldRef, + options: &[(&str, Option<&str>)], + ) -> Result { + let number_rows = match &input { + ColumnarValue::Array(a) => a.len(), + ColumnarValue::Scalar(_) => 1, + }; + let options = options_scalar(options); + let options_field = Arc::new(Field::new("options", options.data_type(), false)); + let args = ScalarFunctionArgs { + args: vec![input, options], + arg_fields: vec![field, options_field], + number_rows, + return_field: Field::new("to_json", DataType::Utf8, true).into(), + config_options: Arc::new(ConfigOptions::default()), + }; + SparkToJson::new() + .invoke_with_args(args)? + .to_array(number_rows) + } + + /// List> with one row: [{a: 1}, {a: NULL}, NULL] + fn sample_list_of_structs() -> ListArray { + let a = Arc::new(Int32Array::from(vec![Some(1), None, None])) as ArrayRef; + let fields = Fields::from(vec![Field::new("a", DataType::Int32, true)]); + let nulls = NullBuffer::from(vec![true, true, false]); + let structs = StructArray::new(fields, vec![a], Some(nulls)); + let item = Arc::new(Field::new("item", structs.data_type().clone(), true)); + let offsets = OffsetBuffer::new(vec![0, 3].into()); + ListArray::new(item, offsets, Arc::new(structs), None) + } + fn sample_struct() -> StructArray { let a = Arc::new(Int32Array::from(vec![Some(1), Some(2), Some(3)])) as ArrayRef; let b = Arc::new(StringArray::from(vec![Some("x"), None, Some("z")])) as ArrayRef; @@ -1217,7 +1419,195 @@ mod tests { ); let err = SparkToJson::new().return_type(&[]).unwrap_err(); - assert!(err.to_string().contains("exactly one argument")); + assert!(err.to_string().contains("one or two arguments")); + } + + #[test] + fn test_return_type_checks_options_type() { + let func = SparkToJson::new(); + let fields = Fields::from(vec![Field::new("a", DataType::Int32, true)]); + let input = DataType::Struct(fields); + let map_of = |k: DataType, v: DataType| { + let entries = Fields::from(vec![ + Field::new("keys", k, false), + Field::new("values", v, true), + ]); + DataType::Map( + Arc::new(Field::new("entries", DataType::Struct(entries), false)), + false, + ) + }; + + let options = map_of(DataType::Utf8, DataType::Utf8); + assert_eq!( + func.return_type(&[input.clone(), options]).unwrap(), + DataType::Utf8 + ); + + // Spark takes a map of strings and nothing else + for options in [ + map_of(DataType::Utf8, DataType::Int32), + map_of(DataType::Int32, DataType::Utf8), + DataType::Utf8, + DataType::Null, + ] { + let err = func.return_type(&[input.clone(), options]).unwrap_err(); + assert!(err.to_string().contains("map of strings"), "{err}"); + } + + let err = func + .return_type(&[input.clone(), input.clone(), input]) + .unwrap_err(); + assert!(err.to_string().contains("one or two arguments"), "{err}"); + } + + #[test] + fn test_options_default() { + assert!(ToJsonOptions::try_new(None).unwrap().ignore_null_fields); + assert!(parse_options(&[]).unwrap().ignore_null_fields); + } + + #[test] + fn test_ignore_null_fields_option_is_parsed() { + // option names and boolean values are case-insensitive, as in Spark + for (key, value, expected) in [ + ("ignoreNullFields", "false", false), + ("ignoreNullFields", "true", true), + ("IGNORENULLFIELDS", "False", false), + ("ignorenullfields", "TRUE", true), + ] { + let options = parse_options(&[(key, Some(value))]).unwrap(); + assert_eq!(options.ignore_null_fields, expected, "{key}={value}"); + } + } + + #[test] + fn test_invalid_boolean_option_errors() { + // Scala's toBoolean takes true and false only; Spark reads a NULL + // value as the string "null" + for value in [Some("yes"), Some("1"), Some(""), None] { + let err = parse_options(&[("ignoreNullFields", value)]).unwrap_err(); + assert!(err.to_string().contains("must be true or false"), "{err}"); + } + } + + #[test] + fn test_supported_option_names() { + // the names are Spark's, and are matched case-insensitively + assert_eq!(ToJsonOption::IgnoreNullFields.name(), "ignoreNullFields"); + for name in ["ignoreNullFields", "IGNORENULLFIELDS", "ignorenullfields"] { + let option = ToJsonOption::parse(name); + assert_eq!(option, Some(ToJsonOption::IgnoreNullFields)); + } + assert_eq!(ToJsonOption::parse("pretty"), None); + + // every option has its own name + for option in ToJsonOption::ALL { + assert_eq!(ToJsonOption::parse(option.name()), Some(option)); + } + } + + #[test] + fn test_other_options_are_rejected() { + // options of Spark's to_json that are not supported, and a name that + // Spark does not know; the error lists what is supported + for key in ["pretty", "timeZone", "dateFormat", "noSuchOption"] { + let err = parse_options(&[(key, Some("x"))]).unwrap_err(); + let err = err.to_string(); + let needle = format!("unsupported option '{key}'"); + assert!(err.contains(&needle), "{err}"); + assert!(err.contains("options: ignoreNullFields"), "{err}"); + } + } + + #[test] + fn test_options_must_be_a_constant_map_of_strings() { + // a column of maps + let column = ColumnarValue::Array(Arc::new(sample_map())); + let err = ToJsonOptions::try_new(Some(&column)).unwrap_err(); + assert!(err.to_string().contains("constant map"), "{err}"); + + // a map whose values are not strings (sample_map is a Map) + let scalar = ScalarValue::Map(Arc::new(sample_map().slice(0, 1))); + let options = ColumnarValue::Scalar(scalar); + let err = ToJsonOptions::try_new(Some(&options)).unwrap_err(); + assert!(err.to_string().contains("map of strings"), "{err}"); + + // a NULL map + let mut mb = MapBuilder::new(None, StringBuilder::new(), StringBuilder::new()); + mb.append(false).unwrap(); + let null_map = ColumnarValue::Scalar(ScalarValue::Map(Arc::new(mb.finish()))); + let err = ToJsonOptions::try_new(Some(&null_map)).unwrap_err(); + assert!(err.to_string().contains("must not be NULL"), "{err}"); + } + + #[test] + fn test_ignore_null_fields_false_writes_null_struct_fields() { + let input = sample_struct(); + let field = field_of(&input); + let input = ColumnarValue::Array(Arc::new(input)); + let options = [("ignoreNullFields", Some("false"))]; + let out = run_with_options(input, field, &options).unwrap(); + let out = as_string_array(&out).unwrap(); + + assert_eq!(out.value(0), r#"{"a":1,"b":"x","c":true}"#); + // the NULL field is written rather than omitted + assert_eq!(out.value(1), r#"{"a":2,"b":null,"c":false}"#); + // a NULL struct row is NULL whatever the options + assert!(out.is_null(2)); + } + + #[test] + fn test_ignore_null_fields_true_is_the_default() { + let input = sample_struct(); + let field = field_of(&input); + let input = ColumnarValue::Array(Arc::new(input)); + let options = [("ignoreNullFields", Some("true"))]; + let out = run_with_options(input, field, &options).unwrap(); + let out = as_string_array(&out).unwrap(); + + assert_eq!(out.value(1), r#"{"a":2,"c":false}"#); + } + + #[test] + fn test_ignore_null_fields_false_inside_an_array() { + let input = sample_list_of_structs(); + let field = field_of(&input); + let input = ColumnarValue::Array(Arc::new(input)); + let options = [("ignoreNullFields", Some("false"))]; + let out = run_with_options(input, field, &options).unwrap(); + let out = as_string_array(&out).unwrap(); + + assert_eq!(out.value(0), r#"[{"a":1},{"a":null},null]"#); + } + + #[test] + fn test_ignore_null_fields_does_not_apply_to_map_values() { + // NULL map values are written as null with either setting + for value in ["true", "false"] { + let input = sample_map(); + let field = field_of(&input); + let input = ColumnarValue::Array(Arc::new(input)); + let options = [("ignoreNullFields", Some(value))]; + let out = run_with_options(input, field, &options).unwrap(); + let out = as_string_array(&out).unwrap(); + + assert_eq!(out.value(0), r#"{"a":1,"b":null}"#); + } + } + + #[test] + fn test_scalar_input_with_options() { + let input = sample_struct(); + let field = field_of(&input); + let scalar = ScalarValue::Struct(Arc::new(input.slice(1, 1))); + let options = [("ignoreNullFields", Some("false"))]; + let out = run_with_options(ColumnarValue::Scalar(scalar), field, &options); + let out = out.unwrap(); + let out = as_string_array(&out).unwrap(); + + assert_eq!(out.len(), 1); + assert_eq!(out.value(0), r#"{"a":2,"b":null,"c":false}"#); } #[test] @@ -1420,14 +1810,7 @@ mod tests { #[test] fn test_list_of_structs_input() { - // one row: [{a: 1}, {a: NULL}, NULL] - let a = Arc::new(Int32Array::from(vec![Some(1), None, None])) as ArrayRef; - let fields = Fields::from(vec![Field::new("a", DataType::Int32, true)]); - let nulls = NullBuffer::from(vec![true, true, false]); - let structs = StructArray::new(fields, vec![a], Some(nulls)); - let item = Arc::new(Field::new("item", structs.data_type().clone(), true)); - let offsets = OffsetBuffer::new(vec![0, 3].into()); - let input = ListArray::new(item, offsets, Arc::new(structs), None); + let input = sample_list_of_structs(); let field = field_of(&input); let out = run(ColumnarValue::Array(Arc::new(input)), field).unwrap(); diff --git a/crates/sqllogictest/test_files/json/to_json.slt b/crates/sqllogictest/test_files/json/to_json.slt index c0c46e30..0379ef21 100644 --- a/crates/sqllogictest/test_files/json/to_json.slt +++ b/crates/sqllogictest/test_files/json/to_json.slt @@ -536,6 +536,56 @@ FROM (VALUES ('a'::STRING, 1::INT), ('b'::STRING, NULL::INT)) AS t(k, v); {"a":1} {"b":null} +## Options + +# ignoreNullFields: NULL struct fields are written as null rather than omitted +query T +SELECT to_json(named_struct('a', 1::INT, 'b', NULL::STRING), map('ignoreNullFields', 'false')); +---- +{"a":1,"b":null} + +query T +SELECT to_json(named_struct('a', 1::INT, 'b', NULL::STRING), map('ignoreNullFields', 'true')); +---- +{"a":1} + +# option names and boolean values are case-insensitive +query T +SELECT to_json(named_struct('a', NULL::INT), map('IGNORENULLFIELDS', 'False')); +---- +{"a":null} + +# it applies to nested structs and to structs inside an array +query T +SELECT to_json(named_struct('n', named_struct('a', NULL::INT), 's', NULL::STRING), map('ignoreNullFields', 'false')); +---- +{"n":{"a":null},"s":null} + +query T +SELECT to_json(array(named_struct('a', 1::INT, 'b', NULL::STRING)), map('ignoreNullFields', 'false')); +---- +[{"a":1,"b":null}] + +# NULL map values are written as null with either setting +query T +SELECT to_json(map_from_arrays(array('a', 'b'), array(1::INT, NULL)), map('ignoreNullFields', 'true')); +---- +{"a":1,"b":null} + +# a NULL input is NULL whatever the options +query T +SELECT to_json(CASE WHEN false THEN named_struct('a', 1::INT) END, map('ignoreNullFields', 'false')); +---- +NULL + +# column input +query T rowsort +SELECT to_json(named_struct('id', id, 'name', name), map('ignoreNullFields', 'false')) +FROM (VALUES (1::INT, 'a'::STRING), (2::INT, NULL::STRING)) AS t(id, name); +---- +{"id":1,"name":"a"} +{"id":2,"name":null} + ## Errors query error Error during planning: to_json expects a struct, map or array argument, got Utf8 @@ -544,9 +594,30 @@ SELECT to_json('abc'::STRING); query error 'to_json' does not support zero arguments SELECT to_json(); -query error to_json.* but received 2 +# the options must be a map of strings +query error Error during planning: to_json: the options argument must be a map of strings, got SELECT to_json(named_struct('a', 1::INT), named_struct('b', 2::INT)); +query error Error during planning: to_json: the options argument must be a map of strings, got +SELECT to_json(named_struct('a', 1::INT), map('ignoreNullFields', false)); + +# a boolean option is true or false +query error to_json: the ignoreNullFields option must be true or false, got 'maybe' +SELECT to_json(named_struct('a', 1::INT), map('ignoreNullFields', 'maybe')); + +# an option that is not supported is an error rather than ignored +# xfail spark: intended: the pretty option is not supported yet (https://github.com/apache/datafusion-spark/issues/338) +query error to_json: unsupported option 'pretty', supported options: ignoreNullFields +SELECT to_json(named_struct('a', 1::INT), map('pretty', 'true')); + +# xfail spark: intended: to_json rejects every option it does not support, Spark ignores the names it does not know +query error to_json: unsupported option 'noSuchOption', supported options: ignoreNullFields +SELECT to_json(named_struct('a', 1::INT), map('noSuchOption', 'x')); + +# at most two arguments +query error to_json +SELECT to_json(named_struct('a', 1::INT), map('a', 'b'), map('c', 'd')); + # types without a Spark JSON representation are rejected at planning time # xfail spark: intended: Arrow intervals do not record Spark's start and end fields, so to_json rejects them query error Error during planning: to_json: field i has type Interval\(MonthDayNano\), which has no Spark JSON representation From a54ea794d180df68cda622a923fb624feacdd034 Mon Sep 17 00:00:00 2001 From: Sergey Zhukov Date: Sat, 21 Nov 2026 16:46:03 +0100 Subject: [PATCH 6/7] docs: describe the options argument of to_json --- crates/spark/src/function/json/to_json.rs | 23 +++++++++++++++++------ docs/functions/scalar/json.md | 7 +++++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/crates/spark/src/function/json/to_json.rs b/crates/spark/src/function/json/to_json.rs index 5d45bb3f..3cedce1c 100644 --- a/crates/spark/src/function/json/to_json.rs +++ b/crates/spark/src/function/json/to_json.rs @@ -43,10 +43,11 @@ use datafusion_macros::user_doc; /// #[user_doc( doc_section(label = "JSON Functions"), - description = "Serializes a struct, map or array to a JSON string, one per row, as Spark's `to_json` does with \ - its default options: a JSON object for a struct or a map, a JSON array for an array. \ - A NULL input returns NULL. NULL struct fields are omitted (`ignoreNullFields`), while NULL map entries \ - are kept as `\"key\":null` and NULL array elements are written as `null`. \ + description = "Serializes a struct, map or array to a JSON string, one per row, as Spark's `to_json` does: \ + a JSON object for a struct or a map, a JSON array for an array. \ + A NULL input returns NULL. NULL struct fields are omitted unless the `ignoreNullFields` option is \ + `false`, while NULL map entries are always kept as `\"key\":null` and NULL array elements are always \ + written as `null`. \ Nested structs, lists and maps are serialized recursively. \ Floats are written like Java's `Double.toString` (`1.0`, `1.0E-5`; `NaN` and `Infinity` as strings), \ decimals like `BigDecimal.toString` (`1.50`, `0E-10`), binary as Base64, dates as `yyyy-MM-dd`, \ @@ -56,8 +57,8 @@ use datafusion_macros::user_doc; timestamps as microseconds, numbers as above. \ Types with no Spark JSON representation (intervals, durations, times, unions, map keys other than \ strings, booleans, numbers, dates and timestamps) are rejected when the query is planned. \ - The options argument is not supported.", - syntax_example = "to_json(expr)", + The optional second argument takes Spark's JSON options; `ignoreNullFields` is the only one supported.", + syntax_example = "to_json(expr[, options])", sql_example = r#"```sql > SELECT to_json(named_struct('a', 1, 'b', 'x', 'c', CAST(NULL AS INT))); {"a":1,"b":"x"} @@ -69,11 +70,21 @@ use datafusion_macros::user_doc; {"a":1} > SELECT to_json(array(1, NULL, 3)); [1,null,3] +> SELECT to_json(named_struct('a', 1, 'b', CAST(NULL AS INT)), map('ignoreNullFields', 'false')); +{"a":1,"b":null} ```"#, argument( name = "expr", description = "Struct, map or array expression to serialize, e.g. a struct column, a `named_struct` \ call, a map or an array. Struct field names and map keys become the JSON keys. NULL returns NULL." + ), + argument( + name = "options", + description = "Optional constant `map(...)` of string options, as in Spark, e.g. \ + `map('ignoreNullFields', 'false')`. Option names and boolean values are case-insensitive. \ + The only supported option is `ignoreNullFields` (default `true`): set it to `false` to write NULL \ + struct fields as `null` instead of omitting them. Any other option is an error. Spark's options \ + are listed at " ) )] #[derive(Debug, PartialEq, Eq, Hash)] diff --git a/docs/functions/scalar/json.md b/docs/functions/scalar/json.md index 756beeee..6d6a299b 100644 --- a/docs/functions/scalar/json.md +++ b/docs/functions/scalar/json.md @@ -67,15 +67,16 @@ NULL ## `to_json` -Serializes a struct, map or array to a JSON string, one per row, as Spark's `to_json` does with its default options: a JSON object for a struct or a map, a JSON array for an array. A NULL input returns NULL. NULL struct fields are omitted (`ignoreNullFields`), while NULL map entries are kept as `"key":null` and NULL array elements are written as `null`. Nested structs, lists and maps are serialized recursively. Floats are written like Java's `Double.toString` (`1.0`, `1.0E-5`; `NaN` and `Infinity` as strings), decimals like `BigDecimal.toString` (`1.50`, `0E-10`), binary as Base64, dates as `yyyy-MM-dd`, timestamps as `yyyy-MM-dd'T'HH:mm:ss.SSS` with the session time zone offset (`datafusion.execution.time_zone`, `Z` for UTC) and without offset for a timestamp without time zone. Map keys are written as strings holding Spark's internal value: dates as days since the epoch, timestamps as microseconds, numbers as above. Types with no Spark JSON representation (intervals, durations, times, unions, map keys other than strings, booleans, numbers, dates and timestamps) are rejected when the query is planned. The options argument is not supported. +Serializes a struct, map or array to a JSON string, one per row, as Spark's `to_json` does: a JSON object for a struct or a map, a JSON array for an array. A NULL input returns NULL. NULL struct fields are omitted unless the `ignoreNullFields` option is `false`, while NULL map entries are always kept as `"key":null` and NULL array elements are always written as `null`. Nested structs, lists and maps are serialized recursively. Floats are written like Java's `Double.toString` (`1.0`, `1.0E-5`; `NaN` and `Infinity` as strings), decimals like `BigDecimal.toString` (`1.50`, `0E-10`), binary as Base64, dates as `yyyy-MM-dd`, timestamps as `yyyy-MM-dd'T'HH:mm:ss.SSS` with the session time zone offset (`datafusion.execution.time_zone`, `Z` for UTC) and without offset for a timestamp without time zone. Map keys are written as strings holding Spark's internal value: dates as days since the epoch, timestamps as microseconds, numbers as above. Types with no Spark JSON representation (intervals, durations, times, unions, map keys other than strings, booleans, numbers, dates and timestamps) are rejected when the query is planned. The optional second argument takes Spark's JSON options; `ignoreNullFields` is the only one supported. ```sql -to_json(expr) +to_json(expr[, options]) ``` ### Arguments - **expr**: Struct, map or array expression to serialize, e.g. a struct column, a `named_struct` call, a map or an array. Struct field names and map keys become the JSON keys. NULL returns NULL. +- **options**: Optional constant `map(...)` of string options, as in Spark, e.g. `map('ignoreNullFields', 'false')`. Option names and boolean values are case-insensitive. The only supported option is `ignoreNullFields` (default `true`): set it to `false` to write NULL struct fields as `null` instead of omitting them. Any other option is an error. Spark's options are listed at ### Example @@ -90,4 +91,6 @@ to_json(expr) {"a":1} > SELECT to_json(array(1, NULL, 3)); [1,null,3] +> SELECT to_json(named_struct('a', 1, 'b', CAST(NULL AS INT)), map('ignoreNullFields', 'false')); +{"a":1,"b":null} ``` From 3738c1a1c6a4f74ff8c38ed8ee80f266cb5846ac Mon Sep 17 00:00:00 2001 From: Sergey Zhukov Date: Sat, 21 Nov 2026 16:58:40 +0100 Subject: [PATCH 7/7] to_json returns NULL for an untyped NULL, as Spark does --- crates/spark/src/function/json/to_json.rs | 38 ++++++++++++++++++- .../sqllogictest/test_files/json/to_json.slt | 15 ++++++-- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/crates/spark/src/function/json/to_json.rs b/crates/spark/src/function/json/to_json.rs index 3cedce1c..0a70af02 100644 --- a/crates/spark/src/function/json/to_json.rs +++ b/crates/spark/src/function/json/to_json.rs @@ -308,11 +308,13 @@ fn check_options_type(data_type: &DataType) -> Result<()> { } /// Checks the type of the value passed to to_json. Spark accepts a struct, a -/// map or an array; their fields, keys, values and elements are checked with -/// `check_spark_json_type`. +/// map or an array, whose fields, keys, values and elements are checked with +/// `check_spark_json_type`, and an untyped NULL. fn check_input_type(data_type: &DataType) -> Result<()> { use DataType::*; match data_type { + // `to_json(NULL)`: Spark returns NULL + Null => Ok(()), // each field is reported under its own name Struct(fields) => fields.iter().try_for_each(|f| { check_spark_json_type(f.data_type(), &format!("field {}", f.name())) @@ -384,6 +386,11 @@ fn to_json( config: &ConfigOptions, options: &ToJsonOptions, ) -> Result { + // an untyped NULL has nothing to serialize: Spark returns NULL + if matches!(array.data_type(), DataType::Null) { + return Ok(Arc::new(StringArray::new_null(array.len()))); + } + // Spark writes TIMESTAMP values in the session time zone let session_tz: Tz = config .execution @@ -1857,6 +1864,33 @@ mod tests { } } + #[test] + fn test_untyped_null_input() { + // Spark returns NULL for `to_json(NULL)` + let func = SparkToJson::new(); + let return_type = func.return_type(&[DataType::Null]).unwrap(); + assert_eq!(return_type, DataType::Utf8); + + let field: FieldRef = Arc::new(Field::new("n", DataType::Null, true)); + let scalar = ColumnarValue::Scalar(ScalarValue::Null); + let out = run(scalar, Arc::clone(&field)).unwrap(); + assert_eq!(out.len(), 1); + assert!(out.is_null(0)); + + // a column of the NULL type + let input = arrow::array::new_null_array(&DataType::Null, 3); + let out = run(ColumnarValue::Array(input), Arc::clone(&field)).unwrap(); + assert_eq!(out.len(), 3); + assert_eq!(out.null_count(), 3); + + // the options do not change it + let scalar = ColumnarValue::Scalar(ScalarValue::Null); + let options = [("ignoreNullFields", Some("false"))]; + let out = run_with_options(scalar, field, &options).unwrap(); + assert_eq!(out.len(), 1); + assert!(out.is_null(0)); + } + #[test] fn test_nested_struct_and_list() { // {inner: {x: Int64}, tags: List} diff --git a/crates/sqllogictest/test_files/json/to_json.slt b/crates/sqllogictest/test_files/json/to_json.slt index 0379ef21..6e2fce29 100644 --- a/crates/sqllogictest/test_files/json/to_json.slt +++ b/crates/sqllogictest/test_files/json/to_json.slt @@ -436,6 +436,17 @@ SELECT to_json(CASE WHEN false THEN array(1::INT) END); ---- NULL +# so does an untyped NULL +query T +SELECT to_json(NULL); +---- +NULL + +query T +SELECT to_json(NULL, map('ignoreNullFields', 'false')); +---- +NULL + # NULL struct fields are omitted inside an array or a map as well query T SELECT to_json(array(named_struct('a', 1::INT, 'b', NULL::STRING), named_struct('a', NULL::INT, 'b', NULL::STRING))); @@ -637,7 +648,3 @@ SELECT to_json(array(INTERVAL '1' DAY)); # xfail spark: intended: Spark writes an array key as the toString of its internal value, which to_json does not reproduce query error Error during planning: to_json: input has map key type List\(.*\), which is not supported SELECT to_json(map_from_arrays(array(array(1::INT)), array(1::INT))); - -# an untyped NULL is not a struct, map or array -query error Error during planning: to_json expects a struct, map or array argument, got Null -SELECT to_json(NULL);